nnetsauce

 21class AdaBoostClassifier(Boosting, ClassifierMixin):
 22    """AdaBoost Classification (SAMME) model class derived from class Boosting
 23
 24    Parameters:
 25
 26        obj: object
 27            any object containing a method fit (obj.fit()) and a method predict
 28            (obj.predict())
 29
 30        n_estimators: int
 31            number of boosting iterations
 32
 33        learning_rate: float
 34            learning rate of the boosting procedure
 35
 36        n_hidden_features: int
 37            number of nodes in the hidden layer
 38
 39        reg_lambda: float
 40            regularization parameter for weights
 41
 42        reg_alpha: float
 43            controls compromize between l1 and l2 norm of weights
 44
 45        activation_name: str
 46            activation function: 'relu', 'tanh', 'sigmoid', 'prelu' or 'elu'
 47
 48        a: float
 49            hyperparameter for 'prelu' or 'elu' activation function
 50
 51        nodes_sim: str
 52            type of simulation for the nodes: 'sobol', 'hammersley', 'halton',
 53            'uniform'
 54
 55        bias: boolean
 56            indicates if the hidden layer contains a bias term (True) or not
 57            (False)
 58
 59        dropout: float
 60            regularization parameter; (random) percentage of nodes dropped out
 61            of the training
 62
 63        direct_link: boolean
 64            indicates if the original predictors are included (True) in model's
 65            fitting or not (False)
 66
 67        n_clusters: int
 68            number of clusters for 'kmeans' or 'gmm' clustering (could be 0:
 69                no clustering)
 70
 71        cluster_encode: bool
 72            defines how the variable containing clusters is treated (default is one-hot)
 73            if `False`, then labels are used, without one-hot encoding
 74
 75        type_clust: str
 76            type of clustering method: currently k-means ('kmeans') or Gaussian
 77            Mixture Model ('gmm')
 78
 79        type_scaling: a tuple of 3 strings
 80            scaling methods for inputs, hidden layer, and clustering respectively
 81            (and when relevant).
 82            Currently available: standardization ('std') or MinMax scaling ('minmax')
 83
 84        col_sample: float
 85            percentage of covariates randomly chosen for training
 86
 87        row_sample: float
 88            percentage of rows chosen for training, by stratified bootstrapping
 89
 90        seed: int
 91            reproducibility seed for nodes_sim=='uniform'
 92
 93        verbose: int
 94            0 for no output, 1 for a progress bar (default is 1)
 95
 96        method: str
 97            type of Adaboost method, 'SAMME' (discrete) or 'SAMME.R' (real)
 98
 99        backend: str
100            "cpu" or "gpu" or "tpu"
101
102    Attributes:
103
104        alpha_: list
105            AdaBoost coefficients alpha_m
106
107        base_learners_: dict
108            a dictionary containing the base learners
109
110    Examples:
111
112    See also [https://github.com/Techtonique/nnetsauce/blob/master/examples/adaboost_classification.py](https://github.com/Techtonique/nnetsauce/blob/master/examples/adaboost_classification.py)
113
114    ```python
115    import nnetsauce as ns
116    import numpy as np
117    from sklearn.datasets import load_breast_cancer
118    from sklearn.linear_model import LogisticRegression
119    from sklearn.model_selection import train_test_split
120    from sklearn import metrics
121    from time import time
122
123    breast_cancer = load_breast_cancer()
124    Z = breast_cancer.data
125    t = breast_cancer.target
126    np.random.seed(123)
127    X_train, X_test, y_train, y_test = train_test_split(Z, t, test_size=0.2)
128
129    # SAMME.R
130    clf = LogisticRegression(solver='liblinear', multi_class = 'ovr',
131                            random_state=123)
132    fit_obj = ns.AdaBoostClassifier(clf,
133                                    n_hidden_features=int(11.22338867),
134                                    direct_link=True,
135                                    n_estimators=250, learning_rate=0.01126343,
136                                    col_sample=0.72684326, row_sample=0.86429443,
137                                    dropout=0.63078613, n_clusters=2,
138                                    type_clust="gmm",
139                                    verbose=1, seed = 123,
140                                    method="SAMME.R")
141
142    start = time()
143    fit_obj.fit(X_train, y_train)
144    print(f"Elapsed {time() - start}")
145
146    start = time()
147    print(fit_obj.score(X_test, y_test))
148    print(f"Elapsed {time() - start}")
149
150    preds = fit_obj.predict(X_test)
151
152    print(metrics.classification_report(preds, y_test))
153
154    ```
155
156    """
157
158    # construct the object -----
159    _estimator_type = "classifier"
160
161    def __init__(
162        self,
163        obj,
164        n_estimators=10,
165        learning_rate=0.1,
166        n_hidden_features=1,
167        reg_lambda=0,
168        reg_alpha=0.5,
169        activation_name="relu",
170        a=0.01,
171        nodes_sim="sobol",
172        bias=True,
173        dropout=0,
174        direct_link=False,
175        n_clusters=2,
176        cluster_encode=True,
177        type_clust="kmeans",
178        type_scaling=("std", "std", "std"),
179        col_sample=1,
180        row_sample=1,
181        seed=123,
182        verbose=1,
183        method="SAMME",
184        backend="cpu",
185    ):
186        self.type_fit = "classification"
187        self.verbose = verbose
188        self.method = method
189        self.reg_lambda = reg_lambda
190        self.reg_alpha = reg_alpha
191
192        super().__init__(
193            obj=obj,
194            n_estimators=n_estimators,
195            learning_rate=learning_rate,
196            n_hidden_features=n_hidden_features,
197            activation_name=activation_name,
198            a=a,
199            nodes_sim=nodes_sim,
200            bias=bias,
201            dropout=dropout,
202            direct_link=direct_link,
203            n_clusters=n_clusters,
204            cluster_encode=cluster_encode,
205            type_clust=type_clust,
206            type_scaling=type_scaling,
207            col_sample=col_sample,
208            row_sample=row_sample,
209            seed=seed,
210            backend=backend,
211        )
212
213        self.alpha_ = []
214        self.base_learners_ = dict.fromkeys(range(n_estimators))
215
216    def fit(self, X, y, sample_weight=None, **kwargs):
217        """Fit Boosting model to training data (X, y).
218
219        Parameters:
220
221            X: {array-like}, shape = [n_samples, n_features]
222                Training vectors, where n_samples is the number
223                of samples and n_features is the number of features.
224
225            y: array-like, shape = [n_samples]
226                Target values.
227
228            **kwargs: additional parameters to be passed to
229                    self.cook_training_set or self.obj.fit
230
231        Returns:
232
233             self: object
234        """
235
236        assert mx.is_factor(y), "y must contain only integers"
237
238        assert self.method in (
239            "SAMME",
240            "SAMME.R",
241        ), "`method` must be either 'SAMME' or 'SAMME.R'"
242
243        assert (self.reg_lambda <= 1) & (
244            self.reg_lambda >= 0
245        ), "must have self.reg_lambda <= 1 &  self.reg_lambda >= 0"
246
247        assert (self.reg_alpha <= 1) & (
248            self.reg_alpha >= 0
249        ), "must have self.reg_alpha <= 1 &  self.reg_alpha >= 0"
250
251        # training
252        n, p = X.shape
253        self.n_classes = len(np.unique(y))
254        self.classes_ = np.unique(y)  # for compatibility with sklearn
255        self.n_classes_ = len(self.classes_)  # for compatibility with sklearn
256
257        if sample_weight is None:
258            w_m = np.repeat(1.0 / n, n)
259        else:
260            w_m = np.asarray(sample_weight)
261
262        base_learner = CustomClassifier(
263            self.obj,
264            n_hidden_features=self.n_hidden_features,
265            activation_name=self.activation_name,
266            a=self.a,
267            nodes_sim=self.nodes_sim,
268            bias=self.bias,
269            dropout=self.dropout,
270            direct_link=self.direct_link,
271            n_clusters=self.n_clusters,
272            type_clust=self.type_clust,
273            type_scaling=self.type_scaling,
274            col_sample=self.col_sample,
275            row_sample=self.row_sample,
276            seed=self.seed,
277        )
278
279        if self.verbose == 1:
280            pbar = Progbar(self.n_estimators)
281
282        if self.method == "SAMME":
283            err_m = 1e6
284            err_bound = 1 - 1 / self.n_classes
285            self.alpha_.append(1.0)
286            x_range_n = range(n)
287
288            for m in range(self.n_estimators):
289                preds = base_learner.fit(
290                    X, y, sample_weight=w_m.ravel(), **kwargs
291                ).predict(X)
292
293                self.base_learners_.update({m: deepcopy(base_learner)})
294
295                cond = [y[i] != preds[i] for i in x_range_n]
296
297                err_m = max(
298                    sum([elt[0] * elt[1] for elt in zip(cond, w_m)]),
299                    2.220446049250313e-16,
300                )  # sum(w_m) == 1
301
302                if self.reg_lambda > 0:
303                    err_m += self.reg_lambda * (
304                        (1 - self.reg_alpha) * 0.5 * sum([x**2 for x in w_m])
305                        + self.reg_alpha * sum([abs(x) for x in w_m])
306                    )
307
308                err_m = min(err_m, err_bound)
309
310                alpha_m = self.learning_rate * log(
311                    (self.n_classes - 1) * (1 - err_m) / err_m
312                )
313
314                self.alpha_.append(alpha_m)
315
316                w_m_temp = [exp(alpha_m * cond[i]) for i in x_range_n]
317
318                sum_w_m = sum(w_m_temp)
319
320                w_m = np.asarray([w_m_temp[i] / sum_w_m for i in x_range_n])
321
322                base_learner.set_params(seed=self.seed + (m + 1) * 1000)
323
324                if self.verbose == 1:
325                    pbar.update(m)
326
327            if self.verbose == 1:
328                pbar.update(self.n_estimators)
329
330            self.n_estimators = len(self.base_learners_)
331            self.classes_ = np.unique(y)
332
333            return self
334
335        if self.method == "SAMME.R":
336            Y = mo.one_hot_encode2(y, self.n_classes)
337
338            if sample_weight is None:
339                w_m = np.repeat(1.0 / n, n)  # (N, 1)
340
341            else:
342                w_m = np.asarray(sample_weight)
343
344            for m in range(self.n_estimators):
345                probs = base_learner.fit(
346                    X, y, sample_weight=w_m.ravel(), **kwargs
347                ).predict_proba(X)
348
349                np.clip(a=probs, a_min=2.220446049250313e-16, a_max=1.0, out=probs)
350
351                self.base_learners_.update({m: deepcopy(base_learner)})
352
353                w_m *= np.exp(
354                    -1.0
355                    * self.learning_rate
356                    * (1.0 - 1.0 / self.n_classes)
357                    * xlogy(Y, probs).sum(axis=1)
358                )
359
360                w_m /= np.sum(w_m)
361
362                base_learner.set_params(seed=self.seed + (m + 1) * 1000)
363
364                if self.verbose == 1:
365                    pbar.update(m)
366
367            if self.verbose == 1:
368                pbar.update(self.n_estimators)
369
370            self.n_estimators = len(self.base_learners_)
371            self.classes_ = np.unique(y)
372
373            return self
374
375    def predict(self, X, **kwargs):
376        """Predict test data X.
377
378        Parameters:
379
380            X: {array-like}, shape = [n_samples, n_features]
381                Training vectors, where n_samples is the number
382                of samples and n_features is the number of features.
383
384            **kwargs: additional parameters to be passed to
385                  self.cook_test_set
386
387        Returns:
388
389            model predictions: {array-like}
390        """
391        return self.predict_proba(X, **kwargs).argmax(axis=1)
392
393    def predict_proba(self, X, **kwargs):
394        """Predict probabilities for test data X.
395
396        Parameters:
397
398            X: {array-like}, shape = [n_samples, n_features]
399                Training vectors, where n_samples is the number
400                of samples and n_features is the number of features.
401
402            **kwargs: additional parameters to be passed to
403                  self.cook_test_set
404
405        Returns:
406
407            probability estimates for test data: {array-like}
408
409        """
410
411        n_iter = len(self.base_learners_)
412
413        if self.method == "SAMME":
414            ensemble_learner = np.zeros((X.shape[0], self.n_classes))
415
416            # if self.verbose == 1:
417            #    pbar = Progbar(n_iter)
418
419            for idx, base_learner in self.base_learners_.items():
420                preds = base_learner.predict(X, **kwargs)
421
422                ensemble_learner += self.alpha_[idx] * mo.one_hot_encode2(
423                    preds, self.n_classes
424                )
425
426                # if self.verbose == 1:
427                #    pbar.update(idx)
428
429            # if self.verbose == 1:
430            #    pbar.update(n_iter)
431
432            expit_ensemble_learner = expit(ensemble_learner)
433
434            sum_ensemble = expit_ensemble_learner.sum(axis=1)
435
436            return expit_ensemble_learner / sum_ensemble[:, None]
437
438        # if self.method == "SAMME.R":
439        ensemble_learner = 0
440
441        # if self.verbose == 1:
442        #    pbar = Progbar(n_iter)
443
444        for idx, base_learner in self.base_learners_.items():
445            probs = base_learner.predict_proba(X, **kwargs)
446
447            np.clip(a=probs, a_min=2.220446049250313e-16, a_max=1.0, out=probs)
448
449            log_preds_proba = np.log(probs)
450
451            ensemble_learner += log_preds_proba - log_preds_proba.mean(axis=1)[:, None]
452
453            # if self.verbose == 1:
454            #    pbar.update(idx)
455
456        ensemble_learner *= self.n_classes - 1
457
458        # if self.verbose == 1:
459        #    pbar.update(n_iter)
460
461        expit_ensemble_learner = expit(ensemble_learner)
462
463        sum_ensemble = expit_ensemble_learner.sum(axis=1)
464
465        return expit_ensemble_learner / sum_ensemble[:, None]
466
467    @property
468    def _estimator_type(self):
469        return "classifier"            

 46class Base(BaseEstimator):
 47    """Base model from which all the other classes inherit.
 48
 49    This class contains the most important data preprocessing/feature engineering methods.
 50
 51    Parameters:
 52
 53        n_hidden_features: int
 54            number of nodes in the hidden layer
 55
 56        activation_name: str
 57            activation function: 'relu', 'tanh', 'sigmoid', 'prelu' or 'elu'
 58
 59        a: float
 60            hyperparameter for 'prelu' or 'elu' activation function
 61
 62        nodes_sim: str
 63            type of simulation for hidden layer nodes: 'sobol', 'hammersley', 'halton',
 64            'uniform'
 65
 66        bias: boolean
 67            indicates if the hidden layer contains a bias term (True) or
 68            not (False)
 69
 70        dropout: float
 71            regularization parameter; (random) percentage of nodes dropped out
 72            of the training
 73
 74        direct_link: boolean
 75            indicates if the original features are included (True) in model's
 76            fitting or not (False)
 77
 78        n_clusters: int
 79            number of clusters for type_clust='kmeans' or type_clust='gmm'
 80            clustering (could be 0: no clustering)
 81
 82        cluster_encode: bool
 83            defines how the variable containing clusters is treated (default is one-hot);
 84            if `False`, then labels are used, without one-hot encoding
 85
 86        type_clust: str
 87            type of clustering method: currently k-means ('kmeans') or Gaussian
 88            Mixture Model ('gmm')
 89
 90        type_scaling: a tuple of 3 strings
 91            scaling methods for inputs, hidden layer, and clustering respectively
 92            (and when relevant).
 93            Currently available: standardization ('std') or MinMax scaling ('minmax') or robust scaling ('robust') or  max absolute scaling ('maxabs')
 94
 95        col_sample: float
 96            percentage of features randomly chosen for training
 97
 98        row_sample: float
 99            percentage of rows chosen for training, by stratified bootstrapping
100
101        seed: int
102            reproducibility seed for nodes_sim=='uniform', clustering and dropout
103
104        backend: str
105            "cpu" or "gpu" or "tpu"
106
107    """
108
109    # construct the object -----
110
111    def __init__(
112        self,
113        n_hidden_features=5,
114        activation_name="relu",
115        a=0.01,
116        nodes_sim="sobol",
117        bias=True,
118        dropout=0,
119        direct_link=True,
120        n_clusters=2,
121        cluster_encode=True,
122        type_clust="kmeans",
123        type_scaling=("std", "std", "std"),
124        col_sample=1,
125        row_sample=1,
126        seed=123,
127        backend="cpu",
128    ):
129        # input checks -----
130
131        sys_platform = platform.system()
132
133        if (sys_platform == "Windows") and (backend in ("gpu", "tpu")):
134            warnings.warn("No GPU/TPU computing on Windows yet, backend set to 'cpu'")
135            backend = "cpu"
136
137        assert activation_name in (
138            "relu",
139            "tanh",
140            "sigmoid",
141            "prelu",
142            "elu",
143        ), "'activation_name' must be in ('relu', 'tanh', 'sigmoid','prelu', 'elu')"
144
145        assert nodes_sim in (
146            "sobol",
147            "hammersley",
148            "uniform",
149            "halton",
150        ), "'nodes_sim' must be in ('sobol', 'hammersley', 'uniform', 'halton')"
151
152        assert type_clust in (
153            "kmeans",
154            "gmm",
155        ), "'type_clust' must be in ('kmeans', 'gmm')"
156
157        assert (len(type_scaling) == 3) & all(
158            type_scaling[i] in ("minmax", "std", "robust", "maxabs")
159            for i in range(len(type_scaling))
160        ), "'type_scaling' must have length 3, and available scaling methods are 'minmax' scaling, standardization ('std'), robust scaling ('robust') and max absolute ('maxabs')"
161
162        assert (col_sample >= 0) & (
163            col_sample <= 1
164        ), "'col_sample' must be comprised between 0 and 1 (both included)"
165
166        assert backend in (
167            "cpu",
168            "gpu",
169            "tpu",
170        ), "must have 'backend' in ('cpu', 'gpu', 'tpu')"
171
172        self.n_hidden_features = n_hidden_features
173        self.activation_name = activation_name
174        self.a = a
175        self.nodes_sim = nodes_sim
176        self.bias = bias
177        self.seed = seed
178        self.backend = backend
179        self.dropout = dropout
180        self.direct_link = direct_link
181        self.cluster_encode = cluster_encode
182        self.type_clust = type_clust
183        self.type_scaling = type_scaling
184        self.col_sample = col_sample
185        self.row_sample = row_sample
186        self.n_clusters = n_clusters
187        if isinstance(self, RegressorMixin):
188            self.type_fit = "regression"
189        elif isinstance(self, ClassifierMixin):
190            self.type_fit = "classification"
191        self.subsampler_ = None
192        self.index_col_ = None
193        self.index_row_ = True
194        self.clustering_obj_ = None
195        self.clustering_scaler_ = None
196        self.nn_scaler_ = None
197        self.scaler_ = None
198        self.encoder_ = None
199        self.W_ = None
200        self.X_ = None
201        self.y_ = None
202        self.y_mean_ = None
203        self.beta_ = None
204
205        # activation function -----
206        if sys_platform in ("Linux", "Darwin"):
207            activation_options = {
208                "relu": ac.relu if (self.backend == "cpu") else jnn.relu,
209                "tanh": np.tanh if (self.backend == "cpu") else jnp.tanh,
210                "sigmoid": (ac.sigmoid if (self.backend == "cpu") else jnn.sigmoid),
211                "prelu": partial(ac.prelu, a=a),
212                "elu": (
213                    partial(ac.elu, a=a)
214                    if (self.backend == "cpu")
215                    else partial(jnn.elu, a=a)
216                ),
217            }
218        else:  # on Windows currently, no JAX
219            activation_options = {
220                "relu": (ac.relu if (self.backend == "cpu") else NotImplementedError),
221                "tanh": (np.tanh if (self.backend == "cpu") else NotImplementedError),
222                "sigmoid": (
223                    ac.sigmoid if (self.backend == "cpu") else NotImplementedError
224                ),
225                "prelu": partial(ac.prelu, a=a),
226                "elu": (
227                    partial(ac.elu, a=a)
228                    if (self.backend == "cpu")
229                    else NotImplementedError
230                ),
231            }
232        self.activation_func = activation_options[activation_name]
233
234    # "preprocessing" methods to be inherited -----
235
236    def encode_clusters(self, X=None, predict=False, scaler=None, **kwargs):  #
237        """Create new covariates with kmeans or GMM clustering
238
239        Parameters:
240
241            X: {array-like}, shape = [n_samples, n_features]
242                Training vectors, where n_samples is the number
243                of samples and n_features is the number of features.
244
245            predict: boolean
246                is False on training set and True on test set
247
248            scaler: {object} of class StandardScaler, MinMaxScaler, RobustScaler or MaxAbsScaler
249                if scaler has already been fitted on training data (online training), it can be passed here
250
251            **kwargs:
252                additional parameters to be passed to the
253                clustering method
254
255        Returns:
256
257            Clusters' matrix, one-hot encoded: {array-like}
258
259        """
260
261        np.random.seed(self.seed)
262
263        if X is None:
264            X = self.X_
265
266        if isinstance(X, pd.DataFrame):
267            X = copy.deepcopy(X.values.astype(float))
268
269        if len(X.shape) == 1:
270            X = X.reshape(1, -1)
271
272        if predict is False:  # encode training set
273
274            # scale input data before clustering
275            self.clustering_scaler_, scaled_X = mo.scale_covariates(
276                X, choice=self.type_scaling[2], scaler=self.clustering_scaler_
277            )
278
279            self.clustering_obj_, X_clustered = mo.cluster_covariates(
280                scaled_X,
281                self.n_clusters,
282                self.seed,
283                type_clust=self.type_clust,
284                **kwargs
285            )
286
287            if self.cluster_encode == True:
288                return mo.one_hot_encode(X_clustered, self.n_clusters).astype(
289                    np.float16
290                )
291
292            return X_clustered.astype(np.float16)
293
294        # if predict == True, encode test set
295        X_clustered = self.clustering_obj_.predict(self.clustering_scaler_.transform(X))
296
297        if self.cluster_encode == True:
298            return mo.one_hot_encode(X_clustered, self.n_clusters).astype(np.float16)
299
300        return X_clustered.astype(np.float16)
301
302    def create_layer(self, scaled_X, W=None):
303        """Create hidden layer.
304
305        Parameters:
306
307            scaled_X: {array-like}, shape = [n_samples, n_features]
308                Training vectors, where n_samples is the number
309                of samples and n_features is the number of features
310
311            W: {array-like}, shape = [n_features, hidden_features]
312                if provided, constructs the hidden layer with W; otherwise computed internally
313
314        Returns:
315
316            Hidden layer matrix: {array-like}
317
318        """
319
320        n_features = scaled_X.shape[1]
321
322        # hash_sim = {
323        #         "sobol": generate_sobol,
324        #         "hammersley": generate_hammersley,
325        #         "uniform": generate_uniform,
326        #         "halton": generate_halton
327        #     }
328
329        if self.bias is False:  # no bias term in the hidden layer
330            if W is None:
331                if self.nodes_sim == "sobol":
332                    self.W_ = generate_sobol(
333                        n_dims=n_features,
334                        n_points=self.n_hidden_features,
335                        seed=self.seed,
336                    )
337                elif self.nodes_sim == "hammersley":
338                    self.W_ = generate_hammersley(
339                        n_dims=n_features,
340                        n_points=self.n_hidden_features,
341                        seed=self.seed,
342                    )
343                elif self.nodes_sim == "uniform":
344                    self.W_ = generate_uniform(
345                        n_dims=n_features,
346                        n_points=self.n_hidden_features,
347                        seed=self.seed,
348                    )
349                else:
350                    self.W_ = generate_halton(
351                        n_dims=n_features,
352                        n_points=self.n_hidden_features,
353                        seed=self.seed,
354                    )
355
356                assert (
357                    scaled_X.shape[1] == self.W_.shape[0]
358                ), "check dimensions of covariates X and matrix W"
359
360                return mo.dropout(
361                    x=self.activation_func(
362                        mo.safe_sparse_dot(a=scaled_X, b=self.W_, backend=self.backend)
363                    ),
364                    drop_prob=self.dropout,
365                    seed=self.seed,
366                )
367
368            # W is not none
369            assert (
370                scaled_X.shape[1] == W.shape[0]
371            ), "check dimensions of covariates X and matrix W"
372
373            # self.W_ = W
374            return mo.dropout(
375                x=self.activation_func(
376                    mo.safe_sparse_dot(a=scaled_X, b=W, backend=self.backend)
377                ),
378                drop_prob=self.dropout,
379                seed=self.seed,
380            )
381
382        # with bias term in the hidden layer
383        if W is None:
384            n_features_1 = n_features + 1
385
386            if self.nodes_sim == "sobol":
387                self.W_ = generate_sobol(
388                    n_dims=n_features_1,
389                    n_points=self.n_hidden_features,
390                    seed=self.seed,
391                )
392            elif self.nodes_sim == "hammersley":
393                self.W_ = generate_hammersley(
394                    n_dims=n_features_1,
395                    n_points=self.n_hidden_features,
396                    seed=self.seed,
397                )
398            elif self.nodes_sim == "uniform":
399                self.W_ = generate_uniform(
400                    n_dims=n_features_1,
401                    n_points=self.n_hidden_features,
402                    seed=self.seed,
403                )
404            else:
405                self.W_ = generate_halton(
406                    n_dims=n_features_1,
407                    n_points=self.n_hidden_features,
408                    seed=self.seed,
409                )
410
411            # self.W_ = hash_sim[self.nodes_sim](
412            #         n_dims=n_features_1,
413            #         n_points=self.n_hidden_features,
414            #         seed=self.seed,
415            #     )
416
417            return mo.dropout(
418                x=self.activation_func(
419                    mo.safe_sparse_dot(
420                        a=mo.cbind(
421                            np.ones(scaled_X.shape[0]),
422                            scaled_X,
423                            backend=self.backend,
424                        ),
425                        b=self.W_,
426                        backend=self.backend,
427                    )
428                ),
429                drop_prob=self.dropout,
430                seed=self.seed,
431            )
432
433        # W is not None
434        # self.W_ = W
435        return mo.dropout(
436            x=self.activation_func(
437                mo.safe_sparse_dot(
438                    a=mo.cbind(
439                        np.ones(scaled_X.shape[0]),
440                        scaled_X,
441                        backend=self.backend,
442                    ),
443                    b=W,
444                    backend=self.backend,
445                )
446            ),
447            drop_prob=self.dropout,
448            seed=self.seed,
449        )
450
451    def cook_training_set(self, y=None, X=None, W=None, **kwargs):
452        """Create new hidden features for training set, with hidden layer, center the response.
453
454        Parameters:
455
456            y: array-like, shape = [n_samples]
457                Target values
458
459            X: {array-like}, shape = [n_samples, n_features]
460                Training vectors, where n_samples is the number
461                of samples and n_features is the number of features
462
463            W: {array-like}, shape = [n_features, hidden_features]
464                if provided, constructs the hidden layer via W
465
466        Returns:
467
468            (centered response, direct link + hidden layer matrix): {tuple}
469
470        """
471
472        # either X and y are stored or not
473        # assert ((y is None) & (X is None)) | ((y is not None) & (X is not None))
474        if self.n_hidden_features > 0:  # has a hidden layer
475            assert (
476                len(self.type_scaling) >= 2
477            ), "must have len(self.type_scaling) >= 2 when self.n_hidden_features > 0"
478
479        if X is None:
480
481            if self.col_sample == 1:
482                input_X = self.X_
483            else:
484                n_features = self.X_.shape[1]
485                new_n_features = int(np.ceil(n_features * self.col_sample))
486                assert (
487                    new_n_features >= 1
488                ), "check class attribute 'col_sample' and the number of covariates provided for X"
489                np.random.seed(self.seed)
490                index_col = np.random.choice(
491                    range(n_features), size=new_n_features, replace=False
492                )
493                self.index_col_ = index_col
494                input_X = self.X_[:, self.index_col_]
495
496        else:  # X is not None # keep X vs self.X_
497
498            if isinstance(X, pd.DataFrame):
499                X = copy.deepcopy(X.values.astype(float))
500
501            if self.col_sample == 1:
502                input_X = X
503            else:
504                n_features = X.shape[1]
505                new_n_features = int(np.ceil(n_features * self.col_sample))
506                assert (
507                    new_n_features >= 1
508                ), "check class attribute 'col_sample' and the number of covariates provided for X"
509                np.random.seed(self.seed)
510                index_col = np.random.choice(
511                    range(n_features), size=new_n_features, replace=False
512                )
513                self.index_col_ = index_col
514                input_X = X[:, self.index_col_]
515
516        if self.n_clusters <= 0:
517            # data without any clustering: self.n_clusters is None -----
518
519            if self.n_hidden_features > 0:  # with hidden layer
520
521                self.nn_scaler_, scaled_X = mo.scale_covariates(
522                    input_X, choice=self.type_scaling[1], scaler=self.nn_scaler_
523                )
524                Phi_X = (
525                    self.create_layer(scaled_X)
526                    if W is None
527                    else self.create_layer(scaled_X, W=W)
528                )
529                Z = (
530                    mo.cbind(input_X, Phi_X, backend=self.backend)
531                    if self.direct_link is True
532                    else Phi_X
533                )
534                self.scaler_, scaled_Z = mo.scale_covariates(
535                    Z, choice=self.type_scaling[0], scaler=self.scaler_
536                )
537            else:  # no hidden layer
538                Z = input_X
539                self.scaler_, scaled_Z = mo.scale_covariates(
540                    Z, choice=self.type_scaling[0], scaler=self.scaler_
541                )
542
543        else:
544
545            # data with clustering: self.n_clusters is not None ----- # keep
546
547            augmented_X = mo.cbind(
548                input_X,
549                self.encode_clusters(input_X, **kwargs),
550                backend=self.backend,
551            )
552
553            if self.n_hidden_features > 0:  # with hidden layer
554
555                self.nn_scaler_, scaled_X = mo.scale_covariates(
556                    augmented_X,
557                    choice=self.type_scaling[1],
558                    scaler=self.nn_scaler_,
559                )
560                Phi_X = (
561                    self.create_layer(scaled_X)
562                    if W is None
563                    else self.create_layer(scaled_X, W=W)
564                )
565                Z = (
566                    mo.cbind(augmented_X, Phi_X, backend=self.backend)
567                    if self.direct_link is True
568                    else Phi_X
569                )
570                self.scaler_, scaled_Z = mo.scale_covariates(
571                    Z, choice=self.type_scaling[0], scaler=self.scaler_
572                )
573            else:  # no hidden layer
574                Z = augmented_X
575                self.scaler_, scaled_Z = mo.scale_covariates(
576                    Z, choice=self.type_scaling[0], scaler=self.scaler_
577                )
578
579        # Returning model inputs -----
580        if mx.is_factor(y) is False:  # regression
581            # center y
582            if y is None:
583                self.y_mean_, centered_y = mo.center_response(self.y_)
584            else:
585                self.y_mean_, centered_y = mo.center_response(y)
586
587            # y is subsampled
588            if self.row_sample < 1:
589                n, p = Z.shape
590
591                self.subsampler_ = (
592                    SubSampler(y=self.y_, row_sample=self.row_sample, seed=self.seed)
593                    if y is None
594                    else SubSampler(y=y, row_sample=self.row_sample, seed=self.seed)
595                )
596
597                self.index_row_ = self.subsampler_.subsample()
598
599                n_row_sample = len(self.index_row_)
600                # regression
601                return (
602                    centered_y[self.index_row_].reshape(n_row_sample),
603                    self.scaler_.transform(
604                        Z[self.index_row_, :].reshape(n_row_sample, p)
605                    ),
606                )
607            # y is not subsampled
608            # regression
609            return (centered_y, self.scaler_.transform(Z))
610
611        # classification
612        # y is subsampled
613        if self.row_sample < 1:
614            n, p = Z.shape
615
616            self.subsampler_ = (
617                SubSampler(y=self.y_, row_sample=self.row_sample, seed=self.seed)
618                if y is None
619                else SubSampler(y=y, row_sample=self.row_sample, seed=self.seed)
620            )
621
622            self.index_row_ = self.subsampler_.subsample()
623
624            n_row_sample = len(self.index_row_)
625            # classification
626            return (
627                y[self.index_row_].reshape(n_row_sample),
628                self.scaler_.transform(Z[self.index_row_, :].reshape(n_row_sample, p)),
629            )
630        # y is not subsampled
631        # classification
632        return (y, self.scaler_.transform(Z))
633
634    def cook_test_set(self, X, **kwargs):
635        """Transform data from test set, with hidden layer.
636
637        Parameters:
638
639            X: {array-like}, shape = [n_samples, n_features]
640                Training vectors, where n_samples is the number
641                of samples and n_features is the number of features
642
643            **kwargs: additional parameters to be passed to self.encode_cluster
644
645        Returns:
646
647            Transformed test set : {array-like}
648        """
649
650        if isinstance(X, pd.DataFrame):
651            X = copy.deepcopy(X.values.astype(float))
652
653        if len(X.shape) == 1:
654            X = X.reshape(1, -1)
655
656        if (
657            self.n_clusters == 0
658        ):  # data without clustering: self.n_clusters is None -----
659            if self.n_hidden_features > 0:
660                # if hidden layer
661                scaled_X = (
662                    self.nn_scaler_.transform(X)
663                    if (self.col_sample == 1)
664                    else self.nn_scaler_.transform(X[:, self.index_col_])
665                )
666                Phi_X = self.create_layer(scaled_X, self.W_)
667                if self.direct_link == True:
668                    return self.scaler_.transform(
669                        mo.cbind(scaled_X, Phi_X, backend=self.backend)
670                    )
671                # when self.direct_link == False
672                return self.scaler_.transform(Phi_X)
673            # if no hidden layer # self.n_hidden_features == 0
674            return self.scaler_.transform(X)
675
676        # data with clustering: self.n_clusters > 0 -----
677        if self.col_sample == 1:
678            predicted_clusters = self.encode_clusters(X=X, predict=True, **kwargs)
679            augmented_X = mo.cbind(X, predicted_clusters, backend=self.backend)
680        else:
681            predicted_clusters = self.encode_clusters(
682                X=X[:, self.index_col_], predict=True, **kwargs
683            )
684            augmented_X = mo.cbind(
685                X[:, self.index_col_], predicted_clusters, backend=self.backend
686            )
687
688        if self.n_hidden_features > 0:  # if hidden layer
689            scaled_X = self.nn_scaler_.transform(augmented_X)
690            Phi_X = self.create_layer(scaled_X, self.W_)
691            if self.direct_link == True:
692                return self.scaler_.transform(
693                    mo.cbind(augmented_X, Phi_X, backend=self.backend)
694                )
695            return self.scaler_.transform(Phi_X)
696
697        # if no hidden layer
698        return self.scaler_.transform(augmented_X)
699
700    def cross_val_score(
701        self,
702        X,
703        y,
704        cv=5,
705        scoring="accuracy",
706        random_state=42,
707        n_jobs=-1,
708        epsilon=0.5,
709        penalized=True,
710        objective="abs",
711        **kwargs
712    ):
713        """
714        Penalized Cross-validation score for a model.
715
716        Parameters:
717
718            X: {array-like}, shape = [n_samples, n_features]
719                Training vectors, where n_samples is the number
720                of samples and n_features is the number of features
721
722            y: array-like, shape = [n_samples]
723                Target values
724
725            X_test: {array-like}, shape = [n_samples, n_features]
726                Test vectors, where n_samples is the number
727                of samples and n_features is the number of features
728
729            y_test: array-like, shape = [n_samples]
730                Target values
731
732            cv: int
733                Number of folds
734
735            scoring: str
736                Scoring metric
737
738            random_state: int
739                Random state
740
741            n_jobs: int
742                Number of jobs to run in parallel
743
744            epsilon: float
745                Penalty parameter
746
747            penalized: bool
748                Whether to obtain penalized cross-validation score or not
749
750            objective: str
751                'abs': Minimize the absolute difference between cross-validation score and validation score
752                'relative': Minimize the relative difference between cross-validation score and validation score
753        Returns:
754
755            A namedtuple with the following fields:
756                - cv_score: float
757                    cross-validation score
758                - val_score: float
759                    validation score
760                - penalized_score: float
761                    penalized cross-validation score: cv_score / val_score + epsilon*(1/val_score + 1/cv_score)
762                    If higher scoring metric is better, minimize the function result.
763                    If lower scoring metric is better, maximize the function result.
764        """
765        if scoring == "accuracy":
766            scoring_func = accuracy_score
767        elif scoring == "balanced_accuracy":
768            scoring_func = balanced_accuracy_score
769        elif scoring == "f1":
770            scoring_func = f1_score
771        elif scoring == "roc_auc":
772            scoring_func = roc_auc_score
773        elif scoring == "r2":
774            scoring_func = r2_score
775        elif scoring == "mse":
776            scoring_func = mean_squared_error
777        elif scoring == "mae":
778            scoring_func = mean_absolute_error
779        elif scoring == "mape":
780            scoring_func = mean_absolute_percentage_error
781        elif scoring == "rmse":
782
783            def scoring_func(y_true, y_pred):
784                return np.sqrt(mean_squared_error(y_true, y_pred))
785
786        X_train, X_val, y_train, y_val = train_test_split(
787            X, y, test_size=0.2, random_state=random_state
788        )
789
790        res = cross_val_score(
791            self, X_train, y_train, cv=cv, scoring=scoring, n_jobs=n_jobs
792        )  # cross-validation error
793
794        if penalized == False:
795            return res
796
797        DescribeResult = namedtuple(
798            "DescribeResult", ["cv_score", "val_score", "penalized_score"]
799        )
800
801        numerator = res.mean()
802
803        # Evaluate on the (cv+1)-th fold
804        preds_val = self.fit(X_train, y_train).predict(X_val)
805        try:
806            denominator = scoring(y_val, preds_val)  # validation error
807        except Exception as e:
808            denominator = scoring_func(y_val, preds_val)
809
810        # if higher is better
811        if objective == "abs":
812            penalized_score = np.abs(numerator - denominator) + epsilon * (
813                1 / denominator + 1 / numerator
814            )
815        elif objective == "relative":
816            ratio = numerator / denominator
817            penalized_score = np.abs(ratio - 1) + epsilon * (
818                1 / denominator + 1 / numerator
819            )
820
821        return DescribeResult(
822            cv_score=numerator,
823            val_score=denominator,
824            penalized_score=penalized_score,
825        )