tisthemachinelearner

  8class BaseModel(BaseEstimator):
  9    """
 10    Base class for dynamically loading and wrapping scikit-learn models.
 11    """
 12    # Custom parameters that should only be passed to nnetsauce models
 13    CUSTOM_PARAMS = [
 14        'n_hidden_features',
 15        'activation_name',
 16        'bias',
 17        'dropout',
 18        'direct_link',
 19        'n_clusters',
 20        'cluster_encode',
 21        'type_clust'
 22    ]
 23
 24    def __init__(self, base_model, custom=False, **kwargs):
 25        """
 26        Initialize a scikit-learn model dynamically.
 27
 28        Parameters:
 29
 30        - base_model (str): The class name of the scikit-learn model (e.g., 'LogisticRegression').
 31
 32        - custom (bool): Whether the model is a custom nnetsauce model.
 33
 34        - **kwargs: Additional parameters to pass to the model constructor.
 35
 36        """
 37        modules = [
 38            "linear_model",
 39            "ensemble",
 40            "neural_network",
 41            "svm",
 42            "neighbors",
 43            "tree",
 44            "discriminant_analysis",
 45            "gaussian_process",
 46            "naive_bayes",
 47            "kernel_ridge",
 48            "gbdt_regressor",
 49            "gbdt_classifier"
 50        ] 
 51
 52        self.base_model = base_model
 53        self.custom = custom
 54
 55        # Split kwargs into base and custom parameters
 56        self.base_kwargs = {k: v for k, v in kwargs.items() if k not in self.CUSTOM_PARAMS}
 57        self.custom_kwargs = {k: v for k, v in kwargs.items() if k in self.CUSTOM_PARAMS}
 58
 59        # Initialize only the base model here
 60        self.model = self._load_model(base_model, modules)(**self.base_kwargs)
 61        
 62        # Custom model wrapping is handled in derived classes
 63
 64    def _load_model(self, base_model, modules):
 65        """
 66        Load a model class from scikit-learn modules.
 67
 68        Parameters:
 69
 70        - base_model (str): The class name of the scikit-learn model.
 71
 72        - modules (list): List of scikit-learn submodules to search.
 73
 74        Returns:
 75
 76        - class: The loaded scikit-learn model class.
 77
 78        """
 79        for module_name in modules:
 80            try:
 81                module = importlib.import_module(f"sklearn.{module_name}")
 82                return getattr(module, base_model)
 83            except (ImportError, AttributeError) as e:    
 84                try:                     
 85                    return getattr(ub, base_model)
 86                except (ImportError, AttributeError) as e:
 87                    continue        
 88
 89        raise ImportError(f"Model '{base_model}' not found in scikit-learn modules.")
 90
 91    def fit(self, X, y, **kwargs):
 92        """
 93        Fit the model to the training data.
 94
 95        Parameters:
 96
 97        - X (array-like): Training data features.
 98
 99        - y (array-like): Target values.
100
101        - **kwargs: Additional parameters to pass to the 
102        model fit method.
103
104        """
105        self.model.fit(X, y, **kwargs)
106        return self
107
108    def predict(self, X, **kwargs):
109        """
110        Predict using the trained model.
111
112        Parameters:
113
114        - X (array-like): Input data.
115
116        - **kwargs: Additional parameters to pass to the model predict method.
117
118        Returns:
119
120        - array-like: Predictions.
121
122        """
123        return self.model.predict(X, **kwargs)
124
125    def score(self, X, y, **kwargs):
126        """
127        Return the score of the model on the given test data and labels.
128
129        Parameters:
130
131        - X (array-like): Test data features.
132
133        - y (array-like): True labels.
134
135        - **kwargs: Additional parameters to pass to the model score method.
136        
137        Returns:
138        
139        - float: The score.
140        """
141        return self.model.score(X, y, **kwargs)

 8class Classifier(BaseModel, ClassifierMixin):
 9    """
10    Wrapper for scikit-learn classifier models.
11
12    Parameters:
13    - model_name (str): The name of the scikit-learn classifier model.
14    - **kwargs: Additional parameters to pass to the scikit-learn model.
15
16    Examples:
17    
18        ```python
19        from sklearn.model_selection import train_test_split
20        from sklearn.datasets import load_breast_cancer
21        from tisthemachinelearner import Classifier
22
23        X, y = load_breast_cancer(return_X_y=True)
24        X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=42)
25
26        clf = Classifier("LogisticRegression", random_state=42)
27        clf.fit(X_train, y_train)
28        print(clf.predict(X_test))
29        print(clf.score(X_test, y_test))
30
31        clf = Classifier("RandomForestClassifier", n_estimators=100, random_state=42)
32        clf.fit(X_train, y_train)
33        print(clf.predict(X_test))
34        print(clf.score(X_test, y_test))
35        ```
36    """
37    def __init__(self, base_model, custom=False, calibrate=False, **kwargs):
38        super().__init__(base_model, custom, **kwargs)
39        if self.custom:
40            self.model = ns.CustomClassifier(self.model, **self.custom_kwargs)
41        self.calibrate = calibrate
42        if self.calibrate: 
43            raise NotImplementedError
44
45    def fit(self, X, y, **kwargs):
46        """Fit the model."""
47        super().fit(X, y, **kwargs)
48        if self.calibrate:
49            self.model = WrappedCalibratedClassifier(self.model, method='sigmoid', cv='prefit')
50            self.model.fit(X, y)
51        return self
52    
53    def predict_proba(self, X):
54        """
55        Predict class probabilities for the input data.
56
57        Parameters:
58        - X (array-like): Input data features.
59
60        Returns:
61        - array-like: Predicted class probabilities.
62        """
63        return self.model.predict_proba(X)

 7class Regressor(BaseModel, RegressorMixin):
 8    """
 9    Wrapper for scikit-learn regressor models.
10
11    Parameters:
12
13    - model_name (str): The name of the scikit-learn regressor model.
14    
15    - **kwargs: Additional parameters to pass to the scikit-learn model.
16
17    Examples:
18        ```python
19        from sklearn.model_selection import train_test_split
20        from sklearn.datasets import load_diabetes
21        from tisthemachinelearner import Regressor
22
23        X, y = load_diabetes(return_X_y=True)
24        X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=42)
25
26        reg = Regressor("LinearRegression")
27        reg.fit(X_train, y_train)
28        print(reg.predict(X_test))
29        print(reg.score(X_test, y_test))
30
31        reg = Regressor("RidgeCV", alphas=[0.01, 0.1, 1, 10])
32        reg.fit(X_train, y_train)
33        print(reg.predict(X_test))
34        print(np.sqrt(np.mean((reg.predict(X_test) - y_test) ** 2)))
35        ```
36    """
37    def __init__(self, base_model, custom=False, **kwargs):
38        super().__init__(base_model, custom, **kwargs)
39        if self.custom:
40            self.model = ns.CustomRegressor(self.model, **self.custom_kwargs)
41
42    def fit(self, X, y, **kwargs):
43        """Fit the model."""
44        super().fit(X, y, **kwargs)
45        if self.custom:
46            self.model.fit(X, y)
47        return self

 16class FiniteDiffRegressor(BaseModel, RegressorMixin):
 17    """
 18    Finite difference trainer for nnetsauce models.
 19
 20    Parameters
 21    ----------
 22
 23    base_model : str
 24        The name of the base model (e.g., 'RidgeCV').
 25
 26    lr : float, optional
 27        Learning rate for optimization (default=1e-4).
 28
 29    optimizer : {'gd', 'sgd', 'adam', 'cd'}, optional
 30        Optimization algorithm: gradient descent ('gd'), stochastic gradient descent ('sgd'),
 31        Adam ('adam'), or coordinate descent ('cd'). Default is 'gd'.
 32
 33    eps : float, optional
 34        Scaling factor for adaptive finite difference step size (default=1e-3).
 35
 36    batch_size : int, optional
 37        Batch size for 'sgd' optimizer (default=32).
 38
 39    alpha : float, optional
 40        Elastic net penalty strength (default=0.0).
 41
 42    l1_ratio : float, optional
 43        Elastic net mixing parameter (0 = Ridge, 1 = Lasso, default=0.0).
 44
 45    type_loss : {'mse', 'quantile'}, optional
 46        Type of loss function to use (default='mse').
 47
 48    q : float, optional
 49        Quantile for quantile loss (default=0.5).
 50
 51    **kwargs
 52        Additional parameters to pass to the scikit-learn model.
 53
 54    """
 55
 56    def __init__(self, base_model, 
 57        lr=1e-4, optimizer='gd', 
 58        eps=1e-3, batch_size=32, 
 59        alpha=0.0, l1_ratio=0.0, 
 60        type_loss="mse", q=0.5,
 61        **kwargs):
 62        super().__init__(base_model, True, **kwargs)
 63        self.model = ns.CustomRegressor(self.model, **self.custom_kwargs)
 64        assert isinstance(self.model, ns.CustomRegressor),\
 65         "'model' must be of class ns.CustomRegressor"
 66        self.lr = lr
 67        self.optimizer = optimizer
 68        self.eps = eps
 69        self.loss_history_ = []
 70        self.opt_state = None
 71        self.batch_size = batch_size  # for SGD
 72        self.loss_history_ = []
 73        self._cd_index = 0  # For coordinate descent
 74        self.alpha = alpha
 75        self.l1_ratio = l1_ratio
 76        self.type_loss = type_loss
 77        self.q = q
 78
 79    def _loss(self, X, y, **kwargs):
 80        """
 81        Compute the loss (with elastic net penalty) for the current model.
 82
 83        Parameters
 84        ----------
 85
 86        X : array-like of shape (n_samples, n_features)
 87            Input data.
 88
 89        y : array-like of shape (n_samples,)
 90            Target values.
 91
 92        **kwargs
 93            Additional keyword arguments for loss calculation.
 94
 95        Returns
 96        -------
 97        float
 98            The computed loss value.
 99        """
100        y_pred = self.model.predict(X)
101        if self.type_loss == "mse": 
102            loss = np.mean((y - y_pred) ** 2)            
103        elif self.type_loss == "quantile":
104            loss = mean_pinball_loss(y, y_pred, alpha=self.q, **kwargs)
105        W = self.model.W_
106        l1 = np.sum(np.abs(W))
107        l2 = np.sum(W ** 2)
108        return loss + self.alpha * (self.l1_ratio * l1 + 0.5 * (1 - self.l1_ratio) * l2)
109
110    def _compute_grad(self, X, y):
111        """
112        Compute the gradient of the loss with respect to W_ using finite differences.
113
114        Parameters
115        ----------
116
117        X : array-like of shape (n_samples, n_features)
118            Input data.
119
120        y : array-like of shape (n_samples,)
121            Target values.
122
123        Returns
124        -------
125
126        ndarray
127            Gradient array with the same shape as W_.
128        """
129        W = deepcopy(self.model.W_)
130        shape = W.shape
131        W_flat = W.flatten()
132        n_params = W_flat.size
133
134        # Adaptive finite difference step
135        h_vec = self.eps * np.maximum(1.0, np.abs(W_flat))
136        eye = np.eye(n_params)
137
138        loss_plus = np.zeros(n_params)
139        loss_minus = np.zeros(n_params)
140
141        for i in range(n_params):
142            h_i = h_vec[i]
143            Wp = W_flat.copy(); Wp[i] += h_i
144            Wm = W_flat.copy(); Wm[i] -= h_i
145
146            self.model.W_ = Wp.reshape(shape)
147            loss_plus[i] = self._loss(X, y)
148
149            self.model.W_ = Wm.reshape(shape)
150            loss_minus[i] = self._loss(X, y)
151
152        grad = ((loss_plus - loss_minus) / (2 * h_vec)).reshape(shape)
153
154        # Add elastic net gradient
155        l1_grad = self.alpha * self.l1_ratio * np.sign(W)
156        l2_grad = self.alpha * (1 - self.l1_ratio) * W
157        grad += l1_grad + l2_grad
158
159        self.model.W_ = W  # restore original
160        return grad
161
162    def fit(self, X, y, epochs=10, verbose=True, show_progress=True, sample_weight=None, **kwargs):
163        """
164        Fit the model using finite difference optimization.
165
166        Parameters
167        ----------
168
169        X : array-like of shape (n_samples, n_features)
170            Training data.
171
172        y : array-like of shape (n_samples,)
173            Target values.
174
175        epochs : int, optional
176            Number of optimization steps (default=10).
177
178        verbose : bool, optional
179            Whether to print progress messages (default=True).
180
181        show_progress : bool, optional
182            Whether to show tqdm progress bar (default=True).
183
184        sample_weight : array-like, optional
185            Sample weights.
186
187        **kwargs
188            Additional keyword arguments.
189
190        Returns
191        -------
192
193        self : object
194            Returns self.
195        """        
196
197        self.model.fit(X, y)
198
199        iterator = tqdm(range(epochs)) if show_progress else range(epochs)
200
201        for epoch in iterator:
202            grad = self._compute_grad(X, y)
203
204            if self.optimizer == 'gd':
205                self.model.W_ -= self.lr * grad
206                self.model.W_ = np.clip(self.model.W_, 0, 1)
207                #print("self.model.W_", self.model.W_)
208
209            elif self.optimizer == 'sgd':
210                # Sample a mini-batch for stochastic gradient
211                n_samples = X.shape[0]
212                idxs = np.random.choice(n_samples, self.batch_size, replace=False)
213                if isinstance(X, pd.DataFrame):
214                    X_batch = X.iloc[idxs,:]
215                else: 
216                    X_batch = X[idxs,:]
217                y_batch = y[idxs]
218                grad = self._compute_grad(X_batch, y_batch)
219
220                self.model.W_ -= self.lr * grad
221                self.model.W_ = np.clip(self.model.W_, 0, 1)
222
223            elif self.optimizer == 'adam':
224                if self.opt_state is None:
225                    self.opt_state = {'m': np.zeros_like(grad), 'v': np.zeros_like(grad), 't': 0}
226                beta1, beta2, eps = 0.9, 0.999, 1e-8
227                self.opt_state['t'] += 1
228                self.opt_state['m'] = beta1 * self.opt_state['m'] + (1 - beta1) * grad
229                self.opt_state['v'] = beta2 * self.opt_state['v'] + (1 - beta2) * (grad ** 2)
230                m_hat = self.opt_state['m'] / (1 - beta1 ** self.opt_state['t'])
231                v_hat = self.opt_state['v'] / (1 - beta2 ** self.opt_state['t'])
232
233                self.model.W_ -= self.lr * m_hat / (np.sqrt(v_hat) + eps)
234                self.model.W_ = np.clip(self.model.W_, 0, 1)
235                #print("self.model.W_", self.model.W_)
236
237            elif self.optimizer == 'cd':  # coordinate descent
238
239                W_shape = self.model.W_.shape
240                W_flat_size = self.model.W_.size
241                W_flat = self.model.W_.flatten()
242                grad_flat = grad.flatten()
243
244                # Update only one coordinate per epoch (cyclic)
245                idx = self._cd_index % W_flat_size
246                W_flat[idx] -= self.lr * grad_flat[idx]
247                # Clip the updated value
248                W_flat[idx] = np.clip(W_flat[idx], 0, 1)
249
250                # Restore W_
251                self.model.W_ = W_flat.reshape(W_shape)
252
253                self._cd_index += 1
254
255            else:
256                raise ValueError(f"Unsupported optimizer: {self.optimizer}")
257
258            loss = self._loss(X, y)
259            self.loss_history_.append(loss)
260
261            if verbose:
262                print(f"Epoch {epoch+1}: Loss = {loss:.6f}")                
263
264        # if sample_weights, else: (must use self.row_index)
265        if sample_weight in kwargs:
266            self.model.fit(
267                X,
268                y,
269                sample_weight=sample_weight[self.index_row_].ravel(),
270                **kwargs
271            )
272
273            return self
274
275        return self
276
277
278    def predict(self, X, level=95, method='splitconformal', **kwargs):
279        """
280        Predict using the trained model.
281
282        Parameters
283        ----------
284
285        X : array-like of shape (n_samples, n_features)
286            Input data.
287
288        level : int, optional
289            Level of confidence for prediction intervals (default=95).
290
291        method : {'splitconformal', 'localconformal'}, optional
292            Method for conformal prediction (default='splitconformal').
293
294        **kwargs
295            Additional keyword arguments. Use `return_pi=True` for prediction intervals,
296            or `return_std=True` for standard deviation estimates.
297
298        Returns
299        -------
300        
301        array or tuple
302            Model predictions, or a tuple with prediction intervals or standard deviations if requested.
303        """
304        if "return_std" in kwargs:
305
306            alpha = 100 - level
307            pi_multiplier = norm.ppf(1 - alpha / 200)
308
309            if len(X.shape) == 1:
310
311                n_features = X.shape[0]
312                new_X = mo.rbind(
313                    X.reshape(1, n_features),
314                    np.ones(n_features).reshape(1, n_features),
315                )
316
317                mean_, std_ = self.model.predict(
318                    new_X, return_std=True
319                )[0]
320
321                preds =  mean_
322                lower =  (mean_ - pi_multiplier * std_)
323                upper =  (mean_ + pi_multiplier * std_)
324
325                DescribeResults = namedtuple(
326                    "DescribeResults", ["mean", "std", "lower", "upper"]
327                )
328
329                return DescribeResults(preds, std_, lower, upper)
330
331            # len(X.shape) > 1
332            mean_, std_ = self.model.predict(
333                X, return_std=True
334            )
335
336            preds =  mean_
337            lower =  (mean_ - pi_multiplier * std_)
338            upper =  (mean_ + pi_multiplier * std_)
339
340            DescribeResults = namedtuple(
341                "DescribeResults", ["mean", "std", "lower", "upper"]
342            )
343
344            return DescribeResults(preds, std_, lower, upper)
345
346        if "return_pi" in kwargs:
347            assert method in (
348                "splitconformal",
349                "localconformal",
350            ), "method must be in ('splitconformal', 'localconformal')"
351            self.pi = ns.PredictionInterval(
352                obj=self,
353                method=method,
354                level=level,
355                type_pi=self.type_pi,
356                replications=self.replications,
357                kernel=self.kernel,
358            )
359
360            if len(self.X_.shape) == 1:
361                if isinstance(X, pd.DataFrame):
362                    self.X_ = pd.DataFrame(
363                        self.X_.values.reshape(1, -1), columns=self.X_.columns
364                    )
365                else:
366                    self.X_ = self.X_.reshape(1, -1)
367                self.y_ = np.array([self.y_])
368
369            self.pi.fit(self.X_, self.y_)
370            # self.X_ = None # consumes memory to keep, dangerous to delete (side effect)
371            # self.y_ = None # consumes memory to keep, dangerous to delete (side effect)
372            preds = self.pi.predict(X, return_pi=True)
373            return preds
374
375        # "return_std" not in kwargs
376        if len(X.shape) == 1:
377
378            n_features = X.shape[0]
379            new_X = mo.rbind(
380                X.reshape(1, n_features),
381                np.ones(n_features).reshape(1, n_features),
382            )
383
384            return (
385                0
386                + self.model.predict(new_X, **kwargs)
387            )[0]
388
389        # len(X.shape) > 1
390        return  self.model.predict(
391            X, **kwargs
392        )