mlsauce

 19class AdaOpt(BaseEstimator, ClassifierMixin):
 20    """AdaOpt classifier.
 21
 22    Attributes:
 23
 24        n_iterations: int
 25            number of iterations of the optimizer at training time.
 26
 27        learning_rate: float
 28            controls the speed of the optimizer at training time.
 29
 30        reg_lambda: float
 31            L2 regularization parameter for successive errors in the optimizer
 32            (at training time).
 33
 34        reg_alpha: float
 35            L1 regularization parameter for successive errors in the optimizer
 36            (at training time).
 37
 38        eta: float
 39            controls the slope in gradient descent (at training time).
 40
 41        gamma: float
 42            controls the step size in gradient descent (at training time).
 43
 44        k: int
 45            number of nearest neighbors selected at test time for classification.
 46
 47        tolerance: float
 48            controls early stopping in gradient descent (at training time).
 49
 50        n_clusters: int
 51            number of clusters, if MiniBatch k-means is used at test time
 52            (for faster prediction).
 53
 54        batch_size: int
 55            size of the batch, if MiniBatch k-means is used at test time
 56            (for faster prediction).
 57
 58        row_sample: float
 59            percentage of rows chosen from training set (by stratified subsampling,
 60            for faster prediction).
 61
 62        type_dist: str
 63            distance used for finding the nearest neighbors; currently `euclidean-f`
 64            (euclidean distances calculated as whole), `euclidean` (euclidean distances
 65            calculated row by row), `cosine` (cosine distance).
 66
 67        n_jobs: int
 68            number of cpus for parallel processing (default: None)
 69
 70        verbose: int
 71            progress bar for parallel processing (yes = 1) or not (no = 0)
 72
 73        cache: boolean
 74            if the nearest neighbors are cached or not, for faster retrieval in
 75            subsequent calls.
 76
 77        n_clusters_input: int
 78            number of clusters (a priori) for clustering the features
 79
 80        clustering_method: str
 81            clustering method: currently 'kmeans', 'gmm'
 82
 83        cluster_scaling: str
 84            scaling method for clustering: currently 'standard', 'robust', 'minmax'
 85
 86        backend: str
 87            backend for parallel processing: "cpu" or "gpu" or "tpu"
 88
 89        seed: int
 90            reproducibility seed for nodes_sim=='uniform', clustering and dropout.
 91
 92    """
 93
 94    def __init__(
 95        self,
 96        n_iterations=50,
 97        learning_rate=0.3,
 98        reg_lambda=0.1,
 99        reg_alpha=0.5,
100        eta=0.01,
101        gamma=0.01,
102        k=3,
103        tolerance=0,
104        n_clusters=0,
105        batch_size=100,
106        row_sample=0.8,
107        type_dist="euclidean-f",
108        n_jobs=None,
109        verbose=0,
110        cache=True,
111        n_clusters_input=0,
112        clustering_method="kmeans",
113        cluster_scaling="standard",
114        backend="cpu",
115        seed=123,
116    ):
117        if n_clusters_input > 0:
118            assert clustering_method in (
119                "kmeans",
120                "gmm",
121            ), "`clustering_method` must be in ('kmeans', 'gmm')"
122            assert cluster_scaling in (
123                "standard",
124                "robust",
125                "minmax",
126            ), "`cluster_scaling` must be in ('standard', 'robust', 'minmax')"
127
128        assert type_dist in (
129            "euclidean",
130            "manhattan",
131            "euclidean-f",
132            "cosine",
133        ), "must have: `type_dist` in ('euclidean', 'manhattan', 'euclidean-f', 'cosine') "
134
135        self.n_iterations = n_iterations
136        self.learning_rate = learning_rate
137        self.reg_lambda = reg_lambda
138        self.reg_alpha = reg_alpha
139        self.eta = eta
140        self.gamma = gamma
141        self.k = k
142        self.tolerance = tolerance
143        self.n_clusters = n_clusters
144        self.batch_size = batch_size
145        self.row_sample = row_sample
146        self.type_dist = type_dist
147        self.n_jobs = n_jobs
148        self.cache = cache
149        self.verbose = verbose
150        self.n_clusters_input = n_clusters_input
151        self.clustering_method = clustering_method
152        self.cluster_scaling = cluster_scaling
153        self.scaler_, self.label_encoder_, self.clusterer_ = None, None, None
154        self.backend = backend
155        self.seed = seed
156
157    def fit(self, X, y, **kwargs):
158        """Fit AdaOpt to training data (X, y)
159
160        Args:
161
162            X: {array-like}, shape = [n_samples, n_features]
163                Training vectors, where n_samples is the number
164                of samples and n_features is the number of features.
165
166            y: array-like, shape = [n_samples]
167                Target values.
168
169            **kwargs: additional parameters to be passed to self.cook_training_set.
170
171        Returns:
172
173            self: object.
174
175        """
176
177        if self.n_clusters_input > 0:
178            clustered_X, self.scaler_, self.label_encoder_, self.clusterer_ = (
179                cluster(
180                    X,
181                    n_clusters=self.n_clusters_input,
182                    method=self.clustering_method,
183                    type_scaling=self.cluster_scaling,
184                    training=True,
185                    seed=self.seed,
186                )
187            )
188            X = np.column_stack((X.copy(), clustered_X))
189
190        if self.row_sample < 1:
191            index_subsample = subsample(
192                y, row_sample=self.row_sample, seed=self.seed
193            )
194            y_ = y[index_subsample]
195            X_ = X[index_subsample, :]
196        else:
197            y_ = deepcopy(y)
198            X_ = deepcopy(X)
199
200        n, p = X_.shape
201
202        n_classes = len(np.unique(y_))
203
204        assert n == len(y_), "must have X.shape[0] == len(y)"
205
206        res = adaoptc.fit_adaopt(
207            X=np.asarray(X_).astype(np.float64),
208            y=np.asarray(y_).astype(np.int64),
209            n_iterations=self.n_iterations,
210            n_X=n,
211            p_X=p,
212            n_classes=n_classes,
213            learning_rate=self.learning_rate,
214            reg_lambda=self.reg_lambda,
215            reg_alpha=self.reg_alpha,
216            eta=self.eta,
217            gamma=self.gamma,
218            tolerance=self.tolerance,
219            backend=self.backend,
220        )
221
222        self.probs_training = res["probs"]
223        self.training_accuracy = res["training_accuracy"]
224        self.alphas = res["alphas"]
225        self.n_iterations = res["n_iterations"]
226        self.scaled_X_train = np.array(res["scaled_X_train"], dtype=np.float64)
227        self.n_classes_ = len(np.unique(y))  # for compatibility with sklearn
228        return self
229
230    def predict(self, X, **kwargs):
231        """Predict test data X.
232
233        Args:
234
235            X: {array-like}, shape = [n_samples, n_features]
236                Training vectors, where n_samples is the number
237                of samples and n_features is the number of features.
238
239            **kwargs: additional parameters to be passed to `predict_proba`
240
241        Returns:
242
243            model predictions: {array-like}
244
245        """
246
247        return np.argmax(self.predict_proba(X, **kwargs), axis=1)
248
249    def predict_proba(self, X, **kwargs):
250        """Predict probabilities for test data X.
251
252        Args:
253
254            X: {array-like}, shape = [n_samples, n_features]
255                Training vectors, where n_samples is the number
256                of samples and n_features is the number of features.
257
258            **kwargs: additional parameters to be passed to
259                self.cook_test_set
260
261        Returns:
262
263            probability estimates for test data: {array-like}
264
265        """
266
267        n_train, p_train = self.scaled_X_train.shape
268
269        if self.n_clusters_input > 0:
270            X = np.column_stack(
271                (
272                    X.copy(),
273                    cluster(
274                        X,
275                        training=False,
276                        scaler=self.scaler_,
277                        label_encoder=self.label_encoder_,
278                        clusterer=self.clusterer_,
279                        seed=self.seed,
280                    ),
281                )
282            )
283
284        n_test = X.shape[0]
285
286        if self.n_jobs is None:
287            return adaoptc.predict_proba_adaopt(
288                X_test=np.asarray(X, order="C").astype(np.float64),
289                scaled_X_train=np.asarray(
290                    self.scaled_X_train, order="C"
291                ).astype(np.float64),
292                n_test=n_test,
293                n_train=n_train,
294                probs_train=self.probs_training,
295                k=self.k,
296                n_clusters=self.n_clusters,
297                batch_size=self.batch_size,
298                type_dist=self.type_dist,
299                cache=self.cache,
300                seed=self.seed,
301                backend=self.backend,
302            )
303
304        # parallel: self.n_jobs is not None
305        assert self.type_dist in (
306            "euclidean",
307            "manhattan",
308            "cosine",
309        ), "must have: `self.type_dist` in ('euclidean', 'manhattan', 'cosine') "
310
311        scaled_X_test = X / norm(X, ord=2, axis=1)[:, None]
312
313        if self.type_dist == "euclidean":
314
315            @delayed
316            @wrap_non_picklable_objects
317            def multiproc_func(i):
318                dists_test_i = adaoptc.distance_to_mat_euclidean2(
319                    np.asarray(scaled_X_test.astype(np.float64), order="C")[
320                        i, :
321                    ],
322                    np.asarray(
323                        self.scaled_X_train.astype(np.float64), order="C"
324                    ),
325                    np.zeros(n_train),
326                    n_train,
327                    p_train,
328                )
329
330                kmin_test_i = adaoptc.find_kmin_x(
331                    dists_test_i, n_x=n_train, k=self.k, cache=self.cache
332                )
333
334                weights_test_i = adaoptc.calculate_weights(kmin_test_i[0])
335
336                probs_test_i = adaoptc.calculate_probs(
337                    kmin_test_i[1], self.probs_training
338                )
339
340                return adaoptc.average_probs(
341                    probs=probs_test_i, weights=weights_test_i
342                )
343
344        if self.type_dist == "manhattan":
345
346            @delayed
347            @wrap_non_picklable_objects
348            def multiproc_func(i):
349                dists_test_i = adaoptc.distance_to_mat_manhattan2(
350                    np.asarray(scaled_X_test.astype(np.float64), order="C")[
351                        i, :
352                    ],
353                    np.asarray(
354                        self.scaled_X_train.astype(np.float64), order="C"
355                    ),
356                    np.zeros(n_train),
357                    n_train,
358                    p_train,
359                )
360
361                kmin_test_i = adaoptc.find_kmin_x(
362                    dists_test_i, n_x=n_train, k=self.k, cache=self.cache
363                )
364
365                weights_test_i = adaoptc.calculate_weights(kmin_test_i[0])
366
367                probs_test_i = adaoptc.calculate_probs(
368                    kmin_test_i[1], self.probs_training
369                )
370
371                return adaoptc.average_probs(
372                    probs=probs_test_i, weights=weights_test_i
373                )
374
375        if self.type_dist == "cosine":
376
377            @delayed
378            @wrap_non_picklable_objects
379            def multiproc_func(i, *args):
380                dists_test_i = adaoptc.distance_to_mat_cosine2(
381                    np.asarray(scaled_X_test.astype(np.float64), order="C")[
382                        i, :
383                    ],
384                    np.asarray(
385                        self.scaled_X_train.astype(np.float64), order="C"
386                    ),
387                    np.zeros(n_train),
388                    n_train,
389                    p_train,
390                )
391
392                kmin_test_i = adaoptc.find_kmin_x(
393                    dists_test_i, n_x=n_train, k=self.k, cache=self.cache
394                )
395
396                weights_test_i = adaoptc.calculate_weights(kmin_test_i[0])
397
398                probs_test_i = adaoptc.calculate_probs(
399                    kmin_test_i[1], self.probs_training
400                )
401
402                return adaoptc.average_probs(
403                    probs=probs_test_i, weights=weights_test_i
404                )
405
406        if self.verbose == 1:
407            res = Parallel(n_jobs=self.n_jobs, prefer="threads")(
408                (multiproc_func)(m) for m in tqdm(range(n_test))
409            )
410
411        else:
412            res = Parallel(n_jobs=self.n_jobs, prefer="threads")(
413                (multiproc_func)(m) for m in range(n_test)
414            )
415
416        return np.asarray(res)

 19class ConformalBayesianRegressor(BaseEstimator, RegressorMixin):
 20    def __init__(
 21        self,
 22        obj=Ridge,
 23        level=95,
 24        hyperparameter_bounds=None,
 25        n_samples=20,
 26        calibration_fraction=0.2,
 27        scaling_method="standard",
 28        random_state=None,
 29        show_progress=False,
 30        verbose=True,
 31        n_jobs=-1,
 32    ):
 33        self.obj = obj
 34        self.level = level
 35        self.alpha_ = 1 - self.level / 100
 36        self.hyperparameter_bounds = hyperparameter_bounds
 37        self.n_samples = n_samples
 38        self.calibration_fraction = calibration_fraction
 39        self.scaling_method = scaling_method
 40        self.random_state = random_state
 41        self.verbose = verbose
 42        self.show_progress = show_progress
 43        self.n_jobs = n_jobs
 44
 45        self.is_fitted_ = False
 46
 47    def _sample_hyperparameters(self):
 48        # Simple uniform sampling or use fixed bounds
 49        configs = []
 50        for _ in range(self.n_samples):
 51            if self.hyperparameter_bounds:
 52                cfg = {}
 53                for k, v in self.hyperparameter_bounds.items():
 54                    if isinstance(v, list) and len(v) == 2:
 55                        # Always sample as float first
 56                        sampled_value = np.random.uniform(v[0], v[1])
 57
 58                        # If both bounds are integers, assume integer parameter
 59                        if isinstance(v[0], (int, np.integer)) and isinstance(
 60                            v[1], (int, np.integer)
 61                        ):
 62                            cfg[k] = int(sampled_value)
 63                        else:
 64                            cfg[k] = sampled_value
 65                    else:
 66                        # Assume fixed value
 67                        cfg[k] = v
 68            else:
 69                cfg = {}
 70            configs.append(cfg)
 71        return configs
 72
 73    def _train_models_parallel(self, X, y, configs):
 74        def train_model(cfg):
 75            self.obj.set_params(**cfg)
 76            self.obj.fit(X, y)
 77            return self.obj
 78
 79        if self.show_progress == False:
 80            models = joblib.Parallel(n_jobs=self.n_jobs)(
 81                joblib.delayed(train_model)(cfg)
 82                for cfg in tqdm(configs, disable=not self.verbose)
 83            )
 84        else:
 85            models = joblib.Parallel(n_jobs=self.n_jobs)(
 86                joblib.delayed(train_model)(cfg) for cfg in configs
 87            )
 88        return models
 89
 90    def _predict_models(self, models, X):
 91        if self.show_progress == False:
 92            preds = [m.predict(X) for m in models]
 93        else:
 94            preds = [
 95                m.predict(X) for m in tqdm(models, disable=not self.verbose)
 96            ]
 97        return np.column_stack(preds)
 98
 99    def fit(self, X, y):
100        if hasattr(X, "values"):  # keep DataFrame if possible
101            X = X.copy()
102        else:
103            X = np.asarray(X)
104        X = np.asarray(X)
105        y = np.asarray(y)
106        rng = check_random_state(self.random_state)
107        X, y = shuffle(X, y, random_state=rng)
108        # 1. Cluster with GMM (full covariance)
109        n_clusters = min(10, X.shape[0] // 30)
110        gmm = GaussianMixture(
111            n_components=n_clusters,
112            covariance_type="full",
113            random_state=self.random_state,
114        )
115        clusters = gmm.fit_predict(X)
116        # 2. Stratified train/calibration split
117        X_train, X_calib, y_train, y_calib = train_test_split(
118            X,
119            y,
120            test_size=self.calibration_fraction,
121            random_state=self.random_state,
122            stratify=clusters,
123        )
124        # 3. Scale features
125        if self.scaling_method == "standard":
126            scaler = StandardScaler().fit(X_train)
127        else:
128            raise ValueError("Scaling method must be 'standard'")
129        self.scaler_ = scaler
130        X_train_s = scaler.transform(X_train)
131        X_calib_s = scaler.transform(X_calib)
132        # 4. Train ensemble
133        configs = self._sample_hyperparameters()
134        self.models_ = self._train_models_parallel(X_train_s, y_train, configs)
135        # 5. Calibration residuals
136        preds_calib = self._predict_models(self.models_, X_calib_s)
137        self.calibration_residuals_ = y_calib - np.median(preds_calib, axis=1)
138        self.is_fitted_ = True
139        return self
140
141    def predict(self, X, return_pi=False):
142        """Obtain predictions and prediction intervals
143
144        Args:
145
146            X: array-like, shape = [n_samples, n_features];
147                Testing set vectors, where n_samples is the number
148                of samples and n_features is the number of features.
149
150            return_pi: boolean
151                Whether the prediction interval is returned or not.
152                Default is False, for compatibility with other _estimators_.
153                If True, a tuple containing the predictions + lower and upper
154                bounds is returned.
155
156        """
157        if not self.is_fitted_:
158            raise RuntimeError("Fit the model first")
159        X_s = self.scaler_.transform(X)
160        preds = self._predict_models(self.models_, X_s)
161        self.mean_ = np.median(preds, axis=1)
162        if return_pi == False:
163            return self.mean_
164        DescribeResult = namedtuple(
165            "DescribeResult", ("mean", "lower", "upper")
166        )
167        q = np.quantile(self.calibration_residuals_, q=self.alpha_ / 200)
168        return DescribeResult(self.mean_, self.mean_ + q, self.mean_ - q)
169
170    def get_coverage(self, y_true, lower, upper):
171        return np.mean((y_true >= lower) & (y_true <= upper))

 21class ConformalBayesianClassifier(BaseEstimator, ClassifierMixin):
 22
 23    # construct the object -----
 24    _estimator_type = "classifier"
 25
 26    def __init__(
 27        self,
 28        obj=Ridge(),
 29        hyperparameter_bounds=None,
 30        n_samples=20,
 31        calibration_fraction=0.2,
 32        scaling_method="standard",
 33        random_state=None,
 34        verbose=True,
 35        n_jobs=-1,
 36        calibrated=False,
 37        calibration_method="sigmoid",
 38        calibration_cv=3,
 39        use_classifier=True,
 40    ):
 41        """
 42        Conformal Bayesian Classifier with optional probability calibration.
 43
 44        Parameters
 45        ----------
 46        obj : estimator object
 47            Base estimator (default: Ridge() for regression-based, or classifier if use_classifier=True)
 48        hyperparameter_bounds : dict, optional
 49            Bounds for hyperparameter sampling
 50        n_samples : int
 51            Number of ensemble models to train
 52        calibration_fraction : float
 53            Fraction of data to use for conformal calibration
 54        scaling_method : str
 55            Feature scaling method (default: "standard")
 56        random_state : int, optional
 57            Random seed for reproducibility
 58        verbose : bool
 59            Whether to show progress bars
 60        n_jobs : int
 61            Number of parallel jobs (-1 for all cores)
 62        calibrated : bool
 63            Whether to apply probability calibration (default: False)
 64        calibration_method : str
 65            Method for calibration: "sigmoid" or "isotonic" (default: "sigmoid")
 66        calibration_cv : int
 67            Number of CV folds for calibration (default: 3)
 68        use_classifier : bool
 69            If True, use sklearn-like classifier with probability averaging.
 70            If False, use regression-based approach with SimpleMultitaskClassifier (default: False)
 71        """
 72        self.obj_base = obj
 73        self.hyperparameter_bounds = hyperparameter_bounds
 74        self.n_samples = n_samples
 75        self.calibration_fraction = calibration_fraction
 76        self.scaling_method = scaling_method
 77        self.random_state = random_state
 78        self.verbose = verbose
 79        self.n_jobs = n_jobs
 80        self.calibrated = calibrated
 81        self.calibration_method = calibration_method
 82        self.calibration_cv = calibration_cv
 83        self.use_classifier = use_classifier
 84
 85        self.is_fitted_ = False
 86
 87    def _sample_hyperparameters(self):
 88        """Sample hyperparameters for ensemble members"""
 89        configs = []
 90
 91        # Get parameter constraints from the model if available
 92        param_constraints = {}
 93        if hasattr(self.obj_base, "_parameter_constraints"):
 94            param_constraints = self.obj_base._parameter_constraints
 95
 96        for _ in range(self.n_samples):
 97            if self.hyperparameter_bounds:
 98                cfg = {}
 99                for k, v in self.hyperparameter_bounds.items():
100                    if isinstance(v, list) and len(v) == 2:
101                        # If both bounds are integers, assume integer parameter
102                        if isinstance(v[0], (int, np.integer)) and isinstance(
103                            v[1], (int, np.integer)
104                        ):
105                            cfg[k] = np.random.randint(v[0], v[1] + 1)
106                        else:
107                            cfg[k] = np.random.uniform(v[0], v[1])
108                    else:
109                        # Assume fixed value
110                        cfg[k] = v
111            else:
112                cfg = {}
113            configs.append(cfg)
114        return configs
115
116    def _train_classifiers_parallel(self, X, y, configs):
117        """Train ensemble of classifiers in parallel"""
118
119        def train_classifier(cfg):
120            clf = clone(self.obj_base)
121            clf.set_params(**cfg)
122            clf.fit(X, y)
123            return clf
124
125        if self.verbose:
126            models = joblib.Parallel(n_jobs=self.n_jobs)(
127                joblib.delayed(train_classifier)(cfg)
128                for cfg in tqdm(configs, desc="Training classifiers")
129            )
130        else:
131            models = joblib.Parallel(n_jobs=self.n_jobs)(
132                joblib.delayed(train_classifier)(cfg) for cfg in configs
133            )
134        return models
135
136    def fit(self, X, y):
137        """
138        Fit the conformal Bayesian classifier.
139
140        Parameters
141        ----------
142        X : array-like of shape (n_samples, n_features)
143            Training data
144        y : array-like of shape (n_samples,)
145            Target values
146
147        Returns
148        -------
149        self : object
150            Fitted classifier
151        """
152        X = np.asarray(X)
153        y = np.asarray(y)
154
155        # Store classes
156        self.classes_ = np.unique(y)
157        self.n_classes_ = len(self.classes_)
158
159        if self.use_classifier:
160            # Simpler approach: use sklearn-like classifier with probability averaging
161            if not is_classifier(self.obj_base):
162                raise ValueError(
163                    "use_classifier=True requires obj to be a classifier "
164                    "(e.g., LogisticRegression, RandomForestClassifier)"
165                )
166
167            rng = check_random_state(self.random_state)
168            X, y = shuffle(X, y, random_state=rng)
169
170            # Stratified train/calibration split
171            X_train, X_calib, y_train, y_calib = train_test_split(
172                X,
173                y,
174                test_size=self.calibration_fraction,
175                random_state=self.random_state,
176                stratify=y,
177            )
178
179            # Scale features
180            if self.scaling_method == "standard":
181                self.scaler_ = StandardScaler().fit(X_train)
182            else:
183                raise ValueError("Scaling method must be 'standard'")
184
185            X_train_s = self.scaler_.transform(X_train)
186            X_calib_s = self.scaler_.transform(X_calib)
187
188            # Train ensemble of classifiers
189            configs = self._sample_hyperparameters()
190            self.classifiers_ = self._train_classifiers_parallel(
191                X_train_s, y_train, configs
192            )
193
194            # Store calibration data for potential conformal prediction
195            self.X_calib_ = X_calib_s
196            self.y_calib_ = y_calib
197
198            if self.calibrated:
199                # Calibrate each classifier
200                if self.verbose:
201                    print("Calibrating classifiers...")
202
203                self.calibrated_classifiers_ = []
204                for clf in tqdm(
205                    self.classifiers_,
206                    disable=not self.verbose,
207                    desc="Calibrating",
208                ):
209                    cal_clf = CalibratedClassifierCV(
210                        clf,
211                        method=self.calibration_method,
212                        cv="prefit",
213                        ensemble=False,
214                    )
215                    cal_clf.fit(X_calib_s, y_calib)
216                    self.calibrated_classifiers_.append(cal_clf)
217
218        else:
219            # Original approach: use regression-based with SimpleMultitaskClassifier
220            base_regressor = ConformalBayesianRegressor(
221                self.obj_base,
222                hyperparameter_bounds=self.hyperparameter_bounds,
223                n_samples=self.n_samples,
224                calibration_fraction=self.calibration_fraction,
225                scaling_method=self.scaling_method,
226                random_state=self.random_state,
227                verbose=self.verbose,
228                n_jobs=self.n_jobs,
229            )
230
231            # Wrap in multitask classifier
232            self.obj = ns.SimpleMultitaskClassifier(base_regressor)
233
234            if self.calibrated:
235                # Fit the base classifier first
236                self.obj.fit(X, y)
237
238                # Then wrap with calibration
239                self.calibrated_obj_ = CalibratedClassifierCV(
240                    self.obj,
241                    method=self.calibration_method,
242                    cv="prefit",
243                    ensemble=False,
244                )
245
246                # Fit calibration on the same data
247                self.calibrated_obj_.fit(X, y)
248            else:
249                self.obj.fit(X, y)
250
251        self.is_fitted_ = True
252        return self
253
254    def predict_proba(self, X):
255        """
256        Predict class probabilities.
257
258        Parameters
259        ----------
260        X : array-like of shape (n_samples, n_features)
261            Test data
262
263        Returns
264        -------
265        proba : array of shape (n_samples, n_classes)
266            Predicted probabilities for each class
267        """
268        if not self.is_fitted_:
269            raise RuntimeError("Fit the model first")
270
271        X = np.asarray(X)
272
273        if self.use_classifier:
274            # Average probabilities from ensemble
275            X_s = self.scaler_.transform(X)
276
277            if self.calibrated:
278                # Use calibrated classifiers
279                probas = np.array(
280                    [
281                        clf.predict_proba(X_s)
282                        for clf in self.calibrated_classifiers_
283                    ]
284                )
285            else:
286                # Use uncalibrated classifiers
287                probas = np.array(
288                    [clf.predict_proba(X_s) for clf in self.classifiers_]
289                )
290
291            # Average probabilities across ensemble
292            mean_proba = np.mean(probas, axis=0)
293
294            return mean_proba
295        else:
296            # Use regression-based approach
297            if self.calibrated:
298                return self.calibrated_obj_.predict_proba(X)
299            else:
300                return self.obj.predict_proba(X)
301
302    def predict(self, X):
303        """
304        Predict class labels.
305
306        Parameters
307        ----------
308        X : array-like of shape (n_samples, n_features)
309            Test data
310
311        Returns
312        -------
313        y_pred : array of shape (n_samples,)
314            Predicted class labels
315        """
316        proba = self.predict_proba(X)
317        return self.classes_[np.argmax(proba, axis=1)]
318
319    @property
320    def _estimator_type(self):
321        return "classifier"