diff --git a/docs/algebraic/gam.html b/docs/algebraic/gam.html deleted file mode 100644 index a85ae4cb..00000000 --- a/docs/algebraic/gam.html +++ /dev/null @@ -1,311 +0,0 @@ - - - - - - - - - - - - - - - - -
-
-
-
- -Expand source code - -
from copy import deepcopy
-import numpy as np
-import pandas as pd
-from sklearn.base import BaseEstimator
-from sklearn.tree import DecisionTreeRegressor
-from sklearn.utils.validation import check_is_fitted
-from sklearn.utils import check_array
-from sklearn.utils.multiclass import check_classification_targets
-from sklearn.utils.multiclass import type_of_target
-from sklearn.utils.validation import check_X_y
-from sklearn.utils.validation import check_random_state
-from sklearn.utils.validation import column_or_1d
-from sklearn.utils.validation import check_consistent_length
-from sklearn.utils.validation import _check_sample_weight
-from sklearn.ensemble import GradientBoostingClassifier, GradientBoostingRegressor
-from sklearn.datasets import load_breast_cancer
-from sklearn.model_selection import train_test_split
-from sklearn.metrics import accuracy_score, roc_auc_score
-from tqdm import tqdm
-
-
-class TreeGAMClassifier(BaseEstimator):
-    """Tree-based GAM classifier.
-    Uses cyclical boosting to fit a GAM with small trees.
-    Simplified version of the explainable boosting machine described in https://github.com/interpretml/interpret
-    """
-
-    def __init__(
-        self,
-        max_leaf_nodes=3,
-        n_boosting_rounds=20,
-        random_state=None,
-    ):
-        self.max_leaf_nodes = max_leaf_nodes
-        self.random_state = random_state
-        self.n_boosting_rounds = n_boosting_rounds
-
-    def fit(self, X, y, sample_weight=None):
-        X, y = check_X_y(X, y, accept_sparse=False, multi_output=False)
-        check_classification_targets(y)
-        sample_weight = _check_sample_weight(sample_weight, X, dtype=None)
-
-        # cycle through features and fit a tree to each one
-        ests = []
-        for boosting_round in tqdm(range(self.n_boosting_rounds)):
-            for feature_num in range(X.shape[1]):
-                X_ = np.zeros_like(X)
-                X_[:, feature_num] = X[:, feature_num]
-                est = DecisionTreeRegressor(
-                    max_leaf_nodes=self.max_leaf_nodes,
-                    random_state=self.random_state,
-                )
-                est.fit(X_, y, sample_weight=sample_weight)
-                if not est.tree_.feature[0] == feature_num:
-                    # failed to split on this feature
-                    continue
-                ests.append(est)
-                y = y - est.predict(X)
-
-        self.est_ = GradientBoostingRegressor()
-        self.est_.fit(X, y)
-        self.est_.n_estimators_ = len(ests)
-        self.est_.estimators_ = np.array(ests).reshape(-1, 1)
-
-        return self
-
-    def predict_proba(self, X):
-        X = check_array(X, accept_sparse=False, dtype=None)
-        check_is_fitted(self)
-        probs1 = self.est_.predict(X)
-        return np.array([1 - probs1, probs1]).T
-
-    def predict(self, X):
-        X = check_array(X, accept_sparse=False, dtype=None)
-        check_is_fitted(self)
-        return (self.est_.predict(X) > 0.5).astype(int)
-
-
-if __name__ == "__main__":
-    breast = load_breast_cancer()
-    feature_names = list(breast.feature_names)
-    X, y = pd.DataFrame(breast.data, columns=feature_names), breast.target
-    X_train, X_test, y_train, y_test = train_test_split(X, y, random_state=42)
-    gam = TreeGAMClassifier(max_leaf_nodes=2)
-    gam.fit(X_train, y_train)
-
-    # check roc auc score
-    y_pred = gam.predict_proba(X_test)[:, 1]
-    print(
-        "train roc auc score:", roc_auc_score(y_train, gam.predict_proba(X_train)[:, 1])
-    )
-    print("test roc auc score:", roc_auc_score(y_test, y_pred))
-
-
-
-
-
-
-
-
-
-

Classes

-
-
-class TreeGAMClassifier -(max_leaf_nodes=3, n_boosting_rounds=20, random_state=None) -
-
-

Tree-based GAM classifier. -Uses cyclical boosting to fit a GAM with small trees. -Simplified version of the explainable boosting machine described in https://github.com/interpretml/interpret

-
- -Expand source code - -
class TreeGAMClassifier(BaseEstimator):
-    """Tree-based GAM classifier.
-    Uses cyclical boosting to fit a GAM with small trees.
-    Simplified version of the explainable boosting machine described in https://github.com/interpretml/interpret
-    """
-
-    def __init__(
-        self,
-        max_leaf_nodes=3,
-        n_boosting_rounds=20,
-        random_state=None,
-    ):
-        self.max_leaf_nodes = max_leaf_nodes
-        self.random_state = random_state
-        self.n_boosting_rounds = n_boosting_rounds
-
-    def fit(self, X, y, sample_weight=None):
-        X, y = check_X_y(X, y, accept_sparse=False, multi_output=False)
-        check_classification_targets(y)
-        sample_weight = _check_sample_weight(sample_weight, X, dtype=None)
-
-        # cycle through features and fit a tree to each one
-        ests = []
-        for boosting_round in tqdm(range(self.n_boosting_rounds)):
-            for feature_num in range(X.shape[1]):
-                X_ = np.zeros_like(X)
-                X_[:, feature_num] = X[:, feature_num]
-                est = DecisionTreeRegressor(
-                    max_leaf_nodes=self.max_leaf_nodes,
-                    random_state=self.random_state,
-                )
-                est.fit(X_, y, sample_weight=sample_weight)
-                if not est.tree_.feature[0] == feature_num:
-                    # failed to split on this feature
-                    continue
-                ests.append(est)
-                y = y - est.predict(X)
-
-        self.est_ = GradientBoostingRegressor()
-        self.est_.fit(X, y)
-        self.est_.n_estimators_ = len(ests)
-        self.est_.estimators_ = np.array(ests).reshape(-1, 1)
-
-        return self
-
-    def predict_proba(self, X):
-        X = check_array(X, accept_sparse=False, dtype=None)
-        check_is_fitted(self)
-        probs1 = self.est_.predict(X)
-        return np.array([1 - probs1, probs1]).T
-
-    def predict(self, X):
-        X = check_array(X, accept_sparse=False, dtype=None)
-        check_is_fitted(self)
-        return (self.est_.predict(X) > 0.5).astype(int)
-
-

Ancestors

-
    -
  • sklearn.base.BaseEstimator
    • -
-

Methods

-
-
-def fit(self, X, y, sample_weight=None) -
-
-
-
- -Expand source code - -
def fit(self, X, y, sample_weight=None):
-    X, y = check_X_y(X, y, accept_sparse=False, multi_output=False)
-    check_classification_targets(y)
-    sample_weight = _check_sample_weight(sample_weight, X, dtype=None)
-
-    # cycle through features and fit a tree to each one
-    ests = []
-    for boosting_round in tqdm(range(self.n_boosting_rounds)):
-        for feature_num in range(X.shape[1]):
-            X_ = np.zeros_like(X)
-            X_[:, feature_num] = X[:, feature_num]
-            est = DecisionTreeRegressor(
-                max_leaf_nodes=self.max_leaf_nodes,
-                random_state=self.random_state,
-            )
-            est.fit(X_, y, sample_weight=sample_weight)
-            if not est.tree_.feature[0] == feature_num:
-                # failed to split on this feature
-                continue
-            ests.append(est)
-            y = y - est.predict(X)
-
-    self.est_ = GradientBoostingRegressor()
-    self.est_.fit(X, y)
-    self.est_.n_estimators_ = len(ests)
-    self.est_.estimators_ = np.array(ests).reshape(-1, 1)
-
-    return self
-
-
-
-def predict(self, X) -
-
-
-
- -Expand source code - -
def predict(self, X):
-    X = check_array(X, accept_sparse=False, dtype=None)
-    check_is_fitted(self)
-    return (self.est_.predict(X) > 0.5).astype(int)
-
-
-
-def predict_proba(self, X) -
-
-
-
- -Expand source code - -
def predict_proba(self, X):
-    X = check_array(X, accept_sparse=False, dtype=None)
-    check_is_fitted(self)
-    probs1 = self.est_.predict(X)
-    return np.array([1 - probs1, probs1]).T
-
-
-
-
-
-
-
- -
- - - - - - - \ No newline at end of file diff --git a/docs/algebraic/index.html b/docs/algebraic/index.html index 7e990fdd..1903006d 100644 --- a/docs/algebraic/index.html +++ b/docs/algebraic/index.html @@ -29,14 +29,14 @@

Sub-modules

-
imodels.algebraic.gam
-
-
-
imodels.algebraic.slim

Wrapper for sparse, integer linear models …

+
imodels.algebraic.tree_gam
+
+
+
@@ -65,8 +65,8 @@

Index πŸ”

  • Sub-modules

    • diff --git a/docs/algebraic/tree_gam.html b/docs/algebraic/tree_gam.html new file mode 100644 index 00000000..cbf8c2f7 --- /dev/null +++ b/docs/algebraic/tree_gam.html @@ -0,0 +1,872 @@ + + + + + + + + + + + + + + + + +
    +
    +
    +
    + +Expand source code + +
    from copy import deepcopy
+import numpy as np
+import pandas as pd
+from sklearn.base import BaseEstimator
+from sklearn.linear_model import LinearRegression, RidgeCV
+from sklearn.tree import DecisionTreeRegressor
+from sklearn.utils.validation import check_is_fitted
+from sklearn.utils import check_array
+from sklearn.utils.multiclass import check_classification_targets
+from sklearn.utils.validation import check_X_y
+from sklearn.utils.validation import _check_sample_weight
+from sklearn.ensemble import GradientBoostingClassifier, GradientBoostingRegressor
+from sklearn.model_selection import train_test_split
+from sklearn.metrics import accuracy_score, roc_auc_score
+from tqdm import tqdm
+
+import imodels
+
+from sklearn.base import RegressorMixin, ClassifierMixin
+
+
+class TreeGAM(BaseEstimator):
+    """Tree-based GAM classifier.
+    Uses cyclical boosting to fit a GAM with small trees.
+    Simplified version of the explainable boosting machine described in https://github.com/interpretml/interpret
+    Only works for binary classification.
+    Fits a scalar bias to the mean.
+    """
+
+    def __init__(
+        self,
+        n_boosting_rounds=100,
+        max_leaf_nodes=3,
+        reg_param=0.0,
+        learning_rate: float = 0.01,
+        n_boosting_rounds_marginal=0,
+        max_leaf_nodes_marginal=2,
+        reg_param_marginal=0.0,
+        fit_linear_marginal=None,
+        random_state=None,
+    ):
+        """
+        Params
+        ------
+        n_boosting_rounds : int
+            Number of boosting rounds for the cyclic boosting.
+        max_leaf_nodes : int
+            Maximum number of leaf nodes for the trees in the cyclic boosting.
+        reg_param : float
+            Regularization parameter for the cyclic boosting.
+        learning_rate: float
+            Learning rate for the cyclic boosting.
+        n_boosting_rounds_marginal : int
+            Number of boosting rounds for the marginal boosting.
+        max_leaf_nodes_marginal : int
+            Maximum number of leaf nodes for the trees in the marginal boosting.
+        reg_param_marginal : float
+            Regularization parameter for the marginal boosting.
+        fit_linear_marginal : str [None, "None", "ridge", "NNLS"]
+            Whether to fit a linear model to the marginal effects.
+            NNLS for non-negative least squares
+            ridge for ridge regression
+            None for no linear model
+
+        random_state : int
+            Random seed.
+        """
+        self.n_boosting_rounds = n_boosting_rounds
+        self.max_leaf_nodes = max_leaf_nodes
+        self.reg_param = reg_param
+        self.learning_rate = learning_rate
+        self.max_leaf_nodes_marginal = max_leaf_nodes_marginal
+        self.reg_param_marginal = reg_param_marginal
+        self.n_boosting_rounds_marginal = n_boosting_rounds_marginal
+        self.fit_linear_marginal = fit_linear_marginal
+        self.random_state = random_state
+
+    def fit(self, X, y, sample_weight=None, validation_frac=0.15):
+        X, y = check_X_y(X, y, accept_sparse=False, multi_output=False)
+        if isinstance(self, ClassifierMixin):
+            check_classification_targets(y)
+        sample_weight = _check_sample_weight(sample_weight, X, dtype=None)
+
+        # split into train and validation for early stopping
+        (
+            X_train,
+            X_val,
+            y_train,
+            y_val,
+            sample_weight_train,
+            sample_weight_val,
+        ) = train_test_split(
+            X,
+            y,
+            sample_weight,
+            test_size=validation_frac,
+            random_state=self.random_state,
+        )
+
+        self.estimators_marginal = []
+        self.estimators_ = []
+        self.bias_ = np.mean(y)
+
+        if self.n_boosting_rounds_marginal > 0:
+            self._marginal_fit(
+                X_train,
+                y_train,
+                sample_weight_train,
+            )
+
+        if self.n_boosting_rounds > 0:
+            self._cyclic_boost(
+                X_train,
+                y_train,
+                sample_weight_train,
+                X_val,
+                y_val,
+                sample_weight_val,
+            )
+
+        return self
+
+    def _marginal_fit(
+        self,
+        X_train,
+        y_train,
+        sample_weight_train,
+    ):
+        """Fit a gbdt estimator for each feature independently.
+        Store in self.estimators_marginal"""
+        residuals_train = y_train - self.predict_proba(X_train)[:, 1]
+        p = X_train.shape[1]
+        for feature_num in range(p):
+            X_ = np.zeros_like(X_train)
+            X_[:, feature_num] = X_train[:, feature_num]
+            est = GradientBoostingRegressor(
+                max_leaf_nodes=self.max_leaf_nodes_marginal,
+                random_state=self.random_state,
+                n_estimators=self.n_boosting_rounds_marginal,
+            )
+            est.fit(X_, residuals_train, sample_weight=sample_weight_train)
+            if self.reg_param_marginal > 0:
+                est = imodels.HSTreeRegressor(est, reg_param=self.reg_param_marginal)
+            self.estimators_marginal.append(est)
+
+        if (
+            self.fit_linear_marginal is not None
+            and not self.fit_linear_marginal == "None"
+        ):
+            if self.fit_linear_marginal.lower() == "ridge":
+                linear_marginal = RidgeCV(fit_intercept=False)
+            elif self.fit_linear_marginal.lower() == "nnls":
+                linear_marginal = LinearRegression(fit_intercept=False, positive=True)
+            linear_marginal.fit(
+                np.array([est.predict(X_train) for est in self.estimators_marginal]).T,
+                residuals_train,
+                sample_weight_train,
+            )
+            self.marginal_coef_ = linear_marginal.coef_
+            self.lin = linear_marginal
+        else:
+            self.marginal_coef_ = np.ones(p) / p
+
+    def _cyclic_boost(
+        self, X_train, y_train, sample_weight_train, X_val, y_val, sample_weight_val
+    ):
+        """Apply cyclic boosting, storing trees in self.estimators_"""
+
+        residuals_train = y_train - self.predict_proba(X_train)[:, 1]
+        mse_val = np.average(
+            np.square(y_val - self.predict_proba(X_val)[:, 1]),
+            weights=sample_weight_val,
+        )
+        for boosting_round in range(self.n_boosting_rounds):
+            for feature_num in range(X_train.shape[1]):
+                X_ = np.zeros_like(X_train)
+                X_[:, feature_num] = X_train[:, feature_num]
+                est = DecisionTreeRegressor(
+                    max_leaf_nodes=self.max_leaf_nodes,
+                    random_state=self.random_state,
+                )
+                est.fit(X_, residuals_train, sample_weight=sample_weight_train)
+                succesfully_split_on_feature = np.all(
+                    (est.tree_.feature[0] == feature_num) | (est.tree_.feature[0] == -2)
+                )
+                if not succesfully_split_on_feature:
+                    continue
+                if self.reg_param > 0:
+                    est = imodels.HSTreeRegressor(est, reg_param=self.reg_param)
+                self.estimators_.append(est)
+                residuals_train = residuals_train - self.learning_rate * est.predict(
+                    X_train
+                )
+
+            # early stopping if validation error does not decrease
+            mse_val_new = np.average(
+                np.square(y_val - self.predict_proba(X_val)[:, 1]),
+                weights=sample_weight_val,
+            )
+            # print(f"mse val: {mse_val:.4f} mse_val_new: {mse_val_new:.4f}")
+            if mse_val_new >= mse_val:
+                return
+            else:
+                mse_val = mse_val_new
+
+    def predict_proba(self, X):
+        X = check_array(X, accept_sparse=False, dtype=None)
+        check_is_fitted(self)
+        probs1 = np.ones(X.shape[0]) * self.bias_
+        for i, est in enumerate(self.estimators_marginal):
+            probs1 += est.predict(X) * self.marginal_coef_[i]
+        for est in self.estimators_:
+            probs1 += self.learning_rate * est.predict(X)
+        probs1 = np.clip(probs1, a_min=0, a_max=1)
+        return np.array([1 - probs1, probs1]).T
+
+    def predict(self, X):
+        if isinstance(self, RegressorMixin):
+            return self.predict_proba(X)[:, 1]
+        elif isinstance(self, ClassifierMixin):
+            return np.argmax(self.predict_proba(X), axis=1)
+
+    def get_shape_function_vals(self, X, max_evals=100):
+        """Uses predict_proba to compute shape_function
+        Returns
+        -------
+        feature_vals_list : list of arrays
+            The values of each feature for which the shape function is evaluated.
+        shape_function_vals_list : list of arrays
+            The shape function evaluated at each value of the corresponding feature.
+        """
+        p = X.shape[1]
+        dummy_input = np.zeros((1, p))
+        base = self.predict_proba(dummy_input)[:, 1][0]
+        feature_vals_list = []
+        shape_function_vals_list = []
+        for feat_num in range(p):
+            feature_vals = sorted(np.unique(X[:, feat_num]))
+            while len(feature_vals) > max_evals:
+                feature_vals = feature_vals[::2]
+            dummy_input = np.zeros((len(feature_vals), p))
+            dummy_input[:, feat_num] = feature_vals
+            shape_function_vals = self.predict_proba(dummy_input)[:, 1] - base
+            feature_vals_list.append(feature_vals)
+            shape_function_vals_list.append(shape_function_vals.tolist())
+        return feature_vals_list, shape_function_vals_list
+
+
+class TreeGAMRegressor(TreeGAM, RegressorMixin):
+    ...
+
+
+class TreeGAMClassifier(TreeGAM, ClassifierMixin):
+    ...
+
+
+if __name__ == "__main__":
+    X, y, feature_names = imodels.get_clean_dataset("heart")
+    X, X_test, y_train, y_test = train_test_split(X, y, random_state=42)
+    gam = TreeGAMClassifier()
+    gam.fit(X, y_train)
+
+    # check roc auc score
+    y_pred = gam.predict_proba(X_test)[:, 1]
+    print("train roc auc score:", roc_auc_score(y_train, gam.predict_proba(X)[:, 1]))
+    print("test roc auc score:", roc_auc_score(y_test, y_pred))
+    print(
+        "accs",
+        accuracy_score(y_train, gam.predict(X)),
+        accuracy_score(y_test, gam.predict(X_test)),
+        "imb",
+        np.mean(y_train),
+        np.mean(y_test),
+    )
    +
    +
    +
    +
    +
    +
    +
    +
    +
    +

    Classes

    +
    +
    +class TreeGAM +(n_boosting_rounds=100, max_leaf_nodes=3, reg_param=0.0, learning_rate:Β floatΒ =Β 0.01, n_boosting_rounds_marginal=0, max_leaf_nodes_marginal=2, reg_param_marginal=0.0, fit_linear_marginal=None, random_state=None) +
    +
    +

    Tree-based GAM classifier. +Uses cyclical boosting to fit a GAM with small trees. +Simplified version of the explainable boosting machine described in https://github.com/interpretml/interpret +Only works for binary classification. +Fits a scalar bias to the mean.

    +

    Params

    +

    n_boosting_rounds : int +Number of boosting rounds for the cyclic boosting. +max_leaf_nodes : int +Maximum number of leaf nodes for the trees in the cyclic boosting. +reg_param : float +Regularization parameter for the cyclic boosting. +learning_rate: float +Learning rate for the cyclic boosting. +n_boosting_rounds_marginal : int +Number of boosting rounds for the marginal boosting. +max_leaf_nodes_marginal : int +Maximum number of leaf nodes for the trees in the marginal boosting. +reg_param_marginal : float +Regularization parameter for the marginal boosting. +fit_linear_marginal : str [None, "None", "ridge", "NNLS"] +Whether to fit a linear model to the marginal effects. +NNLS for non-negative least squares +ridge for ridge regression +None for no linear model

    +

    random_state : int +Random seed.

    +
    + +Expand source code + +
    class TreeGAM(BaseEstimator):
+    """Tree-based GAM classifier.
+    Uses cyclical boosting to fit a GAM with small trees.
+    Simplified version of the explainable boosting machine described in https://github.com/interpretml/interpret
+    Only works for binary classification.
+    Fits a scalar bias to the mean.
+    """
+
+    def __init__(
+        self,
+        n_boosting_rounds=100,
+        max_leaf_nodes=3,
+        reg_param=0.0,
+        learning_rate: float = 0.01,
+        n_boosting_rounds_marginal=0,
+        max_leaf_nodes_marginal=2,
+        reg_param_marginal=0.0,
+        fit_linear_marginal=None,
+        random_state=None,
+    ):
+        """
+        Params
+        ------
+        n_boosting_rounds : int
+            Number of boosting rounds for the cyclic boosting.
+        max_leaf_nodes : int
+            Maximum number of leaf nodes for the trees in the cyclic boosting.
+        reg_param : float
+            Regularization parameter for the cyclic boosting.
+        learning_rate: float
+            Learning rate for the cyclic boosting.
+        n_boosting_rounds_marginal : int
+            Number of boosting rounds for the marginal boosting.
+        max_leaf_nodes_marginal : int
+            Maximum number of leaf nodes for the trees in the marginal boosting.
+        reg_param_marginal : float
+            Regularization parameter for the marginal boosting.
+        fit_linear_marginal : str [None, "None", "ridge", "NNLS"]
+            Whether to fit a linear model to the marginal effects.
+            NNLS for non-negative least squares
+            ridge for ridge regression
+            None for no linear model
+
+        random_state : int
+            Random seed.
+        """
+        self.n_boosting_rounds = n_boosting_rounds
+        self.max_leaf_nodes = max_leaf_nodes
+        self.reg_param = reg_param
+        self.learning_rate = learning_rate
+        self.max_leaf_nodes_marginal = max_leaf_nodes_marginal
+        self.reg_param_marginal = reg_param_marginal
+        self.n_boosting_rounds_marginal = n_boosting_rounds_marginal
+        self.fit_linear_marginal = fit_linear_marginal
+        self.random_state = random_state
+
+    def fit(self, X, y, sample_weight=None, validation_frac=0.15):
+        X, y = check_X_y(X, y, accept_sparse=False, multi_output=False)
+        if isinstance(self, ClassifierMixin):
+            check_classification_targets(y)
+        sample_weight = _check_sample_weight(sample_weight, X, dtype=None)
+
+        # split into train and validation for early stopping
+        (
+            X_train,
+            X_val,
+            y_train,
+            y_val,
+            sample_weight_train,
+            sample_weight_val,
+        ) = train_test_split(
+            X,
+            y,
+            sample_weight,
+            test_size=validation_frac,
+            random_state=self.random_state,
+        )
+
+        self.estimators_marginal = []
+        self.estimators_ = []
+        self.bias_ = np.mean(y)
+
+        if self.n_boosting_rounds_marginal > 0:
+            self._marginal_fit(
+                X_train,
+                y_train,
+                sample_weight_train,
+            )
+
+        if self.n_boosting_rounds > 0:
+            self._cyclic_boost(
+                X_train,
+                y_train,
+                sample_weight_train,
+                X_val,
+                y_val,
+                sample_weight_val,
+            )
+
+        return self
+
+    def _marginal_fit(
+        self,
+        X_train,
+        y_train,
+        sample_weight_train,
+    ):
+        """Fit a gbdt estimator for each feature independently.
+        Store in self.estimators_marginal"""
+        residuals_train = y_train - self.predict_proba(X_train)[:, 1]
+        p = X_train.shape[1]
+        for feature_num in range(p):
+            X_ = np.zeros_like(X_train)
+            X_[:, feature_num] = X_train[:, feature_num]
+            est = GradientBoostingRegressor(
+                max_leaf_nodes=self.max_leaf_nodes_marginal,
+                random_state=self.random_state,
+                n_estimators=self.n_boosting_rounds_marginal,
+            )
+            est.fit(X_, residuals_train, sample_weight=sample_weight_train)
+            if self.reg_param_marginal > 0:
+                est = imodels.HSTreeRegressor(est, reg_param=self.reg_param_marginal)
+            self.estimators_marginal.append(est)
+
+        if (
+            self.fit_linear_marginal is not None
+            and not self.fit_linear_marginal == "None"
+        ):
+            if self.fit_linear_marginal.lower() == "ridge":
+                linear_marginal = RidgeCV(fit_intercept=False)
+            elif self.fit_linear_marginal.lower() == "nnls":
+                linear_marginal = LinearRegression(fit_intercept=False, positive=True)
+            linear_marginal.fit(
+                np.array([est.predict(X_train) for est in self.estimators_marginal]).T,
+                residuals_train,
+                sample_weight_train,
+            )
+            self.marginal_coef_ = linear_marginal.coef_
+            self.lin = linear_marginal
+        else:
+            self.marginal_coef_ = np.ones(p) / p
+
+    def _cyclic_boost(
+        self, X_train, y_train, sample_weight_train, X_val, y_val, sample_weight_val
+    ):
+        """Apply cyclic boosting, storing trees in self.estimators_"""
+
+        residuals_train = y_train - self.predict_proba(X_train)[:, 1]
+        mse_val = np.average(
+            np.square(y_val - self.predict_proba(X_val)[:, 1]),
+            weights=sample_weight_val,
+        )
+        for boosting_round in range(self.n_boosting_rounds):
+            for feature_num in range(X_train.shape[1]):
+                X_ = np.zeros_like(X_train)
+                X_[:, feature_num] = X_train[:, feature_num]
+                est = DecisionTreeRegressor(
+                    max_leaf_nodes=self.max_leaf_nodes,
+                    random_state=self.random_state,
+                )
+                est.fit(X_, residuals_train, sample_weight=sample_weight_train)
+                succesfully_split_on_feature = np.all(
+                    (est.tree_.feature[0] == feature_num) | (est.tree_.feature[0] == -2)
+                )
+                if not succesfully_split_on_feature:
+                    continue
+                if self.reg_param > 0:
+                    est = imodels.HSTreeRegressor(est, reg_param=self.reg_param)
+                self.estimators_.append(est)
+                residuals_train = residuals_train - self.learning_rate * est.predict(
+                    X_train
+                )
+
+            # early stopping if validation error does not decrease
+            mse_val_new = np.average(
+                np.square(y_val - self.predict_proba(X_val)[:, 1]),
+                weights=sample_weight_val,
+            )
+            # print(f"mse val: {mse_val:.4f} mse_val_new: {mse_val_new:.4f}")
+            if mse_val_new >= mse_val:
+                return
+            else:
+                mse_val = mse_val_new
+
+    def predict_proba(self, X):
+        X = check_array(X, accept_sparse=False, dtype=None)
+        check_is_fitted(self)
+        probs1 = np.ones(X.shape[0]) * self.bias_
+        for i, est in enumerate(self.estimators_marginal):
+            probs1 += est.predict(X) * self.marginal_coef_[i]
+        for est in self.estimators_:
+            probs1 += self.learning_rate * est.predict(X)
+        probs1 = np.clip(probs1, a_min=0, a_max=1)
+        return np.array([1 - probs1, probs1]).T
+
+    def predict(self, X):
+        if isinstance(self, RegressorMixin):
+            return self.predict_proba(X)[:, 1]
+        elif isinstance(self, ClassifierMixin):
+            return np.argmax(self.predict_proba(X), axis=1)
+
+    def get_shape_function_vals(self, X, max_evals=100):
+        """Uses predict_proba to compute shape_function
+        Returns
+        -------
+        feature_vals_list : list of arrays
+            The values of each feature for which the shape function is evaluated.
+        shape_function_vals_list : list of arrays
+            The shape function evaluated at each value of the corresponding feature.
+        """
+        p = X.shape[1]
+        dummy_input = np.zeros((1, p))
+        base = self.predict_proba(dummy_input)[:, 1][0]
+        feature_vals_list = []
+        shape_function_vals_list = []
+        for feat_num in range(p):
+            feature_vals = sorted(np.unique(X[:, feat_num]))
+            while len(feature_vals) > max_evals:
+                feature_vals = feature_vals[::2]
+            dummy_input = np.zeros((len(feature_vals), p))
+            dummy_input[:, feat_num] = feature_vals
+            shape_function_vals = self.predict_proba(dummy_input)[:, 1] - base
+            feature_vals_list.append(feature_vals)
+            shape_function_vals_list.append(shape_function_vals.tolist())
+        return feature_vals_list, shape_function_vals_list
    +
    +

    Ancestors

    +
      +
    • sklearn.base.BaseEstimator
      • +
    +

    Subclasses

    + +

    Methods

    +
    +
    +def fit(self, X, y, sample_weight=None, validation_frac=0.15) +
    +
    +
    +
    + +Expand source code + +
    def fit(self, X, y, sample_weight=None, validation_frac=0.15):
+    X, y = check_X_y(X, y, accept_sparse=False, multi_output=False)
+    if isinstance(self, ClassifierMixin):
+        check_classification_targets(y)
+    sample_weight = _check_sample_weight(sample_weight, X, dtype=None)
+
+    # split into train and validation for early stopping
+    (
+        X_train,
+        X_val,
+        y_train,
+        y_val,
+        sample_weight_train,
+        sample_weight_val,
+    ) = train_test_split(
+        X,
+        y,
+        sample_weight,
+        test_size=validation_frac,
+        random_state=self.random_state,
+    )
+
+    self.estimators_marginal = []
+    self.estimators_ = []
+    self.bias_ = np.mean(y)
+
+    if self.n_boosting_rounds_marginal > 0:
+        self._marginal_fit(
+            X_train,
+            y_train,
+            sample_weight_train,
+        )
+
+    if self.n_boosting_rounds > 0:
+        self._cyclic_boost(
+            X_train,
+            y_train,
+            sample_weight_train,
+            X_val,
+            y_val,
+            sample_weight_val,
+        )
+
+    return self
    +
    +
    +
    +def get_shape_function_vals(self, X, max_evals=100) +
    +
    +

    Uses predict_proba to compute shape_function +Returns

    +
    +
    +
    feature_vals_list : list of arrays
    +
    The values of each feature for which the shape function is evaluated.
    +
    shape_function_vals_list : list of arrays
    +
    The shape function evaluated at each value of the corresponding feature.
    +
    +
    + +Expand source code + +
    def get_shape_function_vals(self, X, max_evals=100):
+    """Uses predict_proba to compute shape_function
+    Returns
+    -------
+    feature_vals_list : list of arrays
+        The values of each feature for which the shape function is evaluated.
+    shape_function_vals_list : list of arrays
+        The shape function evaluated at each value of the corresponding feature.
+    """
+    p = X.shape[1]
+    dummy_input = np.zeros((1, p))
+    base = self.predict_proba(dummy_input)[:, 1][0]
+    feature_vals_list = []
+    shape_function_vals_list = []
+    for feat_num in range(p):
+        feature_vals = sorted(np.unique(X[:, feat_num]))
+        while len(feature_vals) > max_evals:
+            feature_vals = feature_vals[::2]
+        dummy_input = np.zeros((len(feature_vals), p))
+        dummy_input[:, feat_num] = feature_vals
+        shape_function_vals = self.predict_proba(dummy_input)[:, 1] - base
+        feature_vals_list.append(feature_vals)
+        shape_function_vals_list.append(shape_function_vals.tolist())
+    return feature_vals_list, shape_function_vals_list
    +
    +
    +
    +def predict(self, X) +
    +
    +
    +
    + +Expand source code + +
    def predict(self, X):
+    if isinstance(self, RegressorMixin):
+        return self.predict_proba(X)[:, 1]
+    elif isinstance(self, ClassifierMixin):
+        return np.argmax(self.predict_proba(X), axis=1)
    +
    +
    +
    +def predict_proba(self, X) +
    +
    +
    +
    + +Expand source code + +
    def predict_proba(self, X):
+    X = check_array(X, accept_sparse=False, dtype=None)
+    check_is_fitted(self)
+    probs1 = np.ones(X.shape[0]) * self.bias_
+    for i, est in enumerate(self.estimators_marginal):
+        probs1 += est.predict(X) * self.marginal_coef_[i]
+    for est in self.estimators_:
+        probs1 += self.learning_rate * est.predict(X)
+    probs1 = np.clip(probs1, a_min=0, a_max=1)
+    return np.array([1 - probs1, probs1]).T
    +
    +
    +
    +
    +
    +class TreeGAMClassifier +(n_boosting_rounds=100, max_leaf_nodes=3, reg_param=0.0, learning_rate:Β floatΒ =Β 0.01, n_boosting_rounds_marginal=0, max_leaf_nodes_marginal=2, reg_param_marginal=0.0, fit_linear_marginal=None, random_state=None) +
    +
    +

    Tree-based GAM classifier. +Uses cyclical boosting to fit a GAM with small trees. +Simplified version of the explainable boosting machine described in https://github.com/interpretml/interpret +Only works for binary classification. +Fits a scalar bias to the mean.

    +

    Params

    +

    n_boosting_rounds : int +Number of boosting rounds for the cyclic boosting. +max_leaf_nodes : int +Maximum number of leaf nodes for the trees in the cyclic boosting. +reg_param : float +Regularization parameter for the cyclic boosting. +learning_rate: float +Learning rate for the cyclic boosting. +n_boosting_rounds_marginal : int +Number of boosting rounds for the marginal boosting. +max_leaf_nodes_marginal : int +Maximum number of leaf nodes for the trees in the marginal boosting. +reg_param_marginal : float +Regularization parameter for the marginal boosting. +fit_linear_marginal : str [None, "None", "ridge", "NNLS"] +Whether to fit a linear model to the marginal effects. +NNLS for non-negative least squares +ridge for ridge regression +None for no linear model

    +

    random_state : int +Random seed.

    +
    + +Expand source code + +
    class TreeGAMClassifier(TreeGAM, ClassifierMixin):
+    ...
    +
    +

    Ancestors

    +
      +
    • TreeGAM
      • +
    • sklearn.base.BaseEstimator
      • +
    • sklearn.base.ClassifierMixin
      • +
    +

    Inherited members

    + +
    +
    +class TreeGAMRegressor +(n_boosting_rounds=100, max_leaf_nodes=3, reg_param=0.0, learning_rate:Β floatΒ =Β 0.01, n_boosting_rounds_marginal=0, max_leaf_nodes_marginal=2, reg_param_marginal=0.0, fit_linear_marginal=None, random_state=None) +
    +
    +

    Tree-based GAM classifier. +Uses cyclical boosting to fit a GAM with small trees. +Simplified version of the explainable boosting machine described in https://github.com/interpretml/interpret +Only works for binary classification. +Fits a scalar bias to the mean.

    +

    Params

    +

    n_boosting_rounds : int +Number of boosting rounds for the cyclic boosting. +max_leaf_nodes : int +Maximum number of leaf nodes for the trees in the cyclic boosting. +reg_param : float +Regularization parameter for the cyclic boosting. +learning_rate: float +Learning rate for the cyclic boosting. +n_boosting_rounds_marginal : int +Number of boosting rounds for the marginal boosting. +max_leaf_nodes_marginal : int +Maximum number of leaf nodes for the trees in the marginal boosting. +reg_param_marginal : float +Regularization parameter for the marginal boosting. +fit_linear_marginal : str [None, "None", "ridge", "NNLS"] +Whether to fit a linear model to the marginal effects. +NNLS for non-negative least squares +ridge for ridge regression +None for no linear model

    +

    random_state : int +Random seed.

    +
    + +Expand source code + +
    class TreeGAMRegressor(TreeGAM, RegressorMixin):
+    ...
    +
    +

    Ancestors

    +
      +
    • TreeGAM
      • +
    • sklearn.base.BaseEstimator
      • +
    • sklearn.base.RegressorMixin
      • +
    +

    Inherited members

    + +
    +
    +
    +
    + +
    + + + + + + + \ No newline at end of file diff --git a/docs/index.html b/docs/index.html index 25c62815..f923b0fd 100644 --- a/docs/index.html +++ b/docs/index.html @@ -158,7 +158,7 @@

    Supported models

    Tree GAM -πŸ—‚οΈ, πŸ”—, πŸ“„ +πŸ—‚οΈ, πŸ”—, πŸ“„ Generalized additive model fit with short boosted trees @@ -363,6 +363,12 @@

    Support for different tasks

    Requires extra dependencies for speed +Tree GAM +TreeGAMClassifier +TreeGAMRegressor + + + Greedy tree sums (FIGS) FIGSClassifier FIGSRegressor @@ -444,6 +450,9 @@

    H

    HS Example. HS applies post-hoc regularization to any decision tree by shrinking each node towards its parent.

    +

    MDI+: A Flexible Random Forest-Based Feature Importance Framework

    +

    πŸ“„ Paper, πŸ“Œ Citation

    +

    MDI+ is a novel feature importance framework, which generalizes the popular mean decrease in impurity (MDI) importance score for random forests. At its core, MDI+ expands upon a recently discovered connection between linear regression and decision trees. In doing so, MDI+ enables practitioners to (1) tailor the feature importance computation to the data/problem structure and (2) incorporate additional features or knowledge to mitigate known biases of decision trees. In both real data case studies and extensive real-data-inspired simulations, MDI+ outperforms commonly used feature importance measures (e.g., MDI, permutation-based scores, and TreeSHAP) by substantional margins.

    References

    Readings @@ -511,7 +520,7 @@

    References

    # Github repo available [here](https://github.com/csinva/imodels) from .algebraic.slim import SLIMRegressor, SLIMClassifier -from .algebraic.gam import TreeGAMClassifier +from .algebraic.tree_gam import TreeGAMClassifier, TreeGAMRegressor from .discretization.discretizer import RFDiscretizer, BasicDiscretizer from .discretization.mdlp import MDLPDiscretizer, BRLDiscretizer from .experimental.bartpy import BART @@ -529,27 +538,64 @@

    References

    from .rule_set.skope_rules import SkopeRulesClassifier from .rule_set.slipper import SlipperClassifier from .tree.c45_tree.c45_tree import C45TreeClassifier -from .tree.cart_ccp import DecisionTreeCCPClassifier, DecisionTreeCCPRegressor, HSDecisionTreeCCPClassifierCV, \ - HSDecisionTreeCCPRegressorCV +from .tree.cart_ccp import ( + DecisionTreeCCPClassifier, + DecisionTreeCCPRegressor, + HSDecisionTreeCCPClassifierCV, + HSDecisionTreeCCPRegressorCV, +) + # from .tree.iterative_random_forest.iterative_random_forest import IRFClassifier # from .tree.optimal_classification_tree import OptimalTreeModel from .tree.cart_wrapper import GreedyTreeClassifier, GreedyTreeRegressor from .tree.figs import FIGSRegressor, FIGSClassifier, FIGSRegressorCV, FIGSClassifierCV from .tree.gosdt.pygosdt import OptimalTreeClassifier -from .tree.gosdt.pygosdt_shrinkage import HSOptimalTreeClassifier, HSOptimalTreeClassifierCV -from .tree.hierarchical_shrinkage import HSTreeRegressor, HSTreeClassifier, HSTreeRegressorCV, HSTreeClassifierCV +from .tree.gosdt.pygosdt_shrinkage import ( + HSOptimalTreeClassifier, + HSOptimalTreeClassifierCV, +) +from .tree.hierarchical_shrinkage import ( + HSTreeRegressor, + HSTreeClassifier, + HSTreeRegressorCV, + HSTreeClassifierCV, +) from .tree.tao import TaoTreeClassifier, TaoTreeRegressor from .util.data_util import get_clean_dataset from .util.distillation import DistilledRegressor from .util.explain_errors import explain_classification_errors -CLASSIFIERS = [BayesianRuleListClassifier, GreedyRuleListClassifier, SkopeRulesClassifier, - BoostedRulesClassifier, SLIMClassifier, SlipperClassifier, BayesianRuleSetClassifier, - C45TreeClassifier, OptimalTreeClassifier, OptimalRuleListClassifier, OneRClassifier, - SlipperClassifier, RuleFitClassifier, TaoTreeClassifier, - FIGSClassifier, HSTreeClassifier, HSTreeClassifierCV] # , IRFClassifier -REGRESSORS = [RuleFitRegressor, SLIMRegressor, GreedyTreeClassifier, FIGSRegressor, - TaoTreeRegressor, HSTreeRegressor, HSTreeRegressorCV, BART] +CLASSIFIERS = [ + BayesianRuleListClassifier, + GreedyRuleListClassifier, + SkopeRulesClassifier, + BoostedRulesClassifier, + SLIMClassifier, + SlipperClassifier, + BayesianRuleSetClassifier, + C45TreeClassifier, + OptimalTreeClassifier, + OptimalRuleListClassifier, + OneRClassifier, + SlipperClassifier, + RuleFitClassifier, + TaoTreeClassifier, + TreeGAMClassifier, + FIGSClassifier, + HSTreeClassifier, + HSTreeClassifierCV, +] # , IRFClassifier +REGRESSORS = [ + RuleFitRegressor, + SLIMRegressor, + GreedyTreeClassifier, + FIGSRegressor, + TaoTreeRegressor, + TreeGAMRegressor, + HSTreeRegressor, + HSTreeRegressorCV, + BART, +] ESTIMATORS = CLASSIFIERS + REGRESSORS DISCRETIZERS = [RFDiscretizer, BasicDiscretizer, MDLPDiscretizer, BRLDiscretizer]
    @@ -609,7 +655,14 @@

    Index πŸ”

  • Support for different tasks -
  • References
    • + +
  • Our favorite models +
    • +
  • References
    • @@ -438,8 +518,12 @@

    Params

    Expand source code 
    class HSTree:
-    def __init__(self, estimator_: BaseEstimator = DecisionTreeClassifier(max_leaf_nodes=20),
-                 reg_param: float = 1, shrinkage_scheme_: str = 'node_based'):
+    def __init__(
+        self,
+        estimator_: BaseEstimator = DecisionTreeClassifier(max_leaf_nodes=20),
+        reg_param: float = 1,
+        shrinkage_scheme_: str = "node_based",
+    ):
         """HSTree (Tree with hierarchical shrinkage applied).
         Hierarchical shinkage is an extremely fast post-hoc regularization method which works on any decision tree (or tree-based ensemble, such as Random Forest).
         It does not modify the tree structure, and instead regularizes the tree by shrinking the prediction over each node towards the sample means of its ancestors (using a single regularization parameter).
@@ -454,11 +538,11 @@ 
    Params
    
 
         reg_param: float
             Higher is more regularization (can be arbitrarily large, should not be < 0)
-        
+
         shrinkage_scheme: str
-            Experimental: Used to experiment with different forms of shrinkage. options are: 
+            Experimental: Used to experiment with different forms of shrinkage. options are:
                 (i) node_based shrinks based on number of samples in parent node
-                (ii) leaf_based only shrinks leaf nodes based on number of leaf samples 
+                (ii) leaf_based only shrinks leaf nodes based on number of leaf samples
                 (iii) constant shrinks every node by a constant lambda
         """
         super().__init__()
@@ -470,41 +554,54 @@ 
    Params
    
 
     def get_params(self, deep=True):
         if deep:
-            return deepcopy({'reg_param': self.reg_param, 'estimator_': self.estimator_,
-                             'shrinkage_scheme_': self.shrinkage_scheme_})
-        return {'reg_param': self.reg_param, 'estimator_': self.estimator_,
-                'shrinkage_scheme_': self.shrinkage_scheme_}
+            return deepcopy(
+                {
+                    "reg_param": self.reg_param,
+                    "estimator_": self.estimator_,
+                    "shrinkage_scheme_": self.shrinkage_scheme_,
+                }
+            )
+        return {
+            "reg_param": self.reg_param,
+            "estimator_": self.estimator_,
+            "shrinkage_scheme_": self.shrinkage_scheme_,
+        }
 
     def fit(self, X, y, sample_weight=None, *args, **kwargs):
         # remove feature_names if it exists (note: only works as keyword-arg)
-        feature_names = kwargs.pop('feature_names', None)  # None returned if not passed
+        feature_names = kwargs.pop("feature_names", None)  # None returned if not passed
         X, y, feature_names = check_fit_arguments(self, X, y, feature_names)
-        self.estimator_ = self.estimator_.fit(X, y, *args, sample_weight=sample_weight, **kwargs)
+        self.estimator_ = self.estimator_.fit(
+            X, y, *args, sample_weight=sample_weight, **kwargs
+        )
         self._shrink()
 
         # compute complexity
-        if hasattr(self.estimator_, 'tree_'):
+        if hasattr(self.estimator_, "tree_"):
             self.complexity_ = compute_tree_complexity(self.estimator_.tree_)
-        elif hasattr(self.estimator_, 'estimators_'):
+        elif hasattr(self.estimator_, "estimators_"):
             self.complexity_ = 0
             for i in range(len(self.estimator_.estimators_)):
                 t = deepcopy(self.estimator_.estimators_[i])
                 if isinstance(t, np.ndarray):
-                    assert t.size == 1, 'multiple trees stored under tree_?'
+                    assert t.size == 1, "multiple trees stored under tree_?"
                     t = t[0]
                 self.complexity_ += compute_tree_complexity(t.tree_)
         return self
 
-    def _shrink_tree(self, tree, reg_param, i=0, parent_val=None, parent_num=None, cum_sum=0):
-        """Shrink the tree
-        """
+    def _shrink_tree(
+        self, tree, reg_param, i=0, parent_val=None, parent_num=None, cum_sum=0
+    ):
+        """Shrink the tree"""
         if reg_param is None:
             reg_param = 1.0
         left = tree.children_left[i]
         right = tree.children_right[i]
         is_leaf = left == right
         n_samples = tree.weighted_n_node_samples[i]
-        if isinstance(self, RegressorMixin) or isinstance(self.estimator_, GradientBoostingClassifier):
+        if isinstance(self, RegressorMixin) or isinstance(
+            self.estimator_, GradientBoostingClassifier
+        ):
             val = deepcopy(tree.value[i, :, :])
         else:  # If classification, normalize to probability vector
             val = tree.value[i, :, :] / n_samples
@@ -516,42 +613,59 @@ 
    Params
    
 
         # if has parent
         else:
-            if self.shrinkage_scheme_ == 'node_based':
+            if self.shrinkage_scheme_ == "node_based":
                 val_new = (val - parent_val) / (1 + reg_param / parent_num)
-            elif self.shrinkage_scheme_ == 'constant':
+            elif self.shrinkage_scheme_ == "constant":
                 val_new = (val - parent_val) / (1 + reg_param)
             else:  # leaf_based
                 val_new = 0
             cum_sum += val_new
 
         # Step 2: Update node values
-        if self.shrinkage_scheme_ == 'node_based' or self.shrinkage_scheme_ == 'constant':
+        if (
+            self.shrinkage_scheme_ == "node_based"
+            or self.shrinkage_scheme_ == "constant"
+        ):
             tree.value[i, :, :] = cum_sum
         else:  # leaf_based
             if is_leaf:  # update node values if leaf_based
                 root_val = tree.value[0, :, :]
-                tree.value[i, :, :] = root_val + (val - root_val) / (1 + reg_param / n_samples)
+                tree.value[i, :, :] = root_val + (val - root_val) / (
+                    1 + reg_param / n_samples
+                )
             else:
                 tree.value[i, :, :] = val
 
                 # Step 3: Recurse if not leaf
         if not is_leaf:
-            self._shrink_tree(tree, reg_param, left,
-                              parent_val=val, parent_num=n_samples, cum_sum=deepcopy(cum_sum))
-            self._shrink_tree(tree, reg_param, right,
-                              parent_val=val, parent_num=n_samples, cum_sum=deepcopy(cum_sum))
+            self._shrink_tree(
+                tree,
+                reg_param,
+                left,
+                parent_val=val,
+                parent_num=n_samples,
+                cum_sum=deepcopy(cum_sum),
+            )
+            self._shrink_tree(
+                tree,
+                reg_param,
+                right,
+                parent_val=val,
+                parent_num=n_samples,
+                cum_sum=deepcopy(cum_sum),
+            )
 
             # edit the non-leaf nodes for later visualization (doesn't effect predictions)
 
         return tree
 
     def _shrink(self):
-        if hasattr(self.estimator_, 'tree_'):
+        if hasattr(self.estimator_, "tree_"):
             self._shrink_tree(self.estimator_.tree_, self.reg_param)
-        elif hasattr(self.estimator_, 'estimators_'):
+        elif hasattr(self.estimator_, "estimators_"):
             for t in self.estimator_.estimators_:
                 if isinstance(t, np.ndarray):
-                    assert t.size == 1, 'multiple trees stored under tree_?'
+                    assert t.size == 1, "multiple trees stored under tree_?"
                     t = t[0]
                 self._shrink_tree(t.tree_, self.reg_param)
 
@@ -559,24 +673,26 @@ 
    Params
    
         return self.estimator_.predict(X, *args, **kwargs)
 
     def predict_proba(self, X, *args, **kwargs):
-        if hasattr(self.estimator_, 'predict_proba'):
+        if hasattr(self.estimator_, "predict_proba"):
             return self.estimator_.predict_proba(X, *args, **kwargs)
         else:
             return NotImplemented
 
     def score(self, X, y, *args, **kwargs):
-        if hasattr(self.estimator_, 'score'):
+        if hasattr(self.estimator_, "score"):
             return self.estimator_.score(X, y, *args, **kwargs)
         else:
             return NotImplemented
 
     def __str__(self):
-        s = '> ------------------------------\n'
-        s += '> Decision Tree with Hierarchical Shrinkage\n'
-        s += '> \tPrediction is made by looking at the value in the appropriate leaf of the tree\n'
-        s += '> ------------------------------' + '\n'
-        if hasattr(self, 'feature_names') and self.feature_names is not None:
-            return s + export_text(self.estimator_, feature_names=self.feature_names, show_weights=True)
+        s = "> ------------------------------\n"
+        s += "> Decision Tree with Hierarchical Shrinkage\n"
+        s += "> \tPrediction is made by looking at the value in the appropriate leaf of the tree\n"
+        s += "> ------------------------------" + "\n"
+        if hasattr(self, "feature_names") and self.feature_names is not None:
+            return s + export_text(
+                self.estimator_, feature_names=self.feature_names, show_weights=True
+            )
         else:
             return s + export_text(self.estimator_, show_weights=True)
 
@@ -619,20 +735,22 @@ 
    Methods
    
 
 
    def fit(self, X, y, sample_weight=None, *args, **kwargs):
     # remove feature_names if it exists (note: only works as keyword-arg)
-    feature_names = kwargs.pop('feature_names', None)  # None returned if not passed
+    feature_names = kwargs.pop("feature_names", None)  # None returned if not passed
     X, y, feature_names = check_fit_arguments(self, X, y, feature_names)
-    self.estimator_ = self.estimator_.fit(X, y, *args, sample_weight=sample_weight, **kwargs)
+    self.estimator_ = self.estimator_.fit(
+        X, y, *args, sample_weight=sample_weight, **kwargs
+    )
     self._shrink()
 
     # compute complexity
-    if hasattr(self.estimator_, 'tree_'):
+    if hasattr(self.estimator_, "tree_"):
         self.complexity_ = compute_tree_complexity(self.estimator_.tree_)
-    elif hasattr(self.estimator_, 'estimators_'):
+    elif hasattr(self.estimator_, "estimators_"):
         self.complexity_ = 0
         for i in range(len(self.estimator_.estimators_)):
             t = deepcopy(self.estimator_.estimators_[i])
             if isinstance(t, np.ndarray):
-                assert t.size == 1, 'multiple trees stored under tree_?'
+                assert t.size == 1, "multiple trees stored under tree_?"
                 t = t[0]
             self.complexity_ += compute_tree_complexity(t.tree_)
     return self
    
@@ -649,10 +767,18 @@ 
    Methods
    
 
 
    def get_params(self, deep=True):
     if deep:
-        return deepcopy({'reg_param': self.reg_param, 'estimator_': self.estimator_,
-                         'shrinkage_scheme_': self.shrinkage_scheme_})
-    return {'reg_param': self.reg_param, 'estimator_': self.estimator_,
-            'shrinkage_scheme_': self.shrinkage_scheme_}
    
+        return deepcopy(
+            {
+                "reg_param": self.reg_param,
+                "estimator_": self.estimator_,
+                "shrinkage_scheme_": self.shrinkage_scheme_,
+            }
+        )
+    return {
+        "reg_param": self.reg_param,
+        "estimator_": self.estimator_,
+        "shrinkage_scheme_": self.shrinkage_scheme_,
+    }
    @@ -678,7 +804,7 @@

    Methods

    Expand source code 
    def predict_proba(self, X, *args, **kwargs):
-    if hasattr(self.estimator_, 'predict_proba'):
+    if hasattr(self.estimator_, "predict_proba"):
         return self.estimator_.predict_proba(X, *args, **kwargs)
     else:
         return NotImplemented
    @@ -694,7 +820,7 @@

    Methods

    Expand source code 
    def score(self, X, y, *args, **kwargs):
-    if hasattr(self.estimator_, 'score'):
+    if hasattr(self.estimator_, "score"):
         return self.estimator_.score(X, y, *args, **kwargs)
     else:
         return NotImplemented
    @@ -729,12 +855,17 @@

    Params

    Expand source code 
    class HSTreeClassifier(HSTree, ClassifierMixin):
-    def __init__(self, estimator_: BaseEstimator = DecisionTreeClassifier(max_leaf_nodes=20),
-                 reg_param: float = 1, shrinkage_scheme_: str = 'node_based'):
-        super().__init__(estimator_=estimator_,
-                         reg_param=reg_param,
-                         shrinkage_scheme_=shrinkage_scheme_,
-                         )
    + def __init__( + self, + estimator_: BaseEstimator = DecisionTreeClassifier(max_leaf_nodes=20), + reg_param: float = 1, + shrinkage_scheme_: str = "node_based", + ): + super().__init__( + estimator_=estimator_, + reg_param=reg_param, + shrinkage_scheme_=shrinkage_scheme_, + )

    Ancestors