2-model.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/grnc13/miniconda3/envs/DataScience/lib/python3.10/site-packages/tqdm/auto.py:22: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n",
      "  from .autonotebook import tqdm as notebook_tqdm\n"
     ]
    }
   ],
   "source": [
    "# Imports for data processing\n",
    "from sklearn.preprocessing import OneHotEncoder, MinMaxScaler, OrdinalEncoder\n",
    "from sklearn.compose import ColumnTransformer\n",
    "from sklearn.model_selection import KFold\n",
    "from imblearn.over_sampling import SMOTE\n",
    "from sklearn.impute import SimpleImputer\n",
    "from imblearn.pipeline import Pipeline\n",
    "\n",
    "# Imports for ML models\n",
    "from sklearn.linear_model import LogisticRegression\n",
    "from sklearn.ensemble import RandomForestClassifier\n",
    "from sklearn.neighbors import KNeighborsClassifier\n",
    "from sklearn.neural_network import MLPClassifier\n",
    "from sklearn.tree import DecisionTreeClassifier\n",
    "from sklearn.naive_bayes import GaussianNB\n",
    "from xgboost import XGBClassifier\n",
    "from sklearn.svm import SVC\n",
    "\n",
    "# Imports for evaluation metrics\n",
    "from sklearn.metrics import roc_auc_score\n",
    "\n",
    "# Imports for hyperparameter optimization\n",
    "import optuna as opt\n",
    "\n",
    "# Other imports\n",
    "from tqdm import tqdm\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "import pickle\n",
    "import sys\n",
    "\n",
    "# Ensure reproducibility\n",
    "random_state = 0"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Import data and perform train and validation split"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>gender</th>\n",
       "      <th>age</th>\n",
       "      <th>hypertension</th>\n",
       "      <th>heart_disease</th>\n",
       "      <th>ever_married</th>\n",
       "      <th>work_type</th>\n",
       "      <th>Residence_type</th>\n",
       "      <th>avg_glucose_level</th>\n",
       "      <th>bmi</th>\n",
       "      <th>smoking_status</th>\n",
       "      <th>stroke</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Male</td>\n",
       "      <td>82.0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>Yes</td>\n",
       "      <td>Private</td>\n",
       "      <td>Urban</td>\n",
       "      <td>144.90</td>\n",
       "      <td>26.4</td>\n",
       "      <td>smokes</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Male</td>\n",
       "      <td>4.0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>No</td>\n",
       "      <td>children</td>\n",
       "      <td>Rural</td>\n",
       "      <td>106.22</td>\n",
       "      <td>16.7</td>\n",
       "      <td>Unknown</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Male</td>\n",
       "      <td>58.0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>Yes</td>\n",
       "      <td>Private</td>\n",
       "      <td>Urban</td>\n",
       "      <td>79.95</td>\n",
       "      <td>25.9</td>\n",
       "      <td>never smoked</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Female</td>\n",
       "      <td>20.0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>No</td>\n",
       "      <td>Private</td>\n",
       "      <td>Rural</td>\n",
       "      <td>96.57</td>\n",
       "      <td>34.1</td>\n",
       "      <td>never smoked</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Female</td>\n",
       "      <td>10.0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>No</td>\n",
       "      <td>children</td>\n",
       "      <td>Rural</td>\n",
       "      <td>69.84</td>\n",
       "      <td>13.7</td>\n",
       "      <td>Unknown</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4083</th>\n",
       "      <td>Female</td>\n",
       "      <td>51.0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>Yes</td>\n",
       "      <td>Self-employed</td>\n",
       "      <td>Urban</td>\n",
       "      <td>232.89</td>\n",
       "      <td>34.0</td>\n",
       "      <td>smokes</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4084</th>\n",
       "      <td>Male</td>\n",
       "      <td>64.0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>Yes</td>\n",
       "      <td>Private</td>\n",
       "      <td>Urban</td>\n",
       "      <td>191.61</td>\n",
       "      <td>37.5</td>\n",
       "      <td>smokes</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4085</th>\n",
       "      <td>Male</td>\n",
       "      <td>37.0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>Yes</td>\n",
       "      <td>Self-employed</td>\n",
       "      <td>Rural</td>\n",
       "      <td>82.43</td>\n",
       "      <td>39.1</td>\n",
       "      <td>Unknown</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4086</th>\n",
       "      <td>Female</td>\n",
       "      <td>22.0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>No</td>\n",
       "      <td>Private</td>\n",
       "      <td>Rural</td>\n",
       "      <td>62.00</td>\n",
       "      <td>32.7</td>\n",
       "      <td>smokes</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4087</th>\n",
       "      <td>Female</td>\n",
       "      <td>19.0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>No</td>\n",
       "      <td>Private</td>\n",
       "      <td>Rural</td>\n",
       "      <td>57.40</td>\n",
       "      <td>22.9</td>\n",
       "      <td>Unknown</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>4088 rows × 11 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "      gender   age  hypertension  heart_disease ever_married      work_type  \\\n",
       "0       Male  82.0             0              1          Yes        Private   \n",
       "1       Male   4.0             0              0           No       children   \n",
       "2       Male  58.0             0              0          Yes        Private   \n",
       "3     Female  20.0             0              0           No        Private   \n",
       "4     Female  10.0             0              0           No       children   \n",
       "...      ...   ...           ...            ...          ...            ...   \n",
       "4083  Female  51.0             0              0          Yes  Self-employed   \n",
       "4084    Male  64.0             0              1          Yes        Private   \n",
       "4085    Male  37.0             0              0          Yes  Self-employed   \n",
       "4086  Female  22.0             0              0           No        Private   \n",
       "4087  Female  19.0             0              0           No        Private   \n",
       "\n",
       "     Residence_type  avg_glucose_level   bmi smoking_status  stroke  \n",
       "0             Urban             144.90  26.4         smokes       1  \n",
       "1             Rural             106.22  16.7        Unknown       0  \n",
       "2             Urban              79.95  25.9   never smoked       0  \n",
       "3             Rural              96.57  34.1   never smoked       0  \n",
       "4             Rural              69.84  13.7        Unknown       0  \n",
       "...             ...                ...   ...            ...     ...  \n",
       "4083          Urban             232.89  34.0         smokes       0  \n",
       "4084          Urban             191.61  37.5         smokes       1  \n",
       "4085          Rural              82.43  39.1        Unknown       0  \n",
       "4086          Rural              62.00  32.7         smokes       0  \n",
       "4087          Rural              57.40  22.9        Unknown       0  \n",
       "\n",
       "[4088 rows x 11 columns]"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Import data\n",
    "datapath = \"data/healthcare-dataset-stroke-data-train.csv\"\n",
    "df = pd.read_csv(datapath)\n",
    "df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Split the data into features and label\n",
    "features = [col for col in list(df.columns) if col!=\"stroke\"]\n",
    "X, y = df[features], df[\"stroke\"]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Create the data preprocessor"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "The following steps are performed for preprocessing the data:\n",
    "- Impute missing values with mean for the numerical columns\n",
    "- Ordinal encoding for binary categorical columns\n",
    "- One hot encoding for remaining categorical columns\n",
    "- MinMaxNormalization for numerical columns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Create the mean imputer and feature scaler\n",
    "num_features = [col for col in features if df[col].dtype in [\"int64\", \"float64\"]]\n",
    "imputer = SimpleImputer()\n",
    "scaler = MinMaxScaler()\n",
    "\n",
    "# Create numerical preprocessor\n",
    "num_preprocessor = Pipeline(steps=[\n",
    "    (\"imputer\", imputer),\n",
    "    (\"scaler\", scaler)\n",
    "])\n",
    "\n",
    "# Create ordinal encoder\n",
    "cat_features = [col for col in features if col not in num_features]\n",
    "ordinal_features = [col for col in cat_features if df[col].unique().shape[0] == 2]\n",
    "ordinal_encoder = OrdinalEncoder(handle_unknown=\"use_encoded_value\", unknown_value=-1)\n",
    "\n",
    "# Create the one-hot encoder\n",
    "one_hot_features = [col for col in cat_features if col not in ordinal_features]\n",
    "one_hot_encoder = OneHotEncoder(handle_unknown=\"ignore\")\n",
    "\n",
    "# Create categorical preprocessor\n",
    "cat_preprocessor = ColumnTransformer(transformers=[\n",
    "    (\"onehot\", one_hot_encoder, one_hot_features),\n",
    "    (\"ordinal\", ordinal_encoder, ordinal_features)\n",
    "])\n",
    "\n",
    "# Create the preprocessor\n",
    "preprocessor = ColumnTransformer(transformers=[\n",
    "    (\"num_preprocessor\", num_preprocessor, num_features),\n",
    "    (\"cat_preprocessor\", cat_preprocessor, cat_features)\n",
    "])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Finally, before going into the model selection phase, it is important to recall that the used dataset is imbalanced. More specifically, there are a lot fewer datapoints of patients who actually had a stroke. "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "To compensate for their smaller frequency, we can increase their importance by giving them more weight. The following code cell calculates the class weights for the training data."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{1: 20.181347150259068, 0: 1.0}"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "def get_class_weights(X, y):\n",
    "    # Create the class weights due to data imbalance\n",
    "    largest_class_size = max([X[y==c].shape[0] for c in y.unique()])\n",
    "    class_weights = {c: largest_class_size/X[y==c].shape[0] for c in y.unique()}\n",
    "    return class_weights\n",
    "    \n",
    "get_class_weights(X, y)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "However, some models are can't handle this type of argument to produce their results. For a more uniform approach across all models, an over-sampling technique, `SMOTE`, is applied using the `imblearn` package. This technique generates new datapoints for the minority class to balance the dataset."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Create dataset balancer\n",
    "resampler = SMOTE(random_state=random_state)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{1: 1.0, 0: 1.0}"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Check that SMOTE balances the dataset\n",
    "# NOTE: SMOTE here is applied to all data, later the data is split into training and validation, only training data should use SMOTE\n",
    "X_res, y_res = resampler.fit_resample(preprocessor.fit(X, y).transform(X), y)\n",
    "get_class_weights(X_res, y_res)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "As we can see from the class weights, SMOTE does indeed balance our dataset!"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Model selection"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "To perform model selection, we will use cross validation to better evaluate our models on validation data. One thing to note is that we can not use SMOTE in our validation pipeline, only for training. This is because we want to train the model to give the same importance to each class, but to evaluate it in a dataset as close to the test set as possible (imbalaced). To do this, we will write a costum cross validation function:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "def cross_validate(X, y, pipe, n_splits, verbose=False):\n",
    "    # Create folds for cross-validation\n",
    "    kf = KFold(n_splits=n_splits)\n",
    "\n",
    "    # Iterate folds for cross-validation\n",
    "    train_score, val_score = 0, 0\n",
    "    kfold_bar = kf.split(X)\n",
    "    if verbose:\n",
    "        kfold_bar = tqdm(kfold_bar, total=n_splits, desc=\"Cross-validating\", position=1, file=sys.stdout)\n",
    "    for train_idx, val_idx in kfold_bar:\n",
    "        \n",
    "        # Define training and validation sets\n",
    "        X_train, X_val = X.loc[train_idx], X.loc[val_idx]\n",
    "        y_train, y_val = y.loc[train_idx], y.loc[val_idx]\n",
    "        \n",
    "        # Fit pipeline to training data\n",
    "        pipe = pipe.fit(X_train, y_train)\n",
    "        \n",
    "        # Get trained preprocessor and model\n",
    "        preprocessor = pipe.get_params()[\"preprocessor\"]\n",
    "        model = pipe.get_params()[\"model\"]\n",
    "        \n",
    "        # Save train data score\n",
    "        y_train_pred = pipe.predict(X_train)\n",
    "        train_score += roc_auc_score(y_train, y_train_pred) / n_splits\n",
    "        \n",
    "        # Define validation pipeline (remove SMOTE)\n",
    "        val_pipe = Pipeline(steps=[\n",
    "            (\"preprocessor\", preprocessor),\n",
    "            (\"model\", model)\n",
    "        ])\n",
    "        \n",
    "        # Save validation data score\n",
    "        y_val_pred = val_pipe.predict(X_val)\n",
    "        val_score += roc_auc_score(y_val, y_val_pred) / n_splits\n",
    "        \n",
    "    return train_score, val_score"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "To perform model selection, a few models were considered to solve this binary classification problem:\n",
    "- Logistic regression classifier\n",
    "- Support vector classifier\n",
    "- Random forest classifier\n",
    "\n",
    "They are all fitted to the training data according to the class weights and later evaluated on the validation dataset using the area under the ROC curve."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Cross-validating: 100%|██████████| 5/5 [00:00<00:00,  6.89it/s]\n",
      "Model: LR | Train score: 0.776 | Val score: 0.759      \n",
      "Cross-validating: 100%|██████████| 5/5 [00:04<00:00,  1.11it/s]\n",
      "Model: SVM | Train score: 0.829 | Val score: 0.678             \n",
      "Cross-validating: 100%|██████████| 5/5 [00:02<00:00,  1.94it/s]\n",
      "Model: RFC | Train score: 1.000 | Val score: 0.556             \n",
      "Cross-validating: 100%|██████████| 5/5 [00:00<00:00,  7.09it/s]\n",
      "Model: KNN | Train score: 0.937 | Val score: 0.588             \n",
      "Cross-validating: 100%|██████████| 5/5 [00:00<00:00, 20.15it/s]\n",
      "Model: DTC | Train score: 1.000 | Val score: 0.572             \n",
      "Cross-validating: 100%|██████████| 5/5 [00:00<00:00, 39.87it/s]\n",
      "Model: GNB | Train score: 0.649 | Val score: 0.647             \n",
      "Cross-validating: 100%|██████████| 5/5 [00:01<00:00,  3.02it/s]\n",
      "Model: XGB | Train score: 0.983 | Val score: 0.554             \n",
      "Cross-validating: 100%|██████████| 5/5 [00:12<00:00,  2.56s/it]\n",
      "Model: NN | Train score: 0.826 | Val score: 0.675              \n",
      "Iterating models: 100%|██████████| 8/8 [00:23<00:00,  2.92s/it]\n"
     ]
    }
   ],
   "source": [
    "# Create model pool to select from\n",
    "model_pool = [\n",
    "    {\"name\": \"LR\", \"model\": LogisticRegression(random_state=random_state), \"train_score\": 0, \"val_score\": 0},\n",
    "    {\"name\": \"SVM\", \"model\": SVC(random_state=random_state), \"train_score\": 0, \"val_score\": 0},\n",
    "    {\"name\": \"RFC\", \"model\": RandomForestClassifier(random_state=random_state), \"train_score\": 0, \"val_score\": 0},\n",
    "    {\"name\": \"KNN\", \"model\": KNeighborsClassifier(), \"train_score\": 0, \"val_score\": 0},\n",
    "    {\"name\": \"DTC\", \"model\": DecisionTreeClassifier(random_state=random_state), \"train_score\": 0, \"val_score\": 0},\n",
    "    {\"name\": \"GNB\", \"model\": GaussianNB(), \"train_score\": 0, \"val_score\": 0},\n",
    "    {\"name\": \"XGB\", \"model\": XGBClassifier(), \"train_score\": 0, \"val_score\": 0},\n",
    "    {\"name\": \"NN\", \"model\": MLPClassifier(random_state=random_state, max_iter=500, early_stopping=True), \"train_score\": 0, \"val_score\": 0}\n",
    "]\n",
    "\n",
    "# Define the metric function and number of kfold splits\n",
    "metric = roc_auc_score\n",
    "n_splits = 5\n",
    "\n",
    "# Iterate each model in the pool\n",
    "model_bar = tqdm(model_pool, total=len(model_pool), desc=\"Iterating models\", position=0, file=sys.stdout)\n",
    "for model_dict in model_bar:\n",
    "    \n",
    "    # Get dictionary values\n",
    "    name, model = model_dict[\"name\"], model_dict[\"model\"]\n",
    "    \n",
    "    # Define the training pipeline using the model\n",
    "    train_pipe = Pipeline(steps=[\n",
    "        (\"preprocessor\", preprocessor),\n",
    "        (\"resampler\", resampler),\n",
    "        (\"model\", model)\n",
    "    ])\n",
    "    \n",
    "    # Cross validate\n",
    "    train_score, val_score = cross_validate(X, y, train_pipe, n_splits, True)\n",
    "    model_dict[\"train_score\"], model_dict[\"val_score\"] = train_score, val_score\n",
    "        \n",
    "    # Print final results\n",
    "    tqdm.write(f\"Model: {name} | Train score: {train_score:.3f} | Val score: {val_score:.3f}\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "As we can see from these trained models, the one that faired better was the Logistic Regressor. However, both the Support Vector Machine classifier and the Neural Network also performed quite well. The Neural Network is especially promising, as it is very likely that performing hyperparameter tuning will boost its performance quite significantly."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Hyperparameter optimization"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Now that we know what the best models are, we will try to find their best hyperparameters so that we can get them to perform the best they can. We will be tuning:\n",
    "- Logistic Regression Classifier\n",
    "- Support Vector Machine Classifier\n",
    "- Neural Networks Classifier"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "In order to do so, we will use an optimization package called `optuna`. First of all, let's define funtions to create the models and define their hyperparameter spaces:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "def LR_model(trial):\n",
    "    solver = trial.suggest_categorical(\"solve\", [\"newton-cg\", \"lbfgs\", \"liblinear\", \"sag\", \"saga\"]) # Solver hyperparameter\n",
    "    c = trial.suggest_float(\"c\", 1e-3, 1) # Hyperparameter for regularization strength\n",
    "    model = LogisticRegression(solver=solver, C=c, random_state=random_state)\n",
    "    return model\n",
    "    \n",
    "def SVM_model(trial):\n",
    "    kernel = trial.suggest_categorical(\"kernel\", [\"linear\", \"poly\", \"rbf\", \"sigmoid\"]) # Define kernel\n",
    "    c = trial.suggest_float(\"c\", 1e-3, 1) # Hyperparameter for regularization strength\n",
    "    model = SVC(C=c, kernel=kernel, probability=True, random_state=random_state)\n",
    "    return model\n",
    "\n",
    "def NN_model(trial):\n",
    "    num_hidden_layers = trial.suggest_int(\"num_hidden_layers\", 1, 4) # Define number of hidden layers\n",
    "    hidden_layer_sizes = [trial.suggest_int(f\"hidden_layer_{i}\", 5, 20) for i in range(num_hidden_layers)] # Define features per hidden layer\n",
    "    model = MLPClassifier(hidden_layer_sizes=hidden_layer_sizes, random_state=random_state, max_iter=500, early_stopping=True)\n",
    "    return model"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Using these functions, we can now create an objective function, which we will use to optimize the ROC AUC metric for all of our models."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "def objective(trial, name):\n",
    "    # Select model based on its name\n",
    "    if name == \"LR\":\n",
    "        model = LR_model(trial)\n",
    "    elif name == \"SVM\":\n",
    "        model = SVM_model(trial)\n",
    "    else:\n",
    "        model = NN_model(trial)\n",
    "        \n",
    "    # Define training pipeline on the model\n",
    "    train_pipe = Pipeline(steps=[\n",
    "        (\"preprocessor\", preprocessor),\n",
    "        (\"resampler\", resampler),\n",
    "        (\"model\", model)\n",
    "    ])\n",
    "    \n",
    "    # Perform cross validation\n",
    "    n_splits = 5\n",
    "    _, val_score = cross_validate(X, y, train_pipe, n_splits)\n",
    "    \n",
    "    return val_score"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Now we will create our model pool and perform hyperparameter optimization for each model:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Iterating models:   0%|          | 0/3 [00:00<?, ?it/s, model=LR]"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\u001b[32m[I 2022-10-31 21:16:36,783]\u001b[0m A new study created in memory with name: LR\u001b[0m\n",
      "\u001b[32m[I 2022-10-31 21:16:37,342]\u001b[0m Trial 0 finished with value: 0.7581401653833015 and parameters: {'solve': 'lbfgs', 'c': 0.6462482189535894}. Best is trial 0 with value: 0.7581401653833015.\u001b[0m\n",
      "\u001b[32m[I 2022-10-31 21:16:37,785]\u001b[0m Trial 1 finished with value: 0.7653168630553093 and parameters: {'solve': 'liblinear', 'c': 0.5293660248331515}. Best is trial 1 with value: 0.7653168630553093.\u001b[0m\n",
      "\u001b[32m[I 2022-10-31 21:16:38,485]\u001b[0m Trial 2 finished with value: 0.758656456868515 and parameters: {'solve': 'lbfgs', 'c': 0.83278722570239}. Best is trial 1 with value: 0.7653168630553093.\u001b[0m\n",
      "\u001b[32m[I 2022-10-31 21:16:38,957]\u001b[0m Trial 3 finished with value: 0.7576260189209498 and parameters: {'solve': 'liblinear', 'c': 0.780748647110169}. Best is trial 1 with value: 0.7653168630553093.\u001b[0m\n",
      "\u001b[32m[I 2022-10-31 21:16:39,345]\u001b[0m Trial 4 finished with value: 0.7605686766291944 and parameters: {'solve': 'sag', 'c': 0.4152472780505331}. Best is trial 1 with value: 0.7653168630553093.\u001b[0m\n",
      "\u001b[32m[I 2022-10-31 21:16:39,990]\u001b[0m Trial 5 finished with value: 0.7581404941149399 and parameters: {'solve': 'lbfgs', 'c': 0.6180178615788012}. Best is trial 1 with value: 0.7653168630553093.\u001b[0m\n",
      "\u001b[32m[I 2022-10-31 21:16:40,432]\u001b[0m Trial 6 finished with value: 0.7651893120349014 and parameters: {'solve': 'liblinear', 'c': 0.4375949218455421}. Best is trial 1 with value: 0.7653168630553093.\u001b[0m\n",
      "\u001b[32m[I 2022-10-31 21:16:40,983]\u001b[0m Trial 7 finished with value: 0.7631342239903722 and parameters: {'solve': 'newton-cg', 'c': 0.12979737135719846}. Best is trial 1 with value: 0.7653168630553093.\u001b[0m\n",
      "\u001b[32m[I 2022-10-31 21:16:41,471]\u001b[0m Trial 8 finished with value: 0.7622380908458386 and parameters: {'solve': 'saga', 'c': 0.10294276593728004}. Best is trial 1 with value: 0.7653168630553093.\u001b[0m\n",
      "\u001b[32m[I 2022-10-31 21:16:41,925]\u001b[0m Trial 9 finished with value: 0.7664078152111748 and parameters: {'solve': 'liblinear', 'c': 0.24518116640960114}. Best is trial 9 with value: 0.7664078152111748.\u001b[0m\n",
      "\u001b[32m[I 2022-10-31 21:16:42,460]\u001b[0m Trial 10 finished with value: 0.761916869526809 and parameters: {'solve': 'newton-cg', 'c': 0.2829295786575749}. Best is trial 9 with value: 0.7664078152111748.\u001b[0m\n",
      "\u001b[32m[I 2022-10-31 21:16:42,920]\u001b[0m Trial 11 finished with value: 0.7674339592548689 and parameters: {'solve': 'liblinear', 'c': 0.3191852813529867}. Best is trial 11 with value: 0.7674339592548689.\u001b[0m\n",
      "\u001b[32m[I 2022-10-31 21:16:43,427]\u001b[0m Trial 12 finished with value: 0.7662802641907667 and parameters: {'solve': 'liblinear', 'c': 0.23213823775318385}. Best is trial 11 with value: 0.7674339592548689.\u001b[0m\n",
      "\u001b[32m[I 2022-10-31 21:16:43,859]\u001b[0m Trial 13 finished with value: 0.7671775485770079 and parameters: {'solve': 'liblinear', 'c': 0.2745958629168817}. Best is trial 11 with value: 0.7674339592548689.\u001b[0m\n",
      "\u001b[32m[I 2022-10-31 21:16:44,426]\u001b[0m Trial 14 finished with value: 0.758656456868515 and parameters: {'solve': 'sag', 'c': 0.9939252984221902}. Best is trial 11 with value: 0.7674339592548689.\u001b[0m\n",
      "\u001b[32m[I 2022-10-31 21:16:44,884]\u001b[0m Trial 15 finished with value: 0.7598067693894505 and parameters: {'solve': 'saga', 'c': 0.0397841064470294}. Best is trial 11 with value: 0.7674339592548689.\u001b[0m\n",
      "\u001b[32m[I 2022-10-31 21:16:45,289]\u001b[0m Trial 16 finished with value: 0.7674339592548689 and parameters: {'solve': 'liblinear', 'c': 0.30712540252984194}. Best is trial 11 with value: 0.7674339592548689.\u001b[0m\n",
      "\u001b[32m[I 2022-10-31 21:16:45,746]\u001b[0m Trial 17 finished with value: 0.7673067432880276 and parameters: {'solve': 'liblinear', 'c': 0.3525166853478301}. Best is trial 11 with value: 0.7674339592548689.\u001b[0m\n",
      "\u001b[32m[I 2022-10-31 21:16:46,211]\u001b[0m Trial 18 finished with value: 0.7653168630553093 and parameters: {'solve': 'liblinear', 'c': 0.5154960265675308}. Best is trial 11 with value: 0.7674339592548689.\u001b[0m\n",
      "\u001b[32m[I 2022-10-31 21:16:46,670]\u001b[0m Trial 19 finished with value: 0.7636465566577016 and parameters: {'solve': 'sag', 'c': 0.17085237635647832}. Best is trial 11 with value: 0.7674339592548689.\u001b[0m\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Iterating models:  33%|███▎      | 1/3 [00:09<00:19,  9.89s/it, model=SVM]"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\u001b[32m[I 2022-10-31 21:16:46,672]\u001b[0m A new study created in memory with name: SVM\u001b[0m\n",
      "\u001b[32m[I 2022-10-31 21:16:59,392]\u001b[0m Trial 0 finished with value: 0.6982987188640801 and parameters: {'kernel': 'poly', 'c': 0.4242311445395658}. Best is trial 0 with value: 0.6982987188640801.\u001b[0m\n",
      "\u001b[32m[I 2022-10-31 21:17:33,676]\u001b[0m Trial 1 finished with value: 0.5630916620467079 and parameters: {'kernel': 'sigmoid', 'c': 0.3840580773069519}. Best is trial 0 with value: 0.6982987188640801.\u001b[0m\n",
      "\u001b[32m[I 2022-10-31 21:18:14,356]\u001b[0m Trial 2 finished with value: 0.5817470400624714 and parameters: {'kernel': 'sigmoid', 'c': 0.07196502213968906}. Best is trial 0 with value: 0.6982987188640801.\u001b[0m\n",
      "\u001b[32m[I 2022-10-31 21:18:30,695]\u001b[0m Trial 3 finished with value: 0.688945537929285 and parameters: {'kernel': 'rbf', 'c': 0.8701421360985724}. Best is trial 0 with value: 0.6982987188640801.\u001b[0m\n",
      "\u001b[32m[I 2022-10-31 21:18:43,613]\u001b[0m Trial 4 finished with value: 0.7658166360725196 and parameters: {'kernel': 'linear', 'c': 0.11915615144306428}. Best is trial 4 with value: 0.7658166360725196.\u001b[0m\n",
      "\u001b[32m[I 2022-10-31 21:19:01,689]\u001b[0m Trial 5 finished with value: 0.7034013281126316 and parameters: {'kernel': 'rbf', 'c': 0.4152472780505331}. Best is trial 4 with value: 0.7658166360725196.\u001b[0m\n",
      "\u001b[32m[I 2022-10-31 21:19:17,725]\u001b[0m Trial 6 finished with value: 0.7281984786553728 and parameters: {'kernel': 'poly', 'c': 0.01977101063591879}. Best is trial 4 with value: 0.7658166360725196.\u001b[0m\n",
      "\u001b[32m[I 2022-10-31 21:19:50,253]\u001b[0m Trial 7 finished with value: 0.5602168843533515 and parameters: {'kernel': 'sigmoid', 'c': 0.6821384788043799}. Best is trial 4 with value: 0.7658166360725196.\u001b[0m\n",
      "\u001b[32m[I 2022-10-31 21:20:07,122]\u001b[0m Trial 8 finished with value: 0.6973061967478776 and parameters: {'kernel': 'rbf', 'c': 0.667099948730222}. Best is trial 4 with value: 0.7658166360725196.\u001b[0m\n",
      "\u001b[32m[I 2022-10-31 21:20:19,884]\u001b[0m Trial 9 finished with value: 0.7669595056432692 and parameters: {'kernel': 'linear', 'c': 0.36434706017168}. Best is trial 9 with value: 0.7669595056432692.\u001b[0m\n",
      "\u001b[32m[I 2022-10-31 21:20:32,450]\u001b[0m Trial 10 finished with value: 0.7632452970136578 and parameters: {'kernel': 'linear', 'c': 0.24409807687658105}. Best is trial 9 with value: 0.7669595056432692.\u001b[0m\n",
      "\u001b[32m[I 2022-10-31 21:20:45,277]\u001b[0m Trial 11 finished with value: 0.7627321406043688 and parameters: {'kernel': 'linear', 'c': 0.20873419727359604}. Best is trial 9 with value: 0.7669595056432692.\u001b[0m\n",
      "\u001b[32m[I 2022-10-31 21:20:57,930]\u001b[0m Trial 12 finished with value: 0.7624757299265077 and parameters: {'kernel': 'linear', 'c': 0.22351342186693604}. Best is trial 9 with value: 0.7669595056432692.\u001b[0m\n",
      "\u001b[32m[I 2022-10-31 21:21:11,193]\u001b[0m Trial 13 finished with value: 0.7678595913161783 and parameters: {'kernel': 'linear', 'c': 0.5878550544040214}. Best is trial 13 with value: 0.7678595913161783.\u001b[0m\n",
      "\u001b[32m[I 2022-10-31 21:21:24,637]\u001b[0m Trial 14 finished with value: 0.7682447055204658 and parameters: {'kernel': 'linear', 'c': 0.6192883230393825}. Best is trial 14 with value: 0.7682447055204658.\u001b[0m\n",
      "\u001b[32m[I 2022-10-31 21:21:37,719]\u001b[0m Trial 15 finished with value: 0.768115673262401 and parameters: {'kernel': 'linear', 'c': 0.6130396350510509}. Best is trial 14 with value: 0.7682447055204658.\u001b[0m\n",
      "\u001b[32m[I 2022-10-31 21:21:51,377]\u001b[0m Trial 16 finished with value: 0.7710104382840288 and parameters: {'kernel': 'linear', 'c': 0.8549785503094043}. Best is trial 16 with value: 0.7710104382840288.\u001b[0m\n",
      "\u001b[32m[I 2022-10-31 21:22:05,319]\u001b[0m Trial 17 finished with value: 0.7687583518824579 and parameters: {'kernel': 'linear', 'c': 0.9855631217288162}. Best is trial 16 with value: 0.7710104382840288.\u001b[0m\n",
      "\u001b[32m[I 2022-10-31 21:22:17,592]\u001b[0m Trial 18 finished with value: 0.6663293870670154 and parameters: {'kernel': 'poly', 'c': 0.9736326418450958}. Best is trial 16 with value: 0.7710104382840288.\u001b[0m\n",
      "\u001b[32m[I 2022-10-31 21:22:31,140]\u001b[0m Trial 19 finished with value: 0.7707530363212146 and parameters: {'kernel': 'linear', 'c': 0.807506408481276}. Best is trial 16 with value: 0.7710104382840288.\u001b[0m\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Iterating models:  67%|██████▋   | 2/3 [05:54<03:26, 206.70s/it, model=NN] "
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\u001b[32m[I 2022-10-31 21:22:31,142]\u001b[0m A new study created in memory with name: NN\u001b[0m\n",
      "\u001b[32m[I 2022-10-31 21:22:35,584]\u001b[0m Trial 0 finished with value: 0.698303555454844 and parameters: {'num_hidden_layers': 3, 'hidden_layer_0': 16, 'hidden_layer_1': 14, 'hidden_layer_2': 13}. Best is trial 0 with value: 0.698303555454844.\u001b[0m\n",
      "\u001b[32m[I 2022-10-31 21:22:38,231]\u001b[0m Trial 1 finished with value: 0.7304941459762713 and parameters: {'num_hidden_layers': 2, 'hidden_layer_0': 15, 'hidden_layer_1': 12}. Best is trial 1 with value: 0.7304941459762713.\u001b[0m\n",
      "\u001b[32m[I 2022-10-31 21:22:43,106]\u001b[0m Trial 2 finished with value: 0.6225403157490155 and parameters: {'num_hidden_layers': 4, 'hidden_layer_0': 20, 'hidden_layer_1': 11, 'hidden_layer_2': 17, 'hidden_layer_3': 13}. Best is trial 1 with value: 0.7304941459762713.\u001b[0m\n",
      "\u001b[32m[I 2022-10-31 21:22:47,577]\u001b[0m Trial 3 finished with value: 0.6956147124140829 and parameters: {'num_hidden_layers': 3, 'hidden_layer_0': 19, 'hidden_layer_1': 6, 'hidden_layer_2': 6}. Best is trial 1 with value: 0.7304941459762713.\u001b[0m\n",
      "\u001b[32m[I 2022-10-31 21:22:51,422]\u001b[0m Trial 4 finished with value: 0.7234829461773848 and parameters: {'num_hidden_layers': 1, 'hidden_layer_0': 18}. Best is trial 1 with value: 0.7304941459762713.\u001b[0m\n",
      "\u001b[32m[I 2022-10-31 21:22:56,742]\u001b[0m Trial 5 finished with value: 0.6399752536403426 and parameters: {'num_hidden_layers': 4, 'hidden_layer_0': 18, 'hidden_layer_1': 20, 'hidden_layer_2': 17, 'hidden_layer_3': 12}. Best is trial 1 with value: 0.7304941459762713.\u001b[0m\n",
      "\u001b[32m[I 2022-10-31 21:23:00,508]\u001b[0m Trial 6 finished with value: 0.7281468690532522 and parameters: {'num_hidden_layers': 4, 'hidden_layer_0': 6, 'hidden_layer_1': 15, 'hidden_layer_2': 7, 'hidden_layer_3': 20}. Best is trial 1 with value: 0.7304941459762713.\u001b[0m\n",
      "\u001b[32m[I 2022-10-31 21:23:04,781]\u001b[0m Trial 7 finished with value: 0.7117968322555666 and parameters: {'num_hidden_layers': 3, 'hidden_layer_0': 11, 'hidden_layer_1': 9, 'hidden_layer_2': 17}. Best is trial 1 with value: 0.7304941459762713.\u001b[0m\n",
      "\u001b[32m[I 2022-10-31 21:23:06,772]\u001b[0m Trial 8 finished with value: 0.648082771378667 and parameters: {'num_hidden_layers': 2, 'hidden_layer_0': 14, 'hidden_layer_1': 5}. Best is trial 1 with value: 0.7304941459762713.\u001b[0m\n",
      "\u001b[32m[I 2022-10-31 21:23:11,368]\u001b[0m Trial 9 finished with value: 0.690251380459524 and parameters: {'num_hidden_layers': 3, 'hidden_layer_0': 14, 'hidden_layer_1': 14, 'hidden_layer_2': 20}. Best is trial 1 with value: 0.7304941459762713.\u001b[0m\n",
      "\u001b[32m[I 2022-10-31 21:23:13,693]\u001b[0m Trial 10 finished with value: 0.7478922076719962 and parameters: {'num_hidden_layers': 1, 'hidden_layer_0': 10}. Best is trial 10 with value: 0.7478922076719962.\u001b[0m\n",
      "\u001b[32m[I 2022-10-31 21:23:15,920]\u001b[0m Trial 11 finished with value: 0.7478922076719962 and parameters: {'num_hidden_layers': 1, 'hidden_layer_0': 10}. Best is trial 10 with value: 0.7478922076719962.\u001b[0m\n",
      "\u001b[32m[I 2022-10-31 21:23:18,169]\u001b[0m Trial 12 finished with value: 0.7478922076719962 and parameters: {'num_hidden_layers': 1, 'hidden_layer_0': 10}. Best is trial 10 with value: 0.7478922076719962.\u001b[0m\n",
      "\u001b[32m[I 2022-10-31 21:23:19,900]\u001b[0m Trial 13 finished with value: 0.7408218320567288 and parameters: {'num_hidden_layers': 1, 'hidden_layer_0': 8}. Best is trial 10 with value: 0.7478922076719962.\u001b[0m\n",
      "\u001b[32m[I 2022-10-31 21:23:23,833]\u001b[0m Trial 14 finished with value: 0.7512093551529518 and parameters: {'num_hidden_layers': 2, 'hidden_layer_0': 9, 'hidden_layer_1': 19}. Best is trial 14 with value: 0.7512093551529518.\u001b[0m\n",
      "\u001b[32m[I 2022-10-31 21:23:26,497]\u001b[0m Trial 15 finished with value: 0.7333468088078752 and parameters: {'num_hidden_layers': 2, 'hidden_layer_0': 6, 'hidden_layer_1': 20}. Best is trial 14 with value: 0.7512093551529518.\u001b[0m\n",
      "\u001b[32m[I 2022-10-31 21:23:30,477]\u001b[0m Trial 16 finished with value: 0.7111240874997486 and parameters: {'num_hidden_layers': 2, 'hidden_layer_0': 8, 'hidden_layer_1': 17}. Best is trial 14 with value: 0.7512093551529518.\u001b[0m\n",
      "\u001b[32m[I 2022-10-31 21:23:32,636]\u001b[0m Trial 17 finished with value: 0.7340791043727795 and parameters: {'num_hidden_layers': 1, 'hidden_layer_0': 12}. Best is trial 14 with value: 0.7512093551529518.\u001b[0m\n",
      "\u001b[32m[I 2022-10-31 21:23:36,683]\u001b[0m Trial 18 finished with value: 0.7223933329876409 and parameters: {'num_hidden_layers': 2, 'hidden_layer_0': 8, 'hidden_layer_1': 18}. Best is trial 14 with value: 0.7512093551529518.\u001b[0m\n",
      "\u001b[32m[I 2022-10-31 21:23:38,465]\u001b[0m Trial 19 finished with value: 0.717369710195984 and parameters: {'num_hidden_layers': 1, 'hidden_layer_0': 5}. Best is trial 14 with value: 0.7512093551529518.\u001b[0m\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Iterating models: 100%|██████████| 3/3 [07:01<00:00, 140.56s/it, model=NN]\n"
     ]
    }
   ],
   "source": [
    "# Create model pool\n",
    "model_pool = {\n",
    "    \"LR\": None,\n",
    "    \"SVM\": None,\n",
    "    \"NN\": None\n",
    "}\n",
    "\n",
    "# Iterate each model in the pool\n",
    "model_bar = tqdm(model_pool, total=len(model_pool), desc=\"Iterating models\", position=0, file=sys.stdout)\n",
    "for name in model_bar:\n",
    "    \n",
    "    # Set tqdm bar postfix to model name\n",
    "    model_bar.set_postfix(model=name)\n",
    "    \n",
    "    # Create optuna study\n",
    "    sampler = opt.samplers.TPESampler(seed=random_state)\n",
    "    study = opt.create_study(study_name=name, direction=\"maximize\", sampler=sampler)\n",
    "    \n",
    "    # Optimize\n",
    "    foo = lambda trial: objective(trial, name)\n",
    "    study.optimize(foo, n_trials=20)\n",
    "    model_pool[name] = study"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "As we can see from the hyperparameter optimization, the following were the best hyperparameters for each model:\n",
    "- Logistic Regression: solver = liblinear and c = 0.3191852813529867\n",
    "- Support Vector Machine: kernel = linear and c = 0.8549785503094043\n",
    "- Neural Network: hidden_layer_sizes = [9, 19]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Training the models with the best hyperparameters"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Now that the best model hyperparameters have been tuned using cross-validation, we can discard the validation data to train these models together with their pipelines in the entire training dataset (including the previous validation data). After the training is completed, we can save them so that we don't have to retrain the pipelines and can just load them later for inference:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training best models: 100%|██████████| 3/3 [00:04<00:00,  1.65s/it]\n"
     ]
    }
   ],
   "source": [
    "# Create list of best models\n",
    "best_models = {\n",
    "    (\"lr\", LogisticRegression(solver=\"liblinear\", C=0.3191852813529867, random_state=random_state)),\n",
    "    (\"svc\", SVC(kernel=\"linear\", C=0.8549785503094043, probability=True, random_state=random_state)),\n",
    "    (\"nn\", MLPClassifier(hidden_layer_sizes=[9, 19], random_state=random_state, max_iter=500, early_stopping=True))\n",
    "}\n",
    "\n",
    "# Iterate each one of them\n",
    "model_bar = tqdm(best_models, total=len(best_models), desc=\"Training best models\", position=0, file=sys.stdout)\n",
    "for name, model in model_bar:\n",
    "    \n",
    "    # Define training pipeline on the model\n",
    "    train_pipe = Pipeline(steps=[\n",
    "        (\"preprocessor\", preprocessor),\n",
    "        (\"resampler\", resampler),\n",
    "        (\"model\", model)\n",
    "    ])\n",
    "    \n",
    "    # Fit the training pipeline on the ENTIRE training dataset (no validation)\n",
    "    train_pipe = train_pipe.fit(X, y)\n",
    "    \n",
    "    # Create test pipe without the resampler\n",
    "    trained_preprocessor = train_pipe.get_params()[\"preprocessor\"]\n",
    "    trained_model = train_pipe.get_params()[\"model\"]\n",
    "    test_pipe = Pipeline(steps=[\n",
    "        (\"preprocessor\", trained_preprocessor),\n",
    "        (\"model\", trained_model)\n",
    "    ])\n",
    "    \n",
    "    # Save the trained test pipeline in the models folder\n",
    "    with open(f\"models/{name}.pkl\", \"wb\") as f:\n",
    "        pickle.dump(test_pipe, f, protocol=pickle.HIGHEST_PROTOCOL)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3.10.6 ('DataScience')",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.6"
  },
  "orig_nbformat": 4,
  "vscode": {
   "interpreter": {
    "hash": "66c20332868a9aca664e5924ef91efac344eda67c73f1c04e9ba2cded2db297f"
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}