6.3.4-Tuning-LightGBM-Models-with-Custom-Tuner.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab_type": "code"
   },
   "outputs": [],
   "source": [
    "!pip install -r https://raw.githubusercontent.com/datamllab/automl-in-action-notebooks/master/requirements.txt\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text"
   },
   "source": [
    "### Load the California housing price prediction dataset\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab_type": "code"
   },
   "outputs": [],
   "source": [
    "from sklearn.datasets import fetch_california_housing\n",
    "\n",
    "house_dataset = fetch_california_housing()\n",
    "\n",
    "# Import pandas package to format the data\n",
    "import pandas as pd\n",
    "\n",
    "# Extract features with their names into the a dataframe format\n",
    "data = pd.DataFrame(house_dataset.data, columns=house_dataset.feature_names)\n",
    "\n",
    "# Extract target with their names into a pd.Series object with name MEDV\n",
    "target = pd.Series(house_dataset.target, name=\"MEDV\")\n",
    "\n",
    "from sklearn.model_selection import train_test_split\n",
    "\n",
    "X_train, X_test, y_train, y_test = train_test_split(\n",
    "    data, target, test_size=0.2, random_state=42\n",
    ")\n",
    "X_train, X_val, y_train, y_val = train_test_split(\n",
    "    X_train, y_train, test_size=0.2, shuffle=False\n",
    ")\n",
    "\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text"
   },
   "source": [
    "### Use LightGBM GBDT model to do regression\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab_type": "code"
   },
   "outputs": [],
   "source": [
    "import lightgbm as lgb\n",
    "from sklearn.metrics import mean_squared_error\n",
    "\n",
    "gbdt_model = lgb.LGBMRegressor(\n",
    "    boosting_type=\"gbdt\", num_leaves=31, learning_rate=0.05, n_estimators=10\n",
    ")  # create model\n",
    "\n",
    "validation_data = (X_val, y_val)\n",
    "gbdt_model.fit(\n",
    "    X_train,\n",
    "    y_train,\n",
    "    eval_set=[validation_data],\n",
    "    eval_metric=\"mse\",\n",
    "    early_stopping_rounds=5,\n",
    ")  # fit the model\n",
    "\n",
    "# evalaute model\n",
    "y_pred_gbdt = gbdt_model.predict(X_test, num_iteration=gbdt_model.best_iteration_)\n",
    "test_mse_1 = mean_squared_error(y_test, y_pred_gbdt)\n",
    "print(\"The GBDT prediction MSE on test set: {}\".format(test_mse_1))\n",
    "\n",
    "# save, load, and evaluate the model\n",
    "fname = \"gbdt_model.txt\"\n",
    "gbdt_model.booster_.save_model(fname, num_iteration=gbdt_model.best_iteration_)\n",
    "\n",
    "gbdt_model_2 = lgb.Booster(model_file=fname)\n",
    "gbdt_model_2.predict(X_test)\n",
    "test_mse_2 = mean_squared_error(y_test, y_pred_gbdt)\n",
    "print(\"The reloaded GBDT prediction MSE on test set: {}\".format(test_mse_2))\n",
    "\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text"
   },
   "source": [
    "### Create the LightGBM model building function\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab_type": "code"
   },
   "outputs": [],
   "source": [
    "\n",
    "def build_model(hp):\n",
    "    model = lgb.LGBMRegressor(\n",
    "        boosting_type=\"gbdt\",\n",
    "        num_leaves=hp.Choice(\"num_leaves\", [15, 31, 63], default=31),\n",
    "        learning_rate=hp.Float(\"learning_rate\", 1e-3, 10, sampling=\"log\", default=0.05),\n",
    "        n_estimators=hp.Int(\"n_estimators\", 10, 200, step=10),\n",
    "    )\n",
    "\n",
    "    return model\n",
    "\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text"
   },
   "source": [
    "### Customize the LightGBM tuner\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab_type": "code"
   },
   "outputs": [],
   "source": [
    "import os\n",
    "import pickle\n",
    "import tensorflow as tf\n",
    "import kerastuner as kt\n",
    "import lightgbm as lgb\n",
    "from sklearn.metrics import mean_squared_error\n",
    "\n",
    "\n",
    "class LightGBMTuner(kt.Tuner):\n",
    "    def run_trial(self, trial, X, y, validation_data):\n",
    "        model = self.hypermodel.build(trial.hyperparameters)  # build the model\n",
    "        model.fit(\n",
    "            X_train,\n",
    "            y_train,\n",
    "            eval_set=[validation_data],\n",
    "            eval_metric=\"mse\",\n",
    "            early_stopping_rounds=5,\n",
    "        )  # fit the model\n",
    "        X_val, y_val = validation_data\n",
    "        y_pred = model.predict(\n",
    "            X_val, num_iteration=model.best_iteration_\n",
    "        )  # evaluate the model\n",
    "        eval_mse = mean_squared_error(y_val, y_pred)\n",
    "        self.save_model(trial.trial_id, model)  # save the model to disk\n",
    "        return {\"mse\": eval_mse}\n",
    "\n",
    "    def save_model(self, trial_id, model, step=0):\n",
    "        fname = os.path.join(self.get_trial_dir(trial_id), \"model.txt\")\n",
    "        model.booster_.save_model(fname, num_iteration=model.best_iteration_)\n",
    "\n",
    "    def load_model(self, trial):\n",
    "        fname = os.path.join(self.get_trial_dir(trial.trial_id), \"model.txt\")\n",
    "        model = lgb.Booster(model_file=fname)\n",
    "        return model\n",
    "\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text"
   },
   "source": [
    "### Run the tuner to select a LightGBM models for the housing price prediction\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab_type": "code"
   },
   "outputs": [],
   "source": [
    "my_lightgbm_tuner = LightGBMTuner(\n",
    "    oracle=kt.oracles.RandomSearch(\n",
    "        objective=kt.Objective(\"mse\", \"min\"), max_trials=10, seed=42\n",
    "    ),\n",
    "    hypermodel=build_model,\n",
    "    overwrite=True,\n",
    "    project_name=\"my_lightgbm_tuner\",\n",
    ")\n",
    "\n",
    "my_lightgbm_tuner.search(X_train, y_train, validation_data=(X_val, y_val))\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text"
   },
   "source": [
    "### Evaluate the best discovered model\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab_type": "code"
   },
   "outputs": [],
   "source": [
    "from sklearn.metrics import mean_squared_error\n",
    "\n",
    "best_model = my_lightgbm_tuner.get_best_models(1)[0]\n",
    "y_pred_test = best_model.predict(X_test)\n",
    "test_mse = mean_squared_error(y_test, y_pred_test)\n",
    "print(\"The prediction MSE on test set: {}\".format(test_mse))\n",
    "\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text"
   },
   "source": [
    "### Check the best model\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab_type": "code"
   },
   "outputs": [],
   "source": [
    "my_lightgbm_tuner.get_best_models(1)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab_type": "code"
   },
   "outputs": [],
   "source": [
    "my_lightgbm_tuner.results_summary(1)\n"
   ]
  }
 ],
 "metadata": {
  "colab": {
   "collapsed_sections": [],
   "name": "6.3.4-Tuning-LightGBM-Models-with-Custom-Tuner",
   "private_outputs": false,
   "provenance": [],
   "toc_visible": true
  },
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.0"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}