Model Exploration.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "view-in-github"
   },
   "source": [
    "<a href=\"https://colab.research.google.com/github/KenzaxTazi/Agri-Risk/blob/master/Model_exploration_Raghul_pre_damon_final.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Machine Learning Model Exploration\n",
    "This Notebook contains the code to train three ensemble methods- Random Forest, Extra Trees and XGBoost on the crop data.\n",
    "\n",
    "It initially tries a naive fitting of the algorithms to make sure they can overfit the training set. Then a more exhaustive hyper-parameter search is conducted."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "ZA5OTap12UVr"
   },
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "import xgboost as xgb\n",
    "from sklearn.model_selection import RandomizedSearchCV\n",
    "from scipy.stats import randint as sp_randint\n",
    "from scipy.stats import uniform\n",
    "from sklearn.model_selection import train_test_split\n",
    "from tqdm import tqdm\n",
    "tqdm.pandas()\n",
    "import xarray as xr\n",
    "import random\n",
    "import pickle\n",
    "import json\n",
    "from sklearn.ensemble import RandomForestRegressor, ExtraTreesRegressor"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "test_fraction = 0.3\n",
    "random_seed = 42\n",
    "target = 'maize_a_2010'"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "qglC5r2-AKDC"
   },
   "source": [
    "## Process the data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "YX0R-IM9kKuc"
   },
   "outputs": [],
   "source": [
    "# Load up the selected features. (Chosen from Recursive Feature Elimination)\n",
    "with open('json_data/selected_features.json', 'r') as json_file:\n",
    "    features_to_select = json.load(json_file)\n",
    "    \n",
    "features_to_select = features_to_select['features']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "kpJYJJQAlDlT"
   },
   "outputs": [],
   "source": [
    "# Choose the appropriate column names here\n",
    "non_climatic_features = [\"soil_types\",\n",
    "                         \"climate_zones\",\n",
    "                         \"elevation\",\n",
    "                         \"slope\",\n",
    "                         \"irrigation\"]\n",
    "\n",
    "target = [\"maize_a_2010\"]\n",
    "categorical_variables = [\"climate_zones\", \"soil_types\"]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "-Mt6gLh3lyjW"
   },
   "outputs": [],
   "source": [
    "columns_to_keep = features_to_select + non_climatic_features + target"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "AKJyUO6sDp9N"
   },
   "outputs": [],
   "source": [
    "dataset = pd.read_csv(\"..data/climate_monthly_seasonal_2005_2010_with_spatial_attributes_filtered.csv\") #replace this with the file you want to use\n",
    "#you should import the csv which is already processed (nans removed) and with only the columns we want but not one-hot encoded\n",
    "dataset = dataset[columns_to_keep]\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "8GBMUv4rDFKu"
   },
   "outputs": [],
   "source": [
    "def one_hot(df, columns):\n",
    "    '''\n",
    "    one hot encode columns and concat to original df\n",
    "    ----------\n",
    "    Parameters: \n",
    "    df (pandas dataframe)\n",
    "    columns (list of strings): columns to one hot encode\n",
    "    \n",
    "    -------\n",
    "    Return:\n",
    "     One pandas dataframe with the one hot encoded columns \n",
    "    \n",
    "    '''\n",
    "    return pd.concat((df, pd.get_dummies(df[columns])), axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "0R2YzuouDF7c"
   },
   "outputs": [],
   "source": [
    "def format_col_to_str(df, cols):\n",
    "    '''\n",
    "    Convert multiple columns to str format\n",
    "    ---------------------\n",
    "    Parameters:\n",
    "    df (pandas DataFrame)\n",
    "    cols (list of strings): cols to convert\n",
    "    '''\n",
    "    \n",
    "    return df[cols].astype(str)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "55usFvQXGc0P"
   },
   "outputs": [],
   "source": [
    "# Create a copy of the dataset to add the one-hot encoded variables\n",
    "dataset_copy = dataset.copy()\n",
    "dataset_copy[categorical_variables] = format_col_to_str(dataset_copy, categorical_variables)\n",
    "dataset_copy = one_hot(dataset_1, columns=categorical_variables)\n",
    "dataset_copy.drop(categorical_variables, axis=1,inplace=True)\n",
    "X = dataset_copy.drop(target,axis=1)\n",
    "y = dataset_copy[[target]]\n",
    "train_set_x, test_set_x, train_set_y, test_set_y = train_test_split(X,\n",
    "                                                                    y,\n",
    "                                                                    test_size=test_fraction,\n",
    "                                                                    random_state=random_seed,\n",
    "                                                                    shuffle=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "d2ss8SnsIVGg"
   },
   "source": [
    "# Model exploration # "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "KluTcmZTbHJ1"
   },
   "source": [
    "## Which models can at least fit the training data? ##"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "f4KKy4beESyD"
   },
   "source": [
    "They need to at least get a high R2 on the training set in order to have a chance at the test set (after sufficient regularisation)."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "g_ogXEQ2bLVE"
   },
   "source": [
    "### Random Forests ###\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 394
    },
    "colab_type": "code",
    "id": "HLFNZB5TIUCC",
    "outputId": "bd28d3db-75b3-437d-8ae2-9a8f887b4f8b"
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local/lib/python3.6/dist-packages/ipykernel_launcher.py:5: DataConversionWarning: A column-vector y was passed when a 1d array was expected. Please change the shape of y to (n_samples,), for example using ravel().\n",
      "  \"\"\"\n",
      "[Parallel(n_jobs=-1)]: Using backend ThreadingBackend with 4 concurrent workers.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "building tree 1 of 15building tree 2 of 15building tree 3 of 15building tree 4 of 15\n",
      "\n",
      "\n",
      "\n",
      "building tree 5 of 15\n",
      "building tree 6 of 15\n",
      "building tree 7 of 15\n",
      "building tree 8 of 15\n",
      "building tree 9 of 15\n",
      "building tree 10 of 15\n",
      "building tree 11 of 15\n",
      "building tree 12 of 15\n",
      "building tree 13 of 15\n",
      "building tree 14 of 15\n",
      "building tree 15 of 15\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=-1)]: Done  15 out of  15 | elapsed:  7.2min finished\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 26min 46s, sys: 478 ms, total: 26min 47s\n",
      "Wall time: 7min 14s\n"
     ]
    }
   ],
   "source": [
    "%%time \n",
    "rnd_reg = RandomForestRegressor(n_estimators=15, n_jobs=-1, verbose=3)\n",
    "rnd_reg.fit(train_set_x,train_set_y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 68
    },
    "colab_type": "code",
    "id": "hpOcxA0GIi6H",
    "outputId": "8d0fe431-3cca-4bbb-958a-da0d3e97408e"
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=4)]: Using backend ThreadingBackend with 4 concurrent workers.\n",
      "[Parallel(n_jobs=4)]: Done  15 out of  15 | elapsed:    1.0s finished\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "0.9924742079280336"
      ]
     },
     "execution_count": 44,
     "metadata": {
      "tags": []
     },
     "output_type": "execute_result"
    }
   ],
   "source": [
    "rnd_reg.score(train_set_x,train_set_y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 68
    },
    "colab_type": "code",
    "id": "NzUH2ROsESAs",
    "outputId": "ea9442e8-11c1-4c13-831e-af3f68b04137"
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=4)]: Using backend ThreadingBackend with 4 concurrent workers.\n",
      "[Parallel(n_jobs=4)]: Done  15 out of  15 | elapsed:    0.5s finished\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "0.9564778689335509"
      ]
     },
     "execution_count": 45,
     "metadata": {
      "tags": []
     },
     "output_type": "execute_result"
    }
   ],
   "source": [
    "rnd_reg.score(test_set_x,test_set_y)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "N4aJWb-eW5_Q"
   },
   "source": [
    "### Extra trees ###\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 326
    },
    "colab_type": "code",
    "id": "m185zE8MTB3i",
    "outputId": "d7336ee8-9ca8-47ca-99f7-c584fc3917b5"
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local/lib/python3.6/dist-packages/ipykernel_launcher.py:5: DataConversionWarning: A column-vector y was passed when a 1d array was expected. Please change the shape of y to (n_samples,), for example using ravel().\n",
      "  \"\"\"\n",
      "[Parallel(n_jobs=-1)]: Using backend ThreadingBackend with 4 concurrent workers.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "building tree 1 of 10building tree 2 of 10building tree 3 of 10building tree 4 of 10\n",
      "\n",
      "\n",
      "\n",
      "building tree 5 of 10\n",
      "building tree 6 of 10\n",
      "building tree 7 of 10\n",
      "building tree 8 of 10\n",
      "building tree 9 of 10\n",
      "building tree 10 of 10\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=-1)]: Done   7 out of  10 | elapsed:  1.2min remaining:   31.6s\n",
      "[Parallel(n_jobs=-1)]: Done  10 out of  10 | elapsed:  1.6min finished\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 5min 31s, sys: 97.3 ms, total: 5min 31s\n",
      "Wall time: 1min 36s\n"
     ]
    }
   ],
   "source": [
    "%%time \n",
    "\n",
    "extra_trees = ExtraTreesRegressor(n_jobs=-1, verbose=3, n_estimators=10)\n",
    "extra_trees = extra_trees.fit(train_set_x, train_set_y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 85
    },
    "colab_type": "code",
    "id": "aiCZh-RuTeS9",
    "outputId": "670679f6-0c94-42dc-d783-c6cacb8b2694"
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=4)]: Using backend ThreadingBackend with 4 concurrent workers.\n",
      "[Parallel(n_jobs=4)]: Done   7 out of  10 | elapsed:    0.7s remaining:    0.3s\n",
      "[Parallel(n_jobs=4)]: Done  10 out of  10 | elapsed:    0.9s finished\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "1.0"
      ]
     },
     "execution_count": 48,
     "metadata": {
      "tags": []
     },
     "output_type": "execute_result"
    }
   ],
   "source": [
    "extra_trees.score(train_set_x,train_set_y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 85
    },
    "colab_type": "code",
    "id": "vi5NMI54dnTF",
    "outputId": "2da1036a-36b0-4f46-9cdf-146d07fa2f35"
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=4)]: Using backend ThreadingBackend with 4 concurrent workers.\n",
      "[Parallel(n_jobs=4)]: Done   7 out of  10 | elapsed:    0.3s remaining:    0.1s\n",
      "[Parallel(n_jobs=4)]: Done  10 out of  10 | elapsed:    0.4s finished\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "0.959570610665564"
      ]
     },
     "execution_count": 47,
     "metadata": {
      "tags": []
     },
     "output_type": "execute_result"
    }
   ],
   "source": [
    "extra_trees.score(test_set_x,test_set_y)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "5XkeEpdVXpGT"
   },
   "source": [
    "### Extreme Boosting ###"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "TIVZ60uIee3q"
   },
   "outputs": [],
   "source": [
    "xgbmodel = xgb.XGBRegressor(n_jobs=-1, verboxity=2, random_state=random_seed, n_estimators=100, objective=\"reg:squarederror\", max_depth=20)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 143
    },
    "colab_type": "code",
    "id": "HuUaTOhKfamI",
    "outputId": "7a7c64ba-5c18-4dfa-acd9-22485f59ab44"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "XGBRegressor(base_score=0.5, booster='gbtree', colsample_bylevel=1,\n",
       "             colsample_bynode=1, colsample_bytree=1, gamma=0,\n",
       "             importance_type='gain', learning_rate=0.1, max_delta_step=0,\n",
       "             max_depth=20, min_child_weight=1, missing=None, n_estimators=100,\n",
       "             n_jobs=-1, nthread=None, objective='reg:squarederror',\n",
       "             random_state=42, reg_alpha=0, reg_lambda=1, scale_pos_weight=1,\n",
       "             seed=None, silent=None, subsample=1, verbosity=1, verboxity=2)"
      ]
     },
     "execution_count": 40,
     "metadata": {
      "tags": []
     },
     "output_type": "execute_result"
    }
   ],
   "source": [
    "xgbmodel.fit(train_set_x,train_set_y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 35
    },
    "colab_type": "code",
    "id": "NPDdv2N3gWak",
    "outputId": "68a0f0ef-898b-4b76-a421-66221c062ffb"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.9975928970862603"
      ]
     },
     "execution_count": 41,
     "metadata": {
      "tags": []
     },
     "output_type": "execute_result"
    }
   ],
   "source": [
    "xgbmodel.score(train_set_y, xgbmodel.predict(train_set_x))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "3gqPy36kTVHt"
   },
   "source": [
    "All three models have capacity to fit the data."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "rkU6llLbNsgB"
   },
   "source": [
    "## Hyperparameter tuning: search of parameter space ## "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "7lqMjBUITIH1"
   },
   "source": [
    "### Random Forest ### "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "AjQZTgMiMTPn"
   },
   "source": [
    "Example hyperparameter random search\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "qo1eEcX69Y7J"
   },
   "outputs": [],
   "source": [
    "parameter_grid = {\n",
    "  'n_estimators': sp_randint(8,40),\n",
    "  'max_depth':  sp_randint(800,10000),\n",
    "  'min_samples_split': uniform(0.0001,0.8)\n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "FMCkEGiS9b70"
   },
   "outputs": [],
   "source": [
    "rnd_search = RandomForestRegressor()\n",
    "random_search_rf = RandomizedSearchCV(rnd_search, parameter_grid, cv=5, n_jobs=-1, random_state=random_seed, verbose=10)\n",
    "\n",
    "random_search_rf.fit(train_set_x,train_set_y)\n",
    "result = pd.concat([pd.DataFrame(random_search_rf.cv_results_[\"params\"]),pd.DataFrame(random_search_rf.cv_results_[\"mean_test_score\"], columns=[\"Score\"])],axis=1)\n",
    "result.sort_values(by=\"Score\", ascending=False, inplace=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "IT8r9NKeXml7"
   },
   "source": [
    "Selection of best RF model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "bkJCVYUbRwlF"
   },
   "outputs": [],
   "source": [
    "rf = RandomForestRegressor(bootstrap=True, ccp_alpha=0.0, criterion='mse',\n",
    "                      max_depth=9734, max_features='auto', max_leaf_nodes=None,\n",
    "                      max_samples=None, min_impurity_decrease=0.0,\n",
    "                      min_impurity_split=None, min_samples_leaf=1,\n",
    "                      min_samples_split=3, min_weight_fraction_leaf=0.0,\n",
    "                      n_estimators=48, n_jobs=None, oob_score=False,\n",
    "                      random_state=random_seed, verbose=3, warm_start=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "9o_Pp2YTR0rR"
   },
   "outputs": [],
   "source": [
    "%%time\n",
    "\n",
    "rf.fit(train_set_x, train_set_y)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "ZZsWj6X2Xt6s"
   },
   "source": [
    "Final RF has score of R2=0.958"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 102
    },
    "colab_type": "code",
    "id": "K32CK5cyR70P",
    "outputId": "81a88f1b-c596-4b17-dc9b-200717dcb0fe"
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done   1 out of   1 | elapsed:    0.1s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done   2 out of   2 | elapsed:    0.2s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  48 out of  48 | elapsed:    4.4s finished\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "0.958166398421789"
      ]
     },
     "execution_count": 30,
     "metadata": {
      "tags": []
     },
     "output_type": "execute_result"
    }
   ],
   "source": [
    "rf.score(test_set_x_prepared, test_set_y)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "sWA-acyiNz_S"
   },
   "source": [
    "### Extra Trees ###"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "f_WmbcojuyKU"
   },
   "outputs": [],
   "source": [
    "parameter_grid = {\n",
    "  'n_estimators': sp_randint(1,50),\n",
    "  'max_depth':  sp_randint(1,40),\n",
    "  'min_samples_split': uniform(0.01,0.199)\n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "PllVnTe8Pmn6"
   },
   "outputs": [],
   "source": [
    "extra_trees_search = ExtraTreesRegressor()\n",
    "random_search = RandomizedSearchCV(extra_trees_search, parameter_grid, cv=5, n_jobs=-1, random_state=random_seed)\n",
    "random_search.fit(train_set_x,train_set_y)\n",
    "\n",
    "result = pd.concat([pd.DataFrame(random_search.cv_results_[\"params\"]),pd.DataFrame(random_search.cv_results_[\"mean_test_score\"], columns=[\"Score\"])],axis=1)\n",
    "result.sort_values(by=\"Score\", ascending=False, inplace=True)\n",
    "result"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "1AEhPTk4W1AS"
   },
   "source": [
    "Selecting final extra trees model\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "mstAQXRvT_2O"
   },
   "outputs": [],
   "source": [
    "extra_trees_final = ExtraTreesRegressor(bootstrap=False, ccp_alpha=0.0, criterion='mse',\n",
    "                    max_depth=None, max_features='auto', max_leaf_nodes=None,\n",
    "                    max_samples=None, min_impurity_decrease=0.0,\n",
    "                    min_impurity_split=None, min_samples_leaf=1,\n",
    "                    min_samples_split=2, min_weight_fraction_leaf=0.0,\n",
    "                    n_estimators=10, n_jobs=-1, oob_score=False,\n",
    "                    random_state=None, verbose=3, warm_start=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "Rga4cDi1UDHe"
   },
   "outputs": [],
   "source": [
    "%%time\n",
    "extra_trees_final.fit(train_set_x, train_set_y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 85
    },
    "colab_type": "code",
    "id": "LpWY-spaVatd",
    "outputId": "49f0d0be-ebba-4711-d2db-33805bd7cdc1"
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=4)]: Using backend ThreadingBackend with 4 concurrent workers.\n",
      "[Parallel(n_jobs=4)]: Done   7 out of  10 | elapsed:    0.3s remaining:    0.1s\n",
      "[Parallel(n_jobs=4)]: Done  10 out of  10 | elapsed:    0.4s finished\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "0.9598494134785569"
      ]
     },
     "execution_count": 61,
     "metadata": {
      "tags": []
     },
     "output_type": "execute_result"
    }
   ],
   "source": [
    "extra_trees_final.score(test_set_x, test_set_y)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "95r3nUU6WVzS"
   },
   "source": [
    "Final extra trees score is 0.960"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "bBuNvsRJUbQZ"
   },
   "source": [
    "### Extreme Boosting ###"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "_6BlZMHCHC-V"
   },
   "source": [
    "Example random search over hyperparameters"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 938
    },
    "colab_type": "code",
    "id": "mtag2-MxaN6C",
    "outputId": "2a239d6b-df63-44e3-80a9-c8adeb4d42eb"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fitting 3 folds for each of 5 candidates, totalling 15 fits\n",
      "[CV] colsample_bytree=0.7399632095077889, gamma=1.2609507143064098, max_depth=15, min_child_weight=16, n_estimators=237, subsample=0.8719370031589297 \n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[CV]  colsample_bytree=0.7399632095077889, gamma=1.2609507143064098, max_depth=15, min_child_weight=16, n_estimators=237, subsample=0.8719370031589297, score=0.959, total=  26.7s\n",
      "[CV] colsample_bytree=0.7399632095077889, gamma=1.2609507143064098, max_depth=15, min_child_weight=16, n_estimators=237, subsample=0.8719370031589297 \n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=1)]: Done   1 out of   1 | elapsed:   26.7s remaining:    0.0s\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[CV]  colsample_bytree=0.7399632095077889, gamma=1.2609507143064098, max_depth=15, min_child_weight=16, n_estimators=237, subsample=0.8719370031589297, score=0.958, total=  26.7s\n",
      "[CV] colsample_bytree=0.7399632095077889, gamma=1.2609507143064098, max_depth=15, min_child_weight=16, n_estimators=237, subsample=0.8719370031589297 \n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=1)]: Done   2 out of   2 | elapsed:   53.5s remaining:    0.0s\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[CV]  colsample_bytree=0.7399632095077889, gamma=1.2609507143064098, max_depth=15, min_child_weight=16, n_estimators=237, subsample=0.8719370031589297, score=0.959, total=  26.2s\n",
      "[CV] colsample_bytree=0.7456666202282872, gamma=1.260099974915818, max_depth=15, min_child_weight=16, n_estimators=229, subsample=0.8720223002348642 \n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=1)]: Done   3 out of   3 | elapsed:  1.3min remaining:    0.0s\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[CV]  colsample_bytree=0.7456666202282872, gamma=1.260099974915818, max_depth=15, min_child_weight=16, n_estimators=229, subsample=0.8720223002348642, score=0.959, total=  26.2s\n",
      "[CV] colsample_bytree=0.7456666202282872, gamma=1.260099974915818, max_depth=15, min_child_weight=16, n_estimators=229, subsample=0.8720223002348642 \n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=1)]: Done   4 out of   4 | elapsed:  1.8min remaining:    0.0s\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[CV]  colsample_bytree=0.7456666202282872, gamma=1.260099974915818, max_depth=15, min_child_weight=16, n_estimators=229, subsample=0.8720223002348642, score=0.957, total=  25.9s\n",
      "[CV] colsample_bytree=0.7456666202282872, gamma=1.260099974915818, max_depth=15, min_child_weight=16, n_estimators=229, subsample=0.8720223002348642 \n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=1)]: Done   5 out of   5 | elapsed:  2.2min remaining:    0.0s\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[CV]  colsample_bytree=0.7456666202282872, gamma=1.260099974915818, max_depth=15, min_child_weight=16, n_estimators=229, subsample=0.8720223002348642, score=0.959, total=  25.8s\n",
      "[CV] colsample_bytree=0.7666458062236836, gamma=1.2600205844942958, max_depth=14, min_child_weight=16, n_estimators=236, subsample=0.878771054180315 \n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=1)]: Done   6 out of   6 | elapsed:  2.6min remaining:    0.0s\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[CV]  colsample_bytree=0.7666458062236836, gamma=1.2600205844942958, max_depth=14, min_child_weight=16, n_estimators=236, subsample=0.878771054180315, score=0.959, total=  22.8s\n",
      "[CV] colsample_bytree=0.7666458062236836, gamma=1.2600205844942958, max_depth=14, min_child_weight=16, n_estimators=236, subsample=0.878771054180315 \n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=1)]: Done   7 out of   7 | elapsed:  3.0min remaining:    0.0s\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[CV]  colsample_bytree=0.7666458062236836, gamma=1.2600205844942958, max_depth=14, min_child_weight=16, n_estimators=236, subsample=0.878771054180315, score=0.957, total=  22.6s\n",
      "[CV] colsample_bytree=0.7666458062236836, gamma=1.2600205844942958, max_depth=14, min_child_weight=16, n_estimators=236, subsample=0.878771054180315 \n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=1)]: Done   8 out of   8 | elapsed:  3.4min remaining:    0.0s\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[CV]  colsample_bytree=0.7666458062236836, gamma=1.2600205844942958, max_depth=14, min_child_weight=16, n_estimators=236, subsample=0.878771054180315, score=0.959, total=  22.6s\n",
      "[CV] colsample_bytree=0.7100623012672811, gamma=1.2609922115592913, max_depth=13, min_child_weight=16, n_estimators=234, subsample=0.8704951286326448 \n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=1)]: Done   9 out of   9 | elapsed:  3.8min remaining:    0.0s\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[CV]  colsample_bytree=0.7100623012672811, gamma=1.2609922115592913, max_depth=13, min_child_weight=16, n_estimators=234, subsample=0.8704951286326448, score=0.958, total=  19.0s\n",
      "[CV] colsample_bytree=0.7100623012672811, gamma=1.2609922115592913, max_depth=13, min_child_weight=16, n_estimators=234, subsample=0.8704951286326448 \n",
      "[CV]  colsample_bytree=0.7100623012672811, gamma=1.2609922115592913, max_depth=13, min_child_weight=16, n_estimators=234, subsample=0.8704951286326448, score=0.957, total=  18.9s\n",
      "[CV] colsample_bytree=0.7100623012672811, gamma=1.2609922115592913, max_depth=13, min_child_weight=16, n_estimators=234, subsample=0.8704951286326448 \n",
      "[CV]  colsample_bytree=0.7100623012672811, gamma=1.2609922115592913, max_depth=13, min_child_weight=16, n_estimators=234, subsample=0.8704951286326448, score=0.959, total=  18.8s\n",
      "[CV] colsample_bytree=0.7445556014913692, gamma=1.260291229140198, max_depth=15, min_child_weight=15, n_estimators=234, subsample=0.8609333132642722 \n",
      "[CV]  colsample_bytree=0.7445556014913692, gamma=1.260291229140198, max_depth=15, min_child_weight=15, n_estimators=234, subsample=0.8609333132642722, score=0.958, total=  27.0s\n",
      "[CV] colsample_bytree=0.7445556014913692, gamma=1.260291229140198, max_depth=15, min_child_weight=15, n_estimators=234, subsample=0.8609333132642722 \n",
      "[CV]  colsample_bytree=0.7445556014913692, gamma=1.260291229140198, max_depth=15, min_child_weight=15, n_estimators=234, subsample=0.8609333132642722, score=0.957, total=  26.7s\n",
      "[CV] colsample_bytree=0.7445556014913692, gamma=1.260291229140198, max_depth=15, min_child_weight=15, n_estimators=234, subsample=0.8609333132642722 \n",
      "[CV]  colsample_bytree=0.7445556014913692, gamma=1.260291229140198, max_depth=15, min_child_weight=15, n_estimators=234, subsample=0.8609333132642722, score=0.959, total=  26.6s\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=1)]: Done  15 out of  15 | elapsed:  6.0min finished\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>colsample_bytree</th>\n",
       "      <th>gamma</th>\n",
       "      <th>max_depth</th>\n",
       "      <th>min_child_weight</th>\n",
       "      <th>n_estimators</th>\n",
       "      <th>subsample</th>\n",
       "      <th>Score</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0.745667</td>\n",
       "      <td>1.260100</td>\n",
       "      <td>15</td>\n",
       "      <td>16</td>\n",
       "      <td>229</td>\n",
       "      <td>0.872022</td>\n",
       "      <td>0.958369</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0.766646</td>\n",
       "      <td>1.260021</td>\n",
       "      <td>14</td>\n",
       "      <td>16</td>\n",
       "      <td>236</td>\n",
       "      <td>0.878771</td>\n",
       "      <td>0.958349</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0.739963</td>\n",
       "      <td>1.260951</td>\n",
       "      <td>15</td>\n",
       "      <td>16</td>\n",
       "      <td>237</td>\n",
       "      <td>0.871937</td>\n",
       "      <td>0.958315</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0.744556</td>\n",
       "      <td>1.260291</td>\n",
       "      <td>15</td>\n",
       "      <td>15</td>\n",
       "      <td>234</td>\n",
       "      <td>0.860933</td>\n",
       "      <td>0.958272</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0.710062</td>\n",
       "      <td>1.260992</td>\n",
       "      <td>13</td>\n",
       "      <td>16</td>\n",
       "      <td>234</td>\n",
       "      <td>0.870495</td>\n",
       "      <td>0.957775</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   colsample_bytree     gamma  max_depth  ...  n_estimators  subsample     Score\n",
       "1          0.745667  1.260100         15  ...           229   0.872022  0.958369\n",
       "2          0.766646  1.260021         14  ...           236   0.878771  0.958349\n",
       "0          0.739963  1.260951         15  ...           237   0.871937  0.958315\n",
       "4          0.744556  1.260291         15  ...           234   0.860933  0.958272\n",
       "3          0.710062  1.260992         13  ...           234   0.870495  0.957775\n",
       "\n",
       "[5 rows x 7 columns]"
      ]
     },
     "execution_count": 22,
     "metadata": {
      "tags": []
     },
     "output_type": "execute_result"
    }
   ],
   "source": [
    "parameter_grid4 = {\n",
    "  'min_child_weight': sp_randint(15,17),\n",
    "  'max_depth':  sp_randint(13,16),\n",
    "  'gamma': uniform(1.260,0.001),\n",
    "  'subsample': uniform(0.86, 0.02),\n",
    " 'colsample_bytree': uniform(0.71,0.08),\n",
    "  'n_estimators': sp_randint(225,240)\n",
    "}\n",
    "\n",
    "xgb_model4 = xgb.XGBRegressor(tree_method='gpu_hist', objective=\"reg:squarederror\") #tree method gpu_hist allows gpu usage\n",
    "\n",
    "random_search_xgb4 = RandomizedSearchCV(xgb_model4, parameter_grid4, cv=5, random_state=random_seed, verbose=10, error_score='raise', n_iter=5)\n",
    "\n",
    "random_search_xgb4.fit(train_set_x,train_set_y)\n",
    "\n",
    "result = pd.concat([pd.DataFrame(random_search_xgb4.cv_results_[\"params\"]),pd.DataFrame(random_search_xgb4.cv_results_[\"mean_test_score\"], columns=[\"Score\"])],axis=1)\n",
    "result.sort_values(by=\"Score\", ascending=False, inplace=True)\n",
    "result"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "RA615l8HV-mA"
   },
   "source": [
    "Final model\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "-TlsGr2OhO-n"
   },
   "outputs": [],
   "source": [
    "xgb_model_final = xgb.XGBRegressor(tree_method='gpu_hist', objective=\"reg:squarederror\",\n",
    "                                   colsample_bytree=0.745667, gamma=1.260100, max_depth=15,\n",
    "                                   min_child_weight=16, n_estimators=229, subsample=0.872022,\n",
    "                                   verbosity=3, random_state=random_seed, seed=random_seed)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 153
    },
    "colab_type": "code",
    "id": "tQEMJLmfiArU",
    "outputId": "93a594d3-2ff2-4df5-9ea2-90d537d198be"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "XGBRegressor(base_score=0.5, booster='gbtree', colsample_bylevel=1,\n",
       "             colsample_bynode=1, colsample_bytree=0.745667, gamma=1.2601,\n",
       "             importance_type='gain', learning_rate=0.1, max_delta_step=0,\n",
       "             max_depth=15, min_child_weight=16, missing=None, n_estimators=229,\n",
       "             n_jobs=1, nthread=None, objective='reg:squarederror',\n",
       "             random_state=42, reg_alpha=0, reg_lambda=1, scale_pos_weight=1,\n",
       "             seed=None, silent=None, subsample=0.872022, tree_method='gpu_hist',\n",
       "             verbosity=3)"
      ]
     },
     "execution_count": 65,
     "metadata": {
      "tags": []
     },
     "output_type": "execute_result"
    }
   ],
   "source": [
    "xgb_model_final.fit(train_set_x,train_set_y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 34
    },
    "colab_type": "code",
    "id": "j_74aGJyid1q",
    "outputId": "e435103c-8517-4917-ed77-97539987c1e2"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.9627299228249108"
      ]
     },
     "execution_count": 15,
     "metadata": {
      "tags": []
     },
     "output_type": "execute_result"
    }
   ],
   "source": [
    "xgb_model_final.score(test_set_x, test_set_y)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "cMtbLt_MWCBS"
   },
   "source": [
    "Final XGBoost test score is 0.962\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "AoB6XJOxHGjE"
   },
   "outputs": [],
   "source": [
    "xgb_model_final.save_model(f'final_model_seeded_{random_state}.json')"
   ]
  }
 ],
 "metadata": {
  "colab": {
   "authorship_tag": "ABX9TyPGdgX8XMUQ2mxQqklOJVPm",
   "collapsed_sections": [],
   "include_colab_link": true,
   "machine_shape": "hm",
   "mount_file_id": "14G0Plb1oiIIrM0gA6OqMXuYWQXXowgWJ",
   "name": "Model_exploration_Raghul_pre_damon_final.ipynb",
   "provenance": [],
   "toc_visible": true
  },
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.0"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}