2 - Model Parameter Tuning.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "from model_funcs import gsearch, rsearch, data_split\n",
    "from sklearn.model_selection import RandomizedSearchCV, GridSearchCV\n",
    "import pandas as pd\n",
    "import numpy as np"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Density</th>\n",
       "      <th>YP</th>\n",
       "      <th>Temperature</th>\n",
       "      <th>ROP</th>\n",
       "      <th>Pipe rotation</th>\n",
       "      <th>Flow rate</th>\n",
       "      <th>Inclination</th>\n",
       "      <th>Eccentricity</th>\n",
       "      <th>Concentration</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>8.314</td>\n",
       "      <td>20</td>\n",
       "      <td>80</td>\n",
       "      <td>30</td>\n",
       "      <td>0</td>\n",
       "      <td>100</td>\n",
       "      <td>90</td>\n",
       "      <td>0.881</td>\n",
       "      <td>0.35</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>8.314</td>\n",
       "      <td>20</td>\n",
       "      <td>80</td>\n",
       "      <td>30</td>\n",
       "      <td>0</td>\n",
       "      <td>200</td>\n",
       "      <td>90</td>\n",
       "      <td>0.881</td>\n",
       "      <td>0.14</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>8.314</td>\n",
       "      <td>20</td>\n",
       "      <td>80</td>\n",
       "      <td>30</td>\n",
       "      <td>80</td>\n",
       "      <td>100</td>\n",
       "      <td>90</td>\n",
       "      <td>0.881</td>\n",
       "      <td>0.06</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>8.314</td>\n",
       "      <td>20</td>\n",
       "      <td>80</td>\n",
       "      <td>30</td>\n",
       "      <td>80</td>\n",
       "      <td>200</td>\n",
       "      <td>90</td>\n",
       "      <td>0.881</td>\n",
       "      <td>0.02</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>8.314</td>\n",
       "      <td>20</td>\n",
       "      <td>180</td>\n",
       "      <td>30</td>\n",
       "      <td>0</td>\n",
       "      <td>100</td>\n",
       "      <td>90</td>\n",
       "      <td>0.881</td>\n",
       "      <td>0.11</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   Density  YP  Temperature  ROP  Pipe rotation  Flow rate  Inclination  \\\n",
       "0    8.314  20           80   30              0        100           90   \n",
       "1    8.314  20           80   30              0        200           90   \n",
       "2    8.314  20           80   30             80        100           90   \n",
       "3    8.314  20           80   30             80        200           90   \n",
       "4    8.314  20          180   30              0        100           90   \n",
       "\n",
       "   Eccentricity  Concentration  \n",
       "0         0.881           0.35  \n",
       "1         0.881           0.14  \n",
       "2         0.881           0.06  \n",
       "3         0.881           0.02  \n",
       "4         0.881           0.11  "
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data = pd.read_csv('files/cleaned_data.csv')\n",
    "data.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "X_train, X_test, y_train, y_test = data_split(data)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Parameter Tuning"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.linear_model import Ridge\n",
    "from sklearn.ensemble import RandomForestRegressor, GradientBoostingRegressor, AdaBoostRegressor\n",
    "from sklearn.tree import DecisionTreeRegressor"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Random Forest"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "forest_rparams = {'n_estimators': np.logspace(1, 4, 50).astype(int), 'max_depth': np.logspace(1, 3, 20).astype(int), 'bootstrap': [True, False]}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "c:\\Users\\Thompson\\AppData\\Local\\Programs\\Python\\Python39\\lib\\site-packages\\sklearn\\ensemble\\_forest.py:396: FutureWarning: Criterion 'mse' was deprecated in v1.0 and will be removed in version 1.2. Use `criterion='squared_error'` which is equivalent.\n",
      "  warn(\n"
     ]
    }
   ],
   "source": [
    "forest_rsearch = rsearch(iter = 200, est = RandomForestRegressor(criterion='mse', random_state=42), params=forest_rparams, x=X_train, y=y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>param_n_estimators</th>\n",
       "      <th>param_max_depth</th>\n",
       "      <th>param_bootstrap</th>\n",
       "      <th>params</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>197</th>\n",
       "      <td>71</td>\n",
       "      <td>12</td>\n",
       "      <td>True</td>\n",
       "      <td>{'n_estimators': 71, 'max_depth': 12, 'bootstr...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>199</th>\n",
       "      <td>47</td>\n",
       "      <td>26</td>\n",
       "      <td>True</td>\n",
       "      <td>{'n_estimators': 47, 'max_depth': 26, 'bootstr...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>104</th>\n",
       "      <td>35</td>\n",
       "      <td>615</td>\n",
       "      <td>True</td>\n",
       "      <td>{'n_estimators': 35, 'max_depth': 615, 'bootst...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>83</th>\n",
       "      <td>26</td>\n",
       "      <td>183</td>\n",
       "      <td>True</td>\n",
       "      <td>{'n_estimators': 26, 'max_depth': 183, 'bootst...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>191</th>\n",
       "      <td>62</td>\n",
       "      <td>10</td>\n",
       "      <td>True</td>\n",
       "      <td>{'n_estimators': 62, 'max_depth': 10, 'bootstr...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>189</th>\n",
       "      <td>62</td>\n",
       "      <td>33</td>\n",
       "      <td>True</td>\n",
       "      <td>{'n_estimators': 62, 'max_depth': 33, 'bootstr...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>910</td>\n",
       "      <td>12</td>\n",
       "      <td>True</td>\n",
       "      <td>{'n_estimators': 910, 'max_depth': 12, 'bootst...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>24</th>\n",
       "      <td>517</td>\n",
       "      <td>483</td>\n",
       "      <td>True</td>\n",
       "      <td>{'n_estimators': 517, 'max_depth': 483, 'boots...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>153</th>\n",
       "      <td>686</td>\n",
       "      <td>10</td>\n",
       "      <td>True</td>\n",
       "      <td>{'n_estimators': 686, 'max_depth': 10, 'bootst...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>176</th>\n",
       "      <td>109</td>\n",
       "      <td>42</td>\n",
       "      <td>True</td>\n",
       "      <td>{'n_estimators': 109, 'max_depth': 42, 'bootst...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "    param_n_estimators param_max_depth param_bootstrap  \\\n",
       "197                 71              12            True   \n",
       "199                 47              26            True   \n",
       "104                 35             615            True   \n",
       "83                  26             183            True   \n",
       "191                 62              10            True   \n",
       "189                 62              33            True   \n",
       "14                 910              12            True   \n",
       "24                 517             483            True   \n",
       "153                686              10            True   \n",
       "176                109              42            True   \n",
       "\n",
       "                                                params  \n",
       "197  {'n_estimators': 71, 'max_depth': 12, 'bootstr...  \n",
       "199  {'n_estimators': 47, 'max_depth': 26, 'bootstr...  \n",
       "104  {'n_estimators': 35, 'max_depth': 615, 'bootst...  \n",
       "83   {'n_estimators': 26, 'max_depth': 183, 'bootst...  \n",
       "191  {'n_estimators': 62, 'max_depth': 10, 'bootstr...  \n",
       "189  {'n_estimators': 62, 'max_depth': 33, 'bootstr...  \n",
       "14   {'n_estimators': 910, 'max_depth': 12, 'bootst...  \n",
       "24   {'n_estimators': 517, 'max_depth': 483, 'boots...  \n",
       "153  {'n_estimators': 686, 'max_depth': 10, 'bootst...  \n",
       "176  {'n_estimators': 109, 'max_depth': 42, 'bootst...  "
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pd.DataFrame(forest_rsearch.cv_results_).sort_values(by = 'mean_test_score', ascending= False).head(10).iloc[:, 4:8]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "RandomForestRegressor(criterion='mse', max_depth=12, n_estimators=71)"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "forest_rsearch.best_estimator_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "forest_gparams = {'max_depth': np.arange(20, 200, 5), 'n_estimators': np.arange(5, 70, 2)}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "forest_gsearch = gsearch(est = RandomForestRegressor(criterion = 'squared_error', bootstrap=True, random_state=42), \n",
    "                    params=forest_gparams, x=X_train, y=y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>param_max_depth</th>\n",
       "      <th>param_n_estimators</th>\n",
       "      <th>params</th>\n",
       "      <th>split0_test_score</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>1152</th>\n",
       "      <td>190</td>\n",
       "      <td>65</td>\n",
       "      <td>{'max_depth': 190, 'n_estimators': 65}</td>\n",
       "      <td>-0.041979</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>30</th>\n",
       "      <td>20</td>\n",
       "      <td>65</td>\n",
       "      <td>{'max_depth': 20, 'n_estimators': 65}</td>\n",
       "      <td>-0.041979</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>921</th>\n",
       "      <td>155</td>\n",
       "      <td>65</td>\n",
       "      <td>{'max_depth': 155, 'n_estimators': 65}</td>\n",
       "      <td>-0.041979</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>888</th>\n",
       "      <td>150</td>\n",
       "      <td>65</td>\n",
       "      <td>{'max_depth': 150, 'n_estimators': 65}</td>\n",
       "      <td>-0.041979</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>855</th>\n",
       "      <td>145</td>\n",
       "      <td>65</td>\n",
       "      <td>{'max_depth': 145, 'n_estimators': 65}</td>\n",
       "      <td>-0.041979</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>822</th>\n",
       "      <td>140</td>\n",
       "      <td>65</td>\n",
       "      <td>{'max_depth': 140, 'n_estimators': 65}</td>\n",
       "      <td>-0.041979</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>63</th>\n",
       "      <td>25</td>\n",
       "      <td>65</td>\n",
       "      <td>{'max_depth': 25, 'n_estimators': 65}</td>\n",
       "      <td>-0.041979</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1185</th>\n",
       "      <td>195</td>\n",
       "      <td>65</td>\n",
       "      <td>{'max_depth': 195, 'n_estimators': 65}</td>\n",
       "      <td>-0.041979</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>789</th>\n",
       "      <td>135</td>\n",
       "      <td>65</td>\n",
       "      <td>{'max_depth': 135, 'n_estimators': 65}</td>\n",
       "      <td>-0.041979</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>756</th>\n",
       "      <td>130</td>\n",
       "      <td>65</td>\n",
       "      <td>{'max_depth': 130, 'n_estimators': 65}</td>\n",
       "      <td>-0.041979</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "     param_max_depth param_n_estimators  \\\n",
       "1152             190                 65   \n",
       "30                20                 65   \n",
       "921              155                 65   \n",
       "888              150                 65   \n",
       "855              145                 65   \n",
       "822              140                 65   \n",
       "63                25                 65   \n",
       "1185             195                 65   \n",
       "789              135                 65   \n",
       "756              130                 65   \n",
       "\n",
       "                                      params  split0_test_score  \n",
       "1152  {'max_depth': 190, 'n_estimators': 65}          -0.041979  \n",
       "30     {'max_depth': 20, 'n_estimators': 65}          -0.041979  \n",
       "921   {'max_depth': 155, 'n_estimators': 65}          -0.041979  \n",
       "888   {'max_depth': 150, 'n_estimators': 65}          -0.041979  \n",
       "855   {'max_depth': 145, 'n_estimators': 65}          -0.041979  \n",
       "822   {'max_depth': 140, 'n_estimators': 65}          -0.041979  \n",
       "63     {'max_depth': 25, 'n_estimators': 65}          -0.041979  \n",
       "1185  {'max_depth': 195, 'n_estimators': 65}          -0.041979  \n",
       "789   {'max_depth': 135, 'n_estimators': 65}          -0.041979  \n",
       "756   {'max_depth': 130, 'n_estimators': 65}          -0.041979  "
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pd.DataFrame(forest_gsearch.cv_results_).sort_values(by = 'mean_test_score', ascending= False).head(10).iloc[:, 4:8]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [],
   "source": [
    "forest_gparams2 = {'max_depth': np.arange(150, 200), 'n_estimators': np.arange(35, 70)}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [],
   "source": [
    "forest_gsearch2 = gsearch(est = RandomForestRegressor(criterion = 'squared_error', random_state=42, bootstrap=True), \n",
    "                    params=forest_gparams2, x=X_train, y=y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 141,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>param_max_depth</th>\n",
       "      <th>param_n_estimators</th>\n",
       "      <th>params</th>\n",
       "      <th>split0_test_score</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>24</th>\n",
       "      <td>150</td>\n",
       "      <td>59</td>\n",
       "      <td>{'max_depth': 150, 'n_estimators': 59}</td>\n",
       "      <td>-0.039787</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>70</th>\n",
       "      <td>152</td>\n",
       "      <td>35</td>\n",
       "      <td>{'max_depth': 152, 'n_estimators': 35}</td>\n",
       "      <td>-0.038164</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>681</th>\n",
       "      <td>169</td>\n",
       "      <td>51</td>\n",
       "      <td>{'max_depth': 169, 'n_estimators': 51}</td>\n",
       "      <td>-0.036818</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1564</th>\n",
       "      <td>194</td>\n",
       "      <td>59</td>\n",
       "      <td>{'max_depth': 194, 'n_estimators': 59}</td>\n",
       "      <td>-0.037362</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1689</th>\n",
       "      <td>198</td>\n",
       "      <td>44</td>\n",
       "      <td>{'max_depth': 198, 'n_estimators': 44}</td>\n",
       "      <td>-0.038176</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1186</th>\n",
       "      <td>183</td>\n",
       "      <td>66</td>\n",
       "      <td>{'max_depth': 183, 'n_estimators': 66}</td>\n",
       "      <td>-0.039807</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1664</th>\n",
       "      <td>197</td>\n",
       "      <td>54</td>\n",
       "      <td>{'max_depth': 197, 'n_estimators': 54}</td>\n",
       "      <td>-0.043121</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1590</th>\n",
       "      <td>195</td>\n",
       "      <td>50</td>\n",
       "      <td>{'max_depth': 195, 'n_estimators': 50}</td>\n",
       "      <td>-0.037107</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1379</th>\n",
       "      <td>189</td>\n",
       "      <td>49</td>\n",
       "      <td>{'max_depth': 189, 'n_estimators': 49}</td>\n",
       "      <td>-0.036948</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>215</th>\n",
       "      <td>156</td>\n",
       "      <td>40</td>\n",
       "      <td>{'max_depth': 156, 'n_estimators': 40}</td>\n",
       "      <td>-0.043675</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "     param_max_depth param_n_estimators  \\\n",
       "24               150                 59   \n",
       "70               152                 35   \n",
       "681              169                 51   \n",
       "1564             194                 59   \n",
       "1689             198                 44   \n",
       "1186             183                 66   \n",
       "1664             197                 54   \n",
       "1590             195                 50   \n",
       "1379             189                 49   \n",
       "215              156                 40   \n",
       "\n",
       "                                      params  split0_test_score  \n",
       "24    {'max_depth': 150, 'n_estimators': 59}          -0.039787  \n",
       "70    {'max_depth': 152, 'n_estimators': 35}          -0.038164  \n",
       "681   {'max_depth': 169, 'n_estimators': 51}          -0.036818  \n",
       "1564  {'max_depth': 194, 'n_estimators': 59}          -0.037362  \n",
       "1689  {'max_depth': 198, 'n_estimators': 44}          -0.038176  \n",
       "1186  {'max_depth': 183, 'n_estimators': 66}          -0.039807  \n",
       "1664  {'max_depth': 197, 'n_estimators': 54}          -0.043121  \n",
       "1590  {'max_depth': 195, 'n_estimators': 50}          -0.037107  \n",
       "1379  {'max_depth': 189, 'n_estimators': 49}          -0.036948  \n",
       "215   {'max_depth': 156, 'n_estimators': 40}          -0.043675  "
      ]
     },
     "execution_count": 141,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pd.DataFrame(forest_gsearch2.cv_results_).sort_values(by = 'mean_test_score', ascending= False).head(10).iloc[:, 4:8]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [],
   "source": [
    "forest_gparams3 = {'max_depth': np.arange(150, 200), 'n_estimators': np.arange(49, 68)}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [],
   "source": [
    "forest_gsearch3 = gsearch(est = RandomForestRegressor(criterion = 'squared_error', random_state = 42,  \n",
    "                    bootstrap=True), params=forest_gparams3, x=X_train, y=y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 135,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>param_max_depth</th>\n",
       "      <th>param_n_estimators</th>\n",
       "      <th>params</th>\n",
       "      <th>split0_test_score</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>492</th>\n",
       "      <td>175</td>\n",
       "      <td>66</td>\n",
       "      <td>{'max_depth': 175, 'n_estimators': 66}</td>\n",
       "      <td>-0.041804</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>302</th>\n",
       "      <td>165</td>\n",
       "      <td>66</td>\n",
       "      <td>{'max_depth': 165, 'n_estimators': 66}</td>\n",
       "      <td>-0.041804</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>549</th>\n",
       "      <td>178</td>\n",
       "      <td>66</td>\n",
       "      <td>{'max_depth': 178, 'n_estimators': 66}</td>\n",
       "      <td>-0.041804</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>815</th>\n",
       "      <td>192</td>\n",
       "      <td>66</td>\n",
       "      <td>{'max_depth': 192, 'n_estimators': 66}</td>\n",
       "      <td>-0.041804</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>625</th>\n",
       "      <td>182</td>\n",
       "      <td>66</td>\n",
       "      <td>{'max_depth': 182, 'n_estimators': 66}</td>\n",
       "      <td>-0.041804</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>321</th>\n",
       "      <td>166</td>\n",
       "      <td>66</td>\n",
       "      <td>{'max_depth': 166, 'n_estimators': 66}</td>\n",
       "      <td>-0.041804</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>207</th>\n",
       "      <td>160</td>\n",
       "      <td>66</td>\n",
       "      <td>{'max_depth': 160, 'n_estimators': 66}</td>\n",
       "      <td>-0.041804</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>929</th>\n",
       "      <td>198</td>\n",
       "      <td>66</td>\n",
       "      <td>{'max_depth': 198, 'n_estimators': 66}</td>\n",
       "      <td>-0.041804</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>416</th>\n",
       "      <td>171</td>\n",
       "      <td>66</td>\n",
       "      <td>{'max_depth': 171, 'n_estimators': 66}</td>\n",
       "      <td>-0.041804</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>36</th>\n",
       "      <td>151</td>\n",
       "      <td>66</td>\n",
       "      <td>{'max_depth': 151, 'n_estimators': 66}</td>\n",
       "      <td>-0.041804</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "    param_max_depth param_n_estimators  \\\n",
       "492             175                 66   \n",
       "302             165                 66   \n",
       "549             178                 66   \n",
       "815             192                 66   \n",
       "625             182                 66   \n",
       "321             166                 66   \n",
       "207             160                 66   \n",
       "929             198                 66   \n",
       "416             171                 66   \n",
       "36              151                 66   \n",
       "\n",
       "                                     params  split0_test_score  \n",
       "492  {'max_depth': 175, 'n_estimators': 66}          -0.041804  \n",
       "302  {'max_depth': 165, 'n_estimators': 66}          -0.041804  \n",
       "549  {'max_depth': 178, 'n_estimators': 66}          -0.041804  \n",
       "815  {'max_depth': 192, 'n_estimators': 66}          -0.041804  \n",
       "625  {'max_depth': 182, 'n_estimators': 66}          -0.041804  \n",
       "321  {'max_depth': 166, 'n_estimators': 66}          -0.041804  \n",
       "207  {'max_depth': 160, 'n_estimators': 66}          -0.041804  \n",
       "929  {'max_depth': 198, 'n_estimators': 66}          -0.041804  \n",
       "416  {'max_depth': 171, 'n_estimators': 66}          -0.041804  \n",
       "36   {'max_depth': 151, 'n_estimators': 66}          -0.041804  "
      ]
     },
     "execution_count": 135,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pd.DataFrame(forest_gsearch3.cv_results_).sort_values(by = 'mean_test_score', ascending= False).head(10).iloc[:, 4:8]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 136,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "RandomForestRegressor(max_depth=150, n_estimators=66, random_state=42)"
      ]
     },
     "execution_count": 136,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "forest_gsearch3.best_estimator_"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Gradient Boosting"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [],
   "source": [
    "gb_rparams = {'loss':['squared_error', 'lad', 'huber'],\n",
    "             'learning_rate': np.logspace(-3, 0, 10),\n",
    "             'n_estimators': np.arange(50, 200, 5),\n",
    "             'subsample': [.6, .7, .8, .9, 1],\n",
    "             'max_depth': np.arange(2, 11)}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "gb_reg = GradientBoostingRegressor(random_state = 42, n_iter_no_change = 5, validation_fraction = .15)\n",
    "gb_rsearch = rsearch(iter = 4050, est = gb_reg, params=gb_rparams, x=X_train, y=y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>param_subsample</th>\n",
       "      <th>param_n_estimators</th>\n",
       "      <th>param_max_depth</th>\n",
       "      <th>param_loss</th>\n",
       "      <th>param_learning_rate</th>\n",
       "      <th>params</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>1632</th>\n",
       "      <td>0.9</td>\n",
       "      <td>155</td>\n",
       "      <td>2</td>\n",
       "      <td>ls</td>\n",
       "      <td>0.464159</td>\n",
       "      <td>{'subsample': 0.9, 'n_estimators': 155, 'max_d...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2402</th>\n",
       "      <td>0.9</td>\n",
       "      <td>130</td>\n",
       "      <td>2</td>\n",
       "      <td>ls</td>\n",
       "      <td>0.464159</td>\n",
       "      <td>{'subsample': 0.9, 'n_estimators': 130, 'max_d...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3021</th>\n",
       "      <td>0.9</td>\n",
       "      <td>185</td>\n",
       "      <td>2</td>\n",
       "      <td>ls</td>\n",
       "      <td>0.464159</td>\n",
       "      <td>{'subsample': 0.9, 'n_estimators': 185, 'max_d...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>605</th>\n",
       "      <td>0.9</td>\n",
       "      <td>60</td>\n",
       "      <td>2</td>\n",
       "      <td>ls</td>\n",
       "      <td>0.464159</td>\n",
       "      <td>{'subsample': 0.9, 'n_estimators': 60, 'max_de...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3433</th>\n",
       "      <td>0.9</td>\n",
       "      <td>150</td>\n",
       "      <td>2</td>\n",
       "      <td>ls</td>\n",
       "      <td>0.464159</td>\n",
       "      <td>{'subsample': 0.9, 'n_estimators': 150, 'max_d...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>182</th>\n",
       "      <td>1</td>\n",
       "      <td>180</td>\n",
       "      <td>2</td>\n",
       "      <td>huber</td>\n",
       "      <td>0.464159</td>\n",
       "      <td>{'subsample': 1, 'n_estimators': 180, 'max_dep...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1865</th>\n",
       "      <td>1</td>\n",
       "      <td>75</td>\n",
       "      <td>2</td>\n",
       "      <td>huber</td>\n",
       "      <td>0.464159</td>\n",
       "      <td>{'subsample': 1, 'n_estimators': 75, 'max_dept...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3864</th>\n",
       "      <td>0.9</td>\n",
       "      <td>155</td>\n",
       "      <td>2</td>\n",
       "      <td>ls</td>\n",
       "      <td>0.215443</td>\n",
       "      <td>{'subsample': 0.9, 'n_estimators': 155, 'max_d...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3892</th>\n",
       "      <td>0.9</td>\n",
       "      <td>85</td>\n",
       "      <td>2</td>\n",
       "      <td>ls</td>\n",
       "      <td>0.215443</td>\n",
       "      <td>{'subsample': 0.9, 'n_estimators': 85, 'max_de...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1319</th>\n",
       "      <td>0.9</td>\n",
       "      <td>160</td>\n",
       "      <td>2</td>\n",
       "      <td>ls</td>\n",
       "      <td>0.215443</td>\n",
       "      <td>{'subsample': 0.9, 'n_estimators': 160, 'max_d...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "     param_subsample param_n_estimators param_max_depth param_loss  \\\n",
       "1632             0.9                155               2         ls   \n",
       "2402             0.9                130               2         ls   \n",
       "3021             0.9                185               2         ls   \n",
       "605              0.9                 60               2         ls   \n",
       "3433             0.9                150               2         ls   \n",
       "182                1                180               2      huber   \n",
       "1865               1                 75               2      huber   \n",
       "3864             0.9                155               2         ls   \n",
       "3892             0.9                 85               2         ls   \n",
       "1319             0.9                160               2         ls   \n",
       "\n",
       "     param_learning_rate                                             params  \n",
       "1632            0.464159  {'subsample': 0.9, 'n_estimators': 155, 'max_d...  \n",
       "2402            0.464159  {'subsample': 0.9, 'n_estimators': 130, 'max_d...  \n",
       "3021            0.464159  {'subsample': 0.9, 'n_estimators': 185, 'max_d...  \n",
       "605             0.464159  {'subsample': 0.9, 'n_estimators': 60, 'max_de...  \n",
       "3433            0.464159  {'subsample': 0.9, 'n_estimators': 150, 'max_d...  \n",
       "182             0.464159  {'subsample': 1, 'n_estimators': 180, 'max_dep...  \n",
       "1865            0.464159  {'subsample': 1, 'n_estimators': 75, 'max_dept...  \n",
       "3864            0.215443  {'subsample': 0.9, 'n_estimators': 155, 'max_d...  \n",
       "3892            0.215443  {'subsample': 0.9, 'n_estimators': 85, 'max_de...  \n",
       "1319            0.215443  {'subsample': 0.9, 'n_estimators': 160, 'max_d...  "
      ]
     },
     "execution_count": 53,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pd.DataFrame(gb_rsearch.cv_results_).sort_values(by = 'mean_test_score', ascending= False).head(10).iloc[:, 4:10]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 86,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "GradientBoostingRegressor(learning_rate=0.46415888336127775, loss='ls',\n",
       "                          max_depth=2, n_estimators=60, n_iter_no_change=5,\n",
       "                          random_state=42, subsample=0.9,\n",
       "                          validation_fraction=0.15)"
      ]
     },
     "execution_count": 86,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "gb_rsearch.best_estimator_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {},
   "outputs": [],
   "source": [
    "gb_gparams = {'learning_rate': np.logspace(-1.5, 0, 10),\n",
    "             'n_estimators': np.arange(60, 160, 5),\n",
    "             'subsample': [.75, .8, .85, .9, .95],\n",
    "             'max_depth': np.arange(2, 9)}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {},
   "outputs": [],
   "source": [
    "gb_reg2 = GradientBoostingRegressor(random_state = 42, n_iter_no_change = 5, validation_fraction = .15, loss = 'squared_error')\n",
    "gb_gsearch = gsearch(est = gb_reg2, params=gb_gparams, x=X_train, y=y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 94,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>param_learning_rate</th>\n",
       "      <th>param_max_depth</th>\n",
       "      <th>param_n_estimators</th>\n",
       "      <th>param_subsample</th>\n",
       "      <th>params</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>4903</th>\n",
       "      <td>0.464159</td>\n",
       "      <td>2</td>\n",
       "      <td>60</td>\n",
       "      <td>0.9</td>\n",
       "      <td>{'learning_rate': 0.46415888336127775, 'max_de...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4923</th>\n",
       "      <td>0.464159</td>\n",
       "      <td>2</td>\n",
       "      <td>80</td>\n",
       "      <td>0.9</td>\n",
       "      <td>{'learning_rate': 0.46415888336127775, 'max_de...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4948</th>\n",
       "      <td>0.464159</td>\n",
       "      <td>2</td>\n",
       "      <td>105</td>\n",
       "      <td>0.9</td>\n",
       "      <td>{'learning_rate': 0.46415888336127775, 'max_de...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4913</th>\n",
       "      <td>0.464159</td>\n",
       "      <td>2</td>\n",
       "      <td>70</td>\n",
       "      <td>0.9</td>\n",
       "      <td>{'learning_rate': 0.46415888336127775, 'max_de...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4933</th>\n",
       "      <td>0.464159</td>\n",
       "      <td>2</td>\n",
       "      <td>90</td>\n",
       "      <td>0.9</td>\n",
       "      <td>{'learning_rate': 0.46415888336127775, 'max_de...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4983</th>\n",
       "      <td>0.464159</td>\n",
       "      <td>2</td>\n",
       "      <td>140</td>\n",
       "      <td>0.9</td>\n",
       "      <td>{'learning_rate': 0.46415888336127775, 'max_de...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4973</th>\n",
       "      <td>0.464159</td>\n",
       "      <td>2</td>\n",
       "      <td>130</td>\n",
       "      <td>0.9</td>\n",
       "      <td>{'learning_rate': 0.46415888336127775, 'max_de...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4938</th>\n",
       "      <td>0.464159</td>\n",
       "      <td>2</td>\n",
       "      <td>95</td>\n",
       "      <td>0.9</td>\n",
       "      <td>{'learning_rate': 0.46415888336127775, 'max_de...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4968</th>\n",
       "      <td>0.464159</td>\n",
       "      <td>2</td>\n",
       "      <td>125</td>\n",
       "      <td>0.9</td>\n",
       "      <td>{'learning_rate': 0.46415888336127775, 'max_de...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4963</th>\n",
       "      <td>0.464159</td>\n",
       "      <td>2</td>\n",
       "      <td>120</td>\n",
       "      <td>0.9</td>\n",
       "      <td>{'learning_rate': 0.46415888336127775, 'max_de...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "     param_learning_rate param_max_depth param_n_estimators param_subsample  \\\n",
       "4903            0.464159               2                 60             0.9   \n",
       "4923            0.464159               2                 80             0.9   \n",
       "4948            0.464159               2                105             0.9   \n",
       "4913            0.464159               2                 70             0.9   \n",
       "4933            0.464159               2                 90             0.9   \n",
       "4983            0.464159               2                140             0.9   \n",
       "4973            0.464159               2                130             0.9   \n",
       "4938            0.464159               2                 95             0.9   \n",
       "4968            0.464159               2                125             0.9   \n",
       "4963            0.464159               2                120             0.9   \n",
       "\n",
       "                                                 params  \n",
       "4903  {'learning_rate': 0.46415888336127775, 'max_de...  \n",
       "4923  {'learning_rate': 0.46415888336127775, 'max_de...  \n",
       "4948  {'learning_rate': 0.46415888336127775, 'max_de...  \n",
       "4913  {'learning_rate': 0.46415888336127775, 'max_de...  \n",
       "4933  {'learning_rate': 0.46415888336127775, 'max_de...  \n",
       "4983  {'learning_rate': 0.46415888336127775, 'max_de...  \n",
       "4973  {'learning_rate': 0.46415888336127775, 'max_de...  \n",
       "4938  {'learning_rate': 0.46415888336127775, 'max_de...  \n",
       "4968  {'learning_rate': 0.46415888336127775, 'max_de...  \n",
       "4963  {'learning_rate': 0.46415888336127775, 'max_de...  "
      ]
     },
     "execution_count": 94,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pd.DataFrame(gb_gsearch.cv_results_).sort_values(by = 'mean_test_score', ascending= False).head(10).iloc[:, 4:9]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "metadata": {},
   "outputs": [],
   "source": [
    "gb_gparams2 = {'learning_rate': np.logspace(-1, 0, 10),\n",
    "             'n_estimators': np.arange(60, 160, 5),\n",
    "             'subsample': [ .84, .86, .88, .9, .92, .94, .96, .98],\n",
    "             'max_depth': np.arange(2, 5)}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "metadata": {},
   "outputs": [],
   "source": [
    "gb_gsearch2 = gsearch(est = gb_reg2, params=gb_gparams2, x=X_train, y=y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 93,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>param_learning_rate</th>\n",
       "      <th>param_max_depth</th>\n",
       "      <th>param_n_estimators</th>\n",
       "      <th>param_subsample</th>\n",
       "      <th>params</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>2485</th>\n",
       "      <td>0.359381</td>\n",
       "      <td>2</td>\n",
       "      <td>110</td>\n",
       "      <td>0.94</td>\n",
       "      <td>{'learning_rate': 0.35938136638046275, 'max_de...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2549</th>\n",
       "      <td>0.359381</td>\n",
       "      <td>2</td>\n",
       "      <td>150</td>\n",
       "      <td>0.94</td>\n",
       "      <td>{'learning_rate': 0.35938136638046275, 'max_de...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2469</th>\n",
       "      <td>0.359381</td>\n",
       "      <td>2</td>\n",
       "      <td>100</td>\n",
       "      <td>0.94</td>\n",
       "      <td>{'learning_rate': 0.35938136638046275, 'max_de...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2493</th>\n",
       "      <td>0.359381</td>\n",
       "      <td>2</td>\n",
       "      <td>115</td>\n",
       "      <td>0.94</td>\n",
       "      <td>{'learning_rate': 0.35938136638046275, 'max_de...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2509</th>\n",
       "      <td>0.359381</td>\n",
       "      <td>2</td>\n",
       "      <td>125</td>\n",
       "      <td>0.94</td>\n",
       "      <td>{'learning_rate': 0.35938136638046275, 'max_de...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2429</th>\n",
       "      <td>0.359381</td>\n",
       "      <td>2</td>\n",
       "      <td>75</td>\n",
       "      <td>0.94</td>\n",
       "      <td>{'learning_rate': 0.35938136638046275, 'max_de...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2557</th>\n",
       "      <td>0.359381</td>\n",
       "      <td>2</td>\n",
       "      <td>155</td>\n",
       "      <td>0.94</td>\n",
       "      <td>{'learning_rate': 0.35938136638046275, 'max_de...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2437</th>\n",
       "      <td>0.359381</td>\n",
       "      <td>2</td>\n",
       "      <td>80</td>\n",
       "      <td>0.94</td>\n",
       "      <td>{'learning_rate': 0.35938136638046275, 'max_de...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2461</th>\n",
       "      <td>0.359381</td>\n",
       "      <td>2</td>\n",
       "      <td>95</td>\n",
       "      <td>0.94</td>\n",
       "      <td>{'learning_rate': 0.35938136638046275, 'max_de...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2517</th>\n",
       "      <td>0.359381</td>\n",
       "      <td>2</td>\n",
       "      <td>130</td>\n",
       "      <td>0.94</td>\n",
       "      <td>{'learning_rate': 0.35938136638046275, 'max_de...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "     param_learning_rate param_max_depth param_n_estimators param_subsample  \\\n",
       "2485            0.359381               2                110            0.94   \n",
       "2549            0.359381               2                150            0.94   \n",
       "2469            0.359381               2                100            0.94   \n",
       "2493            0.359381               2                115            0.94   \n",
       "2509            0.359381               2                125            0.94   \n",
       "2429            0.359381               2                 75            0.94   \n",
       "2557            0.359381               2                155            0.94   \n",
       "2437            0.359381               2                 80            0.94   \n",
       "2461            0.359381               2                 95            0.94   \n",
       "2517            0.359381               2                130            0.94   \n",
       "\n",
       "                                                 params  \n",
       "2485  {'learning_rate': 0.35938136638046275, 'max_de...  \n",
       "2549  {'learning_rate': 0.35938136638046275, 'max_de...  \n",
       "2469  {'learning_rate': 0.35938136638046275, 'max_de...  \n",
       "2493  {'learning_rate': 0.35938136638046275, 'max_de...  \n",
       "2509  {'learning_rate': 0.35938136638046275, 'max_de...  \n",
       "2429  {'learning_rate': 0.35938136638046275, 'max_de...  \n",
       "2557  {'learning_rate': 0.35938136638046275, 'max_de...  \n",
       "2437  {'learning_rate': 0.35938136638046275, 'max_de...  \n",
       "2461  {'learning_rate': 0.35938136638046275, 'max_de...  \n",
       "2517  {'learning_rate': 0.35938136638046275, 'max_de...  "
      ]
     },
     "execution_count": 93,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pd.DataFrame(gb_gsearch2.cv_results_).sort_values(by = 'mean_test_score', ascending= False).head(10).iloc[:, 4:9]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 75,
   "metadata": {},
   "outputs": [],
   "source": [
    "gb_gparams3 = {'learning_rate': np.logspace(-0.8, -0.2, 20),\n",
    "             'n_estimators': np.arange(60, 160, 2),\n",
    "             'subsample': np.arange(0.9, 1, 0.01),\n",
    "             'max_depth': [2,3]}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 76,
   "metadata": {},
   "outputs": [],
   "source": [
    "gb_gsearch3 = gsearch(est = gb_reg2, params=gb_gparams3, x=X_train, y=y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 100,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>param_learning_rate</th>\n",
       "      <th>param_max_depth</th>\n",
       "      <th>param_n_estimators</th>\n",
       "      <th>param_subsample</th>\n",
       "      <th>params</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>11360</th>\n",
       "      <td>0.35267</td>\n",
       "      <td>2</td>\n",
       "      <td>132</td>\n",
       "      <td>0.9</td>\n",
       "      <td>{'learning_rate': 0.352669921417466, 'max_dept...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11130</th>\n",
       "      <td>0.35267</td>\n",
       "      <td>2</td>\n",
       "      <td>86</td>\n",
       "      <td>0.9</td>\n",
       "      <td>{'learning_rate': 0.352669921417466, 'max_dept...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11140</th>\n",
       "      <td>0.35267</td>\n",
       "      <td>2</td>\n",
       "      <td>88</td>\n",
       "      <td>0.9</td>\n",
       "      <td>{'learning_rate': 0.352669921417466, 'max_dept...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11141</th>\n",
       "      <td>0.35267</td>\n",
       "      <td>2</td>\n",
       "      <td>88</td>\n",
       "      <td>0.91</td>\n",
       "      <td>{'learning_rate': 0.352669921417466, 'max_dept...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11150</th>\n",
       "      <td>0.35267</td>\n",
       "      <td>2</td>\n",
       "      <td>90</td>\n",
       "      <td>0.9</td>\n",
       "      <td>{'learning_rate': 0.352669921417466, 'max_dept...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11151</th>\n",
       "      <td>0.35267</td>\n",
       "      <td>2</td>\n",
       "      <td>90</td>\n",
       "      <td>0.91</td>\n",
       "      <td>{'learning_rate': 0.352669921417466, 'max_dept...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11160</th>\n",
       "      <td>0.35267</td>\n",
       "      <td>2</td>\n",
       "      <td>92</td>\n",
       "      <td>0.9</td>\n",
       "      <td>{'learning_rate': 0.352669921417466, 'max_dept...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11161</th>\n",
       "      <td>0.35267</td>\n",
       "      <td>2</td>\n",
       "      <td>92</td>\n",
       "      <td>0.91</td>\n",
       "      <td>{'learning_rate': 0.352669921417466, 'max_dept...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11170</th>\n",
       "      <td>0.35267</td>\n",
       "      <td>2</td>\n",
       "      <td>94</td>\n",
       "      <td>0.9</td>\n",
       "      <td>{'learning_rate': 0.352669921417466, 'max_dept...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11171</th>\n",
       "      <td>0.35267</td>\n",
       "      <td>2</td>\n",
       "      <td>94</td>\n",
       "      <td>0.91</td>\n",
       "      <td>{'learning_rate': 0.352669921417466, 'max_dept...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "      param_learning_rate param_max_depth param_n_estimators param_subsample  \\\n",
       "11360             0.35267               2                132             0.9   \n",
       "11130             0.35267               2                 86             0.9   \n",
       "11140             0.35267               2                 88             0.9   \n",
       "11141             0.35267               2                 88            0.91   \n",
       "11150             0.35267               2                 90             0.9   \n",
       "11151             0.35267               2                 90            0.91   \n",
       "11160             0.35267               2                 92             0.9   \n",
       "11161             0.35267               2                 92            0.91   \n",
       "11170             0.35267               2                 94             0.9   \n",
       "11171             0.35267               2                 94            0.91   \n",
       "\n",
       "                                                  params  \n",
       "11360  {'learning_rate': 0.352669921417466, 'max_dept...  \n",
       "11130  {'learning_rate': 0.352669921417466, 'max_dept...  \n",
       "11140  {'learning_rate': 0.352669921417466, 'max_dept...  \n",
       "11141  {'learning_rate': 0.352669921417466, 'max_dept...  \n",
       "11150  {'learning_rate': 0.352669921417466, 'max_dept...  \n",
       "11151  {'learning_rate': 0.352669921417466, 'max_dept...  \n",
       "11160  {'learning_rate': 0.352669921417466, 'max_dept...  \n",
       "11161  {'learning_rate': 0.352669921417466, 'max_dept...  \n",
       "11170  {'learning_rate': 0.352669921417466, 'max_dept...  \n",
       "11171  {'learning_rate': 0.352669921417466, 'max_dept...  "
      ]
     },
     "execution_count": 100,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pd.DataFrame(gb_gsearch3.cv_results_).sort_values(by = 'mean_test_score', ascending= False).head(10).iloc[:, 4:9]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 90,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "GradientBoostingRegressor(learning_rate=0.352669921417466, max_depth=2,\n",
       "                          n_estimators=60, n_iter_no_change=5, random_state=42,\n",
       "                          subsample=0.9, validation_fraction=0.15)"
      ]
     },
     "execution_count": 90,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "gb_gsearch3.best_estimator_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 109,
   "metadata": {},
   "outputs": [],
   "source": [
    "gb_gparams4 = {'learning_rate': np.arange(0.3, 0.4, 0.005),\n",
    "             'n_estimators': np.arange(60, 140, 1),\n",
    "             'subsample': [0.92, 0.93]}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 110,
   "metadata": {},
   "outputs": [],
   "source": [
    "gb_reg3 = GradientBoostingRegressor(random_state = 42, n_iter_no_change = 5, validation_fraction = .15, loss = 'squared_error', max_depth = 2)\n",
    "gb_gsearch4 = gsearch(est = gb_reg3, params=gb_gparams4, x=X_train, y=y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 114,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>param_learning_rate</th>\n",
       "      <th>param_n_estimators</th>\n",
       "      <th>param_subsample</th>\n",
       "      <th>params</th>\n",
       "      <th>split0_test_score</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>2401</th>\n",
       "      <td>0.375</td>\n",
       "      <td>60</td>\n",
       "      <td>0.93</td>\n",
       "      <td>{'learning_rate': 0.37500000000000006, 'n_esti...</td>\n",
       "      <td>-0.04029</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2400</th>\n",
       "      <td>0.375</td>\n",
       "      <td>60</td>\n",
       "      <td>0.92</td>\n",
       "      <td>{'learning_rate': 0.37500000000000006, 'n_esti...</td>\n",
       "      <td>-0.04029</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2502</th>\n",
       "      <td>0.375</td>\n",
       "      <td>111</td>\n",
       "      <td>0.92</td>\n",
       "      <td>{'learning_rate': 0.37500000000000006, 'n_esti...</td>\n",
       "      <td>-0.04029</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2503</th>\n",
       "      <td>0.375</td>\n",
       "      <td>111</td>\n",
       "      <td>0.93</td>\n",
       "      <td>{'learning_rate': 0.37500000000000006, 'n_esti...</td>\n",
       "      <td>-0.04029</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2504</th>\n",
       "      <td>0.375</td>\n",
       "      <td>112</td>\n",
       "      <td>0.92</td>\n",
       "      <td>{'learning_rate': 0.37500000000000006, 'n_esti...</td>\n",
       "      <td>-0.04029</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2505</th>\n",
       "      <td>0.375</td>\n",
       "      <td>112</td>\n",
       "      <td>0.93</td>\n",
       "      <td>{'learning_rate': 0.37500000000000006, 'n_esti...</td>\n",
       "      <td>-0.04029</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2506</th>\n",
       "      <td>0.375</td>\n",
       "      <td>113</td>\n",
       "      <td>0.92</td>\n",
       "      <td>{'learning_rate': 0.37500000000000006, 'n_esti...</td>\n",
       "      <td>-0.04029</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2507</th>\n",
       "      <td>0.375</td>\n",
       "      <td>113</td>\n",
       "      <td>0.93</td>\n",
       "      <td>{'learning_rate': 0.37500000000000006, 'n_esti...</td>\n",
       "      <td>-0.04029</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2508</th>\n",
       "      <td>0.375</td>\n",
       "      <td>114</td>\n",
       "      <td>0.92</td>\n",
       "      <td>{'learning_rate': 0.37500000000000006, 'n_esti...</td>\n",
       "      <td>-0.04029</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2509</th>\n",
       "      <td>0.375</td>\n",
       "      <td>114</td>\n",
       "      <td>0.93</td>\n",
       "      <td>{'learning_rate': 0.37500000000000006, 'n_esti...</td>\n",
       "      <td>-0.04029</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "     param_learning_rate param_n_estimators param_subsample  \\\n",
       "2401               0.375                 60            0.93   \n",
       "2400               0.375                 60            0.92   \n",
       "2502               0.375                111            0.92   \n",
       "2503               0.375                111            0.93   \n",
       "2504               0.375                112            0.92   \n",
       "2505               0.375                112            0.93   \n",
       "2506               0.375                113            0.92   \n",
       "2507               0.375                113            0.93   \n",
       "2508               0.375                114            0.92   \n",
       "2509               0.375                114            0.93   \n",
       "\n",
       "                                                 params  split0_test_score  \n",
       "2401  {'learning_rate': 0.37500000000000006, 'n_esti...           -0.04029  \n",
       "2400  {'learning_rate': 0.37500000000000006, 'n_esti...           -0.04029  \n",
       "2502  {'learning_rate': 0.37500000000000006, 'n_esti...           -0.04029  \n",
       "2503  {'learning_rate': 0.37500000000000006, 'n_esti...           -0.04029  \n",
       "2504  {'learning_rate': 0.37500000000000006, 'n_esti...           -0.04029  \n",
       "2505  {'learning_rate': 0.37500000000000006, 'n_esti...           -0.04029  \n",
       "2506  {'learning_rate': 0.37500000000000006, 'n_esti...           -0.04029  \n",
       "2507  {'learning_rate': 0.37500000000000006, 'n_esti...           -0.04029  \n",
       "2508  {'learning_rate': 0.37500000000000006, 'n_esti...           -0.04029  \n",
       "2509  {'learning_rate': 0.37500000000000006, 'n_esti...           -0.04029  "
      ]
     },
     "execution_count": 114,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pd.DataFrame(gb_gsearch4.cv_results_).sort_values(by = 'mean_test_score', ascending= False).head(10).iloc[:, 4:9]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 112,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "GradientBoostingRegressor(learning_rate=0.37500000000000006, max_depth=2,\n",
       "                          n_estimators=60, n_iter_no_change=5, random_state=42,\n",
       "                          subsample=0.92, validation_fraction=0.15)"
      ]
     },
     "execution_count": 112,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "gb_gsearch4.best_estimator_"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Adaptive Boosting (AdaBoost)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 115,
   "metadata": {},
   "outputs": [],
   "source": [
    "adb_rparams = {'learning_rate': np.logspace(-3, 1, 12),\n",
    "          'loss' : ['linear', 'square', 'exponential'],\n",
    "          'n_estimators': np.arange(5,200, 5),\n",
    "          'base_estimator': [GradientBoostingRegressor(random_state = 42), Ridge(alpha=0.01, random_state=6, solver='svd'), DecisionTreeRegressor(max_depth = 7)]\n",
    "          }"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 116,
   "metadata": {},
   "outputs": [],
   "source": [
    "adb_rsearch = rsearch(iter = 750, est = AdaBoostRegressor(random_state = 42), params=adb_rparams, x=X_train, y=y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 127,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>param_n_estimators</th>\n",
       "      <th>param_loss</th>\n",
       "      <th>param_learning_rate</th>\n",
       "      <th>param_base_estimator</th>\n",
       "      <th>params</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>688</th>\n",
       "      <td>60</td>\n",
       "      <td>exponential</td>\n",
       "      <td>0.001</td>\n",
       "      <td>GradientBoostingRegressor(random_state=42)</td>\n",
       "      <td>{'n_estimators': 60, 'loss': 'exponential', 'l...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>420</th>\n",
       "      <td>125</td>\n",
       "      <td>exponential</td>\n",
       "      <td>0.001</td>\n",
       "      <td>GradientBoostingRegressor(random_state=42)</td>\n",
       "      <td>{'n_estimators': 125, 'loss': 'exponential', '...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>575</th>\n",
       "      <td>120</td>\n",
       "      <td>exponential</td>\n",
       "      <td>0.00231</td>\n",
       "      <td>GradientBoostingRegressor(random_state=42)</td>\n",
       "      <td>{'n_estimators': 120, 'loss': 'exponential', '...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>298</th>\n",
       "      <td>135</td>\n",
       "      <td>exponential</td>\n",
       "      <td>0.00231</td>\n",
       "      <td>GradientBoostingRegressor(random_state=42)</td>\n",
       "      <td>{'n_estimators': 135, 'loss': 'exponential', '...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>244</th>\n",
       "      <td>130</td>\n",
       "      <td>exponential</td>\n",
       "      <td>0.00231</td>\n",
       "      <td>GradientBoostingRegressor(random_state=42)</td>\n",
       "      <td>{'n_estimators': 130, 'loss': 'exponential', '...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>686</th>\n",
       "      <td>165</td>\n",
       "      <td>exponential</td>\n",
       "      <td>0.001</td>\n",
       "      <td>GradientBoostingRegressor(random_state=42)</td>\n",
       "      <td>{'n_estimators': 165, 'loss': 'exponential', '...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>230</th>\n",
       "      <td>40</td>\n",
       "      <td>exponential</td>\n",
       "      <td>0.00231</td>\n",
       "      <td>GradientBoostingRegressor(random_state=42)</td>\n",
       "      <td>{'n_estimators': 40, 'loss': 'exponential', 'l...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>330</th>\n",
       "      <td>160</td>\n",
       "      <td>exponential</td>\n",
       "      <td>0.001</td>\n",
       "      <td>GradientBoostingRegressor(random_state=42)</td>\n",
       "      <td>{'n_estimators': 160, 'loss': 'exponential', '...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>592</th>\n",
       "      <td>100</td>\n",
       "      <td>exponential</td>\n",
       "      <td>0.00231</td>\n",
       "      <td>GradientBoostingRegressor(random_state=42)</td>\n",
       "      <td>{'n_estimators': 100, 'loss': 'exponential', '...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>190</td>\n",
       "      <td>exponential</td>\n",
       "      <td>0.00231</td>\n",
       "      <td>GradientBoostingRegressor(random_state=42)</td>\n",
       "      <td>{'n_estimators': 190, 'loss': 'exponential', '...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "    param_n_estimators   param_loss param_learning_rate  \\\n",
       "688                 60  exponential               0.001   \n",
       "420                125  exponential               0.001   \n",
       "575                120  exponential             0.00231   \n",
       "298                135  exponential             0.00231   \n",
       "244                130  exponential             0.00231   \n",
       "686                165  exponential               0.001   \n",
       "230                 40  exponential             0.00231   \n",
       "330                160  exponential               0.001   \n",
       "592                100  exponential             0.00231   \n",
       "3                  190  exponential             0.00231   \n",
       "\n",
       "                           param_base_estimator  \\\n",
       "688  GradientBoostingRegressor(random_state=42)   \n",
       "420  GradientBoostingRegressor(random_state=42)   \n",
       "575  GradientBoostingRegressor(random_state=42)   \n",
       "298  GradientBoostingRegressor(random_state=42)   \n",
       "244  GradientBoostingRegressor(random_state=42)   \n",
       "686  GradientBoostingRegressor(random_state=42)   \n",
       "230  GradientBoostingRegressor(random_state=42)   \n",
       "330  GradientBoostingRegressor(random_state=42)   \n",
       "592  GradientBoostingRegressor(random_state=42)   \n",
       "3    GradientBoostingRegressor(random_state=42)   \n",
       "\n",
       "                                                params  \n",
       "688  {'n_estimators': 60, 'loss': 'exponential', 'l...  \n",
       "420  {'n_estimators': 125, 'loss': 'exponential', '...  \n",
       "575  {'n_estimators': 120, 'loss': 'exponential', '...  \n",
       "298  {'n_estimators': 135, 'loss': 'exponential', '...  \n",
       "244  {'n_estimators': 130, 'loss': 'exponential', '...  \n",
       "686  {'n_estimators': 165, 'loss': 'exponential', '...  \n",
       "230  {'n_estimators': 40, 'loss': 'exponential', 'l...  \n",
       "330  {'n_estimators': 160, 'loss': 'exponential', '...  \n",
       "592  {'n_estimators': 100, 'loss': 'exponential', '...  \n",
       "3    {'n_estimators': 190, 'loss': 'exponential', '...  "
      ]
     },
     "execution_count": 127,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pd.DataFrame(adb_rsearch.cv_results_).sort_values(by = 'mean_test_score', ascending= False).head(10).iloc[:, 4:9]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 118,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "AdaBoostRegressor(base_estimator=GradientBoostingRegressor(random_state=42),\n",
       "                  learning_rate=0.001, loss='exponential', n_estimators=60,\n",
       "                  random_state=42)"
      ]
     },
     "execution_count": 118,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "adb_rsearch.best_estimator_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 130,
   "metadata": {},
   "outputs": [],
   "source": [
    "adb_gparams = gparams = {'learning_rate': np.logspace(-4, -2, 10),\n",
    "          'n_estimators': np.arange(50, 160, 5)}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 131,
   "metadata": {},
   "outputs": [],
   "source": [
    "adb_gsearch = gsearch(est = AdaBoostRegressor(loss = 'exponential', base_estimator = GradientBoostingRegressor(random_state=42), \n",
    "                random_state = 42), params=adb_gparams, x=X_train, y=y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 134,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>param_learning_rate</th>\n",
       "      <th>param_n_estimators</th>\n",
       "      <th>params</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>79</th>\n",
       "      <td>0.000464</td>\n",
       "      <td>115</td>\n",
       "      <td>{'learning_rate': 0.00046415888336127773, 'n_e...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>78</th>\n",
       "      <td>0.000464</td>\n",
       "      <td>110</td>\n",
       "      <td>{'learning_rate': 0.00046415888336127773, 'n_e...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>84</th>\n",
       "      <td>0.000464</td>\n",
       "      <td>140</td>\n",
       "      <td>{'learning_rate': 0.00046415888336127773, 'n_e...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>60</th>\n",
       "      <td>0.000278</td>\n",
       "      <td>130</td>\n",
       "      <td>{'learning_rate': 0.0002782559402207126, 'n_es...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>0.0001</td>\n",
       "      <td>125</td>\n",
       "      <td>{'learning_rate': 0.0001, 'n_estimators': 125}</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>81</th>\n",
       "      <td>0.000464</td>\n",
       "      <td>125</td>\n",
       "      <td>{'learning_rate': 0.00046415888336127773, 'n_e...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>37</th>\n",
       "      <td>0.000167</td>\n",
       "      <td>125</td>\n",
       "      <td>{'learning_rate': 0.0001668100537200059, 'n_es...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>0.0001</td>\n",
       "      <td>115</td>\n",
       "      <td>{'learning_rate': 0.0001, 'n_estimators': 115}</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>45</th>\n",
       "      <td>0.000278</td>\n",
       "      <td>55</td>\n",
       "      <td>{'learning_rate': 0.0002782559402207126, 'n_es...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>82</th>\n",
       "      <td>0.000464</td>\n",
       "      <td>130</td>\n",
       "      <td>{'learning_rate': 0.00046415888336127773, 'n_e...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   param_learning_rate param_n_estimators  \\\n",
       "79            0.000464                115   \n",
       "78            0.000464                110   \n",
       "84            0.000464                140   \n",
       "60            0.000278                130   \n",
       "15              0.0001                125   \n",
       "81            0.000464                125   \n",
       "37            0.000167                125   \n",
       "13              0.0001                115   \n",
       "45            0.000278                 55   \n",
       "82            0.000464                130   \n",
       "\n",
       "                                               params  \n",
       "79  {'learning_rate': 0.00046415888336127773, 'n_e...  \n",
       "78  {'learning_rate': 0.00046415888336127773, 'n_e...  \n",
       "84  {'learning_rate': 0.00046415888336127773, 'n_e...  \n",
       "60  {'learning_rate': 0.0002782559402207126, 'n_es...  \n",
       "15     {'learning_rate': 0.0001, 'n_estimators': 125}  \n",
       "81  {'learning_rate': 0.00046415888336127773, 'n_e...  \n",
       "37  {'learning_rate': 0.0001668100537200059, 'n_es...  \n",
       "13     {'learning_rate': 0.0001, 'n_estimators': 115}  \n",
       "45  {'learning_rate': 0.0002782559402207126, 'n_es...  \n",
       "82  {'learning_rate': 0.00046415888336127773, 'n_e...  "
      ]
     },
     "execution_count": 134,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pd.DataFrame(adb_gsearch.cv_results_).sort_values(by = 'mean_test_score', ascending= False).head(10).iloc[:, 4:7]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 129,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "AdaBoostRegressor(base_estimator=GradientBoostingRegressor(random_state=42),\n",
       "                  learning_rate=0.001, loss='exponential', n_estimators=60,\n",
       "                  random_state=42)"
      ]
     },
     "execution_count": 129,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "adb_gsearch.best_estimator_"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Saving the Results"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "import json"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "random_forest = {'max_depth': 150, 'n_estimators':66, 'random_state':42}\n",
    "gradient_boost = {'learning_rate':0.37500000000000006, 'max_depth':2,  'n_estimators':60, 'n_iter_no_change':5, \n",
    "                    'random_state':42, 'subsample':0.92, 'validation_fraction':0.15}\n",
    "adaboost = {'base_estimator':'GradientBoostingRegressor(random_state=42)', 'learning_rate':0.001, \n",
    "                'loss':'exponential', 'n_estimators':60, 'random_state':42}\n",
    "\n",
    "hyperparams = {'random_forest':random_forest, 'gradient_boost':gradient_boost, 'adaboost':adaboost}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open('files/tuned_hyperparams.json', 'w') as file:\n",
    "    json.dump(hyperparams, file)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3.9.6 64-bit",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.6"
  },
  "orig_nbformat": 4,
  "vscode": {
   "interpreter": {
    "hash": "5ef3141e487b593a8f44045d5bdbf5fb76b4a9107cbb1549cfca054e738b2143"
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}