18_hyperparameter_optimisation.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "from xgboost import XGBClassifier\n",
    "from sklearn import preprocessing\n",
    "from sklearn.model_selection import GridSearchCV"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "outputs": [],
   "source": [
    "X = pd.read_csv(\"data/17_selected_features_trainset.tsv\", sep=\"\\t\")\n",
    "y = X.pop(\"class\")"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "outputs": [
    {
     "data": {
      "text/plain": "        recount3_score  antisense_exon_start_ss  antisense_exon_end_ss  \\\n0                59151                    False                  False   \n1                61021                    False                  False   \n2                   21                    False                  False   \n3                  526                    False                  False   \n4                    0                    False                  False   \n...                ...                      ...                    ...   \n484602               0                    False                  False   \n484603           31374                    False                  False   \n484604              67                    False                  False   \n484605           18958                    False                  False   \n484606               0                    False                  False   \n\n        nearest_alt_start_ss_dist  nearest_alt_end_ss_dist  \\\n0                             170                      362   \n1                              24                      232   \n2                              24                      246   \n3                             322                      232   \n4                             322                      232   \n...                           ...                      ...   \n484602                        487                       25   \n484603                        777                      464   \n484604                        572                     2963   \n484605                       1178                        2   \n484606                        232                       24   \n\n        MaxEntScan_start_ss  MaxEntScan_end_ss  CpG_island  intron_length  \\\n0                      9.09              11.90       False            385   \n1                      7.66               3.89       False            499   \n2                      7.15               7.61       False            277   \n3                      1.14               3.89       False            168   \n4                    -12.10               7.63       False             78   \n...                     ...                ...         ...            ...   \n484602               -19.72               3.66        True            909   \n484603                 9.21              13.74       False            415   \n484604                 8.56               2.38       False           2970   \n484605               -12.64               5.91       False           2358   \n484606                 7.66               3.89       False            498   \n\n        phyloP_score  ...  repeat_features_start_site_Satellite repeats  \\\n0            5.65175  ...                                             0   \n1            1.40575  ...                                             0   \n2            5.75650  ...                                             0   \n3            5.35750  ...                                             0   \n4            3.62350  ...                                             0   \n...              ...  ...                                           ...   \n484602       0.58700  ...                                             0   \n484603       2.74250  ...                                             0   \n484604      -0.89575  ...                                             0   \n484605       2.41200  ...                                             0   \n484606       2.04075  ...                                             0   \n\n        repeat_features_start_site_Unknown  \\\n0                                        0   \n1                                        0   \n2                                        0   \n3                                        0   \n4                                        0   \n...                                    ...   \n484602                                   0   \n484603                                   0   \n484604                                   0   \n484605                                   0   \n484606                                   0   \n\n        repeat_features_start_site_Type II Transposons  \\\n0                                                    0   \n1                                                    0   \n2                                                    0   \n3                                                    0   \n4                                                    0   \n...                                                ...   \n484602                                               0   \n484603                                               0   \n484604                                               0   \n484605                                               0   \n484606                                               0   \n\n        repeat_features_end_site_Tandem repeats  \\\n0                                             0   \n1                                             0   \n2                                             0   \n3                                             0   \n4                                             0   \n...                                         ...   \n484602                                        0   \n484603                                        0   \n484604                                        0   \n484605                                        0   \n484606                                        0   \n\n        repeat_features_end_site_LTRs  \\\n0                                   0   \n1                                   0   \n2                                   0   \n3                                   0   \n4                                   0   \n...                               ...   \n484602                              0   \n484603                              0   \n484604                              0   \n484605                              0   \n484606                              0   \n\n        repeat_features_end_site_Type I Transposons/SINE  \\\n0                                                      0   \n1                                                      0   \n2                                                      0   \n3                                                      0   \n4                                                      0   \n...                                                  ...   \n484602                                                 0   \n484603                                                 0   \n484604                                                 0   \n484605                                                 0   \n484606                                                 0   \n\n        repeat_features_end_site_Satellite repeats  \\\n0                                                0   \n1                                                0   \n2                                                0   \n3                                                0   \n4                                                0   \n...                                            ...   \n484602                                           0   \n484603                                           0   \n484604                                           0   \n484605                                           0   \n484606                                           0   \n\n        repeat_features_end_site_Type I Transposons/LINE  splice_site  \\\n0                                                      0            2   \n1                                                      0            2   \n2                                                      0            2   \n3                                                      0            1   \n4                                                      0            2   \n...                                                  ...          ...   \n484602                                                 0            2   \n484603                                                 0            2   \n484604                                                 0            2   \n484605                                                 0            2   \n484606                                                 0            2   \n\n        repeat_features_start_site_Type I Transposons/SINE  \n0                                                       0   \n1                                                       0   \n2                                                       0   \n3                                                       0   \n4                                                       0   \n...                                                   ...   \n484602                                                  0   \n484603                                                  0   \n484604                                                  0   \n484605                                                  0   \n484606                                                  0   \n\n[484607 rows x 26 columns]",
      "text/html": "<div>\n<style scoped>\n    .dataframe tbody tr th:only-of-type {\n        vertical-align: middle;\n    }\n\n    .dataframe tbody tr th {\n        vertical-align: top;\n    }\n\n    .dataframe thead th {\n        text-align: right;\n    }\n</style>\n<table border=\"1\" class=\"dataframe\">\n  <thead>\n    <tr style=\"text-align: right;\">\n      <th></th>\n      <th>recount3_score</th>\n      <th>antisense_exon_start_ss</th>\n      <th>antisense_exon_end_ss</th>\n      <th>nearest_alt_start_ss_dist</th>\n      <th>nearest_alt_end_ss_dist</th>\n      <th>MaxEntScan_start_ss</th>\n      <th>MaxEntScan_end_ss</th>\n      <th>CpG_island</th>\n      <th>intron_length</th>\n      <th>phyloP_score</th>\n      <th>...</th>\n      <th>repeat_features_start_site_Satellite repeats</th>\n      <th>repeat_features_start_site_Unknown</th>\n      <th>repeat_features_start_site_Type II Transposons</th>\n      <th>repeat_features_end_site_Tandem repeats</th>\n      <th>repeat_features_end_site_LTRs</th>\n      <th>repeat_features_end_site_Type I Transposons/SINE</th>\n      <th>repeat_features_end_site_Satellite repeats</th>\n      <th>repeat_features_end_site_Type I Transposons/LINE</th>\n      <th>splice_site</th>\n      <th>repeat_features_start_site_Type I Transposons/SINE</th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <th>0</th>\n      <td>59151</td>\n      <td>False</td>\n      <td>False</td>\n      <td>170</td>\n      <td>362</td>\n      <td>9.09</td>\n      <td>11.90</td>\n      <td>False</td>\n      <td>385</td>\n      <td>5.65175</td>\n      <td>...</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>2</td>\n      <td>0</td>\n    </tr>\n    <tr>\n      <th>1</th>\n      <td>61021</td>\n      <td>False</td>\n      <td>False</td>\n      <td>24</td>\n      <td>232</td>\n      <td>7.66</td>\n      <td>3.89</td>\n      <td>False</td>\n      <td>499</td>\n      <td>1.40575</td>\n      <td>...</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>2</td>\n      <td>0</td>\n    </tr>\n    <tr>\n      <th>2</th>\n      <td>21</td>\n      <td>False</td>\n      <td>False</td>\n      <td>24</td>\n      <td>246</td>\n      <td>7.15</td>\n      <td>7.61</td>\n      <td>False</td>\n      <td>277</td>\n      <td>5.75650</td>\n      <td>...</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>2</td>\n      <td>0</td>\n    </tr>\n    <tr>\n      <th>3</th>\n      <td>526</td>\n      <td>False</td>\n      <td>False</td>\n      <td>322</td>\n      <td>232</td>\n      <td>1.14</td>\n      <td>3.89</td>\n      <td>False</td>\n      <td>168</td>\n      <td>5.35750</td>\n      <td>...</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>1</td>\n      <td>0</td>\n    </tr>\n    <tr>\n      <th>4</th>\n      <td>0</td>\n      <td>False</td>\n      <td>False</td>\n      <td>322</td>\n      <td>232</td>\n      <td>-12.10</td>\n      <td>7.63</td>\n      <td>False</td>\n      <td>78</td>\n      <td>3.62350</td>\n      <td>...</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>2</td>\n      <td>0</td>\n    </tr>\n    <tr>\n      <th>...</th>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n    </tr>\n    <tr>\n      <th>484602</th>\n      <td>0</td>\n      <td>False</td>\n      <td>False</td>\n      <td>487</td>\n      <td>25</td>\n      <td>-19.72</td>\n      <td>3.66</td>\n      <td>True</td>\n      <td>909</td>\n      <td>0.58700</td>\n      <td>...</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>2</td>\n      <td>0</td>\n    </tr>\n    <tr>\n      <th>484603</th>\n      <td>31374</td>\n      <td>False</td>\n      <td>False</td>\n      <td>777</td>\n      <td>464</td>\n      <td>9.21</td>\n      <td>13.74</td>\n      <td>False</td>\n      <td>415</td>\n      <td>2.74250</td>\n      <td>...</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>2</td>\n      <td>0</td>\n    </tr>\n    <tr>\n      <th>484604</th>\n      <td>67</td>\n      <td>False</td>\n      <td>False</td>\n      <td>572</td>\n      <td>2963</td>\n      <td>8.56</td>\n      <td>2.38</td>\n      <td>False</td>\n      <td>2970</td>\n      <td>-0.89575</td>\n      <td>...</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>2</td>\n      <td>0</td>\n    </tr>\n    <tr>\n      <th>484605</th>\n      <td>18958</td>\n      <td>False</td>\n      <td>False</td>\n      <td>1178</td>\n      <td>2</td>\n      <td>-12.64</td>\n      <td>5.91</td>\n      <td>False</td>\n      <td>2358</td>\n      <td>2.41200</td>\n      <td>...</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>2</td>\n      <td>0</td>\n    </tr>\n    <tr>\n      <th>484606</th>\n      <td>0</td>\n      <td>False</td>\n      <td>False</td>\n      <td>232</td>\n      <td>24</td>\n      <td>7.66</td>\n      <td>3.89</td>\n      <td>False</td>\n      <td>498</td>\n      <td>2.04075</td>\n      <td>...</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>2</td>\n      <td>0</td>\n    </tr>\n  </tbody>\n</table>\n<p>484607 rows × 26 columns</p>\n</div>"
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "le = preprocessing.LabelEncoder()\n",
    "le.fit(X[\"splice_site\"])\n",
    "X[\"splice_site\"] = le.transform(X[\"splice_site\"])\n",
    "\n",
    "X"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "outputs": [],
   "source": [
    "X = X.fillna(0)"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fitting 3 folds for each of 1296 candidates, totalling 3888 fits\n",
      "CPU times: user 11.5 s, sys: 1.23 s, total: 12.7 s\n",
      "Wall time: 1h 5min 13s\n"
     ]
    },
    {
     "data": {
      "text/plain": "GridSearchCV(cv=3,\n             estimator=XGBClassifier(base_score=None, booster=None,\n                                     callbacks=None, colsample_bylevel=None,\n                                     colsample_bynode=None,\n                                     colsample_bytree=None,\n                                     early_stopping_rounds=None,\n                                     enable_categorical=False, eval_metric=None,\n                                     gamma=None, gpu_id=None, grow_policy=None,\n                                     importance_type=None,\n                                     interaction_constraints=None,\n                                     learning_rate=None, max_bin=None,\n                                     max_ca...\n                                     n_estimators=100, n_jobs=None,\n                                     num_parallel_tree=None, predictor=None,\n                                     random_state=None, reg_alpha=None,\n                                     reg_lambda=None, ...),\n             n_jobs=12,\n             param_grid={'colsample_bytree': [0.8, 0.9, 1.0],\n                         'learning_rate': [0.01, 0.1, 0.2, 0.3],\n                         'max_depth': [5, 6, 7], 'min_child_weight': [1, 3, 5],\n                         'n_estimators': [50, 100, 150, 200],\n                         'subsample': [0.8, 0.9, 1.0]},\n             scoring='matthews_corrcoef', verbose=1)",
      "text/html": "<style>#sk-container-id-1 {color: black;background-color: white;}#sk-container-id-1 pre{padding: 0;}#sk-container-id-1 div.sk-toggleable {background-color: white;}#sk-container-id-1 label.sk-toggleable__label {cursor: pointer;display: block;width: 100%;margin-bottom: 0;padding: 0.3em;box-sizing: border-box;text-align: center;}#sk-container-id-1 label.sk-toggleable__label-arrow:before {content: \"▸\";float: left;margin-right: 0.25em;color: #696969;}#sk-container-id-1 label.sk-toggleable__label-arrow:hover:before {color: black;}#sk-container-id-1 div.sk-estimator:hover label.sk-toggleable__label-arrow:before {color: black;}#sk-container-id-1 div.sk-toggleable__content {max-height: 0;max-width: 0;overflow: hidden;text-align: left;background-color: #f0f8ff;}#sk-container-id-1 div.sk-toggleable__content pre {margin: 0.2em;color: black;border-radius: 0.25em;background-color: #f0f8ff;}#sk-container-id-1 input.sk-toggleable__control:checked~div.sk-toggleable__content {max-height: 200px;max-width: 100%;overflow: auto;}#sk-container-id-1 input.sk-toggleable__control:checked~label.sk-toggleable__label-arrow:before {content: \"▾\";}#sk-container-id-1 div.sk-estimator input.sk-toggleable__control:checked~label.sk-toggleable__label {background-color: #d4ebff;}#sk-container-id-1 div.sk-label input.sk-toggleable__control:checked~label.sk-toggleable__label {background-color: #d4ebff;}#sk-container-id-1 input.sk-hidden--visually {border: 0;clip: rect(1px 1px 1px 1px);clip: rect(1px, 1px, 1px, 1px);height: 1px;margin: -1px;overflow: hidden;padding: 0;position: absolute;width: 1px;}#sk-container-id-1 div.sk-estimator {font-family: monospace;background-color: #f0f8ff;border: 1px dotted black;border-radius: 0.25em;box-sizing: border-box;margin-bottom: 0.5em;}#sk-container-id-1 div.sk-estimator:hover {background-color: #d4ebff;}#sk-container-id-1 div.sk-parallel-item::after {content: \"\";width: 100%;border-bottom: 1px solid gray;flex-grow: 1;}#sk-container-id-1 div.sk-label:hover label.sk-toggleable__label {background-color: #d4ebff;}#sk-container-id-1 div.sk-serial::before {content: \"\";position: absolute;border-left: 1px solid gray;box-sizing: border-box;top: 0;bottom: 0;left: 50%;z-index: 0;}#sk-container-id-1 div.sk-serial {display: flex;flex-direction: column;align-items: center;background-color: white;padding-right: 0.2em;padding-left: 0.2em;position: relative;}#sk-container-id-1 div.sk-item {position: relative;z-index: 1;}#sk-container-id-1 div.sk-parallel {display: flex;align-items: stretch;justify-content: center;background-color: white;position: relative;}#sk-container-id-1 div.sk-item::before, #sk-container-id-1 div.sk-parallel-item::before {content: \"\";position: absolute;border-left: 1px solid gray;box-sizing: border-box;top: 0;bottom: 0;left: 50%;z-index: -1;}#sk-container-id-1 div.sk-parallel-item {display: flex;flex-direction: column;z-index: 1;position: relative;background-color: white;}#sk-container-id-1 div.sk-parallel-item:first-child::after {align-self: flex-end;width: 50%;}#sk-container-id-1 div.sk-parallel-item:last-child::after {align-self: flex-start;width: 50%;}#sk-container-id-1 div.sk-parallel-item:only-child::after {width: 0;}#sk-container-id-1 div.sk-dashed-wrapped {border: 1px dashed gray;margin: 0 0.4em 0.5em 0.4em;box-sizing: border-box;padding-bottom: 0.4em;background-color: white;}#sk-container-id-1 div.sk-label label {font-family: monospace;font-weight: bold;display: inline-block;line-height: 1.2em;}#sk-container-id-1 div.sk-label-container {text-align: center;}#sk-container-id-1 div.sk-container {/* jupyter's `normalize.less` sets `[hidden] { display: none; }` but bootstrap.min.css set `[hidden] { display: none !important; }` so we also need the `!important` here to be able to override the default hidden behavior on the sphinx rendered scikit-learn.org. See: https://github.com/scikit-learn/scikit-learn/issues/21755 */display: inline-block !important;position: relative;}#sk-container-id-1 div.sk-text-repr-fallback {display: none;}</style><div id=\"sk-container-id-1\" class=\"sk-top-container\"><div class=\"sk-text-repr-fallback\"><pre>GridSearchCV(cv=3,\n             estimator=XGBClassifier(base_score=None, booster=None,\n                                     callbacks=None, colsample_bylevel=None,\n                                     colsample_bynode=None,\n                                     colsample_bytree=None,\n                                     early_stopping_rounds=None,\n                                     enable_categorical=False, eval_metric=None,\n                                     gamma=None, gpu_id=None, grow_policy=None,\n                                     importance_type=None,\n                                     interaction_constraints=None,\n                                     learning_rate=None, max_bin=None,\n                                     max_ca...\n                                     n_estimators=100, n_jobs=None,\n                                     num_parallel_tree=None, predictor=None,\n                                     random_state=None, reg_alpha=None,\n                                     reg_lambda=None, ...),\n             n_jobs=12,\n             param_grid={&#x27;colsample_bytree&#x27;: [0.8, 0.9, 1.0],\n                         &#x27;learning_rate&#x27;: [0.01, 0.1, 0.2, 0.3],\n                         &#x27;max_depth&#x27;: [5, 6, 7], &#x27;min_child_weight&#x27;: [1, 3, 5],\n                         &#x27;n_estimators&#x27;: [50, 100, 150, 200],\n                         &#x27;subsample&#x27;: [0.8, 0.9, 1.0]},\n             scoring=&#x27;matthews_corrcoef&#x27;, verbose=1)</pre><b>In a Jupyter environment, please rerun this cell to show the HTML representation or trust the notebook. <br />On GitHub, the HTML representation is unable to render, please try loading this page with nbviewer.org.</b></div><div class=\"sk-container\" hidden><div class=\"sk-item sk-dashed-wrapped\"><div class=\"sk-label-container\"><div class=\"sk-label sk-toggleable\"><input class=\"sk-toggleable__control sk-hidden--visually\" id=\"sk-estimator-id-1\" type=\"checkbox\" ><label for=\"sk-estimator-id-1\" class=\"sk-toggleable__label sk-toggleable__label-arrow\">GridSearchCV</label><div class=\"sk-toggleable__content\"><pre>GridSearchCV(cv=3,\n             estimator=XGBClassifier(base_score=None, booster=None,\n                                     callbacks=None, colsample_bylevel=None,\n                                     colsample_bynode=None,\n                                     colsample_bytree=None,\n                                     early_stopping_rounds=None,\n                                     enable_categorical=False, eval_metric=None,\n                                     gamma=None, gpu_id=None, grow_policy=None,\n                                     importance_type=None,\n                                     interaction_constraints=None,\n                                     learning_rate=None, max_bin=None,\n                                     max_ca...\n                                     n_estimators=100, n_jobs=None,\n                                     num_parallel_tree=None, predictor=None,\n                                     random_state=None, reg_alpha=None,\n                                     reg_lambda=None, ...),\n             n_jobs=12,\n             param_grid={&#x27;colsample_bytree&#x27;: [0.8, 0.9, 1.0],\n                         &#x27;learning_rate&#x27;: [0.01, 0.1, 0.2, 0.3],\n                         &#x27;max_depth&#x27;: [5, 6, 7], &#x27;min_child_weight&#x27;: [1, 3, 5],\n                         &#x27;n_estimators&#x27;: [50, 100, 150, 200],\n                         &#x27;subsample&#x27;: [0.8, 0.9, 1.0]},\n             scoring=&#x27;matthews_corrcoef&#x27;, verbose=1)</pre></div></div></div><div class=\"sk-parallel\"><div class=\"sk-parallel-item\"><div class=\"sk-item\"><div class=\"sk-label-container\"><div class=\"sk-label sk-toggleable\"><input class=\"sk-toggleable__control sk-hidden--visually\" id=\"sk-estimator-id-2\" type=\"checkbox\" ><label for=\"sk-estimator-id-2\" class=\"sk-toggleable__label sk-toggleable__label-arrow\">estimator: XGBClassifier</label><div class=\"sk-toggleable__content\"><pre>XGBClassifier(base_score=None, booster=None, callbacks=None,\n              colsample_bylevel=None, colsample_bynode=None,\n              colsample_bytree=None, early_stopping_rounds=None,\n              enable_categorical=False, eval_metric=None, gamma=None,\n              gpu_id=None, grow_policy=None, importance_type=None,\n              interaction_constraints=None, learning_rate=None, max_bin=None,\n              max_cat_to_onehot=None, max_delta_step=None, max_depth=None,\n              max_leaves=None, min_child_weight=None, missing=nan,\n              monotone_constraints=None, n_estimators=100, n_jobs=None,\n              num_parallel_tree=None, predictor=None, random_state=None,\n              reg_alpha=None, reg_lambda=None, ...)</pre></div></div></div><div class=\"sk-serial\"><div class=\"sk-item\"><div class=\"sk-estimator sk-toggleable\"><input class=\"sk-toggleable__control sk-hidden--visually\" id=\"sk-estimator-id-3\" type=\"checkbox\" ><label for=\"sk-estimator-id-3\" class=\"sk-toggleable__label sk-toggleable__label-arrow\">XGBClassifier</label><div class=\"sk-toggleable__content\"><pre>XGBClassifier(base_score=None, booster=None, callbacks=None,\n              colsample_bylevel=None, colsample_bynode=None,\n              colsample_bytree=None, early_stopping_rounds=None,\n              enable_categorical=False, eval_metric=None, gamma=None,\n              gpu_id=None, grow_policy=None, importance_type=None,\n              interaction_constraints=None, learning_rate=None, max_bin=None,\n              max_cat_to_onehot=None, max_delta_step=None, max_depth=None,\n              max_leaves=None, min_child_weight=None, missing=nan,\n              monotone_constraints=None, n_estimators=100, n_jobs=None,\n              num_parallel_tree=None, predictor=None, random_state=None,\n              reg_alpha=None, reg_lambda=None, ...)</pre></div></div></div></div></div></div></div></div></div></div>"
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "%%time\n",
    "from bayes_opt import BayesianOptimization\n",
    "parameter_grid = {\n",
    "    'n_estimators': [50, 100, 150, 200],\n",
    "    'learning_rate': [0.01, 0.1, 0.2, 0.3],\n",
    "    'max_depth': [5, 6, 7],\n",
    "    'min_child_weight': [1, 3, 5],\n",
    "    'subsample': [0.8, 0.9, 1.0],\n",
    "    'colsample_bytree': [0.8, 0.9, 1.0]\n",
    "}\n",
    "\n",
    "model = XGBClassifier(tree_method=\"gpu_hist\")\n",
    "\n",
    "grid_search = GridSearchCV(model, parameter_grid, scoring=\"matthews_corrcoef\", cv=3, verbose=1, n_jobs=12)\n",
    "\n",
    "grid_search.fit(X, y)"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "outputs": [
    {
     "data": {
      "text/plain": "      mean_fit_time  std_fit_time  mean_score_time  std_score_time  \\\n0          4.502714      0.001570         0.271742        0.012913   \n1          4.463886      0.054376         0.292894        0.015541   \n2          3.944259      0.563497         0.263358        0.014848   \n3          5.736898      0.092272         0.298172        0.028339   \n4          5.430502      0.700245         0.310719        0.002166   \n...             ...           ...              ...             ...   \n1291      18.302688      0.218331         0.497518        0.000931   \n1292      18.025753      0.424667         0.444883        0.030711   \n1293      20.833965      0.317261         0.424439        0.019300   \n1294      17.223778      0.386428         0.412365        0.015024   \n1295      14.639879      0.462648         0.405718        0.011408   \n\n     param_colsample_bytree param_learning_rate param_max_depth  \\\n0                       0.8                0.01               5   \n1                       0.8                0.01               5   \n2                       0.8                0.01               5   \n3                       0.8                0.01               5   \n4                       0.8                0.01               5   \n...                     ...                 ...             ...   \n1291                    1.0                 0.3               7   \n1292                    1.0                 0.3               7   \n1293                    1.0                 0.3               7   \n1294                    1.0                 0.3               7   \n1295                    1.0                 0.3               7   \n\n     param_min_child_weight param_n_estimators param_subsample  \\\n0                         1                 50             0.8   \n1                         1                 50             0.9   \n2                         1                 50             1.0   \n3                         1                100             0.8   \n4                         1                100             0.9   \n...                     ...                ...             ...   \n1291                      5                150             0.9   \n1292                      5                150             1.0   \n1293                      5                200             0.8   \n1294                      5                200             0.9   \n1295                      5                200             1.0   \n\n                                                 params  split0_test_score  \\\n0     {'colsample_bytree': 0.8, 'learning_rate': 0.0...           0.860016   \n1     {'colsample_bytree': 0.8, 'learning_rate': 0.0...           0.859641   \n2     {'colsample_bytree': 0.8, 'learning_rate': 0.0...           0.859675   \n3     {'colsample_bytree': 0.8, 'learning_rate': 0.0...           0.861579   \n4     {'colsample_bytree': 0.8, 'learning_rate': 0.0...           0.861199   \n...                                                 ...                ...   \n1291  {'colsample_bytree': 1.0, 'learning_rate': 0.3...           0.883443   \n1292  {'colsample_bytree': 1.0, 'learning_rate': 0.3...           0.883171   \n1293  {'colsample_bytree': 1.0, 'learning_rate': 0.3...           0.882202   \n1294  {'colsample_bytree': 1.0, 'learning_rate': 0.3...           0.883590   \n1295  {'colsample_bytree': 1.0, 'learning_rate': 0.3...           0.882307   \n\n      split1_test_score  split2_test_score  mean_test_score  std_test_score  \\\n0              0.894865           0.790842         0.848574        0.043231   \n1              0.895526           0.789940         0.848369        0.043836   \n2              0.895387           0.790379         0.848480        0.043594   \n3              0.897349           0.795451         0.851460        0.042211   \n4              0.897245           0.794912         0.851118        0.042381   \n...                 ...                ...              ...             ...   \n1291           0.913408           0.807783         0.868211        0.044446   \n1292           0.913421           0.809499         0.868697        0.043643   \n1293           0.913004           0.807630         0.867612        0.044239   \n1294           0.911889           0.806607         0.867362        0.044487   \n1295           0.912627           0.809116         0.868016        0.043450   \n\n      rank_test_score  \n0                1254  \n1                1260  \n2                1258  \n3                1215  \n4                1230  \n...               ...  \n1291              601  \n1292              474  \n1293              752  \n1294              791  \n1295              655  \n\n[1296 rows x 17 columns]",
      "text/html": "<div>\n<style scoped>\n    .dataframe tbody tr th:only-of-type {\n        vertical-align: middle;\n    }\n\n    .dataframe tbody tr th {\n        vertical-align: top;\n    }\n\n    .dataframe thead th {\n        text-align: right;\n    }\n</style>\n<table border=\"1\" class=\"dataframe\">\n  <thead>\n    <tr style=\"text-align: right;\">\n      <th></th>\n      <th>mean_fit_time</th>\n      <th>std_fit_time</th>\n      <th>mean_score_time</th>\n      <th>std_score_time</th>\n      <th>param_colsample_bytree</th>\n      <th>param_learning_rate</th>\n      <th>param_max_depth</th>\n      <th>param_min_child_weight</th>\n      <th>param_n_estimators</th>\n      <th>param_subsample</th>\n      <th>params</th>\n      <th>split0_test_score</th>\n      <th>split1_test_score</th>\n      <th>split2_test_score</th>\n      <th>mean_test_score</th>\n      <th>std_test_score</th>\n      <th>rank_test_score</th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <th>0</th>\n      <td>4.502714</td>\n      <td>0.001570</td>\n      <td>0.271742</td>\n      <td>0.012913</td>\n      <td>0.8</td>\n      <td>0.01</td>\n      <td>5</td>\n      <td>1</td>\n      <td>50</td>\n      <td>0.8</td>\n      <td>{'colsample_bytree': 0.8, 'learning_rate': 0.0...</td>\n      <td>0.860016</td>\n      <td>0.894865</td>\n      <td>0.790842</td>\n      <td>0.848574</td>\n      <td>0.043231</td>\n      <td>1254</td>\n    </tr>\n    <tr>\n      <th>1</th>\n      <td>4.463886</td>\n      <td>0.054376</td>\n      <td>0.292894</td>\n      <td>0.015541</td>\n      <td>0.8</td>\n      <td>0.01</td>\n      <td>5</td>\n      <td>1</td>\n      <td>50</td>\n      <td>0.9</td>\n      <td>{'colsample_bytree': 0.8, 'learning_rate': 0.0...</td>\n      <td>0.859641</td>\n      <td>0.895526</td>\n      <td>0.789940</td>\n      <td>0.848369</td>\n      <td>0.043836</td>\n      <td>1260</td>\n    </tr>\n    <tr>\n      <th>2</th>\n      <td>3.944259</td>\n      <td>0.563497</td>\n      <td>0.263358</td>\n      <td>0.014848</td>\n      <td>0.8</td>\n      <td>0.01</td>\n      <td>5</td>\n      <td>1</td>\n      <td>50</td>\n      <td>1.0</td>\n      <td>{'colsample_bytree': 0.8, 'learning_rate': 0.0...</td>\n      <td>0.859675</td>\n      <td>0.895387</td>\n      <td>0.790379</td>\n      <td>0.848480</td>\n      <td>0.043594</td>\n      <td>1258</td>\n    </tr>\n    <tr>\n      <th>3</th>\n      <td>5.736898</td>\n      <td>0.092272</td>\n      <td>0.298172</td>\n      <td>0.028339</td>\n      <td>0.8</td>\n      <td>0.01</td>\n      <td>5</td>\n      <td>1</td>\n      <td>100</td>\n      <td>0.8</td>\n      <td>{'colsample_bytree': 0.8, 'learning_rate': 0.0...</td>\n      <td>0.861579</td>\n      <td>0.897349</td>\n      <td>0.795451</td>\n      <td>0.851460</td>\n      <td>0.042211</td>\n      <td>1215</td>\n    </tr>\n    <tr>\n      <th>4</th>\n      <td>5.430502</td>\n      <td>0.700245</td>\n      <td>0.310719</td>\n      <td>0.002166</td>\n      <td>0.8</td>\n      <td>0.01</td>\n      <td>5</td>\n      <td>1</td>\n      <td>100</td>\n      <td>0.9</td>\n      <td>{'colsample_bytree': 0.8, 'learning_rate': 0.0...</td>\n      <td>0.861199</td>\n      <td>0.897245</td>\n      <td>0.794912</td>\n      <td>0.851118</td>\n      <td>0.042381</td>\n      <td>1230</td>\n    </tr>\n    <tr>\n      <th>...</th>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n    </tr>\n    <tr>\n      <th>1291</th>\n      <td>18.302688</td>\n      <td>0.218331</td>\n      <td>0.497518</td>\n      <td>0.000931</td>\n      <td>1.0</td>\n      <td>0.3</td>\n      <td>7</td>\n      <td>5</td>\n      <td>150</td>\n      <td>0.9</td>\n      <td>{'colsample_bytree': 1.0, 'learning_rate': 0.3...</td>\n      <td>0.883443</td>\n      <td>0.913408</td>\n      <td>0.807783</td>\n      <td>0.868211</td>\n      <td>0.044446</td>\n      <td>601</td>\n    </tr>\n    <tr>\n      <th>1292</th>\n      <td>18.025753</td>\n      <td>0.424667</td>\n      <td>0.444883</td>\n      <td>0.030711</td>\n      <td>1.0</td>\n      <td>0.3</td>\n      <td>7</td>\n      <td>5</td>\n      <td>150</td>\n      <td>1.0</td>\n      <td>{'colsample_bytree': 1.0, 'learning_rate': 0.3...</td>\n      <td>0.883171</td>\n      <td>0.913421</td>\n      <td>0.809499</td>\n      <td>0.868697</td>\n      <td>0.043643</td>\n      <td>474</td>\n    </tr>\n    <tr>\n      <th>1293</th>\n      <td>20.833965</td>\n      <td>0.317261</td>\n      <td>0.424439</td>\n      <td>0.019300</td>\n      <td>1.0</td>\n      <td>0.3</td>\n      <td>7</td>\n      <td>5</td>\n      <td>200</td>\n      <td>0.8</td>\n      <td>{'colsample_bytree': 1.0, 'learning_rate': 0.3...</td>\n      <td>0.882202</td>\n      <td>0.913004</td>\n      <td>0.807630</td>\n      <td>0.867612</td>\n      <td>0.044239</td>\n      <td>752</td>\n    </tr>\n    <tr>\n      <th>1294</th>\n      <td>17.223778</td>\n      <td>0.386428</td>\n      <td>0.412365</td>\n      <td>0.015024</td>\n      <td>1.0</td>\n      <td>0.3</td>\n      <td>7</td>\n      <td>5</td>\n      <td>200</td>\n      <td>0.9</td>\n      <td>{'colsample_bytree': 1.0, 'learning_rate': 0.3...</td>\n      <td>0.883590</td>\n      <td>0.911889</td>\n      <td>0.806607</td>\n      <td>0.867362</td>\n      <td>0.044487</td>\n      <td>791</td>\n    </tr>\n    <tr>\n      <th>1295</th>\n      <td>14.639879</td>\n      <td>0.462648</td>\n      <td>0.405718</td>\n      <td>0.011408</td>\n      <td>1.0</td>\n      <td>0.3</td>\n      <td>7</td>\n      <td>5</td>\n      <td>200</td>\n      <td>1.0</td>\n      <td>{'colsample_bytree': 1.0, 'learning_rate': 0.3...</td>\n      <td>0.882307</td>\n      <td>0.912627</td>\n      <td>0.809116</td>\n      <td>0.868016</td>\n      <td>0.043450</td>\n      <td>655</td>\n    </tr>\n  </tbody>\n</table>\n<p>1296 rows × 17 columns</p>\n</div>"
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_grid_search_result = pd.DataFrame(grid_search.cv_results_)\n",
    "df_grid_search_result.to_csv(\"data/18_hyperparameter_opt_result.tsv\", sep=\"\\t\")\n",
    "df_grid_search_result"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Best Parameters:\n",
      " {'colsample_bytree': 0.9, 'learning_rate': 0.1, 'max_depth': 7, 'min_child_weight': 5, 'n_estimators': 200, 'subsample': 0.9}\n"
     ]
    }
   ],
   "source": [
    "print(\"Best Parameters:\\n\", grid_search.best_params_)"
   ],
   "metadata": {
    "collapsed": false
   }
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}