pred.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Это baseline решение для задачи. План в том, чтобы поработать с признками(категориальными), разделить тренировочную выборку на 2 части - на одной из частей мы будем искать гиперпараметры для нашиъ моделей, а на второй сравнивать эти модели."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 82,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "import scipy\n",
    "%matplotlib inline\n",
    "from  matplotlib import pyplot as plt\n",
    "import seaborn as sns\n",
    "from sklearn.preprocessing import OneHotEncoder\n",
    "from sklearn.feature_extraction import DictVectorizer\n",
    "from sklearn.preprocessing import StandardScaler\n",
    "from sklearn.preprocessing import MaxAbsScaler\n",
    "from sklearn.preprocessing import LabelEncoder\n",
    "from sklearn.preprocessing import LabelBinarizer\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.model_selection import cross_val_score\n",
    "from tqdm import tqdm\n",
    "from sklearn.model_selection import GridSearchCV\n",
    "from sklearn.model_selection import StratifiedKFold\n",
    "from sklearn.linear_model import LogisticRegression\n",
    "from sklearn.neighbors import KNeighborsClassifier\n",
    "from sklearn.metrics import make_scorer\n",
    "from sklearn.metrics import accuracy_score\n",
    "from sklearn.ensemble import BaggingClassifier\n",
    "from sklearn.ensemble import RandomForestClassifier\n",
    "from sklearn.ensemble import GradientBoostingClassifier\n",
    "from sklearn.model_selection import RandomizedSearchCV"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Первым делом надо загрузить наши данные и отделить тренировочную выборку от тестовой"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 96,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style>\n",
       "    .dataframe thead tr:only-child th {\n",
       "        text-align: right;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>DATA_TYPE</th>\n",
       "      <th>POLICY_BEGIN_MONTH</th>\n",
       "      <th>POLICY_END_MONTH</th>\n",
       "      <th>POLICY_IS_RENEWED</th>\n",
       "      <th>POLICY_SALES_CHANNEL</th>\n",
       "      <th>POLICY_SALES_CHANNEL_GROUP</th>\n",
       "      <th>POLICY_BRANCH</th>\n",
       "      <th>POLICY_MIN_AGE</th>\n",
       "      <th>POLICY_MIN_DRIVING_EXPERIENCE</th>\n",
       "      <th>VEHICLE_MAKE</th>\n",
       "      <th>...</th>\n",
       "      <th>POLICY_PRV_CLM_GLT_N</th>\n",
       "      <th>CLIENT_HAS_DAGO</th>\n",
       "      <th>CLIENT_HAS_OSAGO</th>\n",
       "      <th>POLICY_COURT_SIGN</th>\n",
       "      <th>CLAIM_AVG_ACC_ST_PRD</th>\n",
       "      <th>POLICY_HAS_COMPLAINTS</th>\n",
       "      <th>POLICY_YEARS_RENEWED_N</th>\n",
       "      <th>POLICY_DEDUCT_VALUE</th>\n",
       "      <th>CLIENT_REGISTRATION_REGION</th>\n",
       "      <th>POLICY_PRICE_CHANGE</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>POLICY_ID</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>TRAIN</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>39</td>\n",
       "      <td>1</td>\n",
       "      <td>Москва</td>\n",
       "      <td>51</td>\n",
       "      <td>12</td>\n",
       "      <td>Land Rover</td>\n",
       "      <td>...</td>\n",
       "      <td>N</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>Тульская</td>\n",
       "      <td>-1.00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>TRAIN</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>50</td>\n",
       "      <td>5</td>\n",
       "      <td>Москва</td>\n",
       "      <td>35</td>\n",
       "      <td>7</td>\n",
       "      <td>Opel</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0</td>\n",
       "      <td>4</td>\n",
       "      <td>0.0</td>\n",
       "      <td>Москва</td>\n",
       "      <td>-0.05</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>TRAIN</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>52</td>\n",
       "      <td>6</td>\n",
       "      <td>Москва</td>\n",
       "      <td>41</td>\n",
       "      <td>6</td>\n",
       "      <td>Kia</td>\n",
       "      <td>...</td>\n",
       "      <td>1L</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>12518.0</td>\n",
       "      <td>Московская</td>\n",
       "      <td>-0.07</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>TRAIN</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>50</td>\n",
       "      <td>5</td>\n",
       "      <td>Москва</td>\n",
       "      <td>36</td>\n",
       "      <td>12</td>\n",
       "      <td>Citroen</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0</td>\n",
       "      <td>6</td>\n",
       "      <td>15000.0</td>\n",
       "      <td>Москва</td>\n",
       "      <td>0.05</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>TRAIN</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>52</td>\n",
       "      <td>6</td>\n",
       "      <td>Санкт-Петербург</td>\n",
       "      <td>42</td>\n",
       "      <td>5</td>\n",
       "      <td>Renault</td>\n",
       "      <td>...</td>\n",
       "      <td>N</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>50000.0</td>\n",
       "      <td>Ленинградская</td>\n",
       "      <td>0.17</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 29 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "          DATA_TYPE  POLICY_BEGIN_MONTH  POLICY_END_MONTH  POLICY_IS_RENEWED  \\\n",
       "POLICY_ID                                                                      \n",
       "1             TRAIN                   1                 1                  1   \n",
       "2             TRAIN                   1                 1                  1   \n",
       "3             TRAIN                   1                 1                  1   \n",
       "4             TRAIN                   1                 1                  1   \n",
       "5             TRAIN                   1                 1                  0   \n",
       "\n",
       "           POLICY_SALES_CHANNEL  POLICY_SALES_CHANNEL_GROUP    POLICY_BRANCH  \\\n",
       "POLICY_ID                                                                      \n",
       "1                            39                           1           Москва   \n",
       "2                            50                           5           Москва   \n",
       "3                            52                           6           Москва   \n",
       "4                            50                           5           Москва   \n",
       "5                            52                           6  Санкт-Петербург   \n",
       "\n",
       "           POLICY_MIN_AGE  POLICY_MIN_DRIVING_EXPERIENCE VEHICLE_MAKE  \\\n",
       "POLICY_ID                                                               \n",
       "1                      51                             12   Land Rover   \n",
       "2                      35                              7         Opel   \n",
       "3                      41                              6          Kia   \n",
       "4                      36                             12      Citroen   \n",
       "5                      42                              5      Renault   \n",
       "\n",
       "                  ...         POLICY_PRV_CLM_GLT_N  CLIENT_HAS_DAGO  \\\n",
       "POLICY_ID         ...                                                 \n",
       "1                 ...                            N                1   \n",
       "2                 ...                            0                1   \n",
       "3                 ...                           1L                0   \n",
       "4                 ...                            0                1   \n",
       "5                 ...                            N                0   \n",
       "\n",
       "           CLIENT_HAS_OSAGO  POLICY_COURT_SIGN CLAIM_AVG_ACC_ST_PRD  \\\n",
       "POLICY_ID                                                             \n",
       "1                         0                  0                  0.0   \n",
       "2                         1                  0                  0.0   \n",
       "3                         0                  0                  0.0   \n",
       "4                         1                  0                  0.0   \n",
       "5                         0                  0                  0.0   \n",
       "\n",
       "          POLICY_HAS_COMPLAINTS POLICY_YEARS_RENEWED_N POLICY_DEDUCT_VALUE  \\\n",
       "POLICY_ID                                                                    \n",
       "1                             0                      0                 0.0   \n",
       "2                             0                      4                 0.0   \n",
       "3                             0                      1             12518.0   \n",
       "4                             0                      6             15000.0   \n",
       "5                             0                      0             50000.0   \n",
       "\n",
       "          CLIENT_REGISTRATION_REGION POLICY_PRICE_CHANGE  \n",
       "POLICY_ID                                                 \n",
       "1                           Тульская               -1.00  \n",
       "2                             Москва               -0.05  \n",
       "3                         Московская               -0.07  \n",
       "4                             Москва                0.05  \n",
       "5                      Ленинградская                0.17  \n",
       "\n",
       "[5 rows x 29 columns]"
      ]
     },
     "execution_count": 96,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data = pd.read_csv(\"data.txt\", delimiter=';', index_col='POLICY_ID')\n",
    "train_df = data[data['DATA_TYPE']=='TRAIN'].drop('DATA_TYPE', axis=1)\n",
    "X_train = train_df.drop('POLICY_IS_RENEWED', axis=1)\n",
    "y_train = train_df['POLICY_IS_RENEWED']\n",
    "test_df = data[data['DATA_TYPE']=='TEST '].drop('DATA_TYPE', axis=1)\n",
    "X_test = test_df.drop('POLICY_IS_RENEWED', axis=1)\n",
    "data.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Затем посмотрим какие признаки у нас имеются и попробуем что-то с ними сделать"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array(['POLICY_BEGIN_MONTH', 'POLICY_END_MONTH', 'POLICY_SALES_CHANNEL',\n",
       "       'POLICY_SALES_CHANNEL_GROUP', 'POLICY_BRANCH', 'POLICY_MIN_AGE',\n",
       "       'POLICY_MIN_DRIVING_EXPERIENCE', 'VEHICLE_MAKE', 'VEHICLE_MODEL',\n",
       "       'VEHICLE_ENGINE_POWER', 'VEHICLE_IN_CREDIT', 'VEHICLE_SUM_INSURED',\n",
       "       'POLICY_INTERMEDIARY', 'INSURER_GENDER', 'POLICY_CLM_N',\n",
       "       'POLICY_CLM_GLT_N', 'POLICY_PRV_CLM_N', 'POLICY_PRV_CLM_GLT_N',\n",
       "       'CLIENT_HAS_DAGO', 'CLIENT_HAS_OSAGO', 'POLICY_COURT_SIGN',\n",
       "       'CLAIM_AVG_ACC_ST_PRD', 'POLICY_HAS_COMPLAINTS',\n",
       "       'POLICY_YEARS_RENEWED_N', 'POLICY_DEDUCT_VALUE',\n",
       "       'CLIENT_REGISTRATION_REGION', 'POLICY_PRICE_CHANGE'], dtype=object)"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X_train.columns.values"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Первые 2 признака - это месяца. Так и хочется узнать, на какой срок заключена страховка. Введём новый признак <i>'POLICY_LEN_MONTHS'</i> и посмотрим на его значения"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "77407\n",
      " 0     75524\n",
      "-1      1847\n",
      " 6        13\n",
      "-6         6\n",
      " 7         6\n",
      " 11        4\n",
      " 8         3\n",
      "-3         1\n",
      "-4         1\n",
      "-5         1\n",
      " 5         1\n",
      "Name: POLICY_LEN_MONTHS, dtype: int64\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAZEAAAEHCAYAAABvHnsJAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMS4wLCBo\ndHRwOi8vbWF0cGxvdGxpYi5vcmcvpW3flQAAIABJREFUeJzt3XFYlXWe///nCWJSQRDinDO6XG6a\nWllKs2NFMJi4B1BEUXGzRr+XlFu7MmNk2kpNygg5V2VlRtMvxp3RabVRWcGmU0lgiaRpMztIatmY\nsuFunGOEB0gRwfv7h1/PL5KM7vE+SL4e1+V1eT7nvj/vz8errte5P/c5n9tmGIaBiIiICVf09ABE\nRKT3UoiIiIhpChERETFNISIiIqYpRERExLTgnh5AoB071tzTQxAR6VWio8O+8T1diYiIiGkKERER\nMU0hIiIipilERETENIWIiIiYphARERHTFCIiImKaQkRERExTiIiIiGkKERERMe2y2/ZEJFDmVK63\ntP81iXdb2r9Id+hKRERETFOIiIiIaQoRERExTSEiIiKmKURERMQ0hYiIiJimEBEREdMUIiIiYppC\nRERETFOIiIiIaQoRERExzbIQOXz4MFOmTPH/+dGPfsSaNWs4fvw4WVlZJCcnk5WVhc/nA8AwDAoK\nCnC5XKSnp7N//35/XyUlJSQnJ5OcnExJSYm/fd++faSnp+NyuSgoKMAwDKumIyIiXbAsRIYMGcKW\nLVvYsmULmzdvpk+fPrhcLoqKioiLi6OsrIy4uDiKiooAqKyspLa2lrKyMvLz88nLywPg+PHjFBYW\nsnHjRjZt2kRhYaE/ePLy8sjPz6esrIza2loqKyutmo6IiHQhIMtZu3btIiYmhkGDBlFRUUFGRgYA\nGRkZlJeXA/jbbTYbsbGxNDU14fV6qaqqIj4+noiICMLDw4mPj2fHjh14vV5aWlqIjY3FZrORkZFB\nRUVFIKYjIiL/T0C2gne73UyaNAmAhoYG7HY7ANHR0TQ0NADg8XhwOp3+c5xOJx6P57x2h8PRZfu5\n47/NgAF9CQ4OuijzEulJ0dFhPT0EEetDpK2tjW3btvHQQw+d957NZsNms1k9hE4aG08EtJ6IVY4d\na+7pIchl4kIfWCxfzqqsrGTkyJFcffXVAERFReH1egHwer1ERkYCZ68w6uvr/efV19fjcDjOa/d4\nPF22nzteREQCx/IQcbvdpKWl+V8nJSVRWloKQGlpKePHj+/UbhgG1dXVhIWFYbfbSUhIoKqqCp/P\nh8/no6qqioSEBOx2O6GhoVRXV2MYRqe+REQkMCxdzjpx4gQ7d+5k2bJl/rb77ruPnJwciouLGThw\nICtXrgRg7NixbN++HZfLRZ8+fVi+fDkAERERzJs3j8zMTACys7OJiIgAYOnSpeTm5tLa2kpiYiKJ\niYlWTkdERL7GZlxmP67QOrIEip6xLt8XPXpPREREvr8UIiIiYppCRERETFOIiIiIaQoRERExTSEi\nIiKmKURERMQ0hYiIiJimEBEREdMUIiIiYppCRERETFOIiIiIaQoRERExTSEiIiKmKURERMQ0hYiI\niJimEBEREdMUIiIiYpqlIdLU1MT8+fNJTU1lwoQJ/OUvf+H48eNkZWWRnJxMVlYWPp8PAMMwKCgo\nwOVykZ6ezv79+/39lJSUkJycTHJyMiUlJf72ffv2kZ6ejsvloqCggMvsSb8iIj3O0hB5/PHH+clP\nfsKbb77Jli1bGDp0KEVFRcTFxVFWVkZcXBxFRUUAVFZWUltbS1lZGfn5+eTl5QFw/PhxCgsL2bhx\nI5s2baKwsNAfPHl5eeTn51NWVkZtbS2VlZVWTkdERL7GshBpbm7m/fffJzMzE4CQkBD69+9PRUUF\nGRkZAGRkZFBeXg7gb7fZbMTGxtLU1ITX66Wqqor4+HgiIiIIDw8nPj6eHTt24PV6aWlpITY2FpvN\nRkZGBhUVFVZNR0REuhBsVcdHjx4lMjKS3NxcPvroI0aOHMmjjz5KQ0MDdrsdgOjoaBoaGgDweDw4\nnU7/+U6nE4/Hc167w+Hosv3c8d9mwIC+BAcHXaxpivSY6Oiwnh6CiHUh0t7ezoEDB3jssccYPXo0\nBQUF/qWrc2w2GzabzaohdKmx8URA64lY5dix5p4eglwmLvSBxbLlLKfTidPpZPTo0QCkpqZy4MAB\noqKi8Hq9AHi9XiIjI4GzVxj19fX+8+vr63E4HOe1ezyeLtvPHS8iIoFjWYhER0fjdDo5fPgwALt2\n7WLo0KEkJSVRWloKQGlpKePHjwfwtxuGQXV1NWFhYdjtdhISEqiqqsLn8+Hz+aiqqiIhIQG73U5o\naCjV1dUYhtGpLxERCQzLlrMAHnvsMRYuXMjp06eJiYnhV7/6FWfOnCEnJ4fi4mIGDhzIypUrARg7\ndizbt2/H5XLRp08fli9fDkBERATz5s3z36DPzs4mIiICgKVLl5Kbm0trayuJiYkkJiZaOR0REfka\nm3GZ/bhC68gSKHMq11va/5rEuy3tX+ScHrknIiIi338KERERMU0hIiIipilERETENIWIiIiYphAR\nERHTFCIiImKaQkRERExTiIiIiGkKERERMU0hIiIipilERETENIWIiIiYphARERHTFCIiImKaQkRE\nRExTiIiIiGkKERERMU0hIiIiplkaIklJSaSnpzNlyhSmTZsGwPHjx8nKyiI5OZmsrCx8Ph8AhmFQ\nUFCAy+UiPT2d/fv3+/spKSkhOTmZ5ORkSkpK/O379u0jPT0dl8tFQUEBl9nj4kVEepzlVyJr165l\ny5YtbN68GYCioiLi4uIoKysjLi6OoqIiACorK6mtraWsrIz8/Hzy8vKAs6FTWFjIxo0b2bRpE4WF\nhf7gycvLIz8/n7KyMmpra6msrLR6OiIi8hUBX86qqKggIyMDgIyMDMrLyzu122w2YmNjaWpqwuv1\nUlVVRXx8PBEREYSHhxMfH8+OHTvwer20tLQQGxuLzWYjIyODioqKQE9HROSyFmx1gXvvvRebzcad\nd97JnXfeSUNDA3a7HYDo6GgaGhoA8Hg8OJ1O/3lOpxOPx3Neu8Ph6LL93PHfZsCAvgQHB12s6Yn0\nmOjosJ4egoi1IfLKK6/gcDhoaGggKyuLIUOGdHrfZrNhs9msHMJ5GhtPBLSeiFWOHWvu6SHIZeJC\nH1gsXc5yOBwAREVF4XK5qKmpISoqCq/XC4DX6yUyMtJ/bH19vf/c+vp6HA7Hee0ej6fL9nPHi4hI\n4FgWIidOnKClpcX/93fffZdhw4aRlJREaWkpAKWlpYwfPx7A324YBtXV1YSFhWG320lISKCqqgqf\nz4fP56OqqoqEhATsdjuhoaFUV1djGEanvkREJDAsW85qaGggOzsbgI6ODiZNmkRiYiI33XQTOTk5\nFBcXM3DgQFauXAnA2LFj2b59Oy6Xiz59+rB8+XIAIiIimDdvHpmZmQBkZ2cTEREBwNKlS8nNzaW1\ntZXExEQSExOtmo6IiHTBZlxmP67QOrIEypzK9Zb2vybxbkv7Fzmnx+6JiIjI95tCRERETFOIiIiI\naQoRERExTSEiIiKmKURERMQ0hYiIiJimEBEREdMUIiIiYlq3QuSBBx7oVpuIiFxeuhUin3766Xlt\nhw8fvuiDERGR3uWCGzBu3LiRDRs2UFtb698AEaC5uZlrrrnG8sGJiMil7YIhEh8fz+DBg8nPz+fh\nhx/2t4eGhjJixAjLByciIpe2C4bIoEGDGDRoEK+99lqgxiMiIr1It54ncvjwYV588UXq6upob2/3\ntxcXF1s2MBERufR1K0QWLFhAamoq06ZNIygoyOoxiYhIL9GtEDlz5gz/8i//YvVYRESkl+nWV3xj\nY2P56KOPrB6LiIj0Mt0KkZqaGjIzM0lPTyczM9P/pzs6OjrIyMjg/vvvB6Curo4ZM2bgcrnIycmh\nra0NgLa2NnJycnC5XMyYMYOjR4/6+3jppZdwuVykpKSwY8cOf3tlZSUpKSm4XC6Kioq6PWkREbk4\nurWc9cgjj5gu8Pvf/56hQ4fS0tICwIoVK5gzZw5paWksWbKE4uJi7r77bjZt2kT//v156623cLvd\nrFixgpUrV3Lo0CHcbjdutxuPx0NWVhZbt24FYNmyZfzud7/D4XCQmZlJUlIS1157remxiojId9Ot\nK5Fbbrmlyz/fpr6+nnfeecd/1WIYBu+99x4pKSkATJ06lYqKCgC2bdvG1KlTAUhJSWHXrl0YhkFF\nRQVpaWmEhIQQExPD4MGDqampoaamhsGDBxMTE0NISAhpaWn+vkREJDC6dSUyffp0bDbbee3f9hXf\n5cuXs2jRIr788ksAGhsb6d+/P8HBZ8s6nU48Hg8AHo+HH/7wh2cHFRxMWFgYjY2NeDweRo8e7e/T\n4XD4z3E6nZ3aa2pqvnUuAwb0JThY3zCT3i86OqynhyDSvRD5t3/7N//fT506hdvtxm63X/Cct99+\nm8jISG688UZ27979t43yImpsPNHTQxC5KI4da+7pIchl4kIfWLoVIl9fukpISOCuu+664Dn/9V//\nxbZt26isrOTUqVO0tLTw+OOP09TURHt7O8HBwdTX1+NwOICzVxKfffYZTqeT9vZ2mpubGTBgAA6H\ng/r6en+/Ho/Hf843tYuISGCYep5IS0sLn3/++QWPeeihh6isrGTbtm0888wz3HbbbTz99NPceuut\n/hvjJSUlJCUlAZCUlERJSQkAW7du5bbbbsNms5GUlITb7aatrY26ujpqa2sZNWoUN910E7W1tdTV\n1dHW1obb7fb3JSIigfGd74mcOXOGo0ePkpWVZargokWLePDBB1m5ciXXX389M2bMACAzM5NFixbh\ncrkIDw/n2WefBWDYsGFMmDCBiRMnEhQUxJIlS/y/ml+yZAlz586lo6OD6dOnM2zYMFNjEhERc2yG\nYRjfdtCePXv8fw8KCiImJuZb74lcqrSOLIEyp3K9pf2vSbzb0v5Fzrko90Ta29s5cuQIAJGRkRdn\nZCIi0qt1K0Q++OAD5s+fT0hICIZh0N7ezvPPP8/IkSOtHp+IiFzCuhUijz/+OMuXLycuLg6AXbt2\nkZ+fzx/+8AdLByciIpe2bn076+TJk/4AAYiLi+PkyZOWDUpERHqHboVInz59Ov1gcM+ePfTp08ey\nQYmISO/QreWsRx991H9PBOD06dOsWrXK0oGJiMilr1sh0tzcTHFxMQ0NDQBERUXx8ccfWzowERG5\n9HVrOevJJ58kMjKS4cOHM3z4cAYMGMCTTz5p9dhEROQS160QMQyj0y6+V1xxBR0dHZYNSkREeodu\nhUi/fv3Yu3ev//XevXvp27evZYMSEZHeoVv3RBYtWkR2drb/qYGHDh2isLDQ0oGJiMilr1shcvPN\nN+N2u6murgYgNjaW8PBwSwcmIiKXvm6FCEB4eDhjx461ciwiItLLmHqeiIiICChERETkb6AQERER\n0xQiIiJimkJERERMsyxETp06RWZmJpMnTyYtLc2/YWNdXR0zZszA5XKRk5NDW1sbAG1tbeTk5OBy\nuZgxYwZHjx719/XSSy/hcrlISUlhx44d/vbKykpSUlJwuVwUFRVZNRUREfkGloVISEgIa9eu5dVX\nX6W0tJQdO3ZQXV3NihUrmDNnDm+99Rb9+/enuLgYgE2bNtG/f3/eeust5syZw4oVK4CzP2x0u924\n3W5Wr17NL3/5Szo6Oujo6GDZsmWsXr0at9vNa6+9xqFDh6yajoiIdMGyELHZbPTr1w+A9vZ22tvb\nsdlsvPfee6SkpAAwdepUKioqANi2bRtTp04FICUlhV27dmEYBhUVFaSlpRESEkJMTAyDBw+mpqaG\nmpoaBg8eTExMDCEhIaSlpfn7EhGRwLD0nkhHRwdTpkzh9ttv5/bbbycmJob+/fsTHHz2N45OpxOP\nxwOAx+Phhz/8IQDBwcGEhYXR2NiIx+PB6XT6+3Q4HHg8nm9sFxGRwOn2L9bNCAoKYsuWLTQ1NZGd\nnc3hw4etLNctAwb0JTg4qKeHIfI3i44O6+khiFgbIuf079+fW2+9lerqapqammhvbyc4OJj6+noc\nDgdw9kris88+w+l00t7eTnNzMwMGDMDhcFBfX+/vy+Px+M/5pvYLaWw8cZFnJ9Izjh1r7ukhyGXi\nQh9YLFvO+uKLL2hqagKgtbWVnTt3MnToUG699Va2bt0KQElJCUlJSQAkJSVRUlICwNatW7ntttuw\n2WwkJSXhdrtpa2ujrq6O2tpaRo0axU033URtbS11dXW0tbXhdrv9fYmISGBYdiXi9XpZvHgxHR0d\nGIZBamoq48aN49prr+XBBx9k5cqVXH/99cyYMQOAzMxMFi1ahMvlIjw8nGeffRaAYcOGMWHCBCZO\nnEhQUBBLliwhKOjsctSSJUuYO3cuHR0dTJ8+nWHDhlk1HRER6YLNMAyjpwcRSFoCkECZU7ne0v7X\nJN5taf8i5/TIcpaIiHz/KURERMQ0hYiIiJimEBEREdMUIiIiYppCRERETFOIiIiIaQoRERExTSEi\nIiKmKURERMQ0hYiIiJimEBEREdMUIiIiYppCRERETFOIiIiIaQoRERExTSEiIiKmKURERMQ0hYiI\niJhmWYh89tlnzJ49m4kTJ5KWlsbatWsBOH78OFlZWSQnJ5OVlYXP5wPAMAwKCgpwuVykp6ezf/9+\nf18lJSUkJyeTnJxMSUmJv33fvn2kp6fjcrkoKCjgMntcvIhIj7MsRIKCgli8eDGvv/46GzZsYP36\n9Rw6dIiioiLi4uIoKysjLi6OoqIiACorK6mtraWsrIz8/Hzy8vKAs6FTWFjIxo0b2bRpE4WFhf7g\nycvLIz8/n7KyMmpra6msrLRqOiIi0gXLQsRutzNy5EgAQkNDGTJkCB6Ph4qKCjIyMgDIyMigvLwc\nwN9us9mIjY2lqakJr9dLVVUV8fHxREREEB4eTnx8PDt27MDr9dLS0kJsbCw2m42MjAwqKiqsmo6I\niHQhOBBFjh49yocffsjo0aNpaGjAbrcDEB0dTUNDAwAejwen0+k/x+l04vF4zmt3OBxdtp87/tsM\nGNCX4OCgizU1kR4THR3W00MQsT5EvvzyS+bPn88jjzxCaGhop/dsNhs2m83qIXTS2HgioPVErHLs\nWHNPD0EuExf6wGLpt7NOnz7N/PnzSU9PJzk5GYCoqCi8Xi8AXq+XyMhI4OwVRn19vf/c+vp6HA7H\nee0ej6fL9nPHi4hI4FgWIoZh8OijjzJkyBCysrL87UlJSZSWlgJQWlrK+PHjO7UbhkF1dTVhYWHY\n7XYSEhKoqqrC5/Ph8/moqqoiISEBu91OaGgo1dXVGIbRqS8REQkMy5az/vznP7NlyxaGDx/OlClT\nAFiwYAH33XcfOTk5FBcXM3DgQFauXAnA2LFj2b59Oy6Xiz59+rB8+XIAIiIimDdvHpmZmQBkZ2cT\nEREBwNKlS8nNzaW1tZXExEQSExOtmo6IiHTBZlxmP67QOrIEypzK9Zb2vybxbkv7Fzmnx+6JiIjI\n95tCRERETFOIiIiIaQoRERExTSEiIiKmKURERMQ0hYiIiJimEBEREdMUIiIiYppCRERETFOIiIiI\naQoRERExTSEiIiKmKURERMQ0hYiIiJimEBEREdMUIiIiYppCRERETLMsRHJzc4mLi2PSpEn+tuPH\nj5OVlUVycjJZWVn4fD4ADMOgoKAAl8tFeno6+/fv959TUlJCcnIyycnJlJSU+Nv37dtHeno6LpeL\ngoICLrOn/IqIXBIsC5Fp06axevXqTm1FRUXExcVRVlZGXFwcRUVFAFRWVlJbW0tZWRn5+fnk5eUB\nZ0OnsLCQjRs3smnTJgoLC/3Bk5eXR35+PmVlZdTW1lJZWWnVVERE5BtYFiJjxowhPDy8U1tFRQUZ\nGRkAZGRkUF5e3qndZrMRGxtLU1MTXq+Xqqoq4uPjiYiIIDw8nPj4eHbs2IHX66WlpYXY2FhsNhsZ\nGRlUVFRYNRUREfkGwYEs1tDQgN1uByA6OpqGhgYAPB4PTqfTf5zT6cTj8ZzX7nA4umw/d3x3DBjQ\nl+DgoIsxHZEeFR0d1tNDEAlsiHyVzWbDZrMFvG5j44mA1xSxwrFjzT09BLlMXOgDS0C/nRUVFYXX\n6wXA6/USGRkJnL3CqK+v9x9XX1+Pw+E4r93j8XTZfu54EREJrICGSFJSEqWlpQCUlpYyfvz4Tu2G\nYVBdXU1YWBh2u52EhASqqqrw+Xz4fD6qqqpISEjAbrcTGhpKdXU1hmF06ktERALHsuWsBQsWsGfP\nHhobG0lMTOTnP/859913Hzk5ORQXFzNw4EBWrlwJwNixY9m+fTsul4s+ffqwfPlyACIiIpg3bx6Z\nmZkAZGdnExERAcDSpUvJzc2ltbWVxMREEhMTrZqKiIh8A5txmf3AQuvIEihzKtdb2v+axLst7V/k\nnEvmnoiIiHy/KERERMQ0hYiIiJimEBEREdMUIiIiYppCRERETFOIiIiIaQoRERExTSEiIiKmKURE\nRMQ0hYiIiJimEBEREdMUIiIiYppCRERETFOIiIiIaQoRERExTSEiIiKmWfZ4XJFLwf/ZucjyGr+/\n/SnLa4hcqnQlIiIipvX6EKmsrCQlJQWXy0VRUVFPD0dE5LLSq0Oko6ODZcuWsXr1atxuN6+99hqH\nDh3q6WGJiFw2enWI1NTUMHjwYGJiYggJCSEtLY2KioqeHpaIyGWjV99Y93g8OJ1O/2uHw0FNTc0F\nz4mODrN6WHIJeWPK/9djtd3T7++x2iKB0quvREREpGf16hBxOBzU19f7X3s8HhwORw+OSETk8tKr\nQ+Smm26itraWuro62tracLvdJCUl9fSwREQuG736nkhwcDBLlixh7ty5dHR0MH36dIYNG9bTwxIR\nuWzYDMMwenoQIiLSO/Xq5SwREelZChERETFNIdJNu3fvZsqUKaSlpTFr1qyA1v2Hf/gHpkyZwpQp\nUygsLAxYbTj7g84bbriBN998M2A1y8vLSU9PZ8qUKUybNo0//elPAan7ySefcOedd3LjjTfy7//+\n7wGpCT27dU9SUlKnf+tAaWpqYv78+aSmpjJhwgT+8pe/BKTu4cOH/f8vTZkyhR/96EesWbMmILXX\nrFlDWloakyZNYsGCBZw6dcqyWrm5ucTFxTFp0iR/2xtvvEFaWhrXXXcdH3zwwcUrZsi38vl8xoQJ\nE4z/+Z//MQzDMD7//POA1X7vvfeM++67L2D1vqq9vd2YPXu2MXfuXOONN94IWN2WlhbjzJkzhmEY\nxocffmikpKQEpO7nn39u7N2713jmmWeM1atXB6Rme3u7MX78eOPTTz81Tp06ZaSnpxt//etfA1Lb\nMAxj3LhxRkNDQ8DqnfPwww8bGzduNAzDME6dOmX4fL6Aj6G9vd24/fbbjaNHj1peq76+3hg3bpxx\n8uRJwzAMY/78+cZ//ud/WlZvz549xr59+4y0tDR/26FDh4xPPvnEmDVrllFTU3PRaulKpBv++Mc/\n4nK5GDhwIABRUVE9PKLAePnll0lJSQn4fPv164fNZgPg5MmT/r9bLSoqilGjRhEcHLgvLV6OW/c0\nNzfz/vvvk5mZCUBISAj9+/cP+Dh27dpFTEwMgwYNCki9jo4OWltbaW9vp7W1FbvdblmtMWPGEB4e\n3qlt6NChDBky5KLXUoh0Q21tLU1NTcyePZtp06ZRWloa0PrV1dVMnjyZuXPn8te//jUgNT0eD+Xl\n5dx1110Bqfd1b731Fqmpqdx///0sX768R8YQCF1t3ePxeAI6hnvvvZdp06axYcOGgNQ7evQokZGR\n5ObmkpGRwaOPPsqJEycCUvur3G53p+UeKzkcDu655x7GjRtHQkICoaGhJCQkBKS21RQi3dDR0cH+\n/ft56aWXWL16Nb/+9a85cuRIQGqPHDmSbdu28eqrrzJ79myys7MDUvfxxx9n4cKFXHFFz/wn4nK5\nePPNN3nhhRd47rnnemQMl4NXXnmFkpISfvOb37Bu3Tref/99y2u2t7dz4MAB7rrrLkpLS+nTp0/A\n7wW1tbWxbds2UlNTA1LP5/NRUVFBRUUFO3bs4OTJk2zZsiUgta2mEPkG69at8998s9vtJCQk0Ldv\nXyIjI/nxj3/MRx99FJDaX375Jf369QNg7NixtLe388UXX1hed9++fSxYsICkpCS2bt3KL3/5S8rL\nyy2p+/XaX/0kPmbMGOrq6gIy50BfAUDPb91zrlZUVBQul+tbNzC9GJxOJ06nk9GjRwOQmprKgQMH\nLK/7VZWVlYwcOZKrr746IPV27tzJ3/3d3xEZGcmVV15JcnJywL5MYLVe/Yt1K/30pz/lpz/9KXD2\nWzvLli2jvb2d06dPU1NTw5w5cwJS+9ixYxiGgc1mo6amhjNnzjBgwADL637V4sWLueOOO/jHf/xH\nS+p+vfZ///d/++e8f/9+2traAj7nQPnq1j0OhwO3283TTz8dkNonTpzgzJkzhIaGcuLECd59913m\nzZtned3o6GicTieHDx9myJAh7Nq1i6FDh1pe96vcbjdpaWkBqzdw4ED27t3LyZMnueqqq9i1axc3\n3nhjwOpbSSHSDUOHDuUnP/kJkydP5oorriAzM5Phw4cHpPbWrVt55ZVXCAoK4qqrruKZZ54J2I3m\nnrJ161a2bNlCcHAwV111Fc8++2xA5nzs2DGmT59OS0sLV1xxBWvXruX1118nNDTUspo9uXVPQ0OD\nf3m0o6ODSZMmkZiYGJDajz32GAsXLuT06dPExMTwq1/9KiB14Wx47ty5k2XLlgWs5ujRo0lJSWHq\n1KkEBwdz/fXXc+edd1pWb8GCBezZs4fGxkYSExP5+c9/TkREBPn5+XzxxRfcf//9XH/99Rflq+za\n9kREREzTPRERETFNISIiIqYpRERExDSFiIiImKYQERER0xQiIiJimkJELmlJSUmkpqYyefJkJk2a\nhNvtBuDjjz/m3nvvxeVyMX78eObPn89nn33mP2/x4sX8x3/8x3n9Pf/88zzxxBP+10eOHCE7O5vx\n48czbdo0Zs6cSXl5OaWlpcycOZMzZ874j33hhRfIycn5xrFu3ryZ+fPnn9e+e/duRo8e3WkL8ocf\nftj/3ogRIzpt+7F79+5v3ZZgENxBAAAFXElEQVT9+eefZ8SIEbzzzjv+ti+//JKbb76507k+n49H\nHnmE8ePHk5KSwsyZMzttrb9582ZGjBjB66+/ft48Dh486B/vHXfcwY9//GP/63Xr1nU537fffpvZ\ns2f7X69bt4709HQmT55Mampqp397+X7Qjw3lkrdq1SqGDx/OgQMHmDlzJnFxcWRlZfGLX/yCCRMm\nAGef1XDPPffw6quvcuWVV3arX6/Xy6xZs1i0aBEvvPACcPYHh++++y4ZGRls3bqV3/3ud9x7770c\nPHiQTZs2UVJSYmoOQ4cOZfPmzV2+Fx0dzdq1a5k5c+Z32s125MiRlJSUcMcddwDw5ptvcs0113Q6\n5oEHHmD48OGUlZURFBTEnj17+NnPfsaGDRsYPHgwAIMGDeK5554jOTm50w7GI0aM8O/vtHnzZt55\n5x1WrVrlf/+b5nNOTU0Na9eupbi4mP79+9PR0RGwDUQlcHQlIr3GDTfcQL9+/Vi/fj233HKLP0AA\n5syZQ2hoqP9KpTvWrVvHrbfeSkZGhr8tOjra/zo/P5+1a9fy8ccfk5ubyy9+8QtLtl+x2+2kpqby\nm9/85judd8stt3Dw4EF8Ph8ApaWlTJ061f/++++/z5EjR1i0aBFBQUH+c6ZPn85LL73kP+7GG2/k\nmmuuobi4+CLM5v/n8XgIDQ2lb9++AAQFBXHddddd1BrS8xQi0mu89957nDp1ij/96U/+zfu+avTo\n0Rw8eLDb/R04cIBRo0Z94/tXX301ixcv5q677uLaa6/9m/YO++STTzotZ339CZX/+q//SnFxMV6v\nt9t92mw2Jk6ciNvtpq6ujhMnTnTajufgwYOMHDnyvCuz2NjY8zYQffDBB3nxxRdpbW39TvPauXNn\np3kVFBT434uPjyc4OJhx48bx0EMPsWHDBk6ePPmd+pdLn5az5JI3f/58fvCDHxAaGsrzzz/PH/7w\nh4DVnjhxIk888QT33HPP39TPhZaz4Gxg/dM//RO//vWvO11hfZupU6eycOFCPv/8805XVADfZUej\nESNGMGbMGF5++eXv9BCy22+/vdMS19tvv81vf/tbAPr27cuGDRv44IMP+POf/8ymTZtYt24dxcXF\nhISEdLuGXNp0JSKXvFWrVrFlyxbWrVtHfHw8I0aMYO/evecdV1NTw4gRI7rd7w033NCtZ00HBQUF\n5Lkqc+fOpby8nE8//bTb55x7IuLGjRvPe8DSddddx/79+zl9+nSn9urq6i7/nR544AHWrFlDc3Oz\nuQl0wWazMWrUKLKysli/fj3/+7//q/si3zMKEel1Zs2axe7du3njjTf8bWvWrKGpqek7be999913\ns2vXLv74xz/62xoaGgL+5MpzwsLCyMrK4sUXX/xO5y1YsICFCxeed79mzJgxDB48mKeeeoqOjg7g\n7H2S4uJi7r///vP6iYmJISUlhbVr15qfxFd88sknfPzxx/7XR44c4fTp052e5Ci9n5azpNeJiIjg\nt7/9LU8++SRPP/00hmH4t7X+6vr/c8891+mrs/n5+Z36cTgcvPzyy6xYsYKVK1fSt29f+vbtyz//\n8z+bHtv27ds7bac+bdo04uLi/PdEzrHb7V3eSJ81axa///3vv1PNm2++mZtvvrnL91atWsUTTzyB\ny+XiyiuvJCIiglWrVvH3f//3XR4/b948099A+7rW1laWL19OQ0MDP/jBDwgKCuKpp576TstlcunT\nVvAiImKalrNERMQ0LWeJfAcNDQ1dflPL5XLxs5/97KLW+vDDD1m8ePF57bNmzWLGjBkXtZaIWVrO\nEhER07ScJSIipilERETENIWIiIiYphARERHT/i9jIXoki0OoYwAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x7fdb3f0dedd8>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "X_train['POLICY_LEN_MONTHS'] = X_train['POLICY_END_MONTH'] - X_train['POLICY_BEGIN_MONTH']\n",
    "print(len(X_train['POLICY_LEN_MONTHS']))\n",
    "sns.set(style=\"darkgrid\")\n",
    "ax = sns.countplot(x=X_train['POLICY_LEN_MONTHS'], data=X_train)\n",
    "lens = X_train['POLICY_LEN_MONTHS'].value_counts()\n",
    "print(lens)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Как видно из графика, практически всегда это ровно год, бывает, что на 11 месяцев и ещё редкие исключения. Поэтому признак <i>'POLICY_LEN_MONTHS'</i> имеет смысл перевести из класса числовых в категориальный. Вопрос в том, сколько должно быть категорий - мне кажется, что 3: ровно год, 11 месяцев и объединённый класс исключений."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0    75524\n",
       "2     1847\n",
       "1       36\n",
       "Name: POLICY_LEN_MONTHS, dtype: int64"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X_train['POLICY_LEN_MONTHS'] = [2 if (x == -1) else \n",
    "                                0 if (x == 0) else\n",
    "                                1 for x in X_train['POLICY_LEN_MONTHS'].values]\n",
    "X_train['POLICY_LEN_MONTHS'].value_counts()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Теперь вопрос состоит в том, что же делать дальше с этими двумя признаками, отвечающими за месяца. Ввиду появления <i>'POLICY_LEN_MONTHS'</i>, я считаю, что месяц заключения нам теперь не особо важен, к тому же он будет весьма сильно коррелировать с месяцем окончания, поэтому его выкинем. Однако месяц окончания сам по себе остаётся категориальным признаком. Чтобы не плодить из него потом кучу малоинформативных признаков(12), я считаю довольно логичным объединить месяца в группы по временам года. Чисто логически я слабо понимаю, как вообще месяц может повлиять на выбор человека, однако мало ли как погода может влиять на выбор людей, поэтому не буду совсем от них отказываться."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "summer    21079\n",
       "spring    20131\n",
       "autumn    18490\n",
       "winter    17707\n",
       "Name: POLICY_END_SEASON, dtype: int64"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X_train['POLICY_END_SEASON'] = ['summer' if (x >= 6) & (x <= 8)\n",
    "                                else 'autumn' if (x >= 9) & (x <= 11)\n",
    "                                else 'spring' if (x >= 3) & (x <= 5)\n",
    "                                else 'winter' for x in X_train['POLICY_END_MONTH'].values]\n",
    "X_train['POLICY_END_SEASON'].value_counts()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Теперь стоит обсудить <i>'POLICY_SALES_CHANNEL'</i>, <i>'POLICY_SALES_CHANNEL_GROUP'</i>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[39 50 52  2 10 53  1  6 55 17 54 20 23 59 15 40  8 41 14 11 62 60 46  5 22\n",
      " 31 51  3 25 27 16 45  4 13 18 26 44  7 29 28  9 47 49 38 35 12 57 21 63 19\n",
      " 43 24 30 34 32 33 61 42 36 48 37 58 56]\n",
      "[1 5 6 4 3 8 7 2]\n"
     ]
    }
   ],
   "source": [
    "print(X_train['POLICY_SALES_CHANNEL'].unique())\n",
    "print(X_train['POLICY_SALES_CHANNEL_GROUP'].unique())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Эти два признака снова являются категориальными. Ввиду того, что это baseline решение, каналов продаж довольно много, и у нас уже изначально имеется группировка каналов, то признак <i>'POLICY_SALES_CHANNEL'</i> предлагаю опустить, хотя он и может оказаться важным, в частности совсем не обязательно создавать все 63 признака, можно взять лишь самые популярные из них."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Теперь на очереди <i>'POLICY_BRANCH'</i>, он принимает всего 2 значения - Москва и Питер, т.е. он бинарный, просто переименуем его значения в 0 и 1 соответственно."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['Москва' 'Санкт-Петербург']\n"
     ]
    }
   ],
   "source": [
    "print(X_train['POLICY_BRANCH'].unique())\n",
    "X_train['POLICY_BRANCH'] = [0 if (x == 'Москва') else 1 for x in X_train['POLICY_BRANCH'].values]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<i>'POLICY_MIN_AGE'</i>, <i>'POLICY_MIN_DRIVING_EXPERIENCE'</i> - числовые признаки, с ними можно ничего не придумывать, разве что в заполнении <i>'POLICY_MIN_DRIVING_EXPERIENCE'</i> есть явные ошибки, когда вместо стажа вождения, введён год начала. Насколько я понимаю это данные 2016 года. При этом число логически возраст клиентов далжен очень сильно влиять на их желание продлять полис или нет. Скорее всего в алгоритмах, основанных на деревьях они будут иметь большое значение, однако регрессия не способна до конца на них настроиться ввиду того, что она монотонна, поэтому предлагаю наплодить из этих числовых признаков несколько категориальных"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[51 35 41 36 42 60 48 27 38 39 64 56 47 44 40 49 50 53 31 32 30 33 62 28 58\n",
      " 26 63 34 25 29 24 45 66 54 37 61 23 52 55 57 46 43 67 59 69 68 75 22 78 18\n",
      " 76 74 65 80 72 19 73 77 71 20 79 70 21 86 82 81 83 84]\n",
      "[  12    7    6    5   40   19   18   16    8   11   45   20   17   13   15\n",
      "   28   29    2   42    3   36    9   10   44   23   14   43   21   25   26\n",
      "    4   32   34    1   22   31   24   27   48   30    0   35 1986   33   37\n",
      " 1988   52   41 2008 2002   39 1996 1995 1991   38 1979   47   46 1994 2014\n",
      " 1968 2004 1998 1999 1980   49   58 1984 1992 1990 1976 2000 1989 2006 1974\n",
      " 2003 1997 1993 1981 1973 2005 2010 2011   50 2013 2007   51   53 1987 1969\n",
      " 2009 2001   54 1982   60 1975 1985   55 1978 2015 1977 2012 1962 1983 1972\n",
      "   56   57 1970 1958 1963]\n"
     ]
    }
   ],
   "source": [
    "print(X_train['POLICY_MIN_AGE'].unique())\n",
    "print(X_train['POLICY_MIN_DRIVING_EXPERIENCE'].unique())\n",
    "X_train['POLICY_MIN_DRIVING_EXPERIENCE'] = [x if (x < 100) else (2016-x) for x in X_train['POLICY_MIN_DRIVING_EXPERIENCE'].values]\n",
    "X_train['AGE_70+'] = [1 if (x >= 70) else 0 for x in X_train['POLICY_MIN_AGE'].values]\n",
    "X_train['AGE_60+'] = [1 if (x >= 60) & (x < 70) else 0 for x in X_train['POLICY_MIN_AGE'].values]\n",
    "X_train['AGE_50+'] = [1 if (x >= 50) & (x < 60) else 0 for x in X_train['POLICY_MIN_AGE'].values]\n",
    "X_train['AGE_38+'] = [1 if (x >= 38) & (x < 50) else 0 for x in X_train['POLICY_MIN_AGE'].values]\n",
    "X_train['AGE_25+'] = [1 if (x >= 25) & (x < 38) else 0 for x in X_train['POLICY_MIN_AGE'].values]\n",
    "X_train['AGE_18+'] = [1 if (x < 25) else 0 for x in X_train['POLICY_MIN_AGE'].values]\n",
    "X_train['EXP_20+'] = [1 if (x >= 20) else 0 for x in X_train['POLICY_MIN_DRIVING_EXPERIENCE'].values]\n",
    "X_train['EXP_10+'] = [1 if (x >= 10) & (x < 20) else 0 for x in X_train['POLICY_MIN_DRIVING_EXPERIENCE'].values]\n",
    "X_train['EXP_5+'] = [1 if (x >= 5) & (x < 10) else 0 for x in X_train['POLICY_MIN_DRIVING_EXPERIENCE'].values]\n",
    "X_train['EXP_LOW'] = [1 if (x < 5) else 0 for x in X_train['POLICY_MIN_DRIVING_EXPERIENCE'].values]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Признаки <i>'VEHICLE_MAKE'</i>, <i>'VEHICLE_MODEL'</i> - снова категориальные, причём на самом деле это один и тот же признак. Подход к таким признакам довольно просто - использовать <b>DictVectorizer</b>, получить из этого довольно разряженную матрицу... и за счёт этого можно выиграть пару процентов. В простейшем случае нам важно знать лишь категорию машины(насколько она дорогая), что как правило коррелирует с её мощностью, но для чистоты эксперимента всё-таки попробуем это сделать."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 6.65 s, sys: 52 ms, total: 6.7 s\n",
      "Wall time: 6.71 s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "vehicle = ['VEHICLE_MAKE', 'VEHICLE_MODEL']\n",
    "dict_vect = DictVectorizer()\n",
    "X_train_vehicle = dict_vect.fit_transform(X_train[vehicle].fillna('-').T.to_dict().values())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<i>'VEHICLE_ENGINE_POWER'</i> - числовой признак, достаточно будет просто нормализовать, среднее вычитать не нужно.\n",
    "<br><i>'VEHICLE_IN_CREDIT'</i> - бинарный признак, ничего делать не надо\n",
    "<br><i>'VEHICLE_SUM_INSURED'</i> - признак, схожий с мощностью двигателя\n",
    "<br><i>'POLICY_INTERMEDIARY'</i> - категориальный признак с кучей категорий, может оказаться важным, но я предлагаю его пропустить, чтобы упростить обучение модели и не переобучиться.\n",
    "<br><i>'INSURER_GENDER'</i> - бинарный признак, который надо будет переделать LabelEncoderом\n",
    "<br><i>'POLICY_CLM_N'</i> - немного странный признак, видимо он категориальный \n",
    "<br><i>'POLICY_CLM_GLT_N'</i> - аналогично\n",
    "<br><i>'POLICY_PRV_CLM_N'</i> - аналогично\n",
    "<br><i>'POLICY_PRV_CLM_GLT_N'</i> - аналогично\n",
    "<br><i>'CLIENT_HAS_DAGO'</i> - бинарный\n",
    "<br><i>'CLIENT_HAS_OSAGO'</i> - бинарный\n",
    "<br><i>'POLICY_COURT_SIGN'</i> - бинарный\n",
    "<br><i>'CLAIM_AVG_ACC_ST_PRD'</i> - числовой\n",
    "<br><i>'POLICY_HAS_COMPLAINTS'</i> - бинарный\n",
    "<br><i>'POLICY_YEARS_RENEWED_N'</i> - числовой, есть пропуски\n",
    "<br><i>'POLICY_DEDUCT_VALUE'</i> - числовой\n",
    "<br><i>'CLIENT_REGISTRATION_REGION'</i> - похож на модели машин\n",
    "<br><i>'POLICY_PRICE_CHANGE'</i> - числовой"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2\n",
      "0      54481\n",
      "1S      8073\n",
      "1L      8000\n",
      "2       4972\n",
      "3       1377\n",
      "4+       458\n",
      "n/d       46\n",
      "Name: POLICY_CLM_N, dtype: int64\n",
      "CPU times: user 6.68 s, sys: 32 ms, total: 6.71 s\n",
      "Wall time: 6.71 s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "print(X_train['INSURER_GENDER'].nunique())\n",
    "print(X_train['POLICY_CLM_N'].value_counts())\n",
    "X_train['POLICY_YEARS_RENEWED_N'] = [1 if x=='N' else x for x in X_train['POLICY_YEARS_RENEWED_N'].values]\n",
    "region = ['POLICY_INTERMEDIARY', 'CLIENT_REGISTRATION_REGION']\n",
    "reg_vect = DictVectorizer()\n",
    "X_train_region = reg_vect.fit_transform(X_train[region].fillna('-').T.to_dict().values())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Теперь надо сгруппировать признаки одного порядка и преобразовать их"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "standard = StandardScaler()\n",
    "maxabs = MaxAbsScaler()\n",
    "label = LabelEncoder()\n",
    "onehot = OneHotEncoder()\n",
    "season = LabelBinarizer()\n",
    "labelbin = LabelBinarizer()\n",
    "absfeature = ['VEHICLE_ENGINE_POWER', 'VEHICLE_SUM_INSURED']\n",
    "scalefeature = ['POLICY_MIN_AGE', 'POLICY_MIN_DRIVING_EXPERIENCE', 'CLAIM_AVG_ACC_ST_PRD', 'POLICY_YEARS_RENEWED_N', \n",
    "                'POLICY_DEDUCT_VALUE', 'POLICY_PRICE_CHANGE']\n",
    "binaryfeature = ['POLICY_BRANCH', 'AGE_70+', 'AGE_60+', 'AGE_50+', 'AGE_38+', 'AGE_25+', 'AGE_18+',\n",
    "                 'EXP_20+', 'EXP_10+', 'EXP_5+', 'EXP_LOW', 'VEHICLE_IN_CREDIT', 'CLIENT_HAS_DAGO', 'CLIENT_HAS_OSAGO',\n",
    "                 'POLICY_COURT_SIGN', 'POLICY_HAS_COMPLAINTS', 'INSURER_GENDER']\n",
    "onehotfeature = ['POLICY_LEN_MONTHS', 'POLICY_SALES_CHANNEL', 'POLICY_SALES_CHANNEL_GROUP']\n",
    "#labelfeature = ['POLICY_END_SEASON', 'POLICY_CLM_N', 'POLICY_CLM_GLT_N', 'POLICY_PRV_CLM_N', 'POLICY_PRV_CLM_GLT_N']\n",
    "X_train_abs = maxabs.fit_transform(X_train[absfeature])\n",
    "X_train_scaled = standard.fit_transform(X_train[scalefeature])\n",
    "X_train['INSURER_GENDER'] = label.fit_transform(X_train['INSURER_GENDER'])\n",
    "X_train_bool = X_train[binaryfeature]\n",
    "X_train_one_hot = onehot.fit_transform(X_train[onehotfeature])\n",
    "X_train_season = season.fit_transform(X_train['POLICY_END_SEASON'])\n",
    "X_train_N = labelbin.fit_transform(X_train['POLICY_CLM_N'])\n",
    "X_train_GLT_N = labelbin.transform(X_train['POLICY_CLM_GLT_N'])\n",
    "X_train_PRV_N = labelbin.transform(X_train['POLICY_PRV_CLM_N'])\n",
    "X_train_PRV_GLT_N = labelbin.transform(X_train['POLICY_PRV_CLM_GLT_N'])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Теперь соединим все признаки воедино и разобьём тренировочную выборку на тренировочную и валидационную"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "X_train_new = scipy.sparse.hstack([X_train_abs, X_train_scaled, X_train_bool, X_train_season, X_train_N,\n",
    "                                   X_train_one_hot, X_train_GLT_N, X_train_PRV_N, X_train_PRV_GLT_N,\n",
    "                                   X_train_region, X_train_vehicle])\n",
    "X_train_2, X_valid, y_train_2, y_valid = train_test_split(X_train_new, y_train,  test_size = 0.2, random_state = 1)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "После разделения выборки на тренировочную и валидационную части, можно начинать обучение моделей. Это будет проходить примерно по такому сценарию: берётся произвольный алгоритм, при помощи кросс валидации подбираем нужные гиперпараметры. А затем на валидационной выборке сравниваем качество алгоритмов и строим некую модель второго порядка, например, взвешенную сумму."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Также стоит обсудить и метрику качества. В условия задачи мы хотим максимизировать TP+TN. Данное число прямо пропорционально метрике <i>accuracy</i>, которая представляется как (TP+TN)/TP+TN+FP+FN, поскольку знаменатель зависит только количества элементов в выборке, про которую нам необходимо получить резуьльтат."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "skf = StratifiedKFold(n_splits = 5, random_state = 1, shuffle = True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Поскольку это задача классификации, то первым алгоритмом будет обычная логистическая регрессия. Во многом из-за неё пришлось столько времени потратить на признаки, ведь она не умеет работать с категориальными признаками - только числовыми или бинарными."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 17%|█▋        | 1/6 [00:01<00:08,  1.76s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.001 0.663189342372\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      " 33%|███▎      | 2/6 [00:05<00:10,  2.52s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.01 0.673992686664\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      " 50%|█████     | 3/6 [00:12<00:12,  4.04s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.1 0.676059735818\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      " 67%|██████▋   | 4/6 [00:27<00:13,  6.94s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1.0 0.673992751853\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      " 83%|████████▎ | 5/6 [01:02<00:12, 12.58s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "10.0 0.670892210067\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "100%|██████████| 6/6 [02:16<00:00, 22.83s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "100.0 0.669697220031\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "C_space = np.logspace(-3, 2, 6)\n",
    "for c in tqdm(C_space):\n",
    "    lr = LogisticRegression(C = c, random_state = 1)\n",
    "    print(c, cross_val_score(lr, X_train_2, y_train_2, scoring='accuracy', cv=skf).mean())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Как видно из обучения, оптимальный парамерт С для нашей первой логистической регресии равен 0.1. Однако возможно на неполном множестве признаков результат окажется лучше? Попробуем выкинуть некоторые группы"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.668502194792\n"
     ]
    }
   ],
   "source": [
    "X_train_new = scipy.sparse.hstack([X_train_abs, X_train_scaled, X_train_bool, X_train_season, X_train_N,\n",
    "                                   X_train_one_hot, X_train_GLT_N, X_train_PRV_N, X_train_PRV_GLT_N,\n",
    "                                   X_train_region])\n",
    "X_train_2, X_valid, y_train_2, y_valid = train_test_split(X_train_new, y_train,  test_size = 0.2, random_state = 1)\n",
    "lr = LogisticRegression(C = c, random_state = 1)\n",
    "print(cross_val_score(lr, X_train_2, y_train_2, scoring='accuracy', cv=skf).mean())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.672135597562\n"
     ]
    }
   ],
   "source": [
    "X_train_new = scipy.sparse.hstack([X_train_abs, X_train_scaled, X_train_bool, X_train_season, X_train_N,\n",
    "                                   X_train_one_hot, X_train_GLT_N, X_train_PRV_N, X_train_PRV_GLT_N,\n",
    "                                   X_train_vehicle])\n",
    "X_train_2, X_valid, y_train_2, y_valid = train_test_split(X_train_new, y_train,  test_size = 0.2, random_state = 1)\n",
    "lr = LogisticRegression(C = 0.1, random_state = 1)\n",
    "print(cross_val_score(lr, X_train_2, y_train_2, scoring='accuracy', cv=skf).mean())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.662107473142\n"
     ]
    }
   ],
   "source": [
    "X_train_new = scipy.sparse.hstack([X_train_abs, X_train_scaled, X_train_bool, X_train_season,\n",
    "                                   X_train_one_hot,\n",
    "                                   X_train_region, X_train_vehicle])\n",
    "X_train_2, X_valid, y_train_2, y_valid = train_test_split(X_train_new, y_train,  test_size = 0.2, random_state = 1)\n",
    "lr = LogisticRegression(C = 0.1, random_state = 1)\n",
    "print(cross_val_score(lr, X_train_2, y_train_2, scoring='accuracy', cv=skf).mean())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.674897012928\n"
     ]
    }
   ],
   "source": [
    "X_train_new = scipy.sparse.hstack([X_train_abs, X_train_scaled, X_train_bool, X_train_season, X_train_N,\n",
    "                                   X_train_GLT_N, X_train_PRV_N, X_train_PRV_GLT_N,\n",
    "                                   X_train_region, X_train_vehicle])\n",
    "X_train_2, X_valid, y_train_2, y_valid = train_test_split(X_train_new, y_train,  test_size = 0.2, random_state = 1)\n",
    "lr = LogisticRegression(C = 0.1, random_state = 1)\n",
    "print(cross_val_score(lr, X_train_2, y_train_2, scoring='accuracy', cv=skf).mean())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.673330613675\n"
     ]
    }
   ],
   "source": [
    "X_train_new = scipy.sparse.hstack([X_train_abs, X_train_scaled, X_train_season, X_train_N,\n",
    "                                   X_train_one_hot, X_train_GLT_N, X_train_PRV_N, X_train_PRV_GLT_N,\n",
    "                                   X_train_region, X_train_vehicle])\n",
    "X_train_2, X_valid, y_train_2, y_valid = train_test_split(X_train_new, y_train,  test_size = 0.2, random_state = 1)\n",
    "C_space = np.logspace(-3, 2, 6)\n",
    "lr = LogisticRegression(C = 0.1, random_state = 1)\n",
    "print(cross_val_score(lr, X_train_2, y_train_2, scoring='accuracy', cv=skf).mean())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.675995119382\n"
     ]
    }
   ],
   "source": [
    "X_train_new = scipy.sparse.hstack([X_train_abs, X_train_scaled, X_train_bool, X_train_N,\n",
    "                                   X_train_one_hot, X_train_GLT_N, X_train_PRV_N, X_train_PRV_GLT_N,\n",
    "                                   X_train_region, X_train_vehicle])\n",
    "X_train_2, X_valid, y_train_2, y_valid = train_test_split(X_train_new, y_train,  test_size = 0.2, random_state = 1)\n",
    "lr = LogisticRegression(C = 0.1, random_state = 1)\n",
    "print(cross_val_score(lr, X_train_2, y_train_2, scoring='accuracy', cv=skf).mean())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.670294693537\n"
     ]
    }
   ],
   "source": [
    "X_train_new = scipy.sparse.hstack([X_train_abs, X_train_bool, X_train_N,  X_train_season,\n",
    "                                   X_train_one_hot, X_train_GLT_N, X_train_PRV_N, X_train_PRV_GLT_N,\n",
    "                                   X_train_region, X_train_vehicle])\n",
    "X_train_2, X_valid, y_train_2, y_valid = train_test_split(X_train_new, y_train,  test_size = 0.2, random_state = 1)\n",
    "lr = LogisticRegression(C = 0.1, random_state = 1)\n",
    "print(cross_val_score(lr, X_train_2, y_train_2, scoring='accuracy', cv=skf).mean())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.676205050755\n"
     ]
    }
   ],
   "source": [
    "X_train_new = scipy.sparse.hstack([X_train_scaled, X_train_bool, X_train_N, X_train_season,\n",
    "                                   X_train_one_hot, X_train_GLT_N, X_train_PRV_N, X_train_PRV_GLT_N,\n",
    "                                   X_train_region, X_train_vehicle])\n",
    "X_train_2, X_valid, y_train_2, y_valid = train_test_split(X_train_new, y_train,  test_size = 0.2, random_state = 1)\n",
    "lr = LogisticRegression(C = 0.1, random_state = 1)\n",
    "print(cross_val_score(lr, X_train_2, y_train_2, scoring='accuracy', cv=skf).mean())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Как видно из нашего исследования признак season оказался и правда не слишком полезным, а единственная модель, которая показал результаты лучше, чем исходная - без признаков, которые мы нормализовали <i>MaxAbsScalerом</i>."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Вторым алгоритмом, который мы собираемся использовать будет KNN. Маловероятно, что он отработает лучше всех, но тем не менее он может быть полезным. Будем пользоваться им на исходном мн-ве признаков(за исключением категорильных с кучей категорий). У метода ближайших соседей достаточно много параметров, поэтому нам понадобится <i>GridSearchCV</i>\n",
    "<p style=\"color:#148814\">update</p>\n",
    "Учить KNN на всей выборке было плохой идеей. Даже учитывая то, что я убрал все категориальные признаки. Поэтому придётся постпуить по-другому. Сначала найдём нужный размер выборки и на нём гиперпараметры, чтобы время обучения было терпимым, а потом воспользуемся беггингом для охвата большего числа элементов."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [],
   "source": [
    "X_train_new = scipy.sparse.hstack([X_train_abs, X_train_scaled, X_train_bool])\n",
    "X_train_2, X_valid, y_train_2, y_valid = train_test_split(X_train_new, y_train,  test_size = 0.2, random_state = 1)\n",
    "X_train_2_short = X_train_2[:10000,:]\n",
    "y_train_2_short = y_train_2[:10000]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fitting 5 folds for each of 12 candidates, totalling 60 fits\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=1)]: Done  60 out of  60 | elapsed:  4.1min finished\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 3min 55s, sys: 11.7 s, total: 4min 7s\n",
      "Wall time: 4min 7s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "knn = KNeighborsClassifier()\n",
    "clf = GridSearchCV(estimator = knn, n_jobs = 1, cv = skf, return_train_score = True, verbose = 1,\n",
    "                   param_grid = {\"n_neighbors\": [1,3,5,10,20,50], \"weights\": ['uniform', 'distance']})\n",
    "clf.fit(X_train_2_short, y_train_2_short)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([ 0.5742,  0.5742,  0.5976,  0.5938,  0.6065,  0.6036,  0.6117,\n",
       "        0.6227,  0.6304,  0.6343,  0.638 ,  0.6408])"
      ]
     },
     "execution_count": 48,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "clf.cv_results_['mean_test_score']"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Отсюда можно сделать вывод, что оптимальных гиперпараметров мы не нашли, продолжим поиск"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 2min 44s, sys: 7.69 s, total: 2min 52s\n",
      "Wall time: 2min 52s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "clf = GridSearchCV(estimator = knn, n_jobs = 1, cv = skf, return_train_score = True, verbose = 0,\n",
    "                   param_grid = {\"n_neighbors\": [30,70,100,150], \"weights\": ['uniform', 'distance']})\n",
    "clf.fit(X_train_2_short, y_train_2_short)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([ 0.6345,  0.6406,  0.639 ,  0.6395,  0.6401,  0.6394,  0.636 ,\n",
       "        0.6385])"
      ]
     },
     "execution_count": 50,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "clf.cv_results_['mean_test_score']"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Как видно наилучший результат показал KNN с 50 соседями и весами 'distance'. Впоследствие и будем использовать эти параметры в нашм будущем bagging классификаторе. Однако перед этим стоит проверить текущую модель на большем подмн-ве признаков"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "knn = KNeighborsClassifier(n_neighbors = 50, weights = 'distance')\n",
    "X_train_new = scipy.sparse.hstack([X_train_abs, X_train_scaled, X_train_bool, X_train_season, X_train_N,\n",
    "                                   X_train_one_hot, X_train_GLT_N, X_train_PRV_N, X_train_PRV_GLT_N])\n",
    "X_train_2, X_valid, y_train_2, y_valid = train_test_split(X_train_new, y_train,  test_size = 0.2, random_state = 1)\n",
    "X_train_2_short = X_train_2[:10000,:]\n",
    "y_train_2_short = y_train_2[:10000]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.6489\n",
      "CPU times: user 4.95 s, sys: 184 ms, total: 5.14 s\n",
      "Wall time: 5.13 s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "print(cross_val_score(knn, X_train_2_short, y_train_2_short, scoring='accuracy', cv=skf).mean())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Точность выросла, видимо вся проблема была в размере выборки. поэтому можно смело пробовать полный набор признаков"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "X_train_new = scipy.sparse.hstack([X_train_abs, X_train_scaled, X_train_bool, X_train_season, X_train_N,\n",
    "                                   X_train_one_hot, X_train_GLT_N, X_train_PRV_N, X_train_PRV_GLT_N,\n",
    "                                   X_train_region, X_train_vehicle])\n",
    "X_train_2, X_valid, y_train_2, y_valid = train_test_split(X_train_new, y_train,  test_size = 0.2, random_state = 1)\n",
    "X_train_2_short = X_train_2[:10000,:]\n",
    "y_train_2_short = y_train_2[:10000]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.6514\n",
      "CPU times: user 4.88 s, sys: 296 ms, total: 5.17 s\n",
      "Wall time: 5.18 s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "print(cross_val_score(knn, X_train_2_short, y_train_2_short, scoring='accuracy', cv=skf).mean())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(61925, 2152)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=1)]: Done   1 out of   1 | elapsed:    0.1s finished\n",
      "[Parallel(n_jobs=1)]: Done   1 out of   1 | elapsed:  1.3min finished\n",
      "[Parallel(n_jobs=1)]: Done   1 out of   1 | elapsed:    0.1s finished\n",
      "[Parallel(n_jobs=1)]: Done   1 out of   1 | elapsed:  1.3min finished\n",
      "[Parallel(n_jobs=1)]: Done   1 out of   1 | elapsed:    0.1s finished\n",
      "[Parallel(n_jobs=1)]: Done   1 out of   1 | elapsed:  1.3min finished\n",
      "[Parallel(n_jobs=1)]: Done   1 out of   1 | elapsed:    0.1s finished\n",
      "[Parallel(n_jobs=1)]: Done   1 out of   1 | elapsed:  1.4min finished\n",
      "[Parallel(n_jobs=1)]: Done   1 out of   1 | elapsed:    0.1s finished\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.655712544188\n",
      "CPU times: user 6min 18s, sys: 30.5 s, total: 6min 48s\n",
      "Wall time: 6min 48s\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=1)]: Done   1 out of   1 | elapsed:  1.4min finished\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "print(X_train_2.shape)\n",
    "bg = BaggingClassifier(base_estimator = knn, max_samples = 10000, random_state = 1, verbose = 1)\n",
    "print(cross_val_score(bg, X_train_2, y_train_2, scoring='accuracy', cv=skf).mean())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Отлично, беггинг над соседями оказался полезен, поэтому запомним этот алгоритм и перейдём к следующему алгоритму - беггингу над решающими деревьями, он же является случайным лесом. Особенность этих алгоритмов заключается в том, что decision trees устойчивы к размеру выборки, т.е. их можно обучать на небольшои под множестве.\n",
    "<br>Что касается гиперпараметров случайного леса, то в качестве критерия разибения стоит выбрать gini(обоснование: \n",
    "<a href=\"https://www.quora.com/Machine-Learning/Are-gini-index-entropy-or-classification-error-measures-causing-any-difference-on-Decision-Tree-classification\">ответ с quora</a>)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.69416229752\n",
      "CPU times: user 8min 8s, sys: 584 ms, total: 8min 8s\n",
      "Wall time: 8min 8s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "rf = RandomForestClassifier(random_state = 1, n_estimators=100, max_depth=1000, oob_score=True, class_weight='balanced')\n",
    "print(cross_val_score(rf, X_train_2, y_train_2, scoring='accuracy', cv=skf).mean())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "ОБычный случайный лес без подбора гиперпараметров уже дал неплохой прирост в качестве предсказания, даже по сравнению с логистической регрессией. Соотвественно дальше есть 2 пути - тюнить RF или вернуться к LR и использовать на ней <i>BaggingClassifier<i>. Начну с RF"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fitting 5 folds for each of 20 candidates, totalling 100 fits\n",
      "[CV] min_samples_leaf=3, max_depth=30, max_features=2 ................\n",
      "[CV] min_samples_leaf=3, max_depth=30, max_features=2 ................\n",
      "[CV] . min_samples_leaf=3, max_depth=30, max_features=2, total=   1.9s\n",
      "[CV] min_samples_leaf=3, max_depth=30, max_features=2 ................\n",
      "[CV] . min_samples_leaf=3, max_depth=30, max_features=2, total=   2.0s\n",
      "[CV] min_samples_leaf=3, max_depth=30, max_features=2 ................\n",
      "[CV] . min_samples_leaf=3, max_depth=30, max_features=2, total=   1.9s\n",
      "[CV] min_samples_leaf=3, max_depth=30, max_features=2 ................\n",
      "[CV] . min_samples_leaf=3, max_depth=30, max_features=2, total=   1.9s\n",
      "[CV] min_samples_leaf=8, max_depth=800, max_features=100 .............\n",
      "[CV] . min_samples_leaf=3, max_depth=30, max_features=2, total=   1.9s\n",
      "[CV] min_samples_leaf=8, max_depth=800, max_features=100 .............\n",
      "[CV]  min_samples_leaf=8, max_depth=800, max_features=100, total=  32.4s\n",
      "[CV] min_samples_leaf=8, max_depth=800, max_features=100 .............\n",
      "[CV]  min_samples_leaf=8, max_depth=800, max_features=100, total=  32.4s\n",
      "[CV] min_samples_leaf=8, max_depth=800, max_features=100 .............\n",
      "[CV]  min_samples_leaf=8, max_depth=800, max_features=100, total=  32.8s\n",
      "[CV] min_samples_leaf=8, max_depth=800, max_features=100 .............\n",
      "[CV]  min_samples_leaf=8, max_depth=800, max_features=100, total=  32.8s\n",
      "[CV] min_samples_leaf=1, max_depth=30, max_features=15 ...............\n",
      "[CV]  min_samples_leaf=1, max_depth=30, max_features=15, total=  17.6s\n",
      "[CV] min_samples_leaf=1, max_depth=30, max_features=15 ...............\n",
      "[CV]  min_samples_leaf=8, max_depth=800, max_features=100, total=  32.5s\n",
      "[CV] min_samples_leaf=1, max_depth=30, max_features=15 ...............\n",
      "[CV]  min_samples_leaf=1, max_depth=30, max_features=15, total=  18.0s\n",
      "[CV] min_samples_leaf=1, max_depth=30, max_features=15 ...............\n",
      "[CV]  min_samples_leaf=1, max_depth=30, max_features=15, total=  17.6s\n",
      "[CV] min_samples_leaf=1, max_depth=30, max_features=15 ...............\n",
      "[CV]  min_samples_leaf=1, max_depth=30, max_features=15, total=  18.1s\n",
      "[CV] min_samples_leaf=8, max_depth=2000, max_features=5 ..............\n",
      "[CV]  min_samples_leaf=8, max_depth=2000, max_features=5, total=   2.4s\n",
      "[CV] min_samples_leaf=8, max_depth=2000, max_features=5 ..............\n",
      "[CV]  min_samples_leaf=8, max_depth=2000, max_features=5, total=   2.4s\n",
      "[CV] min_samples_leaf=8, max_depth=2000, max_features=5 ..............\n",
      "[CV]  min_samples_leaf=8, max_depth=2000, max_features=5, total=   2.3s\n",
      "[CV] min_samples_leaf=8, max_depth=2000, max_features=5 ..............\n",
      "[CV]  min_samples_leaf=8, max_depth=2000, max_features=5, total=   2.4s\n",
      "[CV] min_samples_leaf=8, max_depth=2000, max_features=5 ..............\n",
      "[CV]  min_samples_leaf=1, max_depth=30, max_features=15, total=  18.8s\n",
      "[CV] min_samples_leaf=5, max_depth=300, max_features=15 ..............\n",
      "[CV]  min_samples_leaf=8, max_depth=2000, max_features=5, total=   2.5s\n",
      "[CV] min_samples_leaf=5, max_depth=300, max_features=15 ..............\n",
      "[CV]  min_samples_leaf=5, max_depth=300, max_features=15, total=   8.0s\n",
      "[CV] min_samples_leaf=5, max_depth=300, max_features=15 ..............\n",
      "[CV]  min_samples_leaf=5, max_depth=300, max_features=15, total=   7.8s\n",
      "[CV] min_samples_leaf=5, max_depth=300, max_features=15 ..............\n",
      "[CV]  min_samples_leaf=5, max_depth=300, max_features=15, total=   7.9s\n",
      "[CV] min_samples_leaf=5, max_depth=300, max_features=15 ..............\n",
      "[CV]  min_samples_leaf=5, max_depth=300, max_features=15, total=   7.8s\n",
      "[CV] min_samples_leaf=1, max_depth=30, max_features=5 ................\n",
      "[CV]  min_samples_leaf=5, max_depth=300, max_features=15, total=   7.8s\n",
      "[CV] min_samples_leaf=1, max_depth=30, max_features=5 ................\n",
      "[CV] . min_samples_leaf=1, max_depth=30, max_features=5, total=   8.6s\n",
      "[CV] min_samples_leaf=1, max_depth=30, max_features=5 ................\n",
      "[CV] . min_samples_leaf=1, max_depth=30, max_features=5, total=   9.1s\n",
      "[CV] min_samples_leaf=1, max_depth=30, max_features=5 ................\n",
      "[CV] . min_samples_leaf=1, max_depth=30, max_features=5, total=   8.7s\n",
      "[CV] min_samples_leaf=1, max_depth=30, max_features=5 ................\n",
      "[CV] . min_samples_leaf=1, max_depth=30, max_features=5, total=   8.8s\n",
      "[CV] min_samples_leaf=5, max_depth=30, max_features=15 ...............\n",
      "[CV] . min_samples_leaf=1, max_depth=30, max_features=5, total=   9.1s\n",
      "[CV] min_samples_leaf=5, max_depth=30, max_features=15 ...............\n",
      "[CV]  min_samples_leaf=5, max_depth=30, max_features=15, total=   6.8s\n",
      "[CV] min_samples_leaf=5, max_depth=30, max_features=15 ...............\n",
      "[CV]  min_samples_leaf=5, max_depth=30, max_features=15, total=   7.1s\n",
      "[CV] min_samples_leaf=5, max_depth=30, max_features=15 ...............\n",
      "[CV]  min_samples_leaf=5, max_depth=30, max_features=15, total=   7.0s\n",
      "[CV] min_samples_leaf=5, max_depth=30, max_features=15 ...............\n",
      "[CV]  min_samples_leaf=5, max_depth=30, max_features=15, total=   6.9s\n",
      "[CV] min_samples_leaf=8, max_depth=100, max_features=40 ..............\n",
      "[CV]  min_samples_leaf=5, max_depth=30, max_features=15, total=   6.9s\n",
      "[CV] min_samples_leaf=8, max_depth=100, max_features=40 ..............\n",
      "[CV]  min_samples_leaf=8, max_depth=100, max_features=40, total=  13.6s\n",
      "[CV] min_samples_leaf=8, max_depth=100, max_features=40 ..............\n",
      "[CV]  min_samples_leaf=8, max_depth=100, max_features=40, total=  13.6s\n",
      "[CV] min_samples_leaf=8, max_depth=100, max_features=40 ..............\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=2)]: Done  37 tasks      | elapsed:  3.8min\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[CV]  min_samples_leaf=8, max_depth=100, max_features=40, total=  13.5s\n",
      "[CV] min_samples_leaf=8, max_depth=100, max_features=40 ..............\n",
      "[CV]  min_samples_leaf=8, max_depth=100, max_features=40, total=  13.6s\n",
      "[CV] min_samples_leaf=8, max_depth=100, max_features=15 ..............\n",
      "[CV]  min_samples_leaf=8, max_depth=100, max_features=15, total=   6.4s\n",
      "[CV] min_samples_leaf=8, max_depth=100, max_features=15 ..............\n",
      "[CV]  min_samples_leaf=8, max_depth=100, max_features=40, total=  13.6s\n",
      "[CV] min_samples_leaf=8, max_depth=100, max_features=15 ..............\n",
      "[CV]  min_samples_leaf=8, max_depth=100, max_features=15, total=   6.6s\n",
      "[CV] min_samples_leaf=8, max_depth=100, max_features=15 ..............\n",
      "[CV]  min_samples_leaf=8, max_depth=100, max_features=15, total=   6.5s\n",
      "[CV] min_samples_leaf=8, max_depth=100, max_features=15 ..............\n",
      "[CV]  min_samples_leaf=8, max_depth=100, max_features=15, total=   6.3s\n",
      "[CV] min_samples_leaf=5, max_depth=30, max_features=2 ................\n",
      "[CV]  min_samples_leaf=8, max_depth=100, max_features=15, total=   6.4s\n",
      "[CV] min_samples_leaf=5, max_depth=30, max_features=2 ................\n",
      "[CV] . min_samples_leaf=5, max_depth=30, max_features=2, total=   1.8s\n",
      "[CV] min_samples_leaf=5, max_depth=30, max_features=2 ................\n",
      "[CV] . min_samples_leaf=5, max_depth=30, max_features=2, total=   1.9s\n",
      "[CV] min_samples_leaf=5, max_depth=30, max_features=2 ................\n",
      "[CV] . min_samples_leaf=5, max_depth=30, max_features=2, total=   1.9s\n",
      "[CV] min_samples_leaf=5, max_depth=30, max_features=2 ................\n",
      "[CV] . min_samples_leaf=5, max_depth=30, max_features=2, total=   1.9s\n",
      "[CV] min_samples_leaf=3, max_depth=300, max_features=40 ..............\n",
      "[CV] . min_samples_leaf=5, max_depth=30, max_features=2, total=   1.9s\n",
      "[CV] min_samples_leaf=3, max_depth=300, max_features=40 ..............\n",
      "[CV]  min_samples_leaf=3, max_depth=300, max_features=40, total=  22.5s\n",
      "[CV] min_samples_leaf=3, max_depth=300, max_features=40 ..............\n",
      "[CV]  min_samples_leaf=3, max_depth=300, max_features=40, total=  22.7s\n",
      "[CV] min_samples_leaf=3, max_depth=300, max_features=40 ..............\n",
      "[CV]  min_samples_leaf=3, max_depth=300, max_features=40, total=  22.1s\n",
      "[CV] min_samples_leaf=3, max_depth=300, max_features=40 ..............\n",
      "[CV]  min_samples_leaf=3, max_depth=300, max_features=40, total=  22.7s\n",
      "[CV] min_samples_leaf=1, max_depth=100, max_features=100 .............\n",
      "[CV]  min_samples_leaf=3, max_depth=300, max_features=40, total=  22.5s\n",
      "[CV] min_samples_leaf=1, max_depth=100, max_features=100 .............\n",
      "[CV]  min_samples_leaf=1, max_depth=100, max_features=100, total= 1.9min\n",
      "[CV] min_samples_leaf=1, max_depth=100, max_features=100 .............\n",
      "[CV]  min_samples_leaf=1, max_depth=100, max_features=100, total= 2.0min\n",
      "[CV] min_samples_leaf=1, max_depth=100, max_features=100 .............\n",
      "[CV]  min_samples_leaf=1, max_depth=100, max_features=100, total= 1.9min\n",
      "[CV] min_samples_leaf=1, max_depth=100, max_features=100 .............\n",
      "[CV]  min_samples_leaf=1, max_depth=100, max_features=100, total= 2.0min\n",
      "[CV] min_samples_leaf=8, max_depth=300, max_features=100 .............\n",
      "[CV]  min_samples_leaf=8, max_depth=300, max_features=100, total=  33.7s\n",
      "[CV] min_samples_leaf=8, max_depth=300, max_features=100 .............\n",
      "[CV]  min_samples_leaf=8, max_depth=300, max_features=100, total=  32.9s\n",
      "[CV] min_samples_leaf=8, max_depth=300, max_features=100 .............\n",
      "[CV]  min_samples_leaf=1, max_depth=100, max_features=100, total= 1.9min\n",
      "[CV] min_samples_leaf=8, max_depth=300, max_features=100 .............\n",
      "[CV]  min_samples_leaf=8, max_depth=300, max_features=100, total=  33.1s\n",
      "[CV] min_samples_leaf=8, max_depth=300, max_features=100 .............\n",
      "[CV]  min_samples_leaf=8, max_depth=300, max_features=100, total=  33.1s\n",
      "[CV] min_samples_leaf=1, max_depth=30, max_features=100 ..............\n",
      "[CV]  min_samples_leaf=8, max_depth=300, max_features=100, total=  32.9s\n",
      "[CV] min_samples_leaf=1, max_depth=30, max_features=100 ..............\n",
      "[CV]  min_samples_leaf=1, max_depth=30, max_features=100, total= 1.2min\n",
      "[CV] min_samples_leaf=1, max_depth=30, max_features=100 ..............\n",
      "[CV]  min_samples_leaf=1, max_depth=30, max_features=100, total= 1.2min\n",
      "[CV] min_samples_leaf=1, max_depth=30, max_features=100 ..............\n",
      "[CV]  min_samples_leaf=1, max_depth=30, max_features=100, total= 1.2min\n",
      "[CV] min_samples_leaf=1, max_depth=30, max_features=100 ..............\n",
      "[CV]  min_samples_leaf=1, max_depth=30, max_features=100, total= 1.2min\n",
      "[CV] min_samples_leaf=3, max_depth=30, max_features=40 ...............\n",
      "[CV]  min_samples_leaf=3, max_depth=30, max_features=40, total=  17.0s\n",
      "[CV] min_samples_leaf=3, max_depth=30, max_features=40 ...............\n",
      "[CV]  min_samples_leaf=3, max_depth=30, max_features=40, total=  17.2s\n",
      "[CV] min_samples_leaf=3, max_depth=30, max_features=40 ...............\n",
      "[CV]  min_samples_leaf=3, max_depth=30, max_features=40, total=  17.1s\n",
      "[CV] min_samples_leaf=3, max_depth=30, max_features=40 ...............\n",
      "[CV]  min_samples_leaf=1, max_depth=30, max_features=100, total= 1.2min\n",
      "[CV] min_samples_leaf=3, max_depth=30, max_features=40 ...............\n",
      "[CV]  min_samples_leaf=3, max_depth=30, max_features=40, total=  17.1s\n",
      "[CV] min_samples_leaf=1, max_depth=100, max_features=15 ..............\n",
      "[CV]  min_samples_leaf=3, max_depth=30, max_features=40, total=  17.6s\n",
      "[CV] min_samples_leaf=1, max_depth=100, max_features=15 ..............\n",
      "[CV]  min_samples_leaf=1, max_depth=100, max_features=15, total= 1.6min\n",
      "[CV] min_samples_leaf=1, max_depth=100, max_features=15 ..............\n",
      "[CV]  min_samples_leaf=1, max_depth=100, max_features=15, total= 1.6min\n",
      "[CV] min_samples_leaf=1, max_depth=100, max_features=15 ..............\n",
      "[CV]  min_samples_leaf=1, max_depth=100, max_features=15, total= 1.6min\n",
      "[CV] min_samples_leaf=1, max_depth=100, max_features=15 ..............\n",
      "[CV]  min_samples_leaf=1, max_depth=100, max_features=15, total= 1.6min\n",
      "[CV] min_samples_leaf=8, max_depth=2000, max_features=40 .............\n",
      "[CV]  min_samples_leaf=8, max_depth=2000, max_features=40, total=  13.7s\n",
      "[CV] min_samples_leaf=8, max_depth=2000, max_features=40 .............\n",
      "[CV]  min_samples_leaf=8, max_depth=2000, max_features=40, total=  13.5s\n",
      "[CV] min_samples_leaf=8, max_depth=2000, max_features=40 .............\n",
      "[CV]  min_samples_leaf=8, max_depth=2000, max_features=40, total=  13.4s\n",
      "[CV] min_samples_leaf=8, max_depth=2000, max_features=40 .............\n",
      "[CV]  min_samples_leaf=8, max_depth=2000, max_features=40, total=  13.7s\n",
      "[CV] min_samples_leaf=8, max_depth=2000, max_features=40 .............\n",
      "[CV]  min_samples_leaf=8, max_depth=2000, max_features=40, total=  13.8s\n",
      "[CV] min_samples_leaf=3, max_depth=800, max_features=15 ..............\n",
      "[CV]  min_samples_leaf=3, max_depth=800, max_features=15, total=   9.8s\n",
      "[CV] min_samples_leaf=3, max_depth=800, max_features=15 ..............\n",
      "[CV]  min_samples_leaf=1, max_depth=100, max_features=15, total= 1.6min\n",
      "[CV] min_samples_leaf=3, max_depth=800, max_features=15 ..............\n",
      "[CV]  min_samples_leaf=3, max_depth=800, max_features=15, total=  10.1s\n",
      "[CV] min_samples_leaf=3, max_depth=800, max_features=15 ..............\n",
      "[CV]  min_samples_leaf=3, max_depth=800, max_features=15, total=  10.3s\n",
      "[CV] min_samples_leaf=3, max_depth=800, max_features=15 ..............\n",
      "[CV]  min_samples_leaf=3, max_depth=800, max_features=15, total=   9.7s\n",
      "[CV] min_samples_leaf=3, max_depth=2000, max_features=15 .............\n",
      "[CV]  min_samples_leaf=3, max_depth=800, max_features=15, total=   9.6s\n",
      "[CV] min_samples_leaf=3, max_depth=2000, max_features=15 .............\n",
      "[CV]  min_samples_leaf=3, max_depth=2000, max_features=15, total=   9.7s\n",
      "[CV] min_samples_leaf=3, max_depth=2000, max_features=15 .............\n",
      "[CV]  min_samples_leaf=3, max_depth=2000, max_features=15, total=  10.0s\n",
      "[CV] min_samples_leaf=3, max_depth=2000, max_features=15 .............\n",
      "[CV]  min_samples_leaf=3, max_depth=2000, max_features=15, total=  10.2s\n",
      "[CV] min_samples_leaf=3, max_depth=2000, max_features=15 .............\n",
      "[CV]  min_samples_leaf=3, max_depth=2000, max_features=15, total=   9.7s\n",
      "[CV] min_samples_leaf=8, max_depth=800, max_features=2 ...............\n",
      "[CV]  min_samples_leaf=8, max_depth=800, max_features=2, total=   1.8s\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[CV] min_samples_leaf=8, max_depth=800, max_features=2 ...............\n",
      "[CV]  min_samples_leaf=8, max_depth=800, max_features=2, total=   1.8s\n",
      "[CV] min_samples_leaf=8, max_depth=800, max_features=2 ...............\n",
      "[CV]  min_samples_leaf=8, max_depth=800, max_features=2, total=   1.8s\n",
      "[CV] min_samples_leaf=8, max_depth=800, max_features=2 ...............\n",
      "[CV]  min_samples_leaf=3, max_depth=2000, max_features=15, total=   9.7s\n",
      "[CV] min_samples_leaf=8, max_depth=800, max_features=2 ...............\n",
      "[CV]  min_samples_leaf=8, max_depth=800, max_features=2, total=   1.8s\n",
      "[CV]  min_samples_leaf=8, max_depth=800, max_features=2, total=   1.8s\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=2)]: Done 100 out of 100 | elapsed: 21.3min finished\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.704093661688 {'min_samples_leaf': 1, 'max_depth': 30, 'max_features': 100}\n",
      "CPU times: user 1min 44s, sys: 328 ms, total: 1min 44s\n",
      "Wall time: 23min\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "params = {\"max_depth\": [30, 100, 300, 800, 2000],\n",
    "          \"min_samples_leaf\": [1, 3, 5, 8],\n",
    "          \"max_features\": [2, 5, 15, 40, 100]}\n",
    "rs = RandomizedSearchCV(estimator = rf, n_jobs = 2, cv = skf, verbose = 2, param_distributions = params, \n",
    "                        scoring = 'accuracy', n_iter = 20)\n",
    "rs.fit(X_train_2, y_train_2)\n",
    "print(rs.best_score_, rs.best_params_)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fitting 5 folds for each of 20 candidates, totalling 100 fits\n",
      "[CV] max_samples=0.9, max_features=60 ................................\n",
      "[CV] max_samples=0.9, max_features=60 ................................\n",
      "[CV] ................. max_samples=0.9, max_features=60, total=  15.4s\n",
      "[CV] ................. max_samples=0.9, max_features=60, total=  15.3s\n",
      "[CV] max_samples=0.9, max_features=60 ................................\n",
      "[CV] max_samples=0.9, max_features=60 ................................\n",
      "[CV] ................. max_samples=0.9, max_features=60, total=   9.2s\n",
      "[CV] ................. max_samples=0.9, max_features=60, total=   9.3s\n",
      "[CV] max_samples=0.9, max_features=60 ................................\n",
      "[CV] max_samples=0.9, max_features=35 ................................\n",
      "[CV] ................. max_samples=0.9, max_features=35, total=   7.6s\n",
      "[CV] max_samples=0.9, max_features=35 ................................\n",
      "[CV] ................. max_samples=0.9, max_features=60, total=   9.1s\n",
      "[CV] max_samples=0.9, max_features=35 ................................\n",
      "[CV] ................. max_samples=0.9, max_features=35, total=   7.6s\n",
      "[CV] max_samples=0.9, max_features=35 ................................\n",
      "[CV] ................. max_samples=0.9, max_features=35, total=   7.5s\n",
      "[CV] max_samples=0.9, max_features=35 ................................\n",
      "[CV] ................. max_samples=0.9, max_features=35, total=   7.7s\n",
      "[CV] max_samples=0.5, max_features=100 ...............................\n",
      "[CV] ................. max_samples=0.9, max_features=35, total=   7.5s\n",
      "[CV] max_samples=0.5, max_features=100 ...............................\n",
      "[CV] ................ max_samples=0.5, max_features=100, total=  11.3s\n",
      "[CV] max_samples=0.5, max_features=100 ...............................\n",
      "[CV] ................ max_samples=0.5, max_features=100, total=  11.4s\n",
      "[CV] max_samples=0.5, max_features=100 ...............................\n",
      "[CV] ................ max_samples=0.5, max_features=100, total=  11.5s\n",
      "[CV] max_samples=0.5, max_features=100 ...............................\n",
      "[CV] ................ max_samples=0.5, max_features=100, total=  11.5s\n",
      "[CV] max_samples=0.5, max_features=6 .................................\n",
      "[CV] .................. max_samples=0.5, max_features=6, total=   5.3s\n",
      "[CV] max_samples=0.5, max_features=6 .................................\n",
      "[CV] ................ max_samples=0.5, max_features=100, total=  10.9s\n",
      "[CV] max_samples=0.5, max_features=6 .................................\n",
      "[CV] .................. max_samples=0.5, max_features=6, total=   5.3s\n",
      "[CV] max_samples=0.5, max_features=6 .................................\n",
      "[CV] .................. max_samples=0.5, max_features=6, total=   5.2s\n",
      "[CV] max_samples=0.5, max_features=6 .................................\n",
      "[CV] .................. max_samples=0.5, max_features=6, total=   5.1s\n",
      "[CV] max_samples=0.9, max_features=180 ...............................\n",
      "[CV] .................. max_samples=0.5, max_features=6, total=   5.2s\n",
      "[CV] max_samples=0.9, max_features=180 ...............................\n",
      "[CV] ................ max_samples=0.9, max_features=180, total=  18.3s\n",
      "[CV] max_samples=0.9, max_features=180 ...............................\n",
      "[CV] ................ max_samples=0.9, max_features=180, total=  18.1s\n",
      "[CV] max_samples=0.9, max_features=180 ...............................\n",
      "[CV] ................ max_samples=0.9, max_features=180, total=  18.7s\n",
      "[CV] max_samples=0.9, max_features=180 ...............................\n",
      "[CV] ................ max_samples=0.9, max_features=180, total=  18.1s\n",
      "[CV] max_samples=0.7, max_features=20 ................................\n",
      "[CV] ................. max_samples=0.7, max_features=20, total=   6.7s\n",
      "[CV] max_samples=0.7, max_features=20 ................................\n",
      "[CV] ................ max_samples=0.9, max_features=180, total=  17.2s\n",
      "[CV] max_samples=0.7, max_features=20 ................................\n",
      "[CV] ................. max_samples=0.7, max_features=20, total=   6.7s\n",
      "[CV] max_samples=0.7, max_features=20 ................................\n",
      "[CV] ................. max_samples=0.7, max_features=20, total=   6.3s\n",
      "[CV] max_samples=0.7, max_features=20 ................................\n",
      "[CV] ................. max_samples=0.7, max_features=20, total=   6.3s\n",
      "[CV] max_samples=0.9, max_features=100 ...............................\n",
      "[CV] ................. max_samples=0.7, max_features=20, total=   6.6s\n",
      "[CV] max_samples=0.9, max_features=100 ...............................\n",
      "[CV] ................ max_samples=0.9, max_features=100, total=  12.5s\n",
      "[CV] max_samples=0.9, max_features=100 ...............................\n",
      "[CV] ................ max_samples=0.9, max_features=100, total=  12.9s\n",
      "[CV] max_samples=0.9, max_features=100 ...............................\n",
      "[CV] ................ max_samples=0.9, max_features=100, total=  12.9s\n",
      "[CV] max_samples=0.9, max_features=100 ...............................\n",
      "[CV] ................ max_samples=0.9, max_features=100, total=  12.6s\n",
      "[CV] max_samples=0.5, max_features=180 ...............................\n",
      "[CV] ................ max_samples=0.9, max_features=100, total=  13.0s\n",
      "[CV] max_samples=0.5, max_features=180 ...............................\n",
      "[CV] ................ max_samples=0.5, max_features=180, total=  16.0s\n",
      "[CV] max_samples=0.5, max_features=180 ...............................\n",
      "[CV] ................ max_samples=0.5, max_features=180, total=  16.7s\n",
      "[CV] max_samples=0.5, max_features=180 ...............................\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=2)]: Done  37 tasks      | elapsed:  3.7min\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[CV] ................ max_samples=0.5, max_features=180, total=  16.5s\n",
      "[CV] max_samples=0.5, max_features=180 ...............................\n",
      "[CV] ................ max_samples=0.5, max_features=180, total=  16.5s\n",
      "[CV] max_samples=0.7, max_features=6 .................................\n",
      "[CV] .................. max_samples=0.7, max_features=6, total=   5.7s\n",
      "[CV] max_samples=0.7, max_features=6 .................................\n",
      "[CV] ................ max_samples=0.5, max_features=180, total=  15.4s\n",
      "[CV] max_samples=0.7, max_features=6 .................................\n",
      "[CV] .................. max_samples=0.7, max_features=6, total=   5.3s\n",
      "[CV] max_samples=0.7, max_features=6 .................................\n",
      "[CV] .................. max_samples=0.7, max_features=6, total=   5.2s\n",
      "[CV] max_samples=0.7, max_features=6 .................................\n",
      "[CV] .................. max_samples=0.7, max_features=6, total=   5.2s\n",
      "[CV] max_samples=0.9, max_features=3 .................................\n",
      "[CV] .................. max_samples=0.7, max_features=6, total=   5.2s\n",
      "[CV] max_samples=0.9, max_features=3 .................................\n",
      "[CV] .................. max_samples=0.9, max_features=3, total=   5.0s\n",
      "[CV] max_samples=0.9, max_features=3 .................................\n",
      "[CV] .................. max_samples=0.9, max_features=3, total=   5.0s\n",
      "[CV] max_samples=0.9, max_features=3 .................................\n",
      "[CV] .................. max_samples=0.9, max_features=3, total=   5.1s\n",
      "[CV] max_samples=0.9, max_features=3 .................................\n",
      "[CV] .................. max_samples=0.9, max_features=3, total=   4.9s\n",
      "[CV] max_samples=0.7, max_features=180 ...............................\n",
      "[CV] .................. max_samples=0.9, max_features=3, total=   5.1s\n",
      "[CV] max_samples=0.7, max_features=180 ...............................\n",
      "[CV] ................ max_samples=0.7, max_features=180, total=  17.3s\n",
      "[CV] max_samples=0.7, max_features=180 ...............................\n",
      "[CV] ................ max_samples=0.7, max_features=180, total=  17.4s\n",
      "[CV] max_samples=0.7, max_features=180 ...............................\n",
      "[CV] ................ max_samples=0.7, max_features=180, total=  18.2s\n",
      "[CV] max_samples=0.7, max_features=180 ...............................\n",
      "[CV] ................ max_samples=0.7, max_features=180, total=  17.9s\n",
      "[CV] max_samples=0.5, max_features=10 ................................\n",
      "[CV] ................. max_samples=0.5, max_features=10, total=   6.0s\n",
      "[CV] max_samples=0.5, max_features=10 ................................\n",
      "[CV] ................ max_samples=0.7, max_features=180, total=  16.3s\n",
      "[CV] max_samples=0.5, max_features=10 ................................\n",
      "[CV] ................. max_samples=0.5, max_features=10, total=   5.8s\n",
      "[CV] max_samples=0.5, max_features=10 ................................\n",
      "[CV] ................. max_samples=0.5, max_features=10, total=   5.4s\n",
      "[CV] max_samples=0.5, max_features=10 ................................\n",
      "[CV] ................. max_samples=0.5, max_features=10, total=   5.6s\n",
      "[CV] max_samples=0.7, max_features=100 ...............................\n",
      "[CV] ................. max_samples=0.5, max_features=10, total=   5.6s\n",
      "[CV] max_samples=0.7, max_features=100 ...............................\n",
      "[CV] ................ max_samples=0.7, max_features=100, total=  11.7s\n",
      "[CV] max_samples=0.7, max_features=100 ...............................\n",
      "[CV] ................ max_samples=0.7, max_features=100, total=  12.3s\n",
      "[CV] max_samples=0.7, max_features=100 ...............................\n",
      "[CV] ................ max_samples=0.7, max_features=100, total=  11.9s\n",
      "[CV] max_samples=0.7, max_features=100 ...............................\n",
      "[CV] ................ max_samples=0.7, max_features=100, total=  12.1s\n",
      "[CV] max_samples=0.7, max_features=35 ................................\n",
      "[CV] ................ max_samples=0.7, max_features=100, total=  11.9s\n",
      "[CV] max_samples=0.7, max_features=35 ................................\n",
      "[CV] ................. max_samples=0.7, max_features=35, total=   7.4s\n",
      "[CV] max_samples=0.7, max_features=35 ................................\n",
      "[CV] ................. max_samples=0.7, max_features=35, total=   7.3s\n",
      "[CV] max_samples=0.7, max_features=35 ................................\n",
      "[CV] ................. max_samples=0.7, max_features=35, total=   7.5s\n",
      "[CV] max_samples=0.7, max_features=35 ................................\n",
      "[CV] ................. max_samples=0.7, max_features=35, total=   7.4s\n",
      "[CV] max_samples=0.7, max_features=10 ................................\n",
      "[CV] ................. max_samples=0.7, max_features=35, total=   7.3s\n",
      "[CV] max_samples=0.7, max_features=10 ................................\n",
      "[CV] ................. max_samples=0.7, max_features=10, total=   5.6s\n",
      "[CV] max_samples=0.7, max_features=10 ................................\n",
      "[CV] ................. max_samples=0.7, max_features=10, total=   5.7s\n",
      "[CV] max_samples=0.7, max_features=10 ................................\n",
      "[CV] ................. max_samples=0.7, max_features=10, total=   5.5s\n",
      "[CV] max_samples=0.7, max_features=10 ................................\n",
      "[CV] ................. max_samples=0.7, max_features=10, total=   5.7s\n",
      "[CV] max_samples=0.5, max_features=300 ...............................\n",
      "[CV] ................. max_samples=0.7, max_features=10, total=   5.8s\n",
      "[CV] max_samples=0.5, max_features=300 ...............................\n",
      "[CV] ................ max_samples=0.5, max_features=300, total=  23.6s\n",
      "[CV] max_samples=0.5, max_features=300 ...............................\n",
      "[CV] ................ max_samples=0.5, max_features=300, total=  24.0s\n",
      "[CV] max_samples=0.5, max_features=300 ...............................\n",
      "[CV] ................ max_samples=0.5, max_features=300, total=  24.7s\n",
      "[CV] max_samples=0.5, max_features=300 ...............................\n",
      "[CV] ................ max_samples=0.5, max_features=300, total=  23.9s\n",
      "[CV] max_samples=0.7, max_features=3 .................................\n",
      "[CV] .................. max_samples=0.7, max_features=3, total=   5.4s\n",
      "[CV] max_samples=0.7, max_features=3 .................................\n",
      "[CV] .................. max_samples=0.7, max_features=3, total=   5.3s\n",
      "[CV] max_samples=0.7, max_features=3 .................................\n",
      "[CV] .................. max_samples=0.7, max_features=3, total=   5.2s\n",
      "[CV] max_samples=0.7, max_features=3 .................................\n",
      "[CV] ................ max_samples=0.5, max_features=300, total=  21.3s\n",
      "[CV] max_samples=0.7, max_features=3 .................................\n",
      "[CV] .................. max_samples=0.7, max_features=3, total=   5.1s\n",
      "[CV] max_samples=0.7, max_features=60 ................................\n",
      "[CV] .................. max_samples=0.7, max_features=3, total=   4.9s\n",
      "[CV] max_samples=0.7, max_features=60 ................................\n",
      "[CV] ................. max_samples=0.7, max_features=60, total=   9.0s\n",
      "[CV] max_samples=0.7, max_features=60 ................................\n",
      "[CV] ................. max_samples=0.7, max_features=60, total=   9.3s\n",
      "[CV] max_samples=0.7, max_features=60 ................................\n",
      "[CV] ................. max_samples=0.7, max_features=60, total=   9.0s\n",
      "[CV] max_samples=0.7, max_features=60 ................................\n",
      "[CV] ................. max_samples=0.7, max_features=60, total=   9.0s\n",
      "[CV] max_samples=0.5, max_features=3 .................................\n",
      "[CV] .................. max_samples=0.5, max_features=3, total=   5.1s\n",
      "[CV] max_samples=0.5, max_features=3 .................................\n",
      "[CV] ................. max_samples=0.7, max_features=60, total=   8.6s\n",
      "[CV] max_samples=0.5, max_features=3 .................................\n",
      "[CV] .................. max_samples=0.5, max_features=3, total=   5.0s\n",
      "[CV] max_samples=0.5, max_features=3 .................................\n",
      "[CV] .................. max_samples=0.5, max_features=3, total=   4.9s\n",
      "[CV] max_samples=0.5, max_features=3 .................................\n",
      "[CV] .................. max_samples=0.5, max_features=3, total=   4.8s\n",
      "[CV] max_samples=0.5, max_features=60 ................................\n",
      "[CV] .................. max_samples=0.5, max_features=3, total=   5.0s\n",
      "[CV] max_samples=0.5, max_features=60 ................................\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[CV] ................. max_samples=0.5, max_features=60, total=   8.4s\n",
      "[CV] max_samples=0.5, max_features=60 ................................\n",
      "[CV] ................. max_samples=0.5, max_features=60, total=   8.6s\n",
      "[CV] max_samples=0.5, max_features=60 ................................\n",
      "[CV] ................. max_samples=0.5, max_features=60, total=   8.5s\n",
      "[CV] max_samples=0.5, max_features=60 ................................\n",
      "[CV] ................. max_samples=0.5, max_features=60, total=   8.5s\n",
      "[CV] ................. max_samples=0.5, max_features=60, total=   7.9s\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=2)]: Done 100 out of 100 | elapsed:  9.1min finished\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.638110617683 {'max_samples': 0.5, 'max_features': 300}\n",
      "CPU times: user 29.1 s, sys: 320 ms, total: 29.4 s\n",
      "Wall time: 9min 33s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "lr = LogisticRegression(C = 0.1, random_state = 1)\n",
    "bg_lr = BaggingClassifier(base_estimator = lr, n_estimators = 100, random_state = 1, n_jobs=1)\n",
    "params = {'max_features': [3,6,10,20,35,60,100,180,300], 'max_samples': [0.5, 0.7, 0.9]}\n",
    "rs_lr = RandomizedSearchCV(estimator = bg_lr, n_jobs = 2, cv = skf, verbose = 2, param_distributions = params, \n",
    "                           scoring = 'accuracy', n_iter = 20, random_state=1)\n",
    "rs_lr.fit(X_train_2, y_train_2)\n",
    "print(rs_lr.best_score_, rs_lr.best_params_)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Как видно, случайный лес удалось ещё улучшить, а вот логистическая регрессия лучше всего отработала на полном мн-ве признаков и самплов"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.673072205316\n",
      "CPU times: user 1.04 s, sys: 248 ms, total: 1.29 s\n",
      "Wall time: 33.9 s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "lr = LogisticRegression(C = 0.1, random_state = 1)\n",
    "bg_lr = BaggingClassifier(base_estimator = lr, n_estimators = 10, random_state = 1, n_jobs=2, max_features=0.7)\n",
    "print(cross_val_score(bg_lr, X_train_2, y_train_2, scoring='accuracy', cv=skf).mean())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Время последнего классификатора для данной задачи - Градиентного бустинга."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 65,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fitting 5 folds for each of 50 candidates, totalling 250 fits\n",
      "[CV] n_estimators=200, min_samples_leaf=1, max_depth=10, max_features=160 \n",
      "[CV] n_estimators=200, min_samples_leaf=1, max_depth=10, max_features=160 \n",
      "[CV]  n_estimators=200, min_samples_leaf=1, max_depth=10, max_features=160, total=  30.9s\n",
      "[CV] n_estimators=200, min_samples_leaf=1, max_depth=10, max_features=160 \n",
      "[CV]  n_estimators=200, min_samples_leaf=1, max_depth=10, max_features=160, total=  31.3s\n",
      "[CV] n_estimators=200, min_samples_leaf=1, max_depth=10, max_features=160 \n",
      "[CV]  n_estimators=200, min_samples_leaf=1, max_depth=10, max_features=160, total=  28.5s\n",
      "[CV] n_estimators=200, min_samples_leaf=1, max_depth=10, max_features=160 \n",
      "[CV]  n_estimators=200, min_samples_leaf=1, max_depth=10, max_features=160, total=  28.5s\n",
      "[CV] n_estimators=200, min_samples_leaf=3, max_depth=10, max_features=20 \n",
      "[CV]  n_estimators=200, min_samples_leaf=3, max_depth=10, max_features=20, total=  10.6s\n",
      "[CV] n_estimators=200, min_samples_leaf=3, max_depth=10, max_features=20 \n",
      "[CV]  n_estimators=200, min_samples_leaf=3, max_depth=10, max_features=20, total=  11.1s\n",
      "[CV] n_estimators=200, min_samples_leaf=3, max_depth=10, max_features=20 \n",
      "[CV]  n_estimators=200, min_samples_leaf=1, max_depth=10, max_features=160, total=  28.1s\n",
      "[CV] n_estimators=200, min_samples_leaf=3, max_depth=10, max_features=20 \n",
      "[CV]  n_estimators=200, min_samples_leaf=3, max_depth=10, max_features=20, total=  11.1s\n",
      "[CV] n_estimators=200, min_samples_leaf=3, max_depth=10, max_features=20 \n",
      "[CV]  n_estimators=200, min_samples_leaf=3, max_depth=10, max_features=20, total=  11.0s\n",
      "[CV] n_estimators=1500, min_samples_leaf=3, max_depth=6, max_features=10 \n",
      "[CV]  n_estimators=200, min_samples_leaf=3, max_depth=10, max_features=20, total=  10.9s\n",
      "[CV] n_estimators=1500, min_samples_leaf=3, max_depth=6, max_features=10 \n",
      "[CV]  n_estimators=1500, min_samples_leaf=3, max_depth=6, max_features=10, total=  37.3s\n",
      "[CV] n_estimators=1500, min_samples_leaf=3, max_depth=6, max_features=10 \n",
      "[CV]  n_estimators=1500, min_samples_leaf=3, max_depth=6, max_features=10, total=  38.8s\n",
      "[CV] n_estimators=1500, min_samples_leaf=3, max_depth=6, max_features=10 \n",
      "[CV]  n_estimators=1500, min_samples_leaf=3, max_depth=6, max_features=10, total=  38.8s\n",
      "[CV] n_estimators=1500, min_samples_leaf=3, max_depth=6, max_features=10 \n",
      "[CV]  n_estimators=1500, min_samples_leaf=3, max_depth=6, max_features=10, total=  38.8s\n",
      "[CV] n_estimators=100, min_samples_leaf=1, max_depth=10, max_features=20 \n",
      "[CV]  n_estimators=100, min_samples_leaf=1, max_depth=10, max_features=20, total=   7.0s\n",
      "[CV] n_estimators=100, min_samples_leaf=1, max_depth=10, max_features=20 \n",
      "[CV]  n_estimators=100, min_samples_leaf=1, max_depth=10, max_features=20, total=   7.2s\n",
      "[CV] n_estimators=100, min_samples_leaf=1, max_depth=10, max_features=20 \n",
      "[CV]  n_estimators=100, min_samples_leaf=1, max_depth=10, max_features=20, total=   7.0s\n",
      "[CV] n_estimators=100, min_samples_leaf=1, max_depth=10, max_features=20 \n",
      "[CV]  n_estimators=100, min_samples_leaf=1, max_depth=10, max_features=20, total=   7.0s\n",
      "[CV] n_estimators=100, min_samples_leaf=1, max_depth=10, max_features=20 \n",
      "[CV]  n_estimators=1500, min_samples_leaf=3, max_depth=6, max_features=10, total=  38.1s\n",
      "[CV] n_estimators=1500, min_samples_leaf=8, max_depth=6, max_features=80 \n",
      "[CV]  n_estimators=100, min_samples_leaf=1, max_depth=10, max_features=20, total=   7.1s\n",
      "[CV] n_estimators=1500, min_samples_leaf=8, max_depth=6, max_features=80 \n",
      "[CV]  n_estimators=1500, min_samples_leaf=8, max_depth=6, max_features=80, total= 1.3min\n",
      "[CV] n_estimators=1500, min_samples_leaf=8, max_depth=6, max_features=80 \n",
      "[CV]  n_estimators=1500, min_samples_leaf=8, max_depth=6, max_features=80, total= 1.3min\n",
      "[CV] n_estimators=1500, min_samples_leaf=8, max_depth=6, max_features=80 \n",
      "[CV]  n_estimators=1500, min_samples_leaf=8, max_depth=6, max_features=80, total= 1.3min\n",
      "[CV] n_estimators=1500, min_samples_leaf=8, max_depth=6, max_features=80 \n",
      "[CV]  n_estimators=1500, min_samples_leaf=8, max_depth=6, max_features=80, total= 1.3min\n",
      "[CV] n_estimators=700, min_samples_leaf=1, max_depth=8, max_features=5 \n",
      "[CV]  n_estimators=700, min_samples_leaf=1, max_depth=8, max_features=5, total=  20.0s\n",
      "[CV] n_estimators=700, min_samples_leaf=1, max_depth=8, max_features=5 \n",
      "[CV]  n_estimators=700, min_samples_leaf=1, max_depth=8, max_features=5, total=  20.7s\n",
      "[CV] n_estimators=700, min_samples_leaf=1, max_depth=8, max_features=5 \n",
      "[CV]  n_estimators=700, min_samples_leaf=1, max_depth=8, max_features=5, total=  21.6s\n",
      "[CV] n_estimators=700, min_samples_leaf=1, max_depth=8, max_features=5 \n",
      "[CV]  n_estimators=1500, min_samples_leaf=8, max_depth=6, max_features=80, total= 1.4min\n",
      "[CV] n_estimators=700, min_samples_leaf=1, max_depth=8, max_features=5 \n",
      "[CV]  n_estimators=700, min_samples_leaf=1, max_depth=8, max_features=5, total=  20.5s\n",
      "[CV] n_estimators=1500, min_samples_leaf=5, max_depth=4, max_features=160 \n",
      "[CV]  n_estimators=700, min_samples_leaf=1, max_depth=8, max_features=5, total=  21.2s\n",
      "[CV] n_estimators=1500, min_samples_leaf=5, max_depth=4, max_features=160 \n",
      "[CV]  n_estimators=1500, min_samples_leaf=5, max_depth=4, max_features=160, total= 1.2min\n",
      "[CV] n_estimators=1500, min_samples_leaf=5, max_depth=4, max_features=160 \n",
      "[CV]  n_estimators=1500, min_samples_leaf=5, max_depth=4, max_features=160, total= 1.2min\n",
      "[CV] n_estimators=1500, min_samples_leaf=5, max_depth=4, max_features=160 \n",
      "[CV]  n_estimators=1500, min_samples_leaf=5, max_depth=4, max_features=160, total= 1.2min\n",
      "[CV] n_estimators=1500, min_samples_leaf=5, max_depth=4, max_features=160 \n",
      "[CV]  n_estimators=1500, min_samples_leaf=5, max_depth=4, max_features=160, total= 1.2min\n",
      "[CV] n_estimators=1500, min_samples_leaf=3, max_depth=8, max_features=80 \n",
      "[CV]  n_estimators=1500, min_samples_leaf=5, max_depth=4, max_features=160, total= 1.2min\n",
      "[CV] n_estimators=1500, min_samples_leaf=3, max_depth=8, max_features=80 \n",
      "[CV]  n_estimators=1500, min_samples_leaf=3, max_depth=8, max_features=80, total= 1.8min\n",
      "[CV] n_estimators=1500, min_samples_leaf=3, max_depth=8, max_features=80 \n",
      "[CV]  n_estimators=1500, min_samples_leaf=3, max_depth=8, max_features=80, total= 1.8min\n",
      "[CV] n_estimators=1500, min_samples_leaf=3, max_depth=8, max_features=80 \n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=2)]: Done  37 tasks      | elapsed: 14.0min\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[CV]  n_estimators=1500, min_samples_leaf=3, max_depth=8, max_features=80, total= 1.8min\n",
      "[CV] n_estimators=1500, min_samples_leaf=3, max_depth=8, max_features=80 \n",
      "[CV]  n_estimators=1500, min_samples_leaf=3, max_depth=8, max_features=80, total= 1.9min\n",
      "[CV] n_estimators=700, min_samples_leaf=8, max_depth=6, max_features=20 \n",
      "[CV]  n_estimators=700, min_samples_leaf=8, max_depth=6, max_features=20, total=  23.3s\n",
      "[CV] n_estimators=700, min_samples_leaf=8, max_depth=6, max_features=20 \n",
      "[CV]  n_estimators=1500, min_samples_leaf=3, max_depth=8, max_features=80, total= 1.8min\n",
      "[CV] n_estimators=700, min_samples_leaf=8, max_depth=6, max_features=20 \n",
      "[CV]  n_estimators=700, min_samples_leaf=8, max_depth=6, max_features=20, total=  24.2s\n",
      "[CV] n_estimators=700, min_samples_leaf=8, max_depth=6, max_features=20 \n",
      "[CV]  n_estimators=700, min_samples_leaf=8, max_depth=6, max_features=20, total=  23.7s\n",
      "[CV] n_estimators=700, min_samples_leaf=8, max_depth=6, max_features=20 \n",
      "[CV]  n_estimators=700, min_samples_leaf=8, max_depth=6, max_features=20, total=  24.2s\n",
      "[CV] n_estimators=700, min_samples_leaf=3, max_depth=4, max_features=160 \n",
      "[CV]  n_estimators=700, min_samples_leaf=8, max_depth=6, max_features=20, total=  24.4s\n",
      "[CV] n_estimators=700, min_samples_leaf=3, max_depth=4, max_features=160 \n",
      "[CV]  n_estimators=700, min_samples_leaf=3, max_depth=4, max_features=160, total=  35.2s\n",
      "[CV] n_estimators=700, min_samples_leaf=3, max_depth=4, max_features=160 \n",
      "[CV]  n_estimators=700, min_samples_leaf=3, max_depth=4, max_features=160, total=  35.3s\n",
      "[CV] n_estimators=700, min_samples_leaf=3, max_depth=4, max_features=160 \n",
      "[CV]  n_estimators=700, min_samples_leaf=3, max_depth=4, max_features=160, total=  37.1s\n",
      "[CV] n_estimators=700, min_samples_leaf=3, max_depth=4, max_features=160 \n",
      "[CV]  n_estimators=700, min_samples_leaf=3, max_depth=4, max_features=160, total=  34.8s\n",
      "[CV] n_estimators=200, min_samples_leaf=1, max_depth=2, max_features=40 \n",
      "[CV]  n_estimators=200, min_samples_leaf=1, max_depth=2, max_features=40, total=   3.9s\n",
      "[CV] n_estimators=200, min_samples_leaf=1, max_depth=2, max_features=40 \n",
      "[CV]  n_estimators=200, min_samples_leaf=1, max_depth=2, max_features=40, total=   3.9s\n",
      "[CV] n_estimators=200, min_samples_leaf=1, max_depth=2, max_features=40 \n",
      "[CV]  n_estimators=200, min_samples_leaf=1, max_depth=2, max_features=40, total=   3.9s\n",
      "[CV] n_estimators=200, min_samples_leaf=1, max_depth=2, max_features=40 \n",
      "[CV]  n_estimators=700, min_samples_leaf=3, max_depth=4, max_features=160, total=  35.9s\n",
      "[CV] n_estimators=200, min_samples_leaf=1, max_depth=2, max_features=40 \n",
      "[CV]  n_estimators=200, min_samples_leaf=1, max_depth=2, max_features=40, total=   4.0s\n",
      "[CV] n_estimators=100, min_samples_leaf=3, max_depth=8, max_features=2 \n",
      "[CV]  n_estimators=100, min_samples_leaf=3, max_depth=8, max_features=2, total=   1.5s\n",
      "[CV] n_estimators=100, min_samples_leaf=3, max_depth=8, max_features=2 \n",
      "[CV]  n_estimators=100, min_samples_leaf=3, max_depth=8, max_features=2, total=   1.4s\n",
      "[CV] n_estimators=100, min_samples_leaf=3, max_depth=8, max_features=2 \n",
      "[CV]  n_estimators=200, min_samples_leaf=1, max_depth=2, max_features=40, total=   4.2s\n",
      "[CV] n_estimators=100, min_samples_leaf=3, max_depth=8, max_features=2 \n",
      "[CV]  n_estimators=100, min_samples_leaf=3, max_depth=8, max_features=2, total=   1.5s\n",
      "[CV] n_estimators=100, min_samples_leaf=3, max_depth=8, max_features=2 \n",
      "[CV]  n_estimators=100, min_samples_leaf=3, max_depth=8, max_features=2, total=   1.4s\n",
      "[CV] n_estimators=400, min_samples_leaf=1, max_depth=10, max_features=40 \n",
      "[CV]  n_estimators=100, min_samples_leaf=3, max_depth=8, max_features=2, total=   1.5s\n",
      "[CV] n_estimators=400, min_samples_leaf=1, max_depth=10, max_features=40 \n",
      "[CV]  n_estimators=400, min_samples_leaf=1, max_depth=10, max_features=40, total=  33.5s\n",
      "[CV] n_estimators=400, min_samples_leaf=1, max_depth=10, max_features=40 \n",
      "[CV]  n_estimators=400, min_samples_leaf=1, max_depth=10, max_features=40, total=  32.7s\n",
      "[CV] n_estimators=400, min_samples_leaf=1, max_depth=10, max_features=40 \n",
      "[CV]  n_estimators=400, min_samples_leaf=1, max_depth=10, max_features=40, total=  32.8s\n",
      "[CV] n_estimators=400, min_samples_leaf=1, max_depth=10, max_features=40 \n",
      "[CV]  n_estimators=400, min_samples_leaf=1, max_depth=10, max_features=40, total=  32.8s\n",
      "[CV] n_estimators=700, min_samples_leaf=1, max_depth=2, max_features=5 \n",
      "[CV]  n_estimators=700, min_samples_leaf=1, max_depth=2, max_features=5, total=   8.6s\n",
      "[CV] n_estimators=700, min_samples_leaf=1, max_depth=2, max_features=5 \n",
      "[CV]  n_estimators=700, min_samples_leaf=1, max_depth=2, max_features=5, total=   8.6s\n",
      "[CV] n_estimators=700, min_samples_leaf=1, max_depth=2, max_features=5 \n",
      "[CV]  n_estimators=700, min_samples_leaf=1, max_depth=2, max_features=5, total=   8.7s\n",
      "[CV] n_estimators=700, min_samples_leaf=1, max_depth=2, max_features=5 \n",
      "[CV]  n_estimators=400, min_samples_leaf=1, max_depth=10, max_features=40, total=  33.9s\n",
      "[CV] n_estimators=700, min_samples_leaf=1, max_depth=2, max_features=5 \n",
      "[CV]  n_estimators=700, min_samples_leaf=1, max_depth=2, max_features=5, total=   8.4s\n",
      "[CV] n_estimators=700, min_samples_leaf=5, max_depth=4, max_features=5 \n",
      "[CV]  n_estimators=700, min_samples_leaf=1, max_depth=2, max_features=5, total=   9.1s\n",
      "[CV] n_estimators=700, min_samples_leaf=5, max_depth=4, max_features=5 \n",
      "[CV]  n_estimators=700, min_samples_leaf=5, max_depth=4, max_features=5, total=  10.8s\n",
      "[CV] n_estimators=700, min_samples_leaf=5, max_depth=4, max_features=5 \n",
      "[CV]  n_estimators=700, min_samples_leaf=5, max_depth=4, max_features=5, total=  11.2s\n",
      "[CV] n_estimators=700, min_samples_leaf=5, max_depth=4, max_features=5 \n",
      "[CV]  n_estimators=700, min_samples_leaf=5, max_depth=4, max_features=5, total=  11.3s\n",
      "[CV] n_estimators=700, min_samples_leaf=5, max_depth=4, max_features=5 \n",
      "[CV]  n_estimators=700, min_samples_leaf=5, max_depth=4, max_features=5, total=  11.3s\n",
      "[CV] n_estimators=100, min_samples_leaf=3, max_depth=2, max_features=20 \n",
      "[CV]  n_estimators=100, min_samples_leaf=3, max_depth=2, max_features=20, total=   1.7s\n",
      "[CV] n_estimators=100, min_samples_leaf=3, max_depth=2, max_features=20 \n",
      "[CV]  n_estimators=700, min_samples_leaf=5, max_depth=4, max_features=5, total=  11.1s\n",
      "[CV] n_estimators=100, min_samples_leaf=3, max_depth=2, max_features=20 \n",
      "[CV]  n_estimators=100, min_samples_leaf=3, max_depth=2, max_features=20, total=   1.7s\n",
      "[CV] n_estimators=100, min_samples_leaf=3, max_depth=2, max_features=20 \n",
      "[CV]  n_estimators=100, min_samples_leaf=3, max_depth=2, max_features=20, total=   1.7s\n",
      "[CV] n_estimators=100, min_samples_leaf=3, max_depth=2, max_features=20 \n",
      "[CV]  n_estimators=100, min_samples_leaf=3, max_depth=2, max_features=20, total=   1.7s\n",
      "[CV] n_estimators=100, min_samples_leaf=1, max_depth=8, max_features=20 \n",
      "[CV]  n_estimators=100, min_samples_leaf=3, max_depth=2, max_features=20, total=   1.7s\n",
      "[CV] n_estimators=100, min_samples_leaf=1, max_depth=8, max_features=20 \n",
      "[CV]  n_estimators=100, min_samples_leaf=1, max_depth=8, max_features=20, total=   5.5s\n",
      "[CV] n_estimators=100, min_samples_leaf=1, max_depth=8, max_features=20 \n",
      "[CV]  n_estimators=100, min_samples_leaf=1, max_depth=8, max_features=20, total=   5.5s\n",
      "[CV] n_estimators=100, min_samples_leaf=1, max_depth=8, max_features=20 \n",
      "[CV]  n_estimators=100, min_samples_leaf=1, max_depth=8, max_features=20, total=   5.6s\n",
      "[CV] n_estimators=100, min_samples_leaf=1, max_depth=8, max_features=20 \n",
      "[CV]  n_estimators=100, min_samples_leaf=1, max_depth=8, max_features=20, total=   5.4s\n",
      "[CV] n_estimators=1500, min_samples_leaf=1, max_depth=2, max_features=80 \n",
      "[CV]  n_estimators=100, min_samples_leaf=1, max_depth=8, max_features=20, total=   5.4s\n",
      "[CV] n_estimators=1500, min_samples_leaf=1, max_depth=2, max_features=80 \n",
      "[CV]  n_estimators=1500, min_samples_leaf=1, max_depth=2, max_features=80, total=  33.8s\n",
      "[CV] n_estimators=1500, min_samples_leaf=1, max_depth=2, max_features=80 \n",
      "[CV]  n_estimators=1500, min_samples_leaf=1, max_depth=2, max_features=80, total=  33.6s\n",
      "[CV] n_estimators=1500, min_samples_leaf=1, max_depth=2, max_features=80 \n",
      "[CV]  n_estimators=1500, min_samples_leaf=1, max_depth=2, max_features=80, total=  32.0s\n",
      "[CV] n_estimators=1500, min_samples_leaf=1, max_depth=2, max_features=80 \n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[CV]  n_estimators=1500, min_samples_leaf=1, max_depth=2, max_features=80, total=  32.0s\n",
      "[CV] n_estimators=1500, min_samples_leaf=1, max_depth=10, max_features=160 \n",
      "[CV]  n_estimators=1500, min_samples_leaf=1, max_depth=2, max_features=80, total=  31.7s\n",
      "[CV] n_estimators=1500, min_samples_leaf=1, max_depth=10, max_features=160 \n",
      "[CV]  n_estimators=1500, min_samples_leaf=1, max_depth=10, max_features=160, total= 3.2min\n",
      "[CV] n_estimators=1500, min_samples_leaf=1, max_depth=10, max_features=160 \n",
      "[CV]  n_estimators=1500, min_samples_leaf=1, max_depth=10, max_features=160, total= 3.2min\n",
      "[CV] n_estimators=1500, min_samples_leaf=1, max_depth=10, max_features=160 \n",
      "[CV]  n_estimators=1500, min_samples_leaf=1, max_depth=10, max_features=160, total= 3.2min\n",
      "[CV] n_estimators=1500, min_samples_leaf=1, max_depth=10, max_features=160 \n",
      "[CV]  n_estimators=1500, min_samples_leaf=1, max_depth=10, max_features=160, total= 3.1min\n",
      "[CV] n_estimators=400, min_samples_leaf=8, max_depth=8, max_features=5 \n",
      "[CV]  n_estimators=400, min_samples_leaf=8, max_depth=8, max_features=5, total=   7.9s\n",
      "[CV] n_estimators=400, min_samples_leaf=8, max_depth=8, max_features=5 \n",
      "[CV]  n_estimators=400, min_samples_leaf=8, max_depth=8, max_features=5, total=   8.3s\n",
      "[CV] n_estimators=400, min_samples_leaf=8, max_depth=8, max_features=5 \n",
      "[CV]  n_estimators=400, min_samples_leaf=8, max_depth=8, max_features=5, total=   8.4s\n",
      "[CV] n_estimators=400, min_samples_leaf=8, max_depth=8, max_features=5 \n",
      "[CV]  n_estimators=400, min_samples_leaf=8, max_depth=8, max_features=5, total=   8.3s\n",
      "[CV] n_estimators=400, min_samples_leaf=8, max_depth=8, max_features=5 \n",
      "[CV]  n_estimators=400, min_samples_leaf=8, max_depth=8, max_features=5, total=   8.4s\n",
      "[CV] n_estimators=1500, min_samples_leaf=8, max_depth=8, max_features=80 \n",
      "[CV]  n_estimators=1500, min_samples_leaf=8, max_depth=8, max_features=80, total= 1.8min\n",
      "[CV] n_estimators=1500, min_samples_leaf=8, max_depth=8, max_features=80 \n",
      "[CV]  n_estimators=1500, min_samples_leaf=1, max_depth=10, max_features=160, total= 3.2min\n",
      "[CV] n_estimators=1500, min_samples_leaf=8, max_depth=8, max_features=80 \n",
      "[CV]  n_estimators=1500, min_samples_leaf=8, max_depth=8, max_features=80, total= 1.8min\n",
      "[CV] n_estimators=1500, min_samples_leaf=8, max_depth=8, max_features=80 \n",
      "[CV]  n_estimators=1500, min_samples_leaf=8, max_depth=8, max_features=80, total= 1.8min\n",
      "[CV] n_estimators=1500, min_samples_leaf=8, max_depth=8, max_features=80 \n",
      "[CV]  n_estimators=1500, min_samples_leaf=8, max_depth=8, max_features=80, total= 1.8min\n",
      "[CV] n_estimators=1500, min_samples_leaf=8, max_depth=8, max_features=10 \n",
      "[CV]  n_estimators=1500, min_samples_leaf=8, max_depth=8, max_features=80, total= 1.8min\n",
      "[CV] n_estimators=1500, min_samples_leaf=8, max_depth=8, max_features=10 \n",
      "[CV]  n_estimators=1500, min_samples_leaf=8, max_depth=8, max_features=10, total=  46.2s\n",
      "[CV] n_estimators=1500, min_samples_leaf=8, max_depth=8, max_features=10 \n",
      "[CV]  n_estimators=1500, min_samples_leaf=8, max_depth=8, max_features=10, total=  49.3s\n",
      "[CV] n_estimators=1500, min_samples_leaf=8, max_depth=8, max_features=10 \n",
      "[CV]  n_estimators=1500, min_samples_leaf=8, max_depth=8, max_features=10, total=  47.6s\n",
      "[CV] n_estimators=1500, min_samples_leaf=8, max_depth=8, max_features=10 \n",
      "[CV]  n_estimators=1500, min_samples_leaf=8, max_depth=8, max_features=10, total=  48.3s\n",
      "[CV] n_estimators=400, min_samples_leaf=1, max_depth=10, max_features=20 \n",
      "[CV]  n_estimators=400, min_samples_leaf=1, max_depth=10, max_features=20, total=  25.9s\n",
      "[CV] n_estimators=400, min_samples_leaf=1, max_depth=10, max_features=20 \n",
      "[CV]  n_estimators=1500, min_samples_leaf=8, max_depth=8, max_features=10, total=  48.6s\n",
      "[CV] n_estimators=400, min_samples_leaf=1, max_depth=10, max_features=20 \n",
      "[CV]  n_estimators=400, min_samples_leaf=1, max_depth=10, max_features=20, total=  26.5s\n",
      "[CV] n_estimators=400, min_samples_leaf=1, max_depth=10, max_features=20 \n",
      "[CV]  n_estimators=400, min_samples_leaf=1, max_depth=10, max_features=20, total=  26.2s\n",
      "[CV] n_estimators=400, min_samples_leaf=1, max_depth=10, max_features=20 \n",
      "[CV]  n_estimators=400, min_samples_leaf=1, max_depth=10, max_features=20, total=  25.9s\n",
      "[CV] n_estimators=700, min_samples_leaf=1, max_depth=4, max_features=40 \n",
      "[CV]  n_estimators=400, min_samples_leaf=1, max_depth=10, max_features=20, total=  26.2s\n",
      "[CV] n_estimators=700, min_samples_leaf=1, max_depth=4, max_features=40 \n",
      "[CV]  n_estimators=700, min_samples_leaf=1, max_depth=4, max_features=40, total=  21.5s\n",
      "[CV] n_estimators=700, min_samples_leaf=1, max_depth=4, max_features=40 \n",
      "[CV]  n_estimators=700, min_samples_leaf=1, max_depth=4, max_features=40, total=  22.4s\n",
      "[CV] n_estimators=700, min_samples_leaf=1, max_depth=4, max_features=40 \n",
      "[CV]  n_estimators=700, min_samples_leaf=1, max_depth=4, max_features=40, total=  22.2s\n",
      "[CV] n_estimators=700, min_samples_leaf=1, max_depth=4, max_features=40 \n",
      "[CV]  n_estimators=700, min_samples_leaf=1, max_depth=4, max_features=40, total=  22.3s\n",
      "[CV] n_estimators=200, min_samples_leaf=5, max_depth=6, max_features=5 \n",
      "[CV]  n_estimators=200, min_samples_leaf=5, max_depth=6, max_features=5, total=   3.6s\n",
      "[CV] n_estimators=200, min_samples_leaf=5, max_depth=6, max_features=5 \n",
      "[CV]  n_estimators=200, min_samples_leaf=5, max_depth=6, max_features=5, total=   3.8s\n",
      "[CV] n_estimators=200, min_samples_leaf=5, max_depth=6, max_features=5 \n",
      "[CV]  n_estimators=200, min_samples_leaf=5, max_depth=6, max_features=5, total=   3.8s\n",
      "[CV] n_estimators=200, min_samples_leaf=5, max_depth=6, max_features=5 \n",
      "[CV]  n_estimators=200, min_samples_leaf=5, max_depth=6, max_features=5, total=   3.9s\n",
      "[CV] n_estimators=200, min_samples_leaf=5, max_depth=6, max_features=5 \n",
      "[CV]  n_estimators=700, min_samples_leaf=1, max_depth=4, max_features=40, total=  22.1s\n",
      "[CV] n_estimators=400, min_samples_leaf=5, max_depth=2, max_features=160 \n",
      "[CV]  n_estimators=200, min_samples_leaf=5, max_depth=6, max_features=5, total=   3.7s\n",
      "[CV] n_estimators=400, min_samples_leaf=5, max_depth=2, max_features=160 \n",
      "[CV]  n_estimators=400, min_samples_leaf=5, max_depth=2, max_features=160, total=  11.5s\n",
      "[CV] n_estimators=400, min_samples_leaf=5, max_depth=2, max_features=160 \n",
      "[CV]  n_estimators=400, min_samples_leaf=5, max_depth=2, max_features=160, total=  11.2s\n",
      "[CV] n_estimators=400, min_samples_leaf=5, max_depth=2, max_features=160 \n",
      "[CV]  n_estimators=400, min_samples_leaf=5, max_depth=2, max_features=160, total=  11.5s\n",
      "[CV] n_estimators=400, min_samples_leaf=5, max_depth=2, max_features=160 \n",
      "[CV]  n_estimators=400, min_samples_leaf=5, max_depth=2, max_features=160, total=  11.2s\n",
      "[CV] n_estimators=1500, min_samples_leaf=3, max_depth=2, max_features=20 \n",
      "[CV]  n_estimators=400, min_samples_leaf=5, max_depth=2, max_features=160, total=  11.3s\n",
      "[CV] n_estimators=1500, min_samples_leaf=3, max_depth=2, max_features=20 \n",
      "[CV]  n_estimators=1500, min_samples_leaf=3, max_depth=2, max_features=20, total=  24.4s\n",
      "[CV] n_estimators=1500, min_samples_leaf=3, max_depth=2, max_features=20 \n",
      "[CV]  n_estimators=1500, min_samples_leaf=3, max_depth=2, max_features=20, total=  24.3s\n",
      "[CV] n_estimators=1500, min_samples_leaf=3, max_depth=2, max_features=20 \n",
      "[CV]  n_estimators=1500, min_samples_leaf=3, max_depth=2, max_features=20, total=  24.6s\n",
      "[CV] n_estimators=1500, min_samples_leaf=3, max_depth=2, max_features=20 \n",
      "[CV]  n_estimators=1500, min_samples_leaf=3, max_depth=2, max_features=20, total=  24.7s\n",
      "[CV] n_estimators=1500, min_samples_leaf=8, max_depth=10, max_features=5 \n",
      "[CV]  n_estimators=1500, min_samples_leaf=3, max_depth=2, max_features=20, total=  25.0s\n",
      "[CV] n_estimators=1500, min_samples_leaf=8, max_depth=10, max_features=5 \n",
      "[CV]  n_estimators=1500, min_samples_leaf=8, max_depth=10, max_features=5, total=  33.2s\n",
      "[CV] n_estimators=1500, min_samples_leaf=8, max_depth=10, max_features=5 \n",
      "[CV]  n_estimators=1500, min_samples_leaf=8, max_depth=10, max_features=5, total=  34.4s\n",
      "[CV] n_estimators=1500, min_samples_leaf=8, max_depth=10, max_features=5 \n",
      "[CV]  n_estimators=1500, min_samples_leaf=8, max_depth=10, max_features=5, total=  34.4s\n",
      "[CV] n_estimators=1500, min_samples_leaf=8, max_depth=10, max_features=5 \n",
      "[CV]  n_estimators=1500, min_samples_leaf=8, max_depth=10, max_features=5, total=  34.3s\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[CV] n_estimators=1500, min_samples_leaf=8, max_depth=6, max_features=160 \n",
      "[CV]  n_estimators=1500, min_samples_leaf=8, max_depth=10, max_features=5, total=  34.1s\n",
      "[CV] n_estimators=1500, min_samples_leaf=8, max_depth=6, max_features=160 \n",
      "[CV]  n_estimators=1500, min_samples_leaf=8, max_depth=6, max_features=160, total= 1.8min\n",
      "[CV] n_estimators=1500, min_samples_leaf=8, max_depth=6, max_features=160 \n",
      "[CV]  n_estimators=1500, min_samples_leaf=8, max_depth=6, max_features=160, total= 1.9min\n",
      "[CV] n_estimators=1500, min_samples_leaf=8, max_depth=6, max_features=160 \n",
      "[CV]  n_estimators=1500, min_samples_leaf=8, max_depth=6, max_features=160, total= 1.8min\n",
      "[CV] n_estimators=1500, min_samples_leaf=8, max_depth=6, max_features=160 \n",
      "[CV]  n_estimators=1500, min_samples_leaf=8, max_depth=6, max_features=160, total= 1.8min\n",
      "[CV] n_estimators=400, min_samples_leaf=3, max_depth=2, max_features=10 \n",
      "[CV]  n_estimators=400, min_samples_leaf=3, max_depth=2, max_features=10, total=   5.4s\n",
      "[CV] n_estimators=400, min_samples_leaf=3, max_depth=2, max_features=10 \n",
      "[CV]  n_estimators=400, min_samples_leaf=3, max_depth=2, max_features=10, total=   5.7s\n",
      "[CV] n_estimators=400, min_samples_leaf=3, max_depth=2, max_features=10 \n",
      "[CV]  n_estimators=400, min_samples_leaf=3, max_depth=2, max_features=10, total=   6.0s\n",
      "[CV] n_estimators=400, min_samples_leaf=3, max_depth=2, max_features=10 \n",
      "[CV]  n_estimators=400, min_samples_leaf=3, max_depth=2, max_features=10, total=   5.5s\n",
      "[CV] n_estimators=400, min_samples_leaf=3, max_depth=2, max_features=10 \n",
      "[CV]  n_estimators=400, min_samples_leaf=3, max_depth=2, max_features=10, total=   5.5s\n",
      "[CV] n_estimators=700, min_samples_leaf=3, max_depth=2, max_features=2 \n",
      "[CV]  n_estimators=700, min_samples_leaf=3, max_depth=2, max_features=2, total=   7.4s\n",
      "[CV] n_estimators=700, min_samples_leaf=3, max_depth=2, max_features=2 \n",
      "[CV]  n_estimators=700, min_samples_leaf=3, max_depth=2, max_features=2, total=   7.4s\n",
      "[CV] n_estimators=700, min_samples_leaf=3, max_depth=2, max_features=2 \n",
      "[CV]  n_estimators=700, min_samples_leaf=3, max_depth=2, max_features=2, total=   7.4s\n",
      "[CV] n_estimators=700, min_samples_leaf=3, max_depth=2, max_features=2 \n",
      "[CV]  n_estimators=700, min_samples_leaf=3, max_depth=2, max_features=2, total=   7.3s\n",
      "[CV] n_estimators=700, min_samples_leaf=3, max_depth=2, max_features=2 \n",
      "[CV]  n_estimators=700, min_samples_leaf=3, max_depth=2, max_features=2, total=   7.6s\n",
      "[CV] n_estimators=1500, min_samples_leaf=8, max_depth=2, max_features=10 \n",
      "[CV]  n_estimators=1500, min_samples_leaf=8, max_depth=6, max_features=160, total= 1.8min\n",
      "[CV] n_estimators=1500, min_samples_leaf=8, max_depth=2, max_features=10 \n",
      "[CV]  n_estimators=1500, min_samples_leaf=8, max_depth=2, max_features=10, total=  20.3s\n",
      "[CV] n_estimators=1500, min_samples_leaf=8, max_depth=2, max_features=10 \n",
      "[CV]  n_estimators=1500, min_samples_leaf=8, max_depth=2, max_features=10, total=  21.0s\n",
      "[CV] n_estimators=1500, min_samples_leaf=8, max_depth=2, max_features=10 \n",
      "[CV]  n_estimators=1500, min_samples_leaf=8, max_depth=2, max_features=10, total=  21.7s\n",
      "[CV] n_estimators=1500, min_samples_leaf=8, max_depth=2, max_features=10 \n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=2)]: Done 158 tasks      | elapsed: 50.7min\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[CV]  n_estimators=1500, min_samples_leaf=8, max_depth=2, max_features=10, total=  21.3s\n",
      "[CV] n_estimators=200, min_samples_leaf=5, max_depth=10, max_features=2 \n",
      "[CV]  n_estimators=1500, min_samples_leaf=8, max_depth=2, max_features=10, total=  21.0s\n",
      "[CV] n_estimators=200, min_samples_leaf=5, max_depth=10, max_features=2 \n",
      "[CV]  n_estimators=200, min_samples_leaf=5, max_depth=10, max_features=2, total=   2.5s\n",
      "[CV] n_estimators=200, min_samples_leaf=5, max_depth=10, max_features=2 \n",
      "[CV]  n_estimators=200, min_samples_leaf=5, max_depth=10, max_features=2, total=   2.5s\n",
      "[CV] n_estimators=200, min_samples_leaf=5, max_depth=10, max_features=2 \n",
      "[CV]  n_estimators=200, min_samples_leaf=5, max_depth=10, max_features=2, total=   2.7s\n",
      "[CV] n_estimators=200, min_samples_leaf=5, max_depth=10, max_features=2 \n",
      "[CV]  n_estimators=200, min_samples_leaf=5, max_depth=10, max_features=2, total=   2.5s\n",
      "[CV] n_estimators=100, min_samples_leaf=8, max_depth=10, max_features=40 \n",
      "[CV]  n_estimators=200, min_samples_leaf=5, max_depth=10, max_features=2, total=   2.6s\n",
      "[CV] n_estimators=100, min_samples_leaf=8, max_depth=10, max_features=40 \n",
      "[CV]  n_estimators=100, min_samples_leaf=8, max_depth=10, max_features=40, total=   7.4s\n",
      "[CV] n_estimators=100, min_samples_leaf=8, max_depth=10, max_features=40 \n",
      "[CV]  n_estimators=100, min_samples_leaf=8, max_depth=10, max_features=40, total=   7.7s\n",
      "[CV] n_estimators=100, min_samples_leaf=8, max_depth=10, max_features=40 \n",
      "[CV]  n_estimators=100, min_samples_leaf=8, max_depth=10, max_features=40, total=   7.7s\n",
      "[CV] n_estimators=100, min_samples_leaf=8, max_depth=10, max_features=40 \n",
      "[CV]  n_estimators=100, min_samples_leaf=8, max_depth=10, max_features=40, total=   7.8s\n",
      "[CV] n_estimators=200, min_samples_leaf=8, max_depth=2, max_features=10 \n",
      "[CV]  n_estimators=200, min_samples_leaf=8, max_depth=2, max_features=10, total=   2.7s\n",
      "[CV] n_estimators=200, min_samples_leaf=8, max_depth=2, max_features=10 \n",
      "[CV]  n_estimators=100, min_samples_leaf=8, max_depth=10, max_features=40, total=   7.3s\n",
      "[CV] n_estimators=200, min_samples_leaf=8, max_depth=2, max_features=10 \n",
      "[CV]  n_estimators=200, min_samples_leaf=8, max_depth=2, max_features=10, total=   2.8s\n",
      "[CV] n_estimators=200, min_samples_leaf=8, max_depth=2, max_features=10 \n",
      "[CV]  n_estimators=200, min_samples_leaf=8, max_depth=2, max_features=10, total=   3.0s\n",
      "[CV] n_estimators=200, min_samples_leaf=8, max_depth=2, max_features=10 \n",
      "[CV]  n_estimators=200, min_samples_leaf=8, max_depth=2, max_features=10, total=   3.0s\n",
      "[CV] n_estimators=400, min_samples_leaf=3, max_depth=6, max_features=5 \n",
      "[CV]  n_estimators=200, min_samples_leaf=8, max_depth=2, max_features=10, total=   3.1s\n",
      "[CV] n_estimators=400, min_samples_leaf=3, max_depth=6, max_features=5 \n",
      "[CV]  n_estimators=400, min_samples_leaf=3, max_depth=6, max_features=5, total=   8.2s\n",
      "[CV] n_estimators=400, min_samples_leaf=3, max_depth=6, max_features=5 \n",
      "[CV]  n_estimators=400, min_samples_leaf=3, max_depth=6, max_features=5, total=   8.1s\n",
      "[CV] n_estimators=400, min_samples_leaf=3, max_depth=6, max_features=5 \n",
      "[CV]  n_estimators=400, min_samples_leaf=3, max_depth=6, max_features=5, total=   8.3s\n",
      "[CV] n_estimators=400, min_samples_leaf=3, max_depth=6, max_features=5 \n",
      "[CV]  n_estimators=400, min_samples_leaf=3, max_depth=6, max_features=5, total=   8.0s\n",
      "[CV] n_estimators=700, min_samples_leaf=1, max_depth=6, max_features=80 \n",
      "[CV]  n_estimators=400, min_samples_leaf=3, max_depth=6, max_features=5, total=   8.6s\n",
      "[CV] n_estimators=700, min_samples_leaf=1, max_depth=6, max_features=80 \n",
      "[CV]  n_estimators=700, min_samples_leaf=1, max_depth=6, max_features=80, total=  39.2s\n",
      "[CV] n_estimators=700, min_samples_leaf=1, max_depth=6, max_features=80 \n",
      "[CV]  n_estimators=700, min_samples_leaf=1, max_depth=6, max_features=80, total=  39.1s\n",
      "[CV] n_estimators=700, min_samples_leaf=1, max_depth=6, max_features=80 \n",
      "[CV]  n_estimators=700, min_samples_leaf=1, max_depth=6, max_features=80, total=  38.4s\n",
      "[CV] n_estimators=700, min_samples_leaf=1, max_depth=6, max_features=80 \n",
      "[CV]  n_estimators=700, min_samples_leaf=1, max_depth=6, max_features=80, total=  38.4s\n",
      "[CV] n_estimators=700, min_samples_leaf=8, max_depth=10, max_features=5 \n",
      "[CV]  n_estimators=700, min_samples_leaf=8, max_depth=10, max_features=5, total=  15.7s\n",
      "[CV] n_estimators=700, min_samples_leaf=8, max_depth=10, max_features=5 \n",
      "[CV]  n_estimators=700, min_samples_leaf=1, max_depth=6, max_features=80, total=  39.7s\n",
      "[CV] n_estimators=700, min_samples_leaf=8, max_depth=10, max_features=5 \n",
      "[CV]  n_estimators=700, min_samples_leaf=8, max_depth=10, max_features=5, total=  16.4s\n",
      "[CV] n_estimators=700, min_samples_leaf=8, max_depth=10, max_features=5 \n",
      "[CV]  n_estimators=700, min_samples_leaf=8, max_depth=10, max_features=5, total=  16.0s\n",
      "[CV] n_estimators=700, min_samples_leaf=8, max_depth=10, max_features=5 \n",
      "[CV]  n_estimators=700, min_samples_leaf=8, max_depth=10, max_features=5, total=  16.0s\n",
      "[CV] n_estimators=700, min_samples_leaf=3, max_depth=8, max_features=5 \n",
      "[CV]  n_estimators=700, min_samples_leaf=8, max_depth=10, max_features=5, total=  16.1s\n",
      "[CV] n_estimators=700, min_samples_leaf=3, max_depth=8, max_features=5 \n",
      "[CV]  n_estimators=700, min_samples_leaf=3, max_depth=8, max_features=5, total=  16.0s\n",
      "[CV] n_estimators=700, min_samples_leaf=3, max_depth=8, max_features=5 \n",
      "[CV]  n_estimators=700, min_samples_leaf=3, max_depth=8, max_features=5, total=  17.0s\n",
      "[CV] n_estimators=700, min_samples_leaf=3, max_depth=8, max_features=5 \n",
      "[CV]  n_estimators=700, min_samples_leaf=3, max_depth=8, max_features=5, total=  17.5s\n",
      "[CV] n_estimators=700, min_samples_leaf=3, max_depth=8, max_features=5 \n",
      "[CV]  n_estimators=700, min_samples_leaf=3, max_depth=8, max_features=5, total=  17.0s\n",
      "[CV] n_estimators=1500, min_samples_leaf=8, max_depth=2, max_features=2 \n",
      "[CV]  n_estimators=700, min_samples_leaf=3, max_depth=8, max_features=5, total=  17.3s\n",
      "[CV] n_estimators=1500, min_samples_leaf=8, max_depth=2, max_features=2 \n",
      "[CV]  n_estimators=1500, min_samples_leaf=8, max_depth=2, max_features=2, total=  15.5s\n",
      "[CV] n_estimators=1500, min_samples_leaf=8, max_depth=2, max_features=2 \n",
      "[CV]  n_estimators=1500, min_samples_leaf=8, max_depth=2, max_features=2, total=  15.9s\n",
      "[CV] n_estimators=1500, min_samples_leaf=8, max_depth=2, max_features=2 \n",
      "[CV]  n_estimators=1500, min_samples_leaf=8, max_depth=2, max_features=2, total=  15.8s\n",
      "[CV] n_estimators=1500, min_samples_leaf=8, max_depth=2, max_features=2 \n",
      "[CV]  n_estimators=1500, min_samples_leaf=8, max_depth=2, max_features=2, total=  15.9s\n",
      "[CV] n_estimators=200, min_samples_leaf=1, max_depth=6, max_features=20 \n",
      "[CV]  n_estimators=200, min_samples_leaf=1, max_depth=6, max_features=20, total=   7.6s\n",
      "[CV] n_estimators=200, min_samples_leaf=1, max_depth=6, max_features=20 \n",
      "[CV]  n_estimators=1500, min_samples_leaf=8, max_depth=2, max_features=2, total=  15.9s\n",
      "[CV] n_estimators=200, min_samples_leaf=1, max_depth=6, max_features=20 \n",
      "[CV]  n_estimators=200, min_samples_leaf=1, max_depth=6, max_features=20, total=   7.6s\n",
      "[CV] n_estimators=200, min_samples_leaf=1, max_depth=6, max_features=20 \n",
      "[CV]  n_estimators=200, min_samples_leaf=1, max_depth=6, max_features=20, total=   7.7s\n",
      "[CV] n_estimators=200, min_samples_leaf=1, max_depth=6, max_features=20 \n",
      "[CV]  n_estimators=200, min_samples_leaf=1, max_depth=6, max_features=20, total=   7.5s\n",
      "[CV] n_estimators=1500, min_samples_leaf=8, max_depth=10, max_features=2 \n",
      "[CV]  n_estimators=200, min_samples_leaf=1, max_depth=6, max_features=20, total=   7.8s\n",
      "[CV] n_estimators=1500, min_samples_leaf=8, max_depth=10, max_features=2 \n",
      "[CV]  n_estimators=1500, min_samples_leaf=8, max_depth=10, max_features=2, total=  17.3s\n",
      "[CV] n_estimators=1500, min_samples_leaf=8, max_depth=10, max_features=2 \n",
      "[CV]  n_estimators=1500, min_samples_leaf=8, max_depth=10, max_features=2, total=  18.2s\n",
      "[CV] n_estimators=1500, min_samples_leaf=8, max_depth=10, max_features=2 \n",
      "[CV]  n_estimators=1500, min_samples_leaf=8, max_depth=10, max_features=2, total=  18.1s\n",
      "[CV] n_estimators=1500, min_samples_leaf=8, max_depth=10, max_features=2 \n",
      "[CV]  n_estimators=1500, min_samples_leaf=8, max_depth=10, max_features=2, total=  17.7s\n",
      "[CV] n_estimators=700, min_samples_leaf=3, max_depth=10, max_features=10 \n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[CV]  n_estimators=1500, min_samples_leaf=8, max_depth=10, max_features=2, total=  17.4s\n",
      "[CV] n_estimators=700, min_samples_leaf=3, max_depth=10, max_features=10 \n",
      "[CV]  n_estimators=700, min_samples_leaf=3, max_depth=10, max_features=10, total=  26.2s\n",
      "[CV] n_estimators=700, min_samples_leaf=3, max_depth=10, max_features=10 \n",
      "[CV]  n_estimators=700, min_samples_leaf=3, max_depth=10, max_features=10, total=  28.2s\n",
      "[CV] n_estimators=700, min_samples_leaf=3, max_depth=10, max_features=10 \n",
      "[CV]  n_estimators=700, min_samples_leaf=3, max_depth=10, max_features=10, total=  27.6s\n",
      "[CV] n_estimators=700, min_samples_leaf=3, max_depth=10, max_features=10 \n",
      "[CV]  n_estimators=700, min_samples_leaf=3, max_depth=10, max_features=10, total=  28.4s\n",
      "[CV] n_estimators=400, min_samples_leaf=5, max_depth=4, max_features=5 \n",
      "[CV]  n_estimators=400, min_samples_leaf=5, max_depth=4, max_features=5, total=   6.3s\n",
      "[CV] n_estimators=400, min_samples_leaf=5, max_depth=4, max_features=5 \n",
      "[CV]  n_estimators=400, min_samples_leaf=5, max_depth=4, max_features=5, total=   6.4s\n",
      "[CV] n_estimators=400, min_samples_leaf=5, max_depth=4, max_features=5 \n",
      "[CV]  n_estimators=700, min_samples_leaf=3, max_depth=10, max_features=10, total=  27.2s\n",
      "[CV] n_estimators=400, min_samples_leaf=5, max_depth=4, max_features=5 \n",
      "[CV]  n_estimators=400, min_samples_leaf=5, max_depth=4, max_features=5, total=   6.4s\n",
      "[CV] n_estimators=400, min_samples_leaf=5, max_depth=4, max_features=5 \n",
      "[CV]  n_estimators=400, min_samples_leaf=5, max_depth=4, max_features=5, total=   6.4s\n",
      "[CV] n_estimators=700, min_samples_leaf=1, max_depth=6, max_features=40 \n",
      "[CV]  n_estimators=400, min_samples_leaf=5, max_depth=4, max_features=5, total=   6.4s\n",
      "[CV] n_estimators=700, min_samples_leaf=1, max_depth=6, max_features=40 \n",
      "[CV]  n_estimators=700, min_samples_leaf=1, max_depth=6, max_features=40, total=  32.2s\n",
      "[CV] n_estimators=700, min_samples_leaf=1, max_depth=6, max_features=40 \n",
      "[CV]  n_estimators=700, min_samples_leaf=1, max_depth=6, max_features=40, total=  32.0s\n",
      "[CV] n_estimators=700, min_samples_leaf=1, max_depth=6, max_features=40 \n",
      "[CV]  n_estimators=700, min_samples_leaf=1, max_depth=6, max_features=40, total=  31.2s\n",
      "[CV] n_estimators=700, min_samples_leaf=1, max_depth=6, max_features=40 \n",
      "[CV]  n_estimators=700, min_samples_leaf=1, max_depth=6, max_features=40, total=  32.1s\n",
      "[CV] n_estimators=1500, min_samples_leaf=1, max_depth=4, max_features=40 \n",
      "[CV]  n_estimators=700, min_samples_leaf=1, max_depth=6, max_features=40, total=  31.2s\n",
      "[CV] n_estimators=1500, min_samples_leaf=1, max_depth=4, max_features=40 \n",
      "[CV]  n_estimators=1500, min_samples_leaf=1, max_depth=4, max_features=40, total=  45.7s\n",
      "[CV] n_estimators=1500, min_samples_leaf=1, max_depth=4, max_features=40 \n",
      "[CV]  n_estimators=1500, min_samples_leaf=1, max_depth=4, max_features=40, total=  45.3s\n",
      "[CV] n_estimators=1500, min_samples_leaf=1, max_depth=4, max_features=40 \n",
      "[CV]  n_estimators=1500, min_samples_leaf=1, max_depth=4, max_features=40, total=  45.3s\n",
      "[CV] n_estimators=1500, min_samples_leaf=1, max_depth=4, max_features=40 \n",
      "[CV]  n_estimators=1500, min_samples_leaf=1, max_depth=4, max_features=40, total=  44.5s\n",
      "[CV] n_estimators=200, min_samples_leaf=5, max_depth=2, max_features=80 \n",
      "[CV]  n_estimators=200, min_samples_leaf=5, max_depth=2, max_features=80, total=   4.3s\n",
      "[CV] n_estimators=200, min_samples_leaf=5, max_depth=2, max_features=80 \n",
      "[CV]  n_estimators=200, min_samples_leaf=5, max_depth=2, max_features=80, total=   4.3s\n",
      "[CV] n_estimators=200, min_samples_leaf=5, max_depth=2, max_features=80 \n",
      "[CV]  n_estimators=200, min_samples_leaf=5, max_depth=2, max_features=80, total=   4.3s\n",
      "[CV] n_estimators=200, min_samples_leaf=5, max_depth=2, max_features=80 \n",
      "[CV]  n_estimators=200, min_samples_leaf=5, max_depth=2, max_features=80, total=   4.4s\n",
      "[CV] n_estimators=200, min_samples_leaf=5, max_depth=2, max_features=80 \n",
      "[CV]  n_estimators=200, min_samples_leaf=5, max_depth=2, max_features=80, total=   4.3s\n",
      "[CV]  n_estimators=1500, min_samples_leaf=1, max_depth=4, max_features=40, total=  45.0s\n",
      "[CV] n_estimators=400, min_samples_leaf=8, max_depth=6, max_features=40 \n",
      "[CV] n_estimators=400, min_samples_leaf=8, max_depth=6, max_features=40 \n",
      "[CV]  n_estimators=400, min_samples_leaf=8, max_depth=6, max_features=40, total=  16.8s\n",
      "[CV] n_estimators=400, min_samples_leaf=8, max_depth=6, max_features=40 \n",
      "[CV]  n_estimators=400, min_samples_leaf=8, max_depth=6, max_features=40, total=  16.9s\n",
      "[CV] n_estimators=400, min_samples_leaf=8, max_depth=6, max_features=40 \n",
      "[CV]  n_estimators=400, min_samples_leaf=8, max_depth=6, max_features=40, total=  16.9s\n",
      "[CV] n_estimators=400, min_samples_leaf=8, max_depth=6, max_features=40 \n",
      "[CV]  n_estimators=400, min_samples_leaf=8, max_depth=6, max_features=40, total=  17.1s\n",
      "[CV] n_estimators=400, min_samples_leaf=3, max_depth=8, max_features=5 \n",
      "[CV]  n_estimators=400, min_samples_leaf=3, max_depth=8, max_features=5, total=   9.0s\n",
      "[CV] n_estimators=400, min_samples_leaf=3, max_depth=8, max_features=5 \n",
      "[CV]  n_estimators=400, min_samples_leaf=8, max_depth=6, max_features=40, total=  16.8s\n",
      "[CV] n_estimators=400, min_samples_leaf=3, max_depth=8, max_features=5 \n",
      "[CV]  n_estimators=400, min_samples_leaf=3, max_depth=8, max_features=5, total=   9.5s\n",
      "[CV] n_estimators=400, min_samples_leaf=3, max_depth=8, max_features=5 \n",
      "[CV]  n_estimators=400, min_samples_leaf=3, max_depth=8, max_features=5, total=   9.4s\n",
      "[CV] n_estimators=400, min_samples_leaf=3, max_depth=8, max_features=5 \n",
      "[CV]  n_estimators=400, min_samples_leaf=3, max_depth=8, max_features=5, total=   9.5s\n",
      "[CV] n_estimators=400, min_samples_leaf=5, max_depth=6, max_features=160 \n",
      "[CV]  n_estimators=400, min_samples_leaf=3, max_depth=8, max_features=5, total=   9.4s\n",
      "[CV] n_estimators=400, min_samples_leaf=5, max_depth=6, max_features=160 \n",
      "[CV]  n_estimators=400, min_samples_leaf=5, max_depth=6, max_features=160, total=  29.0s\n",
      "[CV] n_estimators=400, min_samples_leaf=5, max_depth=6, max_features=160 \n",
      "[CV]  n_estimators=400, min_samples_leaf=5, max_depth=6, max_features=160, total=  28.5s\n",
      "[CV] n_estimators=400, min_samples_leaf=5, max_depth=6, max_features=160 \n",
      "[CV]  n_estimators=400, min_samples_leaf=5, max_depth=6, max_features=160, total=  28.4s\n",
      "[CV] n_estimators=400, min_samples_leaf=5, max_depth=6, max_features=160 \n",
      "[CV]  n_estimators=400, min_samples_leaf=5, max_depth=6, max_features=160, total=  28.3s\n",
      "[CV]  n_estimators=400, min_samples_leaf=5, max_depth=6, max_features=160, total=  28.2s\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=2)]: Done 250 out of 250 | elapsed: 64.9min finished\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.71822365765 {'n_estimators': 200, 'min_samples_leaf': 1, 'max_depth': 10, 'max_features': 160}\n",
      "CPU times: user 46.1 s, sys: 660 ms, total: 46.8 s\n",
      "Wall time: 1h 5min 30s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "params = {'n_estimators': [100, 200, 400, 700, 1500],\n",
    "          'max_depth': [2, 4, 6, 8, 10],\n",
    "          'min_samples_leaf': [1, 3, 5, 8],\n",
    "          'max_features': [2, 5, 10, 20, 40, 80, 160]}\n",
    "gb = GradientBoostingClassifier(random_state = 1)\n",
    "rs_gb = RandomizedSearchCV(estimator = gb, n_jobs = 2, cv = skf, verbose = 2, param_distributions = params, \n",
    "                           scoring = 'accuracy', n_iter = 50, random_state=1)\n",
    "rs_gb.fit(X_train_2, y_train_2)\n",
    "print(rs_gb.best_score_, rs_gb.best_params_)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Честно говоря я уже подустал искать гиперпараметры, поэтому воспользуемся тем, что у нас есть. 4 основных алгоритма: логистическая регрессия(lr), беггинг над ближайшими соседями(bg), случайный лес и градиентный бустинг. Для начала обучим эти классификаторы на всей нашей тренировочной выборке и посмотрим на их accuracy_score на валидационной, так получим предсказания для валидационной выборки, чтобы впоследстии пострить модель второго уровня"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 68,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "lr:  0.675235757654\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=1)]: Done   1 out of   1 | elapsed:    0.1s finished\n",
      "[Parallel(n_jobs=1)]: Done   1 out of   1 | elapsed:  2.2min finished\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "bg:  0.653210179563\n",
      "rf:  0.703914222969\n",
      "gb:  0.717413770831\n",
      "CPU times: user 3min 46s, sys: 12.1 s, total: 3min 58s\n",
      "Wall time: 4min 27s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "rf = RandomForestClassifier(random_state = 1, n_estimators=100, max_depth=30, oob_score=True, \n",
    "                            class_weight='balanced', max_features = 100)\n",
    "gb = GradientBoostingClassifier(random_state = 1, n_estimators = 200, min_samples_leaf = 1, max_depth = 10,\n",
    "                                max_features = 160)\n",
    "lr.fit(X_train_2, y_train_2)\n",
    "print(\"lr: \", lr.score(X_valid, y_valid))\n",
    "bg.fit(X_train_2, y_train_2)\n",
    "print(\"bg: \", bg.score(X_valid, y_valid))\n",
    "rf.fit(X_train_2, y_train_2)\n",
    "print(\"rf: \", rf.score(X_valid, y_valid))\n",
    "gb.fit(X_train_2, y_train_2)\n",
    "print(\"gb: \", gb.score(X_valid, y_valid))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Как видно результаты очень похожи на те, которые были на кросс-валидации. Я не думаю, что мы где-то переобучились. Осталось получить предсказания. Но для классификатора второго порядка нам понадобятся вероятности классов, которые легко получить методом predict_proba. Ещё данный метод называется stackingом. <a href=\"https://stackoverflow.com/questions/39979136/stacking-stacked-generalization-algorithm\">Описание stackingа</a>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 74,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=1)]: Done   1 out of   1 | elapsed:  1.9min finished\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 1min 37s, sys: 18.1 s, total: 1min 55s\n",
      "Wall time: 1min 55s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "pred_lr = lr.predict_proba(X_valid)[:,1]\n",
    "pred_bg = bg.predict_proba(X_valid)[:,1]\n",
    "pred_rf = rf.predict_proba(X_valid)[:,1]\n",
    "pred_gb = gb.predict_proba(X_valid)[:,1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 79,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style>\n",
       "    .dataframe thead tr:only-child th {\n",
       "        text-align: right;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>lr</th>\n",
       "      <th>bg</th>\n",
       "      <th>rf</th>\n",
       "      <th>gb</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0.698427</td>\n",
       "      <td>0.718511</td>\n",
       "      <td>0.710828</td>\n",
       "      <td>0.618152</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0.611940</td>\n",
       "      <td>0.567538</td>\n",
       "      <td>0.546134</td>\n",
       "      <td>0.748490</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0.617781</td>\n",
       "      <td>0.531482</td>\n",
       "      <td>0.391762</td>\n",
       "      <td>0.454219</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0.745690</td>\n",
       "      <td>0.752262</td>\n",
       "      <td>0.628586</td>\n",
       "      <td>0.453879</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0.081654</td>\n",
       "      <td>0.556058</td>\n",
       "      <td>0.129263</td>\n",
       "      <td>0.053711</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "         lr        bg        rf        gb\n",
       "0  0.698427  0.718511  0.710828  0.618152\n",
       "1  0.611940  0.567538  0.546134  0.748490\n",
       "2  0.617781  0.531482  0.391762  0.454219\n",
       "3  0.745690  0.752262  0.628586  0.453879\n",
       "4  0.081654  0.556058  0.129263  0.053711"
      ]
     },
     "execution_count": 79,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "meta_features = [pred_lr, pred_bg, pred_rf, pred_gb]\n",
    "meta_X_valid = pd.DataFrame(meta_features).T\n",
    "meta_X_valid.columns = ['lr', 'bg', 'rf', 'gb']\n",
    "meta_X_valid.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "В качестве мета-алгоритма возьмём логистическую регрессию и посмотрим на ещё результат на кросс-валидации"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 91,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.717284940248\n"
     ]
    }
   ],
   "source": [
    "meta_lr = LogisticRegression(random_state = 1)\n",
    "print(cross_val_score(lr, meta_X_valid, y_valid, scoring='accuracy', cv=skf).mean())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 92,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "LogisticRegression(C=1.0, class_weight=None, dual=False, fit_intercept=True,\n",
       "          intercept_scaling=1, max_iter=100, multi_class='ovr', n_jobs=1,\n",
       "          penalty='l2', random_state=1, solver='liblinear', tol=0.0001,\n",
       "          verbose=0, warm_start=False)"
      ]
     },
     "execution_count": 92,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "meta_lr.fit(meta_X_valid, y_valid)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Теперь осталось только получить предсказания на нужном нам мн-ве. К сожалению все преобразования с признаками придётся выполнить заново, поэтому описания я пропущу"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 97,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 3.68 s, sys: 16 ms, total: 3.7 s\n",
      "Wall time: 4.48 s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "X_test['POLICY_LEN_MONTHS'] = X_test['POLICY_END_MONTH'] - X_test['POLICY_BEGIN_MONTH']\n",
    "X_test['POLICY_LEN_MONTHS'] = [2 if (x == -1) else \n",
    "                                0 if (x == 0) else\n",
    "                                1 for x in X_test['POLICY_LEN_MONTHS'].values]\n",
    "X_test['POLICY_END_SEASON'] = ['summer' if (x >= 6) & (x <= 8)\n",
    "                                else 'autumn' if (x >= 9) & (x <= 11)\n",
    "                                else 'spring' if (x >= 3) & (x <= 5)\n",
    "                                else 'winter' for x in X_test['POLICY_END_MONTH'].values]\n",
    "X_test['POLICY_BRANCH'] = [0 if (x == 'Москва') else 1 for x in X_test['POLICY_BRANCH'].values]\n",
    "X_test['POLICY_MIN_DRIVING_EXPERIENCE'] = [x if (x < 100) else (2016-x) for x in X_test['POLICY_MIN_DRIVING_EXPERIENCE'].values]\n",
    "X_test['AGE_70+'] = [1 if (x >= 70) else 0 for x in X_test['POLICY_MIN_AGE'].values]\n",
    "X_test['AGE_60+'] = [1 if (x >= 60) & (x < 70) else 0 for x in X_test['POLICY_MIN_AGE'].values]\n",
    "X_test['AGE_50+'] = [1 if (x >= 50) & (x < 60) else 0 for x in X_test['POLICY_MIN_AGE'].values]\n",
    "X_test['AGE_38+'] = [1 if (x >= 38) & (x < 50) else 0 for x in X_test['POLICY_MIN_AGE'].values]\n",
    "X_test['AGE_25+'] = [1 if (x >= 25) & (x < 38) else 0 for x in X_test['POLICY_MIN_AGE'].values]\n",
    "X_test['AGE_18+'] = [1 if (x < 25) else 0 for x in X_test['POLICY_MIN_AGE'].values]\n",
    "X_test['EXP_20+'] = [1 if (x >= 20) else 0 for x in X_test['POLICY_MIN_DRIVING_EXPERIENCE'].values]\n",
    "X_test['EXP_10+'] = [1 if (x >= 10) & (x < 20) else 0 for x in X_test['POLICY_MIN_DRIVING_EXPERIENCE'].values]\n",
    "X_test['EXP_5+'] = [1 if (x >= 5) & (x < 10) else 0 for x in X_test['POLICY_MIN_DRIVING_EXPERIENCE'].values]\n",
    "X_test['EXP_LOW'] = [1 if (x < 5) else 0 for x in X_test['POLICY_MIN_DRIVING_EXPERIENCE'].values]\n",
    "X_test_vehicle = dict_vect.transform(X_test[vehicle].fillna('-').T.to_dict().values())\n",
    "X_test['POLICY_YEARS_RENEWED_N'] = [1 if x=='N' else x for x in X_test['POLICY_YEARS_RENEWED_N'].values]\n",
    "X_test_region = reg_vect.transform(X_test[region].fillna('-').T.to_dict().values())\n",
    "X_test_abs = maxabs.transform(X_test[absfeature])\n",
    "X_test_scaled = standard.transform(X_test[scalefeature])\n",
    "X_test['INSURER_GENDER'] = label.transform(X_test['INSURER_GENDER'])\n",
    "X_test_bool = X_test[binaryfeature]\n",
    "X_test_one_hot = onehot.transform(X_test[onehotfeature])\n",
    "X_test_season = season.transform(X_test['POLICY_END_SEASON'])\n",
    "X_test_N = labelbin.transform(X_test['POLICY_CLM_N'])\n",
    "X_test_GLT_N = labelbin.transform(X_test['POLICY_CLM_GLT_N'])\n",
    "X_test_PRV_N = labelbin.transform(X_test['POLICY_PRV_CLM_N'])\n",
    "X_test_PRV_GLT_N = labelbin.transform(X_test['POLICY_PRV_CLM_GLT_N'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 98,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=1)]: Done   1 out of   1 | elapsed:  3.5min finished\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 2min 2s, sys: 12.9 s, total: 2min 15s\n",
      "Wall time: 5min 39s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "X_test_new = scipy.sparse.hstack([X_test_abs, X_test_scaled, X_test_bool, X_test_season, X_test_N,\n",
    "                                   X_test_one_hot, X_test_GLT_N, X_test_PRV_N, X_test_PRV_GLT_N,\n",
    "                                   X_test_region, X_test_vehicle])\n",
    "pred_lr = lr.predict_proba(X_test_new)[:,1]\n",
    "pred_bg = bg.predict_proba(X_test_new)[:,1]\n",
    "pred_rf = rf.predict_proba(X_test_new)[:,1]\n",
    "pred_gb = gb.predict_proba(X_test_new)[:,1]\n",
    "meta_features = [pred_lr, pred_bg, pred_rf, pred_gb]\n",
    "meta_X_test = pd.DataFrame(meta_features).T\n",
    "meta_X_test.columns = ['lr', 'bg', 'rf', 'gb']\n",
    "prediction = meta_lr.predict(meta_X_test)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Tсли я нигде не перепутал, то осталось только записать предсказание в csv файл и получить что-то около <b style=\"color: blue\">0.717284940248</b>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 101,
   "metadata": {},
   "outputs": [],
   "source": [
    "def write_to_submission_file(predicted_labels, out_file,\n",
    "                             target='target', index_label=\"index\"):\n",
    "    predicted_df = pd.DataFrame(predicted_labels,\n",
    "                                index = np.arange(1, predicted_labels.shape[0] + 1),\n",
    "                                columns=[target])\n",
    "    predicted_df.to_csv(out_file, index_label=index_label)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 102,
   "metadata": {},
   "outputs": [],
   "source": [
    "write_to_submission_file(prediction, 'prediction.csv')"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}