test1.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "\n",
    "from sklearn.metrics import roc_auc_score\n",
    "from sklearn.model_selection import KFold\n",
    "from sklearn.model_selection import StratifiedKFold,cross_val_score\n",
    "from lightgbm import LGBMClassifier\n",
    "#from xgboost import XGBClassifier\n",
    "import matplotlib.pyplot as plt\n",
    "import seaborn as sns\n",
    "color = sns.color_palette()\n",
    "sns.set_style('darkgrid')\n",
    "import warnings\n",
    "warnings.filterwarnings('ignore')\n",
    "#from bayes_opt import BayesianOptimization\n",
    "from sklearn.ensemble import (RandomForestClassifier, ExtraTreesClassifier)\n",
    "from sklearn.linear_model import LogisticRegression\n",
    "from sklearn.svm import SVC\n",
    "import gc\n",
    "gc.enable()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Read data and test\n",
      "Shapes :  (307511, 122) (48744, 121)\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "35"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "print('Read data and test')\n",
    "data = pd.read_csv('./input/application_train.csv')\n",
    "test = pd.read_csv('./input/application_test.csv')\n",
    "print('Shapes : ', data.shape, test.shape)\n",
    "y = data['TARGET']\n",
    "ID = test['SK_ID_CURR']\n",
    "del data,test\n",
    "gc.collect()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "rf_oof_train = pd.read_csv('./output/rf_oof_train.csv')\n",
    "rf_oof_test = pd.read_csv('./output/rf_oof_test.csv')\n",
    "et_oof_train = pd.read_csv('./output/et_oof_train.csv')\n",
    "et_oof_test = pd.read_csv('./output/et_oof_test.csv')\n",
    "xgb_oof_train2= pd.read_csv('./output/xgb_oof_train2.csv')\n",
    "xgb_oof_test3 = pd.read_csv('./output/xgb_oof_test3.csv')\n",
    "xgb_oof_train3= pd.read_csv('./output/xgb_oof_train3.csv')\n",
    "xgb_oof_test2 = pd.read_csv('./output/xgb_oof_test2.csv')\n",
    "xgb_oof_train = pd.read_csv('./output/xgb_oof_train.csv')\n",
    "xgb_oof_test = pd.read_csv('./output/xgb_oof_test.csv')\n",
    "xgb_oof_train1 = pd.read_csv('./output/xgb_oof_train1.csv')\n",
    "xgb_oof_test1 = pd.read_csv('./output/xgb_oof_test1.csv')\n",
    "log_oof_train = pd.read_csv('./output/log_oof_train.csv')\n",
    "log_oof_test = pd.read_csv('./output/log_oof_test.csv')\n",
    "lgb_oof_train = pd.read_csv('./output/lgb_oof_train.csv')\n",
    "lgb_oof_test = pd.read_csv('./output/lgb_oof_test.csv')\n",
    "lgb_oof_train2 = pd.read_csv('./output/lgb_oof_train2.csv')\n",
    "lgb_oof_test2 = pd.read_csv('./output/lgb_oof_test2.csv')\n",
    "lgb_oof_train4 = pd.read_csv('./output/lgb_oof_train3.csv')\n",
    "lgb_oof_test4 = pd.read_csv('./output/lgb_oof_test3.csv')\n",
    "lgb_oof_train3 = pd.read_csv('./output/lightGBM_out_of_fold_train_predictions.csv')\n",
    "lgb_oof_test3 = pd.read_csv('./output/lightGBM_out_of_fold_test_predictions.csv')\n",
    "lgb_oof_train5 = pd.read_csv('./output/lightGBM_out_of_fold_train_predictions1.csv')\n",
    "lgb_oof_test5 = pd.read_csv('./output/lightGBM_out_of_fold_test_predictions1.csv')\n",
    "jeepy_oof_train = pd.read_csv('./output/jeepy_oof_train.csv')\n",
    "jeepy_oof_test = pd.read_csv('./output/jeepy_oof_test.csv')\n",
    "nn_oof_train = pd.read_csv('./output/nn_oof_train.csv')\n",
    "nn_oof_test = pd.read_csv('./output/nn_oof_test.csv')\n",
    "WEIGHT_AVERAGE_RANK = pd.read_csv('WEIGHT_AVERAGE_RANK.csv')\n",
    "cv_blending3 = pd.read_csv('cv_blending3.csv')\n",
    "cv7994 = pd.read_csv('stacking-cv-7994.csv')\n",
    "cv7997 = pd.read_csv('cv-7997.csv')\n",
    "hope = pd.read_csv('hope.csv')\n",
    "cat_oof_train = pd.read_csv('./output/catboost_out_of_fold_train_predictions.csv')\n",
    "cat_oof_test = pd.read_csv('./output/catboost_out_of_fold_test_predictions.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "data = pd.DataFrame()\n",
    "data['SK_ID_CURR']=rf_oof_train['ID']\n",
    "data['rf_oof']=rf_oof_train['rf_oof_train']\n",
    "data['log_oof'] =log_oof_train['log_oof_train']\n",
    "data['et_oof'] =et_oof_train['et_oof_train']\n",
    "#data['xgb_oof2']=xgb_oof_train2['xgb_oof_train']\n",
    "data['xgb_oof1']=xgb_oof_train1['xgb_oof_train']\n",
    "data['xgb_oof']=xgb_oof_train['xgb_oof_train']\n",
    "data['lgb_oof2']=lgb_oof_train2['lgb_oof_train']\n",
    "data['lgb_oof4']=lgb_oof_train4['lgb_oof_train']\n",
    "data=data.merge(lgb_oof_train3[['SK_ID_CURR','lightGBM_prediction']],on='SK_ID_CURR',how='left')\n",
    "data=data.merge(lgb_oof_train5[['SK_ID_CURR','lightGBM_prediction']],on='SK_ID_CURR',how='left')\n",
    "data['lgb_oof']=lgb_oof_train['lgb_oof_train']\n",
    "data['jeepy_oof']=jeepy_oof_train['jeepy_oof_train']\n",
    "data=data.merge(cat_oof_train[['SK_ID_CURR','catboost_prediction']],on='SK_ID_CURR',how='left')\n",
    "data['nn_oof'] = nn_oof_train['nn_oof_train']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>SK_ID_CURR</th>\n",
       "      <th>log_oof</th>\n",
       "      <th>et_oof</th>\n",
       "      <th>xgb_oof1</th>\n",
       "      <th>xgb_oof</th>\n",
       "      <th>lgb_oof2</th>\n",
       "      <th>lgb_oof4</th>\n",
       "      <th>lightGBM_prediction_x</th>\n",
       "      <th>lightGBM_prediction_y</th>\n",
       "      <th>lgb_oof</th>\n",
       "      <th>jeepy_oof</th>\n",
       "      <th>catboost_prediction</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>SK_ID_CURR</th>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.003959</td>\n",
       "      <td>0.000268</td>\n",
       "      <td>-0.002113</td>\n",
       "      <td>-0.002435</td>\n",
       "      <td>-0.002461</td>\n",
       "      <td>-0.002202</td>\n",
       "      <td>-0.002167</td>\n",
       "      <td>-0.001616</td>\n",
       "      <td>-0.002525</td>\n",
       "      <td>-0.003281</td>\n",
       "      <td>-0.002764</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>log_oof</th>\n",
       "      <td>0.003959</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.535238</td>\n",
       "      <td>0.278652</td>\n",
       "      <td>0.280225</td>\n",
       "      <td>0.266206</td>\n",
       "      <td>0.264152</td>\n",
       "      <td>0.266791</td>\n",
       "      <td>0.266159</td>\n",
       "      <td>0.265846</td>\n",
       "      <td>0.271590</td>\n",
       "      <td>0.272251</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>et_oof</th>\n",
       "      <td>0.000268</td>\n",
       "      <td>0.535238</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.587025</td>\n",
       "      <td>0.590485</td>\n",
       "      <td>0.551448</td>\n",
       "      <td>0.547204</td>\n",
       "      <td>0.549358</td>\n",
       "      <td>0.550679</td>\n",
       "      <td>0.551380</td>\n",
       "      <td>0.563564</td>\n",
       "      <td>0.553158</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>xgb_oof1</th>\n",
       "      <td>-0.002113</td>\n",
       "      <td>0.278652</td>\n",
       "      <td>0.587025</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.987754</td>\n",
       "      <td>0.966239</td>\n",
       "      <td>0.950940</td>\n",
       "      <td>0.923653</td>\n",
       "      <td>0.920291</td>\n",
       "      <td>0.956389</td>\n",
       "      <td>0.904810</td>\n",
       "      <td>0.904283</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>xgb_oof</th>\n",
       "      <td>-0.002435</td>\n",
       "      <td>0.280225</td>\n",
       "      <td>0.590485</td>\n",
       "      <td>0.987754</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.958286</td>\n",
       "      <td>0.959007</td>\n",
       "      <td>0.924290</td>\n",
       "      <td>0.921749</td>\n",
       "      <td>0.964739</td>\n",
       "      <td>0.903967</td>\n",
       "      <td>0.903156</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>lgb_oof2</th>\n",
       "      <td>-0.002461</td>\n",
       "      <td>0.266206</td>\n",
       "      <td>0.551448</td>\n",
       "      <td>0.966239</td>\n",
       "      <td>0.958286</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.975862</td>\n",
       "      <td>0.932551</td>\n",
       "      <td>0.931456</td>\n",
       "      <td>0.981918</td>\n",
       "      <td>0.907505</td>\n",
       "      <td>0.909989</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>lgb_oof4</th>\n",
       "      <td>-0.002202</td>\n",
       "      <td>0.264152</td>\n",
       "      <td>0.547204</td>\n",
       "      <td>0.950940</td>\n",
       "      <td>0.959007</td>\n",
       "      <td>0.975862</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.935736</td>\n",
       "      <td>0.936304</td>\n",
       "      <td>0.989737</td>\n",
       "      <td>0.899550</td>\n",
       "      <td>0.911643</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>lightGBM_prediction_x</th>\n",
       "      <td>-0.002167</td>\n",
       "      <td>0.266791</td>\n",
       "      <td>0.549358</td>\n",
       "      <td>0.923653</td>\n",
       "      <td>0.924290</td>\n",
       "      <td>0.932551</td>\n",
       "      <td>0.935736</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.965185</td>\n",
       "      <td>0.933160</td>\n",
       "      <td>0.890516</td>\n",
       "      <td>0.935818</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>lightGBM_prediction_y</th>\n",
       "      <td>-0.001616</td>\n",
       "      <td>0.266159</td>\n",
       "      <td>0.550679</td>\n",
       "      <td>0.920291</td>\n",
       "      <td>0.921749</td>\n",
       "      <td>0.931456</td>\n",
       "      <td>0.936304</td>\n",
       "      <td>0.965185</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.932511</td>\n",
       "      <td>0.887361</td>\n",
       "      <td>0.957485</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>lgb_oof</th>\n",
       "      <td>-0.002525</td>\n",
       "      <td>0.265846</td>\n",
       "      <td>0.551380</td>\n",
       "      <td>0.956389</td>\n",
       "      <td>0.964739</td>\n",
       "      <td>0.981918</td>\n",
       "      <td>0.989737</td>\n",
       "      <td>0.933160</td>\n",
       "      <td>0.932511</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.904919</td>\n",
       "      <td>0.907723</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>jeepy_oof</th>\n",
       "      <td>-0.003281</td>\n",
       "      <td>0.271590</td>\n",
       "      <td>0.563564</td>\n",
       "      <td>0.904810</td>\n",
       "      <td>0.903967</td>\n",
       "      <td>0.907505</td>\n",
       "      <td>0.899550</td>\n",
       "      <td>0.890516</td>\n",
       "      <td>0.887361</td>\n",
       "      <td>0.904919</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.872844</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>catboost_prediction</th>\n",
       "      <td>-0.002764</td>\n",
       "      <td>0.272251</td>\n",
       "      <td>0.553158</td>\n",
       "      <td>0.904283</td>\n",
       "      <td>0.903156</td>\n",
       "      <td>0.909989</td>\n",
       "      <td>0.911643</td>\n",
       "      <td>0.935818</td>\n",
       "      <td>0.957485</td>\n",
       "      <td>0.907723</td>\n",
       "      <td>0.872844</td>\n",
       "      <td>1.000000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                       SK_ID_CURR   log_oof    et_oof  xgb_oof1   xgb_oof  \\\n",
       "SK_ID_CURR               1.000000  0.003959  0.000268 -0.002113 -0.002435   \n",
       "log_oof                  0.003959  1.000000  0.535238  0.278652  0.280225   \n",
       "et_oof                   0.000268  0.535238  1.000000  0.587025  0.590485   \n",
       "xgb_oof1                -0.002113  0.278652  0.587025  1.000000  0.987754   \n",
       "xgb_oof                 -0.002435  0.280225  0.590485  0.987754  1.000000   \n",
       "lgb_oof2                -0.002461  0.266206  0.551448  0.966239  0.958286   \n",
       "lgb_oof4                -0.002202  0.264152  0.547204  0.950940  0.959007   \n",
       "lightGBM_prediction_x   -0.002167  0.266791  0.549358  0.923653  0.924290   \n",
       "lightGBM_prediction_y   -0.001616  0.266159  0.550679  0.920291  0.921749   \n",
       "lgb_oof                 -0.002525  0.265846  0.551380  0.956389  0.964739   \n",
       "jeepy_oof               -0.003281  0.271590  0.563564  0.904810  0.903967   \n",
       "catboost_prediction     -0.002764  0.272251  0.553158  0.904283  0.903156   \n",
       "\n",
       "                       lgb_oof2  lgb_oof4  lightGBM_prediction_x  \\\n",
       "SK_ID_CURR            -0.002461 -0.002202              -0.002167   \n",
       "log_oof                0.266206  0.264152               0.266791   \n",
       "et_oof                 0.551448  0.547204               0.549358   \n",
       "xgb_oof1               0.966239  0.950940               0.923653   \n",
       "xgb_oof                0.958286  0.959007               0.924290   \n",
       "lgb_oof2               1.000000  0.975862               0.932551   \n",
       "lgb_oof4               0.975862  1.000000               0.935736   \n",
       "lightGBM_prediction_x  0.932551  0.935736               1.000000   \n",
       "lightGBM_prediction_y  0.931456  0.936304               0.965185   \n",
       "lgb_oof                0.981918  0.989737               0.933160   \n",
       "jeepy_oof              0.907505  0.899550               0.890516   \n",
       "catboost_prediction    0.909989  0.911643               0.935818   \n",
       "\n",
       "                       lightGBM_prediction_y   lgb_oof  jeepy_oof  \\\n",
       "SK_ID_CURR                         -0.001616 -0.002525  -0.003281   \n",
       "log_oof                             0.266159  0.265846   0.271590   \n",
       "et_oof                              0.550679  0.551380   0.563564   \n",
       "xgb_oof1                            0.920291  0.956389   0.904810   \n",
       "xgb_oof                             0.921749  0.964739   0.903967   \n",
       "lgb_oof2                            0.931456  0.981918   0.907505   \n",
       "lgb_oof4                            0.936304  0.989737   0.899550   \n",
       "lightGBM_prediction_x               0.965185  0.933160   0.890516   \n",
       "lightGBM_prediction_y               1.000000  0.932511   0.887361   \n",
       "lgb_oof                             0.932511  1.000000   0.904919   \n",
       "jeepy_oof                           0.887361  0.904919   1.000000   \n",
       "catboost_prediction                 0.957485  0.907723   0.872844   \n",
       "\n",
       "                       catboost_prediction  \n",
       "SK_ID_CURR                       -0.002764  \n",
       "log_oof                           0.272251  \n",
       "et_oof                            0.553158  \n",
       "xgb_oof1                          0.904283  \n",
       "xgb_oof                           0.903156  \n",
       "lgb_oof2                          0.909989  \n",
       "lgb_oof4                          0.911643  \n",
       "lightGBM_prediction_x             0.935818  \n",
       "lightGBM_prediction_y             0.957485  \n",
       "lgb_oof                           0.907723  \n",
       "jeepy_oof                         0.872844  \n",
       "catboost_prediction               1.000000  "
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data.corr()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "test=pd.DataFrame()\n",
    "test['SK_ID_CURR']=rf_oof_test['ID']\n",
    "test['rf_oof']=rf_oof_test['rf_oof_test']\n",
    "test['log_oof'] =log_oof_test['log_oof_test']\n",
    "test['et_oof'] =et_oof_test['et_oof_test']\n",
    "#test['xgb_oof2']=xgb_oof_test2['xgb_oof_test']\n",
    "test['xgb_oof1']=xgb_oof_test1['xgb_oof_test']\n",
    "test['xgb_oof']=xgb_oof_test['xgb_oof_test']\n",
    "test['lgb_oof2']=lgb_oof_test2['lgb_oof_test']\n",
    "test['lgb_oof4']=lgb_oof_test4['lgb_oof_test']\n",
    "test['lightGBM_prediction_x']=lgb_oof_test3['lightGBM_prediction']\n",
    "test['lightGBM_prediction_y']=lgb_oof_test5['lightGBM_prediction']\n",
    "test['lgb_oof']=lgb_oof_test['lgb_oof_test'] \n",
    "test['jeepy_oof']=jeepy_oof_test['jeepy_oof_test']\n",
    "test['catboost_prediction']=cat_oof_test['catboost_prediction']\n",
    "test['nn_oof'] = nn_oof_test['nn_oof_test']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>SK_ID_CURR</th>\n",
       "      <th>rf_oof</th>\n",
       "      <th>log_oof</th>\n",
       "      <th>et_oof</th>\n",
       "      <th>xgb_oof1</th>\n",
       "      <th>xgb_oof</th>\n",
       "      <th>lgb_oof2</th>\n",
       "      <th>lgb_oof4</th>\n",
       "      <th>lightGBM_prediction_x</th>\n",
       "      <th>lightGBM_prediction_y</th>\n",
       "      <th>lgb_oof</th>\n",
       "      <th>jeepy_oof</th>\n",
       "      <th>catboost_prediction</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>SK_ID_CURR</th>\n",
       "      <td>1.000000</td>\n",
       "      <td>-0.004862</td>\n",
       "      <td>0.000273</td>\n",
       "      <td>-0.000241</td>\n",
       "      <td>-0.002525</td>\n",
       "      <td>-0.002393</td>\n",
       "      <td>-0.003303</td>\n",
       "      <td>-0.003903</td>\n",
       "      <td>-0.002444</td>\n",
       "      <td>-0.002891</td>\n",
       "      <td>-0.002951</td>\n",
       "      <td>-0.005172</td>\n",
       "      <td>-0.002228</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>rf_oof</th>\n",
       "      <td>-0.004862</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.417734</td>\n",
       "      <td>0.768712</td>\n",
       "      <td>0.789202</td>\n",
       "      <td>0.790716</td>\n",
       "      <td>0.764490</td>\n",
       "      <td>0.756224</td>\n",
       "      <td>0.749374</td>\n",
       "      <td>0.756615</td>\n",
       "      <td>0.760640</td>\n",
       "      <td>0.740642</td>\n",
       "      <td>0.755712</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>log_oof</th>\n",
       "      <td>0.000273</td>\n",
       "      <td>0.417734</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.476745</td>\n",
       "      <td>0.234507</td>\n",
       "      <td>0.233786</td>\n",
       "      <td>0.228577</td>\n",
       "      <td>0.224938</td>\n",
       "      <td>0.233158</td>\n",
       "      <td>0.234935</td>\n",
       "      <td>0.226321</td>\n",
       "      <td>0.244093</td>\n",
       "      <td>0.236859</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>et_oof</th>\n",
       "      <td>-0.000241</td>\n",
       "      <td>0.768712</td>\n",
       "      <td>0.476745</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.565494</td>\n",
       "      <td>0.564737</td>\n",
       "      <td>0.533201</td>\n",
       "      <td>0.525619</td>\n",
       "      <td>0.522422</td>\n",
       "      <td>0.526322</td>\n",
       "      <td>0.528645</td>\n",
       "      <td>0.548017</td>\n",
       "      <td>0.521944</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>xgb_oof1</th>\n",
       "      <td>-0.002525</td>\n",
       "      <td>0.789202</td>\n",
       "      <td>0.234507</td>\n",
       "      <td>0.565494</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.990767</td>\n",
       "      <td>0.975087</td>\n",
       "      <td>0.957694</td>\n",
       "      <td>0.928580</td>\n",
       "      <td>0.924125</td>\n",
       "      <td>0.963934</td>\n",
       "      <td>0.906708</td>\n",
       "      <td>0.903174</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>xgb_oof</th>\n",
       "      <td>-0.002393</td>\n",
       "      <td>0.790716</td>\n",
       "      <td>0.233786</td>\n",
       "      <td>0.564737</td>\n",
       "      <td>0.990767</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.968170</td>\n",
       "      <td>0.966622</td>\n",
       "      <td>0.930501</td>\n",
       "      <td>0.926724</td>\n",
       "      <td>0.973282</td>\n",
       "      <td>0.906920</td>\n",
       "      <td>0.901975</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>lgb_oof2</th>\n",
       "      <td>-0.003303</td>\n",
       "      <td>0.764490</td>\n",
       "      <td>0.228577</td>\n",
       "      <td>0.533201</td>\n",
       "      <td>0.975087</td>\n",
       "      <td>0.968170</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.980739</td>\n",
       "      <td>0.935936</td>\n",
       "      <td>0.934057</td>\n",
       "      <td>0.986808</td>\n",
       "      <td>0.907226</td>\n",
       "      <td>0.905559</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>lgb_oof4</th>\n",
       "      <td>-0.003903</td>\n",
       "      <td>0.756224</td>\n",
       "      <td>0.224938</td>\n",
       "      <td>0.525619</td>\n",
       "      <td>0.957694</td>\n",
       "      <td>0.966622</td>\n",
       "      <td>0.980739</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.938209</td>\n",
       "      <td>0.939481</td>\n",
       "      <td>0.992287</td>\n",
       "      <td>0.898619</td>\n",
       "      <td>0.906271</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>lightGBM_prediction_x</th>\n",
       "      <td>-0.002444</td>\n",
       "      <td>0.749374</td>\n",
       "      <td>0.233158</td>\n",
       "      <td>0.522422</td>\n",
       "      <td>0.928580</td>\n",
       "      <td>0.930501</td>\n",
       "      <td>0.935936</td>\n",
       "      <td>0.938209</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.960780</td>\n",
       "      <td>0.936600</td>\n",
       "      <td>0.888391</td>\n",
       "      <td>0.925776</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>lightGBM_prediction_y</th>\n",
       "      <td>-0.002891</td>\n",
       "      <td>0.756615</td>\n",
       "      <td>0.234935</td>\n",
       "      <td>0.526322</td>\n",
       "      <td>0.924125</td>\n",
       "      <td>0.926724</td>\n",
       "      <td>0.934057</td>\n",
       "      <td>0.939481</td>\n",
       "      <td>0.960780</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.934999</td>\n",
       "      <td>0.882710</td>\n",
       "      <td>0.949820</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>lgb_oof</th>\n",
       "      <td>-0.002951</td>\n",
       "      <td>0.760640</td>\n",
       "      <td>0.226321</td>\n",
       "      <td>0.528645</td>\n",
       "      <td>0.963934</td>\n",
       "      <td>0.973282</td>\n",
       "      <td>0.986808</td>\n",
       "      <td>0.992287</td>\n",
       "      <td>0.936600</td>\n",
       "      <td>0.934999</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.905294</td>\n",
       "      <td>0.901595</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>jeepy_oof</th>\n",
       "      <td>-0.005172</td>\n",
       "      <td>0.740642</td>\n",
       "      <td>0.244093</td>\n",
       "      <td>0.548017</td>\n",
       "      <td>0.906708</td>\n",
       "      <td>0.906920</td>\n",
       "      <td>0.907226</td>\n",
       "      <td>0.898619</td>\n",
       "      <td>0.888391</td>\n",
       "      <td>0.882710</td>\n",
       "      <td>0.905294</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.864336</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>catboost_prediction</th>\n",
       "      <td>-0.002228</td>\n",
       "      <td>0.755712</td>\n",
       "      <td>0.236859</td>\n",
       "      <td>0.521944</td>\n",
       "      <td>0.903174</td>\n",
       "      <td>0.901975</td>\n",
       "      <td>0.905559</td>\n",
       "      <td>0.906271</td>\n",
       "      <td>0.925776</td>\n",
       "      <td>0.949820</td>\n",
       "      <td>0.901595</td>\n",
       "      <td>0.864336</td>\n",
       "      <td>1.000000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                       SK_ID_CURR    rf_oof   log_oof    et_oof  xgb_oof1  \\\n",
       "SK_ID_CURR               1.000000 -0.004862  0.000273 -0.000241 -0.002525   \n",
       "rf_oof                  -0.004862  1.000000  0.417734  0.768712  0.789202   \n",
       "log_oof                  0.000273  0.417734  1.000000  0.476745  0.234507   \n",
       "et_oof                  -0.000241  0.768712  0.476745  1.000000  0.565494   \n",
       "xgb_oof1                -0.002525  0.789202  0.234507  0.565494  1.000000   \n",
       "xgb_oof                 -0.002393  0.790716  0.233786  0.564737  0.990767   \n",
       "lgb_oof2                -0.003303  0.764490  0.228577  0.533201  0.975087   \n",
       "lgb_oof4                -0.003903  0.756224  0.224938  0.525619  0.957694   \n",
       "lightGBM_prediction_x   -0.002444  0.749374  0.233158  0.522422  0.928580   \n",
       "lightGBM_prediction_y   -0.002891  0.756615  0.234935  0.526322  0.924125   \n",
       "lgb_oof                 -0.002951  0.760640  0.226321  0.528645  0.963934   \n",
       "jeepy_oof               -0.005172  0.740642  0.244093  0.548017  0.906708   \n",
       "catboost_prediction     -0.002228  0.755712  0.236859  0.521944  0.903174   \n",
       "\n",
       "                        xgb_oof  lgb_oof2  lgb_oof4  lightGBM_prediction_x  \\\n",
       "SK_ID_CURR            -0.002393 -0.003303 -0.003903              -0.002444   \n",
       "rf_oof                 0.790716  0.764490  0.756224               0.749374   \n",
       "log_oof                0.233786  0.228577  0.224938               0.233158   \n",
       "et_oof                 0.564737  0.533201  0.525619               0.522422   \n",
       "xgb_oof1               0.990767  0.975087  0.957694               0.928580   \n",
       "xgb_oof                1.000000  0.968170  0.966622               0.930501   \n",
       "lgb_oof2               0.968170  1.000000  0.980739               0.935936   \n",
       "lgb_oof4               0.966622  0.980739  1.000000               0.938209   \n",
       "lightGBM_prediction_x  0.930501  0.935936  0.938209               1.000000   \n",
       "lightGBM_prediction_y  0.926724  0.934057  0.939481               0.960780   \n",
       "lgb_oof                0.973282  0.986808  0.992287               0.936600   \n",
       "jeepy_oof              0.906920  0.907226  0.898619               0.888391   \n",
       "catboost_prediction    0.901975  0.905559  0.906271               0.925776   \n",
       "\n",
       "                       lightGBM_prediction_y   lgb_oof  jeepy_oof  \\\n",
       "SK_ID_CURR                         -0.002891 -0.002951  -0.005172   \n",
       "rf_oof                              0.756615  0.760640   0.740642   \n",
       "log_oof                             0.234935  0.226321   0.244093   \n",
       "et_oof                              0.526322  0.528645   0.548017   \n",
       "xgb_oof1                            0.924125  0.963934   0.906708   \n",
       "xgb_oof                             0.926724  0.973282   0.906920   \n",
       "lgb_oof2                            0.934057  0.986808   0.907226   \n",
       "lgb_oof4                            0.939481  0.992287   0.898619   \n",
       "lightGBM_prediction_x               0.960780  0.936600   0.888391   \n",
       "lightGBM_prediction_y               1.000000  0.934999   0.882710   \n",
       "lgb_oof                             0.934999  1.000000   0.905294   \n",
       "jeepy_oof                           0.882710  0.905294   1.000000   \n",
       "catboost_prediction                 0.949820  0.901595   0.864336   \n",
       "\n",
       "                       catboost_prediction  \n",
       "SK_ID_CURR                       -0.002228  \n",
       "rf_oof                            0.755712  \n",
       "log_oof                           0.236859  \n",
       "et_oof                            0.521944  \n",
       "xgb_oof1                          0.903174  \n",
       "xgb_oof                           0.901975  \n",
       "lgb_oof2                          0.905559  \n",
       "lgb_oof4                          0.906271  \n",
       "lightGBM_prediction_x             0.925776  \n",
       "lightGBM_prediction_y             0.949820  \n",
       "lgb_oof                           0.901595  \n",
       "jeepy_oof                         0.864336  \n",
       "catboost_prediction               1.000000  "
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test.corr()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.49902428137948923"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "roc_auc_score(y, lgb_oof_train3['lightGBM_prediction'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "submission = pd.DataFrame()\n",
    "submission['SK_ID_CURR'] = ID\n",
    "submission['TARGET'] = cv7997.TARGET*0.25+cv_blending3.TARGET*0.75\n",
    "submission.to_csv('cv_blending24.csv', index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [],
   "source": [
    "submission = pd.DataFrame()\n",
    "submission['SK_ID_CURR'] = ID\n",
    "submission['TARGET'] = sub_preds\n",
    "submission.to_csv('cv-7997.csv', index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [],
   "source": [
    "folds = KFold(n_splits=5, shuffle=True, random_state=1700)\n",
    "sub_preds = np.zeros(test.shape[0])\n",
    "feats_trn = [f for f in data.columns if f != 'SK_ID_CURR']\n",
    "feats_test = [f for f in test.columns if f != 'SK_ID_CURR']\n",
    "val_auc = None"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training until validation scores don't improve for 300 rounds.\n",
      "[100]\ttraining's auc: 0.802388\tvalid_1's auc: 0.800204\n",
      "[200]\ttraining's auc: 0.803723\tvalid_1's auc: 0.800418\n",
      "[300]\ttraining's auc: 0.805125\tvalid_1's auc: 0.800537\n",
      "[400]\ttraining's auc: 0.806514\tvalid_1's auc: 0.800548\n",
      "[500]\ttraining's auc: 0.807664\tvalid_1's auc: 0.800501\n",
      "[600]\ttraining's auc: 0.808333\tvalid_1's auc: 0.800446\n",
      "Early stopping, best iteration is:\n",
      "[329]\ttraining's auc: 0.80553\tvalid_1's auc: 0.80059\n",
      "Fold  1 AUC : 0.800590\n",
      "Training until validation scores don't improve for 300 rounds.\n",
      "[100]\ttraining's auc: 0.80155\tvalid_1's auc: 0.803941\n",
      "[200]\ttraining's auc: 0.802908\tvalid_1's auc: 0.804013\n",
      "[300]\ttraining's auc: 0.804373\tvalid_1's auc: 0.804104\n",
      "[400]\ttraining's auc: 0.805851\tvalid_1's auc: 0.804167\n",
      "[500]\ttraining's auc: 0.807024\tvalid_1's auc: 0.804093\n",
      "[600]\ttraining's auc: 0.807761\tvalid_1's auc: 0.804048\n",
      "Early stopping, best iteration is:\n",
      "[346]\ttraining's auc: 0.805041\tvalid_1's auc: 0.804187\n",
      "Fold  2 AUC : 0.804187\n",
      "Training until validation scores don't improve for 300 rounds.\n",
      "[100]\ttraining's auc: 0.803169\tvalid_1's auc: 0.797649\n",
      "[200]\ttraining's auc: 0.804438\tvalid_1's auc: 0.79799\n",
      "[300]\ttraining's auc: 0.805911\tvalid_1's auc: 0.798148\n",
      "[400]\ttraining's auc: 0.807283\tvalid_1's auc: 0.798178\n",
      "[500]\ttraining's auc: 0.808256\tvalid_1's auc: 0.798176\n",
      "[600]\ttraining's auc: 0.809059\tvalid_1's auc: 0.798183\n",
      "Early stopping, best iteration is:\n",
      "[356]\ttraining's auc: 0.806709\tvalid_1's auc: 0.798221\n",
      "Fold  3 AUC : 0.798221\n",
      "Training until validation scores don't improve for 300 rounds.\n",
      "[100]\ttraining's auc: 0.802561\tvalid_1's auc: 0.799714\n",
      "[200]\ttraining's auc: 0.803934\tvalid_1's auc: 0.799929\n",
      "[300]\ttraining's auc: 0.805346\tvalid_1's auc: 0.800023\n",
      "[400]\ttraining's auc: 0.8067\tvalid_1's auc: 0.80012\n",
      "[500]\ttraining's auc: 0.807815\tvalid_1's auc: 0.800031\n",
      "[600]\ttraining's auc: 0.808658\tvalid_1's auc: 0.799945\n",
      "Early stopping, best iteration is:\n",
      "[371]\ttraining's auc: 0.806305\tvalid_1's auc: 0.800148\n",
      "Fold  4 AUC : 0.800148\n",
      "Training until validation scores don't improve for 300 rounds.\n",
      "[100]\ttraining's auc: 0.803694\tvalid_1's auc: 0.795177\n",
      "[200]\ttraining's auc: 0.805061\tvalid_1's auc: 0.795307\n",
      "[300]\ttraining's auc: 0.806522\tvalid_1's auc: 0.79541\n",
      "[400]\ttraining's auc: 0.807834\tvalid_1's auc: 0.7954\n",
      "[500]\ttraining's auc: 0.808841\tvalid_1's auc: 0.795274\n",
      "Early stopping, best iteration is:\n",
      "[299]\ttraining's auc: 0.806505\tvalid_1's auc: 0.795419\n",
      "Fold  5 AUC : 0.795419\n",
      "Full AUC score 0.799713\n"
     ]
    }
   ],
   "source": [
    "for n_fold, (trn_idx, val_idx) in enumerate(folds.split(data)):\n",
    "    trn_x, trn_y = data[feats_trn].iloc[trn_idx], y.iloc[trn_idx]\n",
    "    val_x, val_y = data[feats_trn].iloc[val_idx], y.iloc[val_idx] \n",
    "    clf = LGBMClassifier(\n",
    "        n_estimators=10000,\n",
    "        learning_rate=0.01,\n",
    "        num_leaves=30,\n",
    "        colsample_bytree=0.5,\n",
    "        subsample=0.1,\n",
    "        max_depth=6,\n",
    "        reg_alpha=1,\n",
    "        reg_lambda=1,\n",
    "        min_split_gain=0.1,\n",
    "        min_child_weight=30,\n",
    "        random_state=1700,\n",
    "    )\n",
    "    clf.fit(trn_x, trn_y, \n",
    "            eval_set= [(trn_x, trn_y), (val_x, val_y)], \n",
    "            eval_metric='auc', verbose=100, early_stopping_rounds=300  #30\n",
    "           )\n",
    "    oof_preds = clf.predict_proba(val_x)[:, 1]\n",
    "    sub_preds += clf.predict_proba(test[feats_test])[:, 1] / folds.n_splits\n",
    "    print('Fold %2d AUC : %.6f' % (n_fold + 1, roc_auc_score(val_y, oof_preds)))\n",
    "    if n_fold == 0:\n",
    "        val_auc = roc_auc_score(val_y, oof_preds)\n",
    "    else:\n",
    "        val_auc += roc_auc_score(val_y, oof_preds)\n",
    "    del trn_x, trn_y, val_x, val_y\n",
    "    gc.collect()\n",
    "    \n",
    "print('Full AUC score %.6f' % (val_auc/folds.n_splits) )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "folds = KFold(n_splits=5, shuffle=True, random_state=1700)\n",
    "sub_preds = np.zeros(test.shape[0])\n",
    "feats_trn = [f for f in data.columns if f != 'SK_ID_CURR']\n",
    "feats_test = [f for f in test.columns if f != 'SK_ID_CURR']\n",
    "val_auc = None"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training until validation scores don't improve for 300 rounds.\n",
      "[100]\ttraining's auc: 0.802597\tvalid_1's auc: 0.800274\n",
      "[200]\ttraining's auc: 0.804115\tvalid_1's auc: 0.800474\n",
      "[300]\ttraining's auc: 0.805705\tvalid_1's auc: 0.800665\n",
      "[400]\ttraining's auc: 0.807331\tvalid_1's auc: 0.800687\n",
      "[500]\ttraining's auc: 0.808838\tvalid_1's auc: 0.80065\n",
      "[600]\ttraining's auc: 0.809892\tvalid_1's auc: 0.800612\n",
      "Early stopping, best iteration is:\n",
      "[367]\ttraining's auc: 0.806764\tvalid_1's auc: 0.80073\n",
      "Fold  1 AUC : 0.800730\n",
      "Training until validation scores don't improve for 300 rounds.\n",
      "[100]\ttraining's auc: 0.801653\tvalid_1's auc: 0.804028\n",
      "[200]\ttraining's auc: 0.803146\tvalid_1's auc: 0.804152\n",
      "[300]\ttraining's auc: 0.804803\tvalid_1's auc: 0.804257\n",
      "[400]\ttraining's auc: 0.806542\tvalid_1's auc: 0.804262\n",
      "[500]\ttraining's auc: 0.808189\tvalid_1's auc: 0.804104\n",
      "[600]\ttraining's auc: 0.809419\tvalid_1's auc: 0.803978\n",
      "Early stopping, best iteration is:\n",
      "[371]\ttraining's auc: 0.806026\tvalid_1's auc: 0.804287\n",
      "Fold  2 AUC : 0.804287\n",
      "Training until validation scores don't improve for 300 rounds.\n",
      "[100]\ttraining's auc: 0.803245\tvalid_1's auc: 0.79776\n",
      "[200]\ttraining's auc: 0.804801\tvalid_1's auc: 0.797932\n",
      "[300]\ttraining's auc: 0.806428\tvalid_1's auc: 0.797983\n",
      "[400]\ttraining's auc: 0.808117\tvalid_1's auc: 0.798052\n",
      "[500]\ttraining's auc: 0.809678\tvalid_1's auc: 0.79805\n",
      "[600]\ttraining's auc: 0.810808\tvalid_1's auc: 0.798011\n",
      "[700]\ttraining's auc: 0.811699\tvalid_1's auc: 0.797962\n",
      "Early stopping, best iteration is:\n",
      "[471]\ttraining's auc: 0.809271\tvalid_1's auc: 0.798068\n",
      "Fold  3 AUC : 0.798068\n",
      "Training until validation scores don't improve for 300 rounds.\n",
      "[100]\ttraining's auc: 0.802699\tvalid_1's auc: 0.799732\n",
      "[200]\ttraining's auc: 0.804163\tvalid_1's auc: 0.799962\n",
      "[300]\ttraining's auc: 0.805793\tvalid_1's auc: 0.800061\n",
      "[400]\ttraining's auc: 0.807399\tvalid_1's auc: 0.800087\n",
      "[500]\ttraining's auc: 0.808959\tvalid_1's auc: 0.800009\n",
      "[600]\ttraining's auc: 0.81009\tvalid_1's auc: 0.79991\n",
      "Early stopping, best iteration is:\n",
      "[345]\ttraining's auc: 0.806493\tvalid_1's auc: 0.800116\n",
      "Fold  4 AUC : 0.800116\n",
      "Training until validation scores don't improve for 300 rounds.\n",
      "[100]\ttraining's auc: 0.803855\tvalid_1's auc: 0.795365\n",
      "[200]\ttraining's auc: 0.805373\tvalid_1's auc: 0.795461\n",
      "[300]\ttraining's auc: 0.807017\tvalid_1's auc: 0.795516\n",
      "[400]\ttraining's auc: 0.808586\tvalid_1's auc: 0.795494\n",
      "[500]\ttraining's auc: 0.810045\tvalid_1's auc: 0.795441\n",
      "[600]\ttraining's auc: 0.811072\tvalid_1's auc: 0.79533\n",
      "Early stopping, best iteration is:\n",
      "[321]\ttraining's auc: 0.807345\tvalid_1's auc: 0.795551\n",
      "Fold  5 AUC : 0.795551\n",
      "Full AUC score 0.799750\n"
     ]
    }
   ],
   "source": [
    "for n_fold, (trn_idx, val_idx) in enumerate(folds.split(data)):\n",
    "    trn_x, trn_y = data[feats_trn].iloc[trn_idx], y.iloc[trn_idx]\n",
    "    val_x, val_y = data[feats_trn].iloc[val_idx], y.iloc[val_idx] \n",
    "    clf = LGBMClassifier(\n",
    "        n_estimators=10000,\n",
    "        learning_rate=0.01,\n",
    "        num_leaves=30,\n",
    "        colsample_bytree=0.5,\n",
    "        subsample=0.1,\n",
    "        max_depth=8,\n",
    "        reg_alpha=1,\n",
    "        reg_lambda=1,\n",
    "        min_split_gain=0.1,\n",
    "        min_child_weight=30,\n",
    "        random_state=1700,\n",
    "    )\n",
    "    clf.fit(trn_x, trn_y, \n",
    "            eval_set= [(trn_x, trn_y), (val_x, val_y)], \n",
    "            eval_metric='auc', verbose=100, early_stopping_rounds=300  #30\n",
    "           )\n",
    "    oof_preds = clf.predict_proba(val_x)[:, 1]\n",
    "    sub_preds += clf.predict_proba(test[feats_test])[:, 1] / folds.n_splits\n",
    "    print('Fold %2d AUC : %.6f' % (n_fold + 1, roc_auc_score(val_y, oof_preds)))\n",
    "    if n_fold == 0:\n",
    "        val_auc = roc_auc_score(val_y, oof_preds)\n",
    "    else:\n",
    "        val_auc += roc_auc_score(val_y, oof_preds)\n",
    "    del trn_x, trn_y, val_x, val_y\n",
    "    gc.collect()\n",
    "    \n",
    "print('Full AUC score %.6f' % (val_auc/folds.n_splits) )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}