lightgbm_tuning.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "\n",
    "from utils import *\n",
    "\n",
    "from hyperopt import STATUS_OK, Trials, fmin, hp, tpe\n",
    "import lightgbm as lgb\n",
    "\n",
    "%matplotlib inline\n",
    "data_path = 'data/'\n",
    "seed=1204\n",
    "\n",
    "submission_path=data_path+'submission/'\n",
    "fold_path = 'fold_data/'\n",
    "\n",
    "\n",
    "cv_loss_list=[]\n",
    "n_iteration_list=[]\n",
    "def score(params):\n",
    "    print(\"Training with params: \")\n",
    "    print(params)\n",
    "    cv_losses=[]\n",
    "    cv_iteration=[]\n",
    "    for (train_idx,val_idx) in cv:\n",
    "        cv_train = X.iloc[train_idx]\n",
    "        cv_val = X.iloc[val_idx]\n",
    "        cv_y_train = y[train_idx]\n",
    "        cv_y_val = y[val_idx]\n",
    "        lgb_model = lgb.train(params, lgb.Dataset(cv_train, label=cv_y_train), 2000, \n",
    "                          lgb.Dataset(cv_val, label=cv_y_val), verbose_eval=False, \n",
    "                          early_stopping_rounds=100)\n",
    "       \n",
    "        train_pred = lgb_model.predict(cv_train,lgb_model.best_iteration+1)\n",
    "        val_pred = lgb_model.predict(cv_val,lgb_model.best_iteration+1)\n",
    "        \n",
    "        val_loss = root_mean_squared_error(cv_y_val,val_pred)\n",
    "        train_loss = root_mean_squared_error(cv_y_train,train_pred)\n",
    "        print('Train RMSE: {}. Val RMSE: {}'.format(train_loss,val_loss))\n",
    "        print('Best iteration: {}'.format(lgb_model.best_iteration))\n",
    "        cv_losses.append(val_loss)\n",
    "        cv_iteration.append(lgb_model.best_iteration)\n",
    "    print('6 fold results: {}'.format(cv_losses))\n",
    "    cv_loss_list.append(cv_losses)\n",
    "    n_iteration_list.append(cv_iteration)\n",
    "    \n",
    "    mean_cv_loss = np.mean(cv_losses)\n",
    "    print('Average iterations: {}'.format(np.mean(cv_iteration)))\n",
    "    print(\"Mean Cross Validation RMSE: {}\\n\".format(mean_cv_loss))\n",
    "    return {'loss': mean_cv_loss, 'status': STATUS_OK}\n",
    "\n",
    "def optimize(space,seed=seed,max_evals=5):\n",
    "    \n",
    "    best = fmin(score, space, algo=tpe.suggest, \n",
    "        # trials=trials, \n",
    "        max_evals=max_evals)\n",
    "    return best\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 68,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Full dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "all_data = get_all_data(data_path,'new_sales_lag_after12.pickle')\n",
    "X,y = get_X_y(all_data,33)\n",
    "X.drop('date_block_num',axis=1,inplace=True)\n",
    "cv = get_cv_idxs(all_data,28,33)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training with params: \n",
      "{'colsample_bytree': 0.55, 'learning_rate': 0.42500000000000004, 'max_depth': 9, 'metric': 'rmse', 'min_data_in_leaf': 5, 'objective': 'regression', 'seed': 1204, 'subsample': 0.6000000000000001}\n",
      "6 fold results: [0.8559371056198465, 0.7860583345639058, 0.7213393347250958, 0.7782868728041924, 0.8867479825787459, 0.9692538982923226]\n",
      "Mean Cross Validation RMSE: 0.8329372547640181\n",
      "\n",
      "Training with params: \n",
      "{'colsample_bytree': 0.9500000000000001, 'learning_rate': 0.4, 'max_depth': 4, 'metric': 'rmse', 'min_data_in_leaf': 5, 'objective': 'regression', 'seed': 1204, 'subsample': 0.9500000000000001}\n",
      "6 fold results: [0.8613602362805769, 0.7834800365274855, 0.7121466067967884, 0.7877051510186674, 0.8963612784098204, 0.9493900617061085]\n",
      "Mean Cross Validation RMSE: 0.8317405617899079\n",
      "\n",
      "Training with params: \n",
      "{'colsample_bytree': 0.7000000000000001, 'learning_rate': 0.375, 'max_depth': 8, 'metric': 'rmse', 'min_data_in_leaf': 21, 'objective': 'regression', 'seed': 1204, 'subsample': 0.8}\n",
      "6 fold results: [0.8426225728466852, 0.7814816234185451, 0.7093598503581973, 0.7837867352954468, 0.8887731761826249, 0.9216400339767573]\n",
      "Mean Cross Validation RMSE: 0.8212773320130428\n",
      "\n",
      "Training with params: \n",
      "{'colsample_bytree': 0.8500000000000001, 'learning_rate': 0.225, 'max_depth': 14, 'metric': 'rmse', 'min_data_in_leaf': 26, 'objective': 'regression', 'seed': 1204, 'subsample': 0.65}\n",
      "6 fold results: [0.8280581539330364, 0.77803150414228, 0.7116329983730894, 0.7791228593296401, 0.8868469001669173, 0.9223395147180259]\n",
      "Mean Cross Validation RMSE: 0.8176719884438315\n",
      "\n",
      "Training with params: \n",
      "{'colsample_bytree': 0.7000000000000001, 'learning_rate': 0.1, 'max_depth': 5, 'metric': 'rmse', 'min_data_in_leaf': 29, 'objective': 'regression', 'seed': 1204, 'subsample': 1.0}\n",
      "6 fold results: [0.8380084932898015, 0.7835187184967213, 0.7077588676237395, 0.7714714290666131, 0.9007191909781279, 0.9277174001329183]\n",
      "Mean Cross Validation RMSE: 0.8215323499313203\n",
      "\n",
      "Training with params: \n",
      "{'colsample_bytree': 1.0, 'learning_rate': 0.45, 'max_depth': 4, 'metric': 'rmse', 'min_data_in_leaf': 25, 'objective': 'regression', 'seed': 1204, 'subsample': 0.55}\n",
      "6 fold results: [0.8562734334559194, 0.7796947394223758, 0.7240957453811503, 0.7872892867688494, 0.9117303195639154, 0.9500413314993144]\n",
      "Mean Cross Validation RMSE: 0.8348541426819208\n",
      "\n",
      "Training with params: \n",
      "{'colsample_bytree': 1.0, 'learning_rate': 0.05, 'max_depth': 11, 'metric': 'rmse', 'min_data_in_leaf': 26, 'objective': 'regression', 'seed': 1204, 'subsample': 0.9500000000000001}\n",
      "6 fold results: [0.829310609140302, 0.776011167951953, 0.7039583725286682, 0.773664577135017, 0.8898137857075452, 0.9085074016935689]\n",
      "Mean Cross Validation RMSE: 0.8135443190261759\n",
      "\n",
      "Training with params: \n",
      "{'colsample_bytree': 0.8, 'learning_rate': 0.07500000000000001, 'max_depth': 6, 'metric': 'rmse', 'min_data_in_leaf': 24, 'objective': 'regression', 'seed': 1204, 'subsample': 0.65}\n",
      "6 fold results: [0.8406134181746827, 0.7789935235338319, 0.7044449747448568, 0.7629086902415385, 0.8868527435258295, 0.9393524664602833]\n",
      "Mean Cross Validation RMSE: 0.8188609694468371\n",
      "\n",
      "Training with params: \n",
      "{'colsample_bytree': 0.8500000000000001, 'learning_rate': 0.1, 'max_depth': 14, 'metric': 'rmse', 'min_data_in_leaf': 17, 'objective': 'regression', 'seed': 1204, 'subsample': 0.55}\n",
      "6 fold results: [0.8431420052853705, 0.7788473176156735, 0.702512551422099, 0.762886478488595, 0.8815624015093428, 0.9300895186010217]\n",
      "Mean Cross Validation RMSE: 0.8165067121536836\n",
      "\n",
      "Training with params: \n",
      "{'colsample_bytree': 0.6000000000000001, 'learning_rate': 0.42500000000000004, 'max_depth': 7, 'metric': 'rmse', 'min_data_in_leaf': 28, 'objective': 'regression', 'seed': 1204, 'subsample': 0.65}\n",
      "6 fold results: [0.858337218952421, 0.7795748629283729, 0.7111829505073198, 0.7784847052886883, 0.8850272612848872, 0.9158970773885631]\n",
      "Mean Cross Validation RMSE: 0.8214173460583755\n",
      "\n",
      "Training with params: \n",
      "{'colsample_bytree': 0.9500000000000001, 'learning_rate': 0.05, 'max_depth': 11, 'metric': 'rmse', 'min_data_in_leaf': 18, 'objective': 'regression', 'seed': 1204, 'subsample': 0.65}\n",
      "6 fold results: [0.8447424060371859, 0.7717715002249375, 0.7021743383106697, 0.7673125529088459, 0.8846218321136061, 0.9119173274652521]\n",
      "Mean Cross Validation RMSE: 0.8137566595100828\n",
      "\n",
      "Training with params: \n",
      "{'colsample_bytree': 0.9, 'learning_rate': 0.42500000000000004, 'max_depth': 7, 'metric': 'rmse', 'min_data_in_leaf': 8, 'objective': 'regression', 'seed': 1204, 'subsample': 0.9}\n",
      "6 fold results: [0.8544774345014811, 0.7879432316915823, 0.713879382722397, 0.7773041804625644, 0.8923905075521859, 0.936389501741036]\n",
      "Mean Cross Validation RMSE: 0.827064039778541\n",
      "\n",
      "Training with params: \n",
      "{'colsample_bytree': 0.8500000000000001, 'learning_rate': 0.025, 'max_depth': 9, 'metric': 'rmse', 'min_data_in_leaf': 29, 'objective': 'regression', 'seed': 1204, 'subsample': 0.9}\n",
      "6 fold results: [0.8492118723224511, 0.7726309599420884, 0.701412729121619, 0.7681960842142356, 0.8834753424508036, 0.9227759219788416]\n",
      "Mean Cross Validation RMSE: 0.81628381833834\n",
      "\n",
      "Training with params: \n",
      "{'colsample_bytree': 0.6000000000000001, 'learning_rate': 0.15000000000000002, 'max_depth': 6, 'metric': 'rmse', 'min_data_in_leaf': 15, 'objective': 'regression', 'seed': 1204, 'subsample': 0.6000000000000001}\n",
      "6 fold results: [0.8352961827706531, 0.7762151927653751, 0.698593070305192, 0.7695694059705163, 0.8885692443105906, 0.9299169668280572]\n",
      "Mean Cross Validation RMSE: 0.8163600104917307\n",
      "\n",
      "Training with params: \n",
      "{'colsample_bytree': 0.7000000000000001, 'learning_rate': 0.225, 'max_depth': 14, 'metric': 'rmse', 'min_data_in_leaf': 21, 'objective': 'regression', 'seed': 1204, 'subsample': 0.75}\n",
      "6 fold results: [0.8476508056357286, 0.7741434645484868, 0.7070704323181126, 0.7772884765050047, 0.8888921665396864, 0.909495878151709]\n",
      "Mean Cross Validation RMSE: 0.8174235372831213\n",
      "\n",
      "Training with params: \n",
      "{'colsample_bytree': 0.8, 'learning_rate': 0.1, 'max_depth': 13, 'metric': 'rmse', 'min_data_in_leaf': 12, 'objective': 'regression', 'seed': 1204, 'subsample': 0.6000000000000001}\n",
      "6 fold results: [0.8341524971549776, 0.7721560273934693, 0.6991174942373986, 0.7724563308821508, 0.8871432175059848, 0.9174722259137527]\n",
      "Mean Cross Validation RMSE: 0.813749632181289\n",
      "\n",
      "Training with params: \n",
      "{'colsample_bytree': 0.55, 'learning_rate': 0.125, 'max_depth': 14, 'metric': 'rmse', 'min_data_in_leaf': 9, 'objective': 'regression', 'seed': 1204, 'subsample': 0.65}\n",
      "6 fold results: [0.8330289707850356, 0.7655627818325342, 0.7117964899085728, 0.7711424369269936, 0.8969735176617729, 0.927465094140385]\n",
      "Mean Cross Validation RMSE: 0.817661548542549\n",
      "\n",
      "Training with params: \n",
      "{'colsample_bytree': 0.9500000000000001, 'learning_rate': 0.17500000000000002, 'max_depth': 10, 'metric': 'rmse', 'min_data_in_leaf': 28, 'objective': 'regression', 'seed': 1204, 'subsample': 0.9}\n",
      "6 fold results: [0.834793172759823, 0.7688616534488627, 0.7059126349177756, 0.7766521703253778, 0.8898324584409324, 0.9170267772814944]\n",
      "Mean Cross Validation RMSE: 0.8155131445290443\n",
      "\n",
      "Training with params: \n",
      "{'colsample_bytree': 0.7000000000000001, 'learning_rate': 0.325, 'max_depth': 4, 'metric': 'rmse', 'min_data_in_leaf': 18, 'objective': 'regression', 'seed': 1204, 'subsample': 0.8500000000000001}\n",
      "6 fold results: [0.8488755660250716, 0.7704291077081198, 0.7102981151074952, 0.7812752674067887, 0.9268788482794997, 0.942788442035959]\n",
      "Mean Cross Validation RMSE: 0.8300908910938224\n",
      "\n",
      "Training with params: \n",
      "{'colsample_bytree': 0.9, 'learning_rate': 0.025, 'max_depth': 8, 'metric': 'rmse', 'min_data_in_leaf': 23, 'objective': 'regression', 'seed': 1204, 'subsample': 1.0}\n",
      "6 fold results: [0.8473115362198189, 0.7758412124753287, 0.7010746513209511, 0.7698882608125669, 0.8856273839280419, 0.9273919752895087]\n",
      "Mean Cross Validation RMSE: 0.8178558366743695\n",
      "\n",
      "Training with params: \n",
      "{'colsample_bytree': 0.8, 'learning_rate': 0.275, 'max_depth': 13, 'metric': 'rmse', 'min_data_in_leaf': 12, 'objective': 'regression', 'seed': 1204, 'subsample': 0.75}\n",
      "6 fold results: [0.8375498782031612, 0.7900921619472839, 0.7031445727453137, 0.7752211560376722, 0.8847949865340999, 0.9275834306657884]\n",
      "Mean Cross Validation RMSE: 0.8197310310222199\n",
      "\n",
      "Training with params: \n",
      "{'colsample_bytree': 1.0, 'learning_rate': 0.17500000000000002, 'max_depth': 13, 'metric': 'rmse', 'min_data_in_leaf': 12, 'objective': 'regression', 'seed': 1204, 'subsample': 0.5}\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "6 fold results: [0.8338697583720602, 0.778038391117131, 0.7043272195579601, 0.7687462078508754, 0.8779715072437807, 0.9100615914254346]\n",
      "Mean Cross Validation RMSE: 0.8121691125945404\n",
      "\n",
      "Training with params: \n",
      "{'colsample_bytree': 0.9500000000000001, 'learning_rate': 0.2, 'max_depth': 12, 'metric': 'rmse', 'min_data_in_leaf': 20, 'objective': 'regression', 'seed': 1204, 'subsample': 0.5}\n",
      "6 fold results: [0.8403167069808082, 0.7843483237819054, 0.7056007923491691, 0.7645392381524853, 0.8845422571214483, 0.920678923465974]\n",
      "Mean Cross Validation RMSE: 0.8166710403086318\n",
      "\n",
      "Training with params: \n",
      "{'colsample_bytree': 1.0, 'learning_rate': 0.30000000000000004, 'max_depth': 11, 'metric': 'rmse', 'min_data_in_leaf': 26, 'objective': 'regression', 'seed': 1204, 'subsample': 0.8}\n",
      "6 fold results: [0.8324424841116423, 0.7733470936319395, 0.7098935183327856, 0.7793614221794517, 0.8865751752391489, 0.9044887758867802]\n",
      "Mean Cross Validation RMSE: 0.8143514115636248\n",
      "\n",
      "Training with params: \n",
      "{'colsample_bytree': 1.0, 'learning_rate': 0.15000000000000002, 'max_depth': 3, 'metric': 'rmse', 'min_data_in_leaf': 7, 'objective': 'regression', 'seed': 1204, 'subsample': 0.9500000000000001}\n",
      "6 fold results: [0.8627679024730723, 0.7844634980088354, 0.7125735666581614, 0.7797943670649738, 0.9036250693712312, 0.9505002329279958]\n",
      "Mean Cross Validation RMSE: 0.8322874394173784\n",
      "\n",
      "Training with params: \n",
      "{'colsample_bytree': 0.9, 'learning_rate': 0.25, 'max_depth': 13, 'metric': 'rmse', 'min_data_in_leaf': 27, 'objective': 'regression', 'seed': 1204, 'subsample': 0.7000000000000001}\n",
      "6 fold results: [0.8537368154520331, 0.7755526837508476, 0.7079210085371642, 0.7790597805571653, 0.8930397727479257, 0.9224064175376433]\n",
      "Mean Cross Validation RMSE: 0.8219527464304632\n",
      "\n",
      "Training with params: \n",
      "{'colsample_bytree': 1.0, 'learning_rate': 0.17500000000000002, 'max_depth': 11, 'metric': 'rmse', 'min_data_in_leaf': 22, 'objective': 'regression', 'seed': 1204, 'subsample': 0.5}\n",
      "6 fold results: [0.8452287777259699, 0.7708762628938356, 0.7044968779010579, 0.7649644100106132, 0.8918257610357339, 0.9152474453970276]\n",
      "Mean Cross Validation RMSE: 0.8154399224940397\n",
      "\n",
      "Training with params: \n",
      "{'colsample_bytree': 0.75, 'learning_rate': 0.05, 'max_depth': 10, 'metric': 'rmse', 'min_data_in_leaf': 16, 'objective': 'regression', 'seed': 1204, 'subsample': 0.8}\n",
      "6 fold results: [0.8402933530839467, 0.7712211419508266, 0.6960731344157969, 0.7643968988459034, 0.8753564559647653, 0.9240985789441379]\n",
      "Mean Cross Validation RMSE: 0.8119065938675628\n",
      "\n",
      "Training with params: \n",
      "{'colsample_bytree': 0.5, 'learning_rate': 0.325, 'max_depth': 10, 'metric': 'rmse', 'min_data_in_leaf': 16, 'objective': 'regression', 'seed': 1204, 'subsample': 0.8}\n",
      "6 fold results: [0.8487167797426975, 0.7805486782728848, 0.7184509494781411, 0.7765350896586353, 0.9002072404424437, 0.9944840540235874]\n",
      "Mean Cross Validation RMSE: 0.8364904652697316\n",
      "\n",
      "Training with params: \n",
      "{'colsample_bytree': 0.65, 'learning_rate': 0.05, 'max_depth': 10, 'metric': 'rmse', 'min_data_in_leaf': 13, 'objective': 'regression', 'seed': 1204, 'subsample': 0.7000000000000001}\n",
      "6 fold results: [0.8449192373433965, 0.7710053429110351, 0.699150538465398, 0.7641207682826666, 0.8814089236913416, 0.9153628471476911]\n",
      "Mean Cross Validation RMSE: 0.8126612763069215\n",
      "\n",
      "Training with params: \n",
      "{'colsample_bytree': 0.75, 'learning_rate': 0.15000000000000002, 'max_depth': 13, 'metric': 'rmse', 'min_data_in_leaf': 16, 'objective': 'regression', 'seed': 1204, 'subsample': 0.7000000000000001}\n",
      "6 fold results: [0.8408775007918763, 0.7700459855306603, 0.7025269090861348, 0.7658534521447505, 0.8802269375051176, 0.9255147376289337]\n",
      "Mean Cross Validation RMSE: 0.8141742537812456\n",
      "\n",
      "Training with params: \n",
      "{'colsample_bytree': 0.75, 'learning_rate': 0.2, 'max_depth': 10, 'metric': 'rmse', 'min_data_in_leaf': 19, 'objective': 'regression', 'seed': 1204, 'subsample': 0.8500000000000001}\n",
      "6 fold results: [0.843844162465955, 0.7734799976681291, 0.7070799427852538, 0.7736783683895859, 0.8731720476999097, 0.9201837537665319]\n",
      "Mean Cross Validation RMSE: 0.8152397121292275\n",
      "\n",
      "Training with params: \n",
      "{'colsample_bytree': 0.5, 'learning_rate': 0.375, 'max_depth': 3, 'metric': 'rmse', 'min_data_in_leaf': 10, 'objective': 'regression', 'seed': 1204, 'subsample': 0.8500000000000001}\n",
      "6 fold results: [0.8732257089057543, 0.787941738789224, 0.7245431332853244, 0.7963574355414059, 0.907681487354465, 0.9375603362843367]\n",
      "Mean Cross Validation RMSE: 0.8378849733600852\n",
      "\n",
      "Training with params: \n",
      "{'colsample_bytree': 0.65, 'learning_rate': 0.07500000000000001, 'max_depth': 5, 'metric': 'rmse', 'min_data_in_leaf': 14, 'objective': 'regression', 'seed': 1204, 'subsample': 0.75}\n",
      "6 fold results: [0.8356416864057457, 0.7766402990685761, 0.7060653532326431, 0.7744664474897421, 0.894426648998135, 0.9401533615023638]\n",
      "Mean Cross Validation RMSE: 0.8212322994495342\n",
      "\n",
      "Training with params: \n",
      "{'colsample_bytree': 0.55, 'learning_rate': 0.25, 'max_depth': 8, 'metric': 'rmse', 'min_data_in_leaf': 11, 'objective': 'regression', 'seed': 1204, 'subsample': 0.55}\n",
      "6 fold results: [0.8390184269576161, 0.7775975278409462, 0.7081016641093764, 0.7727410793936449, 0.8890631550442378, 0.9312209527074906]\n",
      "Mean Cross Validation RMSE: 0.8196238010088853\n",
      "\n",
      "Training with params: \n",
      "{'colsample_bytree': 0.65, 'learning_rate': 0.47500000000000003, 'max_depth': 12, 'metric': 'rmse', 'min_data_in_leaf': 6, 'objective': 'regression', 'seed': 1204, 'subsample': 0.7000000000000001}\n",
      "6 fold results: [0.867693830234686, 0.7810522992569052, 0.721673969675573, 0.7762324878133243, 0.9091517307366567, 0.9932898793105637]\n",
      "Mean Cross Validation RMSE: 0.8415156995046181\n",
      "\n",
      "Training with params: \n",
      "{'colsample_bytree': 0.8500000000000001, 'learning_rate': 0.125, 'max_depth': 9, 'metric': 'rmse', 'min_data_in_leaf': 12, 'objective': 'regression', 'seed': 1204, 'subsample': 0.8}\n",
      "6 fold results: [0.8468033386459679, 0.7708789139039539, 0.706721372549546, 0.7762801282624023, 0.8775075774111358, 0.9291012929793538]\n",
      "Mean Cross Validation RMSE: 0.8178821039587266\n",
      "\n",
      "Training with params: \n",
      "{'colsample_bytree': 0.6000000000000001, 'learning_rate': 0.07500000000000001, 'max_depth': 5, 'metric': 'rmse', 'min_data_in_leaf': 16, 'objective': 'regression', 'seed': 1204, 'subsample': 0.8500000000000001}\n",
      "6 fold results: [0.8428920429222831, 0.7739117738998419, 0.7084626438630283, 0.7683843227536097, 0.8910643629093566, 0.9392354259304472]\n",
      "Mean Cross Validation RMSE: 0.8206584287130946\n",
      "\n",
      "Training with params: \n",
      "{'colsample_bytree': 0.8, 'learning_rate': 0.125, 'max_depth': 10, 'metric': 'rmse', 'min_data_in_leaf': 5, 'objective': 'regression', 'seed': 1204, 'subsample': 0.9500000000000001}\n",
      "6 fold results: [0.8383863742986557, 0.7711130888747798, 0.7096075432981993, 0.7733532051532168, 0.8867287625602502, 0.917164650159492]\n",
      "Mean Cross Validation RMSE: 0.8160589373907657\n",
      "\n",
      "Training with params: \n",
      "{'colsample_bytree': 0.75, 'learning_rate': 0.375, 'max_depth': 13, 'metric': 'rmse', 'min_data_in_leaf': 25, 'objective': 'regression', 'seed': 1204, 'subsample': 0.55}\n",
      "6 fold results: [0.8472258626432445, 0.785665064476515, 0.7104820931736987, 0.7823422087207089, 0.8878086502923505, 0.9223812938829165]\n",
      "Mean Cross Validation RMSE: 0.822650862198239\n",
      "\n",
      "Training with params: \n",
      "{'colsample_bytree': 0.9, 'learning_rate': 0.2, 'max_depth': 4, 'metric': 'rmse', 'min_data_in_leaf': 24, 'objective': 'regression', 'seed': 1204, 'subsample': 0.75}\n",
      "6 fold results: [0.851310014696448, 0.7893465028971897, 0.7167162321508643, 0.774622996269544, 0.8902198441794841, 0.950214908878365]\n",
      "Mean Cross Validation RMSE: 0.8287384165119825\n",
      "\n",
      "Training with params: \n",
      "{'colsample_bytree': 0.7000000000000001, 'learning_rate': 0.05, 'max_depth': 6, 'metric': 'rmse', 'min_data_in_leaf': 17, 'objective': 'regression', 'seed': 1204, 'subsample': 0.8}\n",
      "6 fold results: [0.8309739305060592, 0.7762052514211369, 0.7024807615020761, 0.7680828515539408, 0.8919256379754346, 0.9345344145758778]\n",
      "Mean Cross Validation RMSE: 0.8173671412557543\n",
      "\n",
      "Training with params: \n",
      "{'colsample_bytree': 0.9500000000000001, 'learning_rate': 0.1, 'max_depth': 7, 'metric': 'rmse', 'min_data_in_leaf': 8, 'objective': 'regression', 'seed': 1204, 'subsample': 0.6000000000000001}\n",
      "6 fold results: [0.8342252599496099, 0.7834889982112799, 0.6990378922403883, 0.772084927517884, 0.891098948607067, 0.9473384101619448]\n",
      "Mean Cross Validation RMSE: 0.8212124061146957\n",
      "\n",
      "Training with params: \n",
      "{'colsample_bytree': 0.8500000000000001, 'learning_rate': 0.275, 'max_depth': 13, 'metric': 'rmse', 'min_data_in_leaf': 22, 'objective': 'regression', 'seed': 1204, 'subsample': 0.7000000000000001}\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "6 fold results: [0.8519570062705876, 0.787223484497298, 0.7120291311452241, 0.781627412766432, 0.8937862175265588, 0.9241544658016585]\n",
      "Mean Cross Validation RMSE: 0.8251296196679597\n",
      "\n",
      "Training with params: \n",
      "{'colsample_bytree': 0.6000000000000001, 'learning_rate': 0.225, 'max_depth': 9, 'metric': 'rmse', 'min_data_in_leaf': 15, 'objective': 'regression', 'seed': 1204, 'subsample': 0.9}\n",
      "6 fold results: [0.8502521457787197, 0.7769392936040398, 0.7092513743205762, 0.7751647285503341, 0.8976515250077235, 0.9394932136664226]\n",
      "Mean Cross Validation RMSE: 0.8247920468213027\n",
      "\n",
      "Training with params: \n",
      "{'colsample_bytree': 0.65, 'learning_rate': 0.17500000000000002, 'max_depth': 3, 'metric': 'rmse', 'min_data_in_leaf': 12, 'objective': 'regression', 'seed': 1204, 'subsample': 1.0}\n",
      "6 fold results: [0.8493952905943163, 0.7767016571774313, 0.7162249562844007, 0.787310912303716, 0.901887453549288, 0.9467244393569322]\n",
      "Mean Cross Validation RMSE: 0.8297074515443473\n",
      "\n",
      "Training with params: \n",
      "{'colsample_bytree': 0.8, 'learning_rate': 0.025, 'max_depth': 10, 'metric': 'rmse', 'min_data_in_leaf': 9, 'objective': 'regression', 'seed': 1204, 'subsample': 0.5}\n",
      "6 fold results: [0.8392711770939685, 0.7675663948455198, 0.6991594181816566, 0.7660360536234245, 0.8913951294500404, 0.9135723486274147]\n",
      "Mean Cross Validation RMSE: 0.8128334203036708\n",
      "\n",
      "Training with params: \n",
      "{'colsample_bytree': 0.55, 'learning_rate': 0.07500000000000001, 'max_depth': 12, 'metric': 'rmse', 'min_data_in_leaf': 21, 'objective': 'regression', 'seed': 1204, 'subsample': 0.65}\n",
      "6 fold results: [0.8326217991342502, 0.768584186988249, 0.7037600279931194, 0.7710920510298068, 0.8892504053079465, 0.9213041828874909]\n",
      "Mean Cross Validation RMSE: 0.8144354422234771\n",
      "\n",
      "Training with params: \n",
      "{'colsample_bytree': 0.9, 'learning_rate': 0.125, 'max_depth': 7, 'metric': 'rmse', 'min_data_in_leaf': 29, 'objective': 'regression', 'seed': 1204, 'subsample': 0.8500000000000001}\n",
      "6 fold results: [0.8475919362564548, 0.7777678727511503, 0.7074583556727848, 0.7677167765418229, 0.8940914478056827, 0.9309584801687996]\n",
      "Mean Cross Validation RMSE: 0.8209308115327825\n",
      "\n",
      "Training with params: \n",
      "{'colsample_bytree': 0.7000000000000001, 'learning_rate': 0.325, 'max_depth': 6, 'metric': 'rmse', 'min_data_in_leaf': 13, 'objective': 'regression', 'seed': 1204, 'subsample': 0.9500000000000001}\n",
      "6 fold results: [0.8499224495009089, 0.7843598392670094, 0.7122190603712555, 0.781486426040167, 0.8927933335430421, 0.9364481613792608]\n",
      "Mean Cross Validation RMSE: 0.8262048783502739\n",
      "\n",
      "Training with params: \n",
      "{'colsample_bytree': 0.8500000000000001, 'learning_rate': 0.35000000000000003, 'max_depth': 14, 'metric': 'rmse', 'min_data_in_leaf': 23, 'objective': 'regression', 'seed': 1204, 'subsample': 0.6000000000000001}\n",
      "6 fold results: [0.8423167889837418, 0.7956346289076967, 0.7154843011102974, 0.7787898183604529, 0.8988300301209472, 0.9093754693968125]\n",
      "Mean Cross Validation RMSE: 0.8234051728133247\n",
      "\n",
      "Training with params: \n",
      "{'colsample_bytree': 0.9500000000000001, 'learning_rate': 0.15000000000000002, 'max_depth': 8, 'metric': 'rmse', 'min_data_in_leaf': 20, 'objective': 'regression', 'seed': 1204, 'subsample': 0.8}\n",
      "6 fold results: [0.8320081178387863, 0.7750993722549954, 0.7028514232425317, 0.7602720915570768, 0.8889181093083969, 0.9283152185711671]\n",
      "Mean Cross Validation RMSE: 0.8145773887954922\n",
      "\n",
      "Training with params: \n",
      "{'colsample_bytree': 0.5, 'learning_rate': 0.1, 'max_depth': 4, 'metric': 'rmse', 'min_data_in_leaf': 7, 'objective': 'regression', 'seed': 1204, 'subsample': 0.9}\n",
      "6 fold results: [0.8556841872649751, 0.7739073655049168, 0.7104821396429569, 0.7801095931728597, 0.8980344256161166, 0.9378831635363359]\n",
      "Mean Cross Validation RMSE: 0.82601681245636\n",
      "\n",
      "Training with params: \n",
      "{'colsample_bytree': 0.75, 'learning_rate': 0.05, 'max_depth': 5, 'metric': 'rmse', 'min_data_in_leaf': 10, 'objective': 'regression', 'seed': 1204, 'subsample': 0.55}\n",
      "6 fold results: [0.8427304945539731, 0.7733470358885636, 0.7074089271921578, 0.7711815626549222, 0.8904391478837745, 0.9448885922566337]\n",
      "Mean Cross Validation RMSE: 0.8216659600716708\n",
      "\n",
      "Training with params: \n",
      "{'colsample_bytree': 0.8, 'learning_rate': 0.225, 'max_depth': 13, 'metric': 'rmse', 'min_data_in_leaf': 19, 'objective': 'regression', 'seed': 1204, 'subsample': 0.65}\n",
      "6 fold results: [0.8449735532903228, 0.7773188527371623, 0.7014037115659426, 0.7775535291889583, 0.8942084512965809, 0.9229537870407575]\n",
      "Mean Cross Validation RMSE: 0.8197353141866207\n",
      "\n",
      "Training with params: \n",
      "{'colsample_bytree': 1.0, 'learning_rate': 0.2, 'max_depth': 10, 'metric': 'rmse', 'min_data_in_leaf': 27, 'objective': 'regression', 'seed': 1204, 'subsample': 0.75}\n",
      "6 fold results: [0.8406031025054932, 0.7828945133973588, 0.700116037695574, 0.7755048922553363, 0.8942366816021304, 0.9116398483365378]\n",
      "Mean Cross Validation RMSE: 0.8174991792987384\n",
      "\n",
      "Training with params: \n",
      "{'colsample_bytree': 0.9, 'learning_rate': 0.5, 'max_depth': 11, 'metric': 'rmse', 'min_data_in_leaf': 28, 'objective': 'regression', 'seed': 1204, 'subsample': 1.0}\n",
      "6 fold results: [0.8505500556586635, 0.7999203131796047, 0.7117370438755767, 0.7846136838117401, 0.9087522345037327, 0.9253170373074415]\n",
      "Mean Cross Validation RMSE: 0.8301483947227931\n",
      "\n",
      "Training with params: \n",
      "{'colsample_bytree': 0.6000000000000001, 'learning_rate': 0.30000000000000004, 'max_depth': 9, 'metric': 'rmse', 'min_data_in_leaf': 16, 'objective': 'regression', 'seed': 1204, 'subsample': 0.9}\n",
      "6 fold results: [0.8565723301831366, 0.7801874340036895, 0.7098761010734708, 0.7788924947627502, 0.890513228738263, 0.9440733929455051]\n",
      "Mean Cross Validation RMSE: 0.8266858302844692\n",
      "\n",
      "Training with params: \n",
      "{'colsample_bytree': 0.8500000000000001, 'learning_rate': 0.17500000000000002, 'max_depth': 14, 'metric': 'rmse', 'min_data_in_leaf': 18, 'objective': 'regression', 'seed': 1204, 'subsample': 0.6000000000000001}\n",
      "6 fold results: [0.8535313411163487, 0.771491034901383, 0.7024021300662245, 0.7677455378267831, 0.9085606999444414, 0.9265812421569709]\n",
      "Mean Cross Validation RMSE: 0.8217186643353586\n",
      "\n",
      "Training with params: \n",
      "{'colsample_bytree': 0.7000000000000001, 'learning_rate': 0.025, 'max_depth': 13, 'metric': 'rmse', 'min_data_in_leaf': 6, 'objective': 'regression', 'seed': 1204, 'subsample': 0.8}\n",
      "6 fold results: [0.8455298633853855, 0.7665224866723047, 0.7000871967259042, 0.767492997813632, 0.8825853807069831, 0.9097454371139129]\n",
      "Mean Cross Validation RMSE: 0.8119938937363537\n",
      "\n",
      "Training with params: \n",
      "{'colsample_bytree': 0.7000000000000001, 'learning_rate': 0.025, 'max_depth': 10, 'metric': 'rmse', 'min_data_in_leaf': 6, 'objective': 'regression', 'seed': 1204, 'subsample': 0.8}\n",
      "6 fold results: [0.8473623805827661, 0.7697416152557996, 0.6980036485610487, 0.7647469706451604, 0.8825645054640681, 0.9129143254245955]\n",
      "Mean Cross Validation RMSE: 0.8125555743222397\n",
      "\n",
      "Training with params: \n",
      "{'colsample_bytree': 0.65, 'learning_rate': 0.07500000000000001, 'max_depth': 6, 'metric': 'rmse', 'min_data_in_leaf': 6, 'objective': 'regression', 'seed': 1204, 'subsample': 0.8500000000000001}\n",
      "6 fold results: [0.8323460971432549, 0.7711708300026613, 0.7093712213734952, 0.7691187143215016, 0.8809832548768778, 0.9213051320999122]\n",
      "Mean Cross Validation RMSE: 0.8140492083029506\n",
      "\n",
      "Training with params: \n",
      "{'colsample_bytree': 0.75, 'learning_rate': 0.1, 'max_depth': 8, 'metric': 'rmse', 'min_data_in_leaf': 14, 'objective': 'regression', 'seed': 1204, 'subsample': 0.75}\n",
      "6 fold results: [0.8378596191018468, 0.7720150167441308, 0.7015838859285354, 0.7592716660711835, 0.8846660522018156, 0.9131804344344644]\n",
      "Mean Cross Validation RMSE: 0.811429445746996\n",
      "\n",
      "Training with params: \n",
      "{'colsample_bytree': 0.75, 'learning_rate': 0.05, 'max_depth': 8, 'metric': 'rmse', 'min_data_in_leaf': 14, 'objective': 'regression', 'seed': 1204, 'subsample': 0.75}\n",
      "6 fold results: [0.8279448644469738, 0.7748060445949626, 0.7005805611224594, 0.763327740168756, 0.8803137049976673, 0.9262307180543251]\n",
      "Mean Cross Validation RMSE: 0.8122006055641906\n",
      "\n",
      "Training with params: \n",
      "{'colsample_bytree': 0.8, 'learning_rate': 0.1, 'max_depth': 8, 'metric': 'rmse', 'min_data_in_leaf': 14, 'objective': 'regression', 'seed': 1204, 'subsample': 0.7000000000000001}\n",
      "6 fold results: [0.827310664606924, 0.7762200051253334, 0.7037543461137516, 0.7655705885325076, 0.8906359985725593, 0.9131820298641022]\n",
      "Mean Cross Validation RMSE: 0.8127789388025297\n",
      "\n",
      "Training with params: \n",
      "{'colsample_bytree': 0.7000000000000001, 'learning_rate': 0.025, 'max_depth': 8, 'metric': 'rmse', 'min_data_in_leaf': 14, 'objective': 'regression', 'seed': 1204, 'subsample': 0.8}\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "6 fold results: [0.8480156851641116, 0.7735837287825399, 0.7005152474942975, 0.7677599667019522, 0.8825882588608417, 0.914994414083581]\n",
      "Mean Cross Validation RMSE: 0.8145762168478874\n",
      "\n",
      "Training with params: \n",
      "{'colsample_bytree': 0.75, 'learning_rate': 0.05, 'max_depth': 8, 'metric': 'rmse', 'min_data_in_leaf': 26, 'objective': 'regression', 'seed': 1204, 'subsample': 0.75}\n",
      "6 fold results: [0.8385301273210609, 0.7735252374929188, 0.7007980989278302, 0.7682896363954811, 0.882205367000955, 0.9282450469396509]\n",
      "Mean Cross Validation RMSE: 0.8152655856796495\n",
      "\n",
      "Training with params: \n",
      "{'colsample_bytree': 0.7000000000000001, 'learning_rate': 0.025, 'max_depth': 13, 'metric': 'rmse', 'min_data_in_leaf': 6, 'objective': 'regression', 'seed': 1204, 'subsample': 0.8500000000000001}\n",
      "6 fold results: [0.8455298633853855, 0.7665224866723047, 0.7000871967259042, 0.767492997813632, 0.8825853807069831, 0.9097454371139129]\n",
      "Mean Cross Validation RMSE: 0.8119938937363537\n",
      "\n",
      "Training with params: \n",
      "{'colsample_bytree': 0.8, 'learning_rate': 0.125, 'max_depth': 3, 'metric': 'rmse', 'min_data_in_leaf': 5, 'objective': 'regression', 'seed': 1204, 'subsample': 0.8}\n",
      "6 fold results: [0.8640902064767632, 0.7861735107592023, 0.7170469957671934, 0.77831918236835, 0.9031839097187578, 0.948892805720854]\n",
      "Mean Cross Validation RMSE: 0.8329511018018535\n",
      "\n",
      "Training with params: \n",
      "{'colsample_bytree': 0.65, 'learning_rate': 0.07500000000000001, 'max_depth': 12, 'metric': 'rmse', 'min_data_in_leaf': 11, 'objective': 'regression', 'seed': 1204, 'subsample': 0.7000000000000001}\n",
      "6 fold results: [0.8396757781476794, 0.7694790206945381, 0.7020791571099565, 0.7764740668493737, 0.8830202245264862, 0.9126065955331911]\n",
      "Mean Cross Validation RMSE: 0.8138891404768708\n",
      "\n",
      "Training with params: \n",
      "{'colsample_bytree': 0.75, 'learning_rate': 0.4, 'max_depth': 7, 'metric': 'rmse', 'min_data_in_leaf': 24, 'objective': 'regression', 'seed': 1204, 'subsample': 0.8500000000000001}\n",
      "6 fold results: [0.8568727514963126, 0.779228983739871, 0.7136991547390981, 0.7857582029657652, 0.8865999786617669, 0.9423508988675802]\n",
      "Mean Cross Validation RMSE: 0.8274183284117322\n",
      "\n",
      "Training with params: \n",
      "{'colsample_bytree': 0.65, 'learning_rate': 0.1, 'max_depth': 4, 'metric': 'rmse', 'min_data_in_leaf': 16, 'objective': 'regression', 'seed': 1204, 'subsample': 0.75}\n",
      "6 fold results: [0.8528143901714534, 0.7746724078492656, 0.7069030461001948, 0.7738308813414311, 0.9030835216811657, 0.9480687418503535]\n",
      "Mean Cross Validation RMSE: 0.8265621648323108\n",
      "\n",
      "Training with params: \n",
      "{'colsample_bytree': 0.7000000000000001, 'learning_rate': 0.125, 'max_depth': 10, 'metric': 'rmse', 'min_data_in_leaf': 25, 'objective': 'regression', 'seed': 1204, 'subsample': 0.9}\n",
      "6 fold results: [0.8429373392708855, 0.7716801881757873, 0.702042019938034, 0.7609558839918669, 0.8827740326417325, 0.9066077144547884]\n",
      "Mean Cross Validation RMSE: 0.8111661964121825\n",
      "\n",
      "Training with params: \n",
      "{'colsample_bytree': 0.8, 'learning_rate': 0.15000000000000002, 'max_depth': 10, 'metric': 'rmse', 'min_data_in_leaf': 25, 'objective': 'regression', 'seed': 1204, 'subsample': 0.9500000000000001}\n",
      "6 fold results: [0.8351458615270989, 0.7767445328778386, 0.6978698721088404, 0.7773795224669693, 0.87028155286662, 0.9192904555607035]\n",
      "Mean Cross Validation RMSE: 0.8127852995680117\n",
      "\n",
      "Training with params: \n",
      "{'colsample_bytree': 0.75, 'learning_rate': 0.15000000000000002, 'max_depth': 10, 'metric': 'rmse', 'min_data_in_leaf': 25, 'objective': 'regression', 'seed': 1204, 'subsample': 0.9}\n",
      "6 fold results: [0.8443183404032408, 0.7706085055732294, 0.7084918722220266, 0.7808216681054573, 0.8844361139518666, 0.9142171038468719]\n",
      "Mean Cross Validation RMSE: 0.8171489340171155\n",
      "\n",
      "Training with params: \n",
      "{'colsample_bytree': 0.6000000000000001, 'learning_rate': 0.225, 'max_depth': 10, 'metric': 'rmse', 'min_data_in_leaf': 25, 'objective': 'regression', 'seed': 1204, 'subsample': 0.9}\n",
      "6 fold results: [0.8500955360916946, 0.777184905634267, 0.708352291561004, 0.7740135482883977, 0.8943259448947178, 0.9354808437899874]\n",
      "Mean Cross Validation RMSE: 0.823242178376678\n",
      "\n",
      "Training with params: \n",
      "{'colsample_bytree': 0.75, 'learning_rate': 0.25, 'max_depth': 10, 'metric': 'rmse', 'min_data_in_leaf': 17, 'objective': 'regression', 'seed': 1204, 'subsample': 0.9500000000000001}\n",
      "6 fold results: [0.8446501862249353, 0.7821214446086217, 0.7017863717223204, 0.78432356496922, 0.8931421860422862, 0.9260261661692983]\n",
      "Mean Cross Validation RMSE: 0.8220083199561136\n",
      "\n",
      "Training with params: \n",
      "{'colsample_bytree': 0.7000000000000001, 'learning_rate': 0.125, 'max_depth': 11, 'metric': 'rmse', 'min_data_in_leaf': 8, 'objective': 'regression', 'seed': 1204, 'subsample': 0.65}\n",
      "6 fold results: [0.8296535099679573, 0.771709487826801, 0.700832173770315, 0.7817288147085448, 0.8891748378135738, 0.9247068345690328]\n",
      "Mean Cross Validation RMSE: 0.8163009431093707\n",
      "\n",
      "Training with params: \n",
      "{'colsample_bytree': 0.8500000000000001, 'learning_rate': 0.1, 'max_depth': 5, 'metric': 'rmse', 'min_data_in_leaf': 15, 'objective': 'regression', 'seed': 1204, 'subsample': 1.0}\n",
      "6 fold results: [0.8576333397674545, 0.7790242093643632, 0.7049186432629414, 0.7690346527415676, 0.8901461070472046, 0.9430940476229034]\n",
      "Mean Cross Validation RMSE: 0.8239751666344057\n",
      "\n",
      "Training with params: \n",
      "{'colsample_bytree': 0.65, 'learning_rate': 0.17500000000000002, 'max_depth': 8, 'metric': 'rmse', 'min_data_in_leaf': 21, 'objective': 'regression', 'seed': 1204, 'subsample': 0.8500000000000001}\n",
      "6 fold results: [0.8306814886347195, 0.7741416944872973, 0.7039855199695229, 0.7745479398206423, 0.8949452868406895, 0.9271873632340243]\n",
      "Mean Cross Validation RMSE: 0.8175815488311492\n",
      "\n",
      "Training with params: \n",
      "{'colsample_bytree': 0.8, 'learning_rate': 0.2, 'max_depth': 10, 'metric': 'rmse', 'min_data_in_leaf': 29, 'objective': 'regression', 'seed': 1204, 'subsample': 0.7000000000000001}\n",
      "6 fold results: [0.8387462568294991, 0.7715150100444083, 0.7106149285413541, 0.7740079605393276, 0.892856737561959, 0.9273205937809733]\n",
      "Mean Cross Validation RMSE: 0.819176914549587\n",
      "\n",
      "Training with params: \n",
      "{'colsample_bytree': 0.7000000000000001, 'learning_rate': 0.05, 'max_depth': 9, 'metric': 'rmse', 'min_data_in_leaf': 23, 'objective': 'regression', 'seed': 1204, 'subsample': 0.8500000000000001}\n",
      "6 fold results: [0.8328394577163981, 0.7724723456455248, 0.6993398697333905, 0.771261446377982, 0.8792819749238967, 0.9156827833650414]\n",
      "Mean Cross Validation RMSE: 0.8118129796270389\n",
      "\n",
      "Training with params: \n",
      "{'colsample_bytree': 0.6000000000000001, 'learning_rate': 0.275, 'max_depth': 9, 'metric': 'rmse', 'min_data_in_leaf': 23, 'objective': 'regression', 'seed': 1204, 'subsample': 0.9500000000000001}\n",
      "6 fold results: [0.838950234011888, 0.7769202613082934, 0.7116060603752189, 0.763846458881088, 0.8806459503839857, 0.9372802127459203]\n",
      "Mean Cross Validation RMSE: 0.8182081962843991\n",
      "\n",
      "Training with params: \n",
      "{'colsample_bytree': 0.55, 'learning_rate': 0.07500000000000001, 'max_depth': 9, 'metric': 'rmse', 'min_data_in_leaf': 23, 'objective': 'regression', 'seed': 1204, 'subsample': 0.8500000000000001}\n",
      "6 fold results: [0.8258866706288855, 0.7763217235075871, 0.7004784541749751, 0.7679014582383181, 0.8922018674923666, 0.928341523665937]\n",
      "Mean Cross Validation RMSE: 0.8151886162846783\n",
      "\n",
      "Training with params: \n",
      "{'colsample_bytree': 0.65, 'learning_rate': 0.125, 'max_depth': 9, 'metric': 'rmse', 'min_data_in_leaf': 23, 'objective': 'regression', 'seed': 1204, 'subsample': 0.9}\n",
      "6 fold results: [0.8340968010075034, 0.7767492396631853, 0.7036901377577405, 0.7666440069976844, 0.891400170343066, 0.9191904799744147]\n",
      "Mean Cross Validation RMSE: 0.8152951392905989\n",
      "\n",
      "Training with params: \n",
      "{'colsample_bytree': 0.7000000000000001, 'learning_rate': 0.15000000000000002, 'max_depth': 9, 'metric': 'rmse', 'min_data_in_leaf': 9, 'objective': 'regression', 'seed': 1204, 'subsample': 0.9}\n",
      "6 fold results: [0.8448196534205162, 0.7714681293306016, 0.702942368098589, 0.7680393888789567, 0.8801728195471166, 0.9316949323310214]\n",
      "Mean Cross Validation RMSE: 0.8165228819344669\n",
      "\n",
      "Training with params: \n",
      "{'colsample_bytree': 0.7000000000000001, 'learning_rate': 0.07500000000000001, 'max_depth': 14, 'metric': 'rmse', 'min_data_in_leaf': 20, 'objective': 'regression', 'seed': 1204, 'subsample': 0.9500000000000001}\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "6 fold results: [0.8396176785266026, 0.7667332221330301, 0.7033051700706668, 0.7707574508845888, 0.8768915505386885, 0.909841677947736]\n",
      "Mean Cross Validation RMSE: 0.8111911250168854\n",
      "\n",
      "Training with params: \n",
      "{'colsample_bytree': 0.55, 'learning_rate': 0.1, 'max_depth': 14, 'metric': 'rmse', 'min_data_in_leaf': 20, 'objective': 'regression', 'seed': 1204, 'subsample': 1.0}\n",
      "6 fold results: [0.8448660298624038, 0.7695865230988751, 0.7077080994616743, 0.7670256401970663, 0.889760542632325, 0.9185605882843448]\n",
      "Mean Cross Validation RMSE: 0.8162512372561149\n",
      "\n",
      "Training with params: \n",
      "{'colsample_bytree': 0.65, 'learning_rate': 0.17500000000000002, 'max_depth': 14, 'metric': 'rmse', 'min_data_in_leaf': 20, 'objective': 'regression', 'seed': 1204, 'subsample': 0.9500000000000001}\n",
      "6 fold results: [0.8467178153157137, 0.7753283594973317, 0.7134694534686378, 0.7768196892659099, 0.8887862005815124, 0.9349567908123078]\n",
      "Mean Cross Validation RMSE: 0.8226797181569022\n",
      "\n",
      "Training with params: \n",
      "{'colsample_bytree': 0.8, 'learning_rate': 0.07500000000000001, 'max_depth': 14, 'metric': 'rmse', 'min_data_in_leaf': 20, 'objective': 'regression', 'seed': 1204, 'subsample': 1.0}\n",
      "6 fold results: [0.840677741072417, 0.769893879596671, 0.7024931715808325, 0.7696340415321963, 0.8771634641888635, 0.9121508543091831]\n",
      "Mean Cross Validation RMSE: 0.8120021920466939\n",
      "\n",
      "Training with params: \n",
      "{'colsample_bytree': 0.6000000000000001, 'learning_rate': 0.2, 'max_depth': 14, 'metric': 'rmse', 'min_data_in_leaf': 7, 'objective': 'regression', 'seed': 1204, 'subsample': 1.0}\n",
      "6 fold results: [0.846513382971596, 0.776314112753358, 0.7074611103855115, 0.7731388950861631, 0.8894941494973949, 0.9259482386994241]\n",
      "Mean Cross Validation RMSE: 0.8198116482322412\n",
      "\n",
      "Training with params: \n",
      "{'colsample_bytree': 0.75, 'learning_rate': 0.225, 'max_depth': 3, 'metric': 'rmse', 'min_data_in_leaf': 13, 'objective': 'regression', 'seed': 1204, 'subsample': 0.9500000000000001}\n",
      "6 fold results: [0.8778062164543753, 0.7804875059966738, 0.7117322877208407, 0.7856955155462683, 0.9025691205629425, 0.948474990080766]\n",
      "Mean Cross Validation RMSE: 0.8344609393936445\n",
      "\n",
      "Training with params: \n",
      "{'colsample_bytree': 0.7000000000000001, 'learning_rate': 0.25, 'max_depth': 12, 'metric': 'rmse', 'min_data_in_leaf': 22, 'objective': 'regression', 'seed': 1204, 'subsample': 0.9500000000000001}\n",
      "6 fold results: [0.8435454784055632, 0.7786190881992598, 0.7123173041254741, 0.7678838976170052, 0.8863042291842471, 0.9020776172937621]\n",
      "Mean Cross Validation RMSE: 0.8151246024708853\n",
      "\n",
      "Training with params: \n",
      "{'colsample_bytree': 0.5, 'learning_rate': 0.125, 'max_depth': 14, 'metric': 'rmse', 'min_data_in_leaf': 28, 'objective': 'regression', 'seed': 1204, 'subsample': 0.9}\n",
      "6 fold results: [0.8417795820426954, 0.7716520093616347, 0.7077140159402759, 0.7713422890607652, 0.8852820846114866, 0.9152411646578227]\n",
      "Mean Cross Validation RMSE: 0.8155018576124468\n",
      "\n",
      "Training with params: \n",
      "{'colsample_bytree': 0.8500000000000001, 'learning_rate': 0.15000000000000002, 'max_depth': 6, 'metric': 'rmse', 'min_data_in_leaf': 27, 'objective': 'regression', 'seed': 1204, 'subsample': 0.9}\n",
      "6 fold results: [0.8419846741946214, 0.7701373428494475, 0.7104453754595714, 0.7711572539553089, 0.8901801420209117, 0.936859963214556]\n",
      "Mean Cross Validation RMSE: 0.8201274586157362\n",
      "\n",
      "Training with params: \n",
      "{'colsample_bytree': 0.6000000000000001, 'learning_rate': 0.45, 'max_depth': 7, 'metric': 'rmse', 'min_data_in_leaf': 14, 'objective': 'regression', 'seed': 1204, 'subsample': 1.0}\n",
      "6 fold results: [0.8527542365951235, 0.7680089508355231, 0.7192642644954065, 0.7901617763691264, 0.8961616204576713, 0.9270720098450279]\n",
      "Mean Cross Validation RMSE: 0.8255704764329798\n",
      "\n",
      "Training with params: \n",
      "{'colsample_bytree': 0.65, 'learning_rate': 0.025, 'max_depth': 8, 'metric': 'rmse', 'min_data_in_leaf': 19, 'objective': 'regression', 'seed': 1204, 'subsample': 0.9500000000000001}\n",
      "6 fold results: [0.8453782040227574, 0.7739807599344228, 0.7034086948338898, 0.7635441125044022, 0.8837467144166977, 0.9237835093448208]\n",
      "Mean Cross Validation RMSE: 0.8156403325094986\n",
      "\n",
      "Training with params: \n",
      "{'colsample_bytree': 0.75, 'learning_rate': 0.05, 'max_depth': 5, 'metric': 'rmse', 'min_data_in_leaf': 10, 'objective': 'regression', 'seed': 1204, 'subsample': 0.9500000000000001}\n",
      "6 fold results: [0.8427304945539731, 0.7733470358885636, 0.7074089271921578, 0.7711815626549222, 0.8904391478837745, 0.9448885922566337]\n",
      "Mean Cross Validation RMSE: 0.8216659600716708\n",
      "\n",
      "Training with params: \n",
      "{'colsample_bytree': 0.7000000000000001, 'learning_rate': 0.1, 'max_depth': 4, 'metric': 'rmse', 'min_data_in_leaf': 25, 'objective': 'regression', 'seed': 1204, 'subsample': 0.55}\n",
      "6 fold results: [0.8602801743859162, 0.7799116418828899, 0.7088350881243575, 0.7754175079937028, 0.8943247566792805, 0.9353032682861101]\n",
      "Mean Cross Validation RMSE: 0.8256787395587094\n",
      "\n",
      "Training with params: \n",
      "{'colsample_bytree': 0.8, 'learning_rate': 0.30000000000000004, 'max_depth': 14, 'metric': 'rmse', 'min_data_in_leaf': 18, 'objective': 'regression', 'seed': 1204, 'subsample': 0.65}\n",
      "6 fold results: [0.8458145205760937, 0.7884761929142099, 0.7084647419734406, 0.785344795183952, 0.881954359925249, 0.9267568842785451]\n",
      "Mean Cross Validation RMSE: 0.8228019158085816\n",
      "\n",
      "Training with params: \n",
      "{'colsample_bytree': 0.55, 'learning_rate': 0.2, 'max_depth': 11, 'metric': 'rmse', 'min_data_in_leaf': 26, 'objective': 'regression', 'seed': 1204, 'subsample': 1.0}\n",
      "6 fold results: [0.8537900247379039, 0.7694440770724658, 0.7076304807709742, 0.7794525211143327, 0.892890455925474, 0.9389097234096841]\n",
      "Mean Cross Validation RMSE: 0.8236862138384723\n",
      "\n",
      "Training with params: \n",
      "{'colsample_bytree': 0.8, 'learning_rate': 0.35000000000000003, 'max_depth': 8, 'metric': 'rmse', 'min_data_in_leaf': 11, 'objective': 'regression', 'seed': 1204, 'subsample': 0.75}\n",
      "6 fold results: [0.8478144573811919, 0.7764906760775301, 0.7087589265967889, 0.7806156338401955, 0.8832932066983893, 0.9349168692450859]\n",
      "Mean Cross Validation RMSE: 0.8219816283065303\n",
      "\n",
      "Training with params: \n",
      "{'colsample_bytree': 0.8500000000000001, 'learning_rate': 0.05, 'max_depth': 3, 'metric': 'rmse', 'min_data_in_leaf': 5, 'objective': 'regression', 'seed': 1204, 'subsample': 0.8}\n",
      "6 fold results: [0.8747249022041282, 0.7822520572342516, 0.7184558293232396, 0.7818172652400932, 0.9041327708294131, 0.9488865404158091]\n",
      "Mean Cross Validation RMSE: 0.8350448942078225\n",
      "\n",
      "Training with params: \n",
      "{'colsample_bytree': 0.65, 'learning_rate': 0.07500000000000001, 'max_depth': 12, 'metric': 'rmse', 'min_data_in_leaf': 14, 'objective': 'regression', 'seed': 1204, 'subsample': 0.6000000000000001}\n",
      "6 fold results: [0.8311727990526505, 0.7716554048250552, 0.6999013196218289, 0.7682726755587004, 0.8860324741541918, 0.925694447605299]\n",
      "Mean Cross Validation RMSE: 0.8137881868029542\n",
      "\n",
      "Training with params: \n",
      "{'colsample_bytree': 0.7000000000000001, 'learning_rate': 0.17500000000000002, 'max_depth': 6, 'metric': 'rmse', 'min_data_in_leaf': 24, 'objective': 'regression', 'seed': 1204, 'subsample': 0.7000000000000001}\n",
      "6 fold results: [0.8367881091468118, 0.7705615717003224, 0.7095753369512411, 0.7695108249053086, 0.8900724915111992, 0.9163867121458926]\n",
      "Mean Cross Validation RMSE: 0.815482507726796\n",
      "\n",
      "Training with params: \n",
      "{'colsample_bytree': 0.75, 'learning_rate': 0.125, 'max_depth': 8, 'metric': 'rmse', 'min_data_in_leaf': 20, 'objective': 'regression', 'seed': 1204, 'subsample': 0.9}\n",
      "6 fold results: [0.8365034646770253, 0.7771307226886648, 0.7035552621018673, 0.7694219817533519, 0.8921121349803102, 0.9336116064921525]\n",
      "Mean Cross Validation RMSE: 0.8187225287822285\n",
      "\n",
      "Training with params: \n",
      "{'colsample_bytree': 0.9, 'learning_rate': 0.15000000000000002, 'max_depth': 4, 'metric': 'rmse', 'min_data_in_leaf': 17, 'objective': 'regression', 'seed': 1204, 'subsample': 1.0}\n",
      "6 fold results: [0.8634377051726724, 0.7876458127164956, 0.7116756953227673, 0.7765611125067193, 0.9043416110168094, 0.9418482948512171]\n",
      "Mean Cross Validation RMSE: 0.8309183719311135\n",
      "\n",
      "Training with params: \n",
      "{'colsample_bytree': 0.7000000000000001, 'learning_rate': 0.275, 'max_depth': 7, 'metric': 'rmse', 'min_data_in_leaf': 8, 'objective': 'regression', 'seed': 1204, 'subsample': 0.8}\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "6 fold results: [0.8462472471126837, 0.7784608330744149, 0.7156520169644724, 0.7760378952062563, 0.913397797212554, 0.9262400007494188]\n",
      "Mean Cross Validation RMSE: 0.8260059650533002\n",
      "\n",
      "Training with params: \n",
      "{'colsample_bytree': 0.75, 'learning_rate': 0.17500000000000002, 'max_depth': 14, 'metric': 'rmse', 'min_data_in_leaf': 25, 'objective': 'regression', 'seed': 1204, 'subsample': 0.8500000000000001}\n",
      "6 fold results: [0.8290108324796218, 0.7758384243251031, 0.7016525635064537, 0.774371242096817, 0.8798590206440836, 0.914399375588786]\n",
      "Mean Cross Validation RMSE: 0.8125219097734776\n",
      "\n",
      "Training with params: \n",
      "{'colsample_bytree': 0.6000000000000001, 'learning_rate': 0.1, 'max_depth': 11, 'metric': 'rmse', 'min_data_in_leaf': 15, 'objective': 'regression', 'seed': 1204, 'subsample': 0.65}\n",
      "6 fold results: [0.8434436198809985, 0.7715471127944168, 0.7058481562775443, 0.771112541021521, 0.8888746592641508, 0.9224196898667042]\n",
      "Mean Cross Validation RMSE: 0.8172076298508891\n",
      "\n",
      "Training with params: \n",
      "{'colsample_bytree': 0.8, 'learning_rate': 0.25, 'max_depth': 8, 'metric': 'rmse', 'min_data_in_leaf': 21, 'objective': 'regression', 'seed': 1204, 'subsample': 0.75}\n",
      "6 fold results: [0.8463821321864193, 0.7798014753979008, 0.7035021168726048, 0.7727076354975815, 0.8945796895148399, 0.9291424997413731]\n",
      "Mean Cross Validation RMSE: 0.8210192582017867\n",
      "\n",
      "Training with params: \n",
      "{'colsample_bytree': 0.65, 'learning_rate': 0.07500000000000001, 'max_depth': 10, 'metric': 'rmse', 'min_data_in_leaf': 12, 'objective': 'regression', 'seed': 1204, 'subsample': 0.9500000000000001}\n",
      "6 fold results: [0.8333199400719219, 0.7690546697059623, 0.7061966521966212, 0.7679229634205682, 0.8851044831213181, 0.9188970922709871]\n",
      "Mean Cross Validation RMSE: 0.8134159667978965\n",
      "\n",
      "Training with params: \n",
      "{'colsample_bytree': 0.75, 'learning_rate': 0.30000000000000004, 'max_depth': 14, 'metric': 'rmse', 'min_data_in_leaf': 29, 'objective': 'regression', 'seed': 1204, 'subsample': 0.9}\n",
      "6 fold results: [0.8552162209368855, 0.7847750958289658, 0.7080226285533405, 0.7753968976652118, 0.8970626778296777, 0.9297878874521721]\n",
      "Mean Cross Validation RMSE: 0.8250435680443755\n",
      "\n",
      "Training with params: \n",
      "{'colsample_bytree': 0.6000000000000001, 'learning_rate': 0.125, 'max_depth': 5, 'metric': 'rmse', 'min_data_in_leaf': 9, 'objective': 'regression', 'seed': 1204, 'subsample': 0.8500000000000001}\n",
      "6 fold results: [0.844190284411145, 0.7622934795534977, 0.7074584719736959, 0.7671524138594552, 0.8933559373602803, 0.9345952303160534]\n",
      "Mean Cross Validation RMSE: 0.8181743029123546\n",
      "\n",
      "Training with params: \n",
      "{'colsample_bytree': 0.7000000000000001, 'learning_rate': 0.4, 'max_depth': 6, 'metric': 'rmse', 'min_data_in_leaf': 14, 'objective': 'regression', 'seed': 1204, 'subsample': 1.0}\n",
      "6 fold results: [0.8326505220940134, 0.7864022371493221, 0.7171182405999832, 0.775584767326495, 0.8929367612908907, 0.9362185624430551]\n",
      "Mean Cross Validation RMSE: 0.8234851818172931\n",
      "\n",
      "Training with params: \n",
      "{'colsample_bytree': 0.8500000000000001, 'learning_rate': 0.225, 'max_depth': 3, 'metric': 'rmse', 'min_data_in_leaf': 7, 'objective': 'regression', 'seed': 1204, 'subsample': 0.8}\n",
      "6 fold results: [0.851360693764802, 0.7832651233094419, 0.7148308206353609, 0.7840574435254561, 0.9037698566074042, 0.9498444763222049]\n",
      "Mean Cross Validation RMSE: 0.8311880690274451\n",
      "\n",
      "Training with params: \n",
      "{'colsample_bytree': 0.55, 'learning_rate': 0.025, 'max_depth': 12, 'metric': 'rmse', 'min_data_in_leaf': 20, 'objective': 'regression', 'seed': 1204, 'subsample': 0.6000000000000001}\n",
      "6 fold results: [0.8418905816249028, 0.7682165606794691, 0.6993824455603764, 0.7684715664898665, 0.8761084108443773, 0.9141332675023643]\n",
      "Mean Cross Validation RMSE: 0.8113671387835595\n",
      "\n",
      "Training with params: \n",
      "{'colsample_bytree': 0.55, 'learning_rate': 0.025, 'max_depth': 12, 'metric': 'rmse', 'min_data_in_leaf': 20, 'objective': 'regression', 'seed': 1204, 'subsample': 0.6000000000000001}\n",
      "6 fold results: [0.8418905816249028, 0.7682165606794691, 0.6993824455603764, 0.7684715664898665, 0.8761084108443773, 0.9141332675023643]\n",
      "Mean Cross Validation RMSE: 0.8113671387835595\n",
      "\n",
      "Training with params: \n",
      "{'colsample_bytree': 0.5, 'learning_rate': 0.05, 'max_depth': 12, 'metric': 'rmse', 'min_data_in_leaf': 20, 'objective': 'regression', 'seed': 1204, 'subsample': 0.55}\n",
      "6 fold results: [0.8319005412629975, 0.7676601464938417, 0.708317924740421, 0.7668697259632504, 0.8854118723688134, 0.919156400475019]\n",
      "Mean Cross Validation RMSE: 0.8132194352173906\n",
      "\n",
      "Training with params: \n",
      "{'colsample_bytree': 0.5, 'learning_rate': 0.07500000000000001, 'max_depth': 12, 'metric': 'rmse', 'min_data_in_leaf': 20, 'objective': 'regression', 'seed': 1204, 'subsample': 0.65}\n",
      "6 fold results: [0.8444546931999264, 0.7679429713481392, 0.7111428680072102, 0.7681417873607568, 0.8795861299138953, 0.9213290605102993]\n",
      "Mean Cross Validation RMSE: 0.8154329183900378\n",
      "\n",
      "Training with params: \n",
      "{'colsample_bytree': 0.55, 'learning_rate': 0.025, 'max_depth': 12, 'metric': 'rmse', 'min_data_in_leaf': 13, 'objective': 'regression', 'seed': 1204, 'subsample': 0.65}\n",
      "6 fold results: [0.8427220987733341, 0.7720866645253043, 0.7041787168292631, 0.7678220777094232, 0.8885733912727155, 0.9175659613769019]\n",
      "Mean Cross Validation RMSE: 0.8154914850811571\n",
      "\n",
      "Training with params: \n",
      "{'colsample_bytree': 0.6000000000000001, 'learning_rate': 0.025, 'max_depth': 12, 'metric': 'rmse', 'min_data_in_leaf': 22, 'objective': 'regression', 'seed': 1204, 'subsample': 0.55}\n",
      "6 fold results: [0.841898762695296, 0.7663066426179678, 0.7000789658415155, 0.7682589819578369, 0.884113078274269, 0.9160345367212348]\n",
      "Mean Cross Validation RMSE: 0.81278182801802\n",
      "\n",
      "Training with params: \n",
      "{'colsample_bytree': 0.55, 'learning_rate': 0.05, 'max_depth': 10, 'metric': 'rmse', 'min_data_in_leaf': 20, 'objective': 'regression', 'seed': 1204, 'subsample': 0.6000000000000001}\n",
      "6 fold results: [0.8319160449499239, 0.7711792679833961, 0.7005060471780673, 0.7664432464504269, 0.8863793526121302, 0.9206810889754717]\n",
      "Mean Cross Validation RMSE: 0.812850841358236\n",
      "\n",
      "Training with params: \n",
      "{'colsample_bytree': 0.5, 'learning_rate': 0.15000000000000002, 'max_depth': 12, 'metric': 'rmse', 'min_data_in_leaf': 25, 'objective': 'regression', 'seed': 1204, 'subsample': 0.7000000000000001}\n",
      "6 fold results: [0.8462511410649382, 0.7691707662225087, 0.7012053362700654, 0.7727085238377579, 0.8967966674787956, 0.9517500346186251]\n",
      "Mean Cross Validation RMSE: 0.8229804115821152\n",
      "\n",
      "Training with params: \n",
      "{'colsample_bytree': 0.65, 'learning_rate': 0.5, 'max_depth': 7, 'metric': 'rmse', 'min_data_in_leaf': 18, 'objective': 'regression', 'seed': 1204, 'subsample': 0.55}\n",
      "6 fold results: [0.8522856290380392, 0.7825153689605658, 0.7247097901313527, 0.7856622142385237, 0.8939552647552651, 0.930263546397767]\n",
      "Mean Cross Validation RMSE: 0.8282319689202522\n",
      "\n",
      "Training with params: \n",
      "{'colsample_bytree': 0.6000000000000001, 'learning_rate': 0.35000000000000003, 'max_depth': 13, 'metric': 'rmse', 'min_data_in_leaf': 28, 'objective': 'regression', 'seed': 1204, 'subsample': 0.9500000000000001}\n",
      "6 fold results: [0.8410911685414861, 0.7862324765919857, 0.7027511322253697, 0.7764014198640865, 0.8932525099178752, 0.9466457306376806]\n",
      "Mean Cross Validation RMSE: 0.8243957396297472\n",
      "\n",
      "Training with params: \n",
      "{'colsample_bytree': 0.5, 'learning_rate': 0.07500000000000001, 'max_depth': 4, 'metric': 'rmse', 'min_data_in_leaf': 11, 'objective': 'regression', 'seed': 1204, 'subsample': 0.9}\n",
      "6 fold results: [0.8543142436004804, 0.7766370319511653, 0.7104728563415174, 0.780113902542952, 0.8934854015322576, 0.9411283611372275]\n",
      "Mean Cross Validation RMSE: 0.8260252995176001\n",
      "\n",
      "Training with params: \n",
      "{'colsample_bytree': 0.55, 'learning_rate': 0.125, 'max_depth': 14, 'metric': 'rmse', 'min_data_in_leaf': 19, 'objective': 'regression', 'seed': 1204, 'subsample': 0.5}\n",
      "6 fold results: [0.8322912141569645, 0.7711302113009241, 0.7006358357655685, 0.766171860450736, 0.8957077374368619, 0.9250613226012077]\n",
      "Mean Cross Validation RMSE: 0.8151663636187104\n",
      "\n",
      "Training with params: \n",
      "{'colsample_bytree': 0.9, 'learning_rate': 0.1, 'max_depth': 5, 'metric': 'rmse', 'min_data_in_leaf': 10, 'objective': 'regression', 'seed': 1204, 'subsample': 0.5}\n",
      "6 fold results: [0.8462129018414544, 0.7788943667098258, 0.7033994848776371, 0.775580360988212, 0.8942782022469986, 0.939715941000825]\n",
      "Mean Cross Validation RMSE: 0.8230135429441588\n",
      "\n",
      "Training with params: \n",
      "{'colsample_bytree': 0.65, 'learning_rate': 0.025, 'max_depth': 11, 'metric': 'rmse', 'min_data_in_leaf': 5, 'objective': 'regression', 'seed': 1204, 'subsample': 0.5}\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "6 fold results: [0.8432502030068321, 0.7657755048702759, 0.7000663240725246, 0.7654476348981701, 0.8812559452906013, 0.9092122034491108]\n",
      "Mean Cross Validation RMSE: 0.8108346359312524\n",
      "\n",
      "Training with params: \n",
      "{'colsample_bytree': 0.65, 'learning_rate': 0.45, 'max_depth': 11, 'metric': 'rmse', 'min_data_in_leaf': 27, 'objective': 'regression', 'seed': 1204, 'subsample': 1.0}\n",
      "6 fold results: [0.8598440755572317, 0.788028814515866, 0.7134043649366609, 0.7856977614310364, 0.924271657956107, 0.9771136928581985]\n",
      "Mean Cross Validation RMSE: 0.8413933945425168\n",
      "\n",
      "Training with params: \n",
      "{'colsample_bytree': 0.7000000000000001, 'learning_rate': 0.47500000000000003, 'max_depth': 11, 'metric': 'rmse', 'min_data_in_leaf': 25, 'objective': 'regression', 'seed': 1204, 'subsample': 0.9500000000000001}\n",
      "6 fold results: [0.8637300722434602, 0.8000298407438035, 0.7198934975827738, 0.7831927267946306, 0.9018609386434406, 0.9272237075054118]\n",
      "Mean Cross Validation RMSE: 0.8326551305855867\n",
      "\n",
      "Training with params: \n",
      "{'colsample_bytree': 0.6000000000000001, 'learning_rate': 0.07500000000000001, 'max_depth': 11, 'metric': 'rmse', 'min_data_in_leaf': 5, 'objective': 'regression', 'seed': 1204, 'subsample': 0.9}\n",
      "6 fold results: [0.8449585731372402, 0.7724047117237888, 0.7008025348564231, 0.7681701099161665, 0.8985479564940069, 0.9178523942659119]\n",
      "Mean Cross Validation RMSE: 0.817122713398923\n",
      "\n",
      "Training with params: \n",
      "{'colsample_bytree': 0.65, 'learning_rate': 0.05, 'max_depth': 11, 'metric': 'rmse', 'min_data_in_leaf': 26, 'objective': 'regression', 'seed': 1204, 'subsample': 0.8500000000000001}\n",
      "6 fold results: [0.8441807930337738, 0.766949027520519, 0.6981637256714763, 0.7674201733215219, 0.8822352910380002, 0.9210305709267578]\n",
      "Mean Cross Validation RMSE: 0.8133299302520082\n",
      "\n",
      "Training with params: \n",
      "{'colsample_bytree': 0.7000000000000001, 'learning_rate': 0.325, 'max_depth': 11, 'metric': 'rmse', 'min_data_in_leaf': 5, 'objective': 'regression', 'seed': 1204, 'subsample': 0.8500000000000001}\n",
      "6 fold results: [0.851033130536064, 0.7803603917059732, 0.7133652780044055, 0.7853907017969102, 0.8984736834386452, 0.9177626050910718]\n",
      "Mean Cross Validation RMSE: 0.8243976317621784\n",
      "\n",
      "Training with params: \n",
      "{'colsample_bytree': 0.7000000000000001, 'learning_rate': 0.2, 'max_depth': 10, 'metric': 'rmse', 'min_data_in_leaf': 5, 'objective': 'regression', 'seed': 1204, 'subsample': 0.9500000000000001}\n",
      "6 fold results: [0.8445433904948491, 0.7744220870681304, 0.7112907209689295, 0.77410464172947, 0.88374614894509, 0.9138997479880105]\n",
      "Mean Cross Validation RMSE: 0.8170011228657467\n",
      "\n",
      "Training with params: \n",
      "{'colsample_bytree': 0.65, 'learning_rate': 0.15000000000000002, 'max_depth': 11, 'metric': 'rmse', 'min_data_in_leaf': 5, 'objective': 'regression', 'seed': 1204, 'subsample': 0.9}\n",
      "6 fold results: [0.8370874153002954, 0.7711127788675921, 0.7035394542339075, 0.7727836865770882, 0.885595227422342, 0.927909284872821]\n",
      "Mean Cross Validation RMSE: 0.8163379745456744\n",
      "\n",
      "Training with params: \n",
      "{'colsample_bytree': 0.75, 'learning_rate': 0.1, 'max_depth': 14, 'metric': 'rmse', 'min_data_in_leaf': 24, 'objective': 'regression', 'seed': 1204, 'subsample': 0.8}\n",
      "6 fold results: [0.8342683710942546, 0.7712968150249659, 0.7052257261589436, 0.7716683530093568, 0.8823940908668092, 0.9188604689224092]\n",
      "Mean Cross Validation RMSE: 0.8139523041794566\n",
      "\n",
      "Training with params: \n",
      "{'colsample_bytree': 0.6000000000000001, 'learning_rate': 0.025, 'max_depth': 10, 'metric': 'rmse', 'min_data_in_leaf': 8, 'objective': 'regression', 'seed': 1204, 'subsample': 0.9500000000000001}\n",
      "6 fold results: [0.8425707322692184, 0.7721150730115051, 0.7014187961848404, 0.7683172132128416, 0.8807795339478321, 0.9143996549972537]\n",
      "Mean Cross Validation RMSE: 0.8132668339372486\n",
      "\n",
      "Training with params: \n",
      "{'colsample_bytree': 0.65, 'learning_rate': 0.17500000000000002, 'max_depth': 3, 'metric': 'rmse', 'min_data_in_leaf': 17, 'objective': 'regression', 'seed': 1204, 'subsample': 1.0}\n",
      "6 fold results: [0.8762272471604544, 0.7837239629878514, 0.7173069146390088, 0.7842453303330381, 0.9068495300572997, 0.9467244393569322]\n",
      "Mean Cross Validation RMSE: 0.8358462374224307\n",
      "\n",
      "Training with params: \n",
      "{'colsample_bytree': 0.7000000000000001, 'learning_rate': 0.225, 'max_depth': 6, 'metric': 'rmse', 'min_data_in_leaf': 15, 'objective': 'regression', 'seed': 1204, 'subsample': 0.9}\n",
      "6 fold results: [0.8378822704511277, 0.7710420506269507, 0.7129187448628667, 0.7736766970078941, 0.8957647056954685, 0.9445476731847446]\n",
      "Mean Cross Validation RMSE: 0.822638690304842\n",
      "\n",
      "Training with params: \n",
      "{'colsample_bytree': 0.8, 'learning_rate': 0.07500000000000001, 'max_depth': 11, 'metric': 'rmse', 'min_data_in_leaf': 12, 'objective': 'regression', 'seed': 1204, 'subsample': 1.0}\n",
      "6 fold results: [0.8376793045333628, 0.7742614483091762, 0.7083331921854495, 0.7700252421906193, 0.8807526040485393, 0.9267916440257572]\n",
      "Mean Cross Validation RMSE: 0.816307239215484\n",
      "\n",
      "Training with params: \n",
      "{'colsample_bytree': 0.75, 'learning_rate': 0.1, 'max_depth': 13, 'metric': 'rmse', 'min_data_in_leaf': 21, 'objective': 'regression', 'seed': 1204, 'subsample': 0.75}\n",
      "6 fold results: [0.8466079542609024, 0.7693984632495835, 0.7039388064393314, 0.7639218198705693, 0.9009229459643757, 0.9218529650873175]\n",
      "Mean Cross Validation RMSE: 0.8177738258120133\n",
      "\n",
      "Training with params: \n",
      "{'colsample_bytree': 0.65, 'learning_rate': 0.125, 'max_depth': 4, 'metric': 'rmse', 'min_data_in_leaf': 29, 'objective': 'regression', 'seed': 1204, 'subsample': 0.5}\n",
      "6 fold results: [0.8507529325098258, 0.7704286086576405, 0.7102683832851695, 0.7775344861837626, 0.898156156344396, 0.9457121740813444]\n",
      "Mean Cross Validation RMSE: 0.8254754568436898\n",
      "\n",
      "Training with params: \n",
      "{'colsample_bytree': 0.6000000000000001, 'learning_rate': 0.05, 'max_depth': 7, 'metric': 'rmse', 'min_data_in_leaf': 16, 'objective': 'regression', 'seed': 1204, 'subsample': 0.8500000000000001}\n",
      "6 fold results: [0.8263015011850939, 0.7747392176759348, 0.703460946756323, 0.7622838872646553, 0.8851548595732268, 0.9287090963000477]\n",
      "Mean Cross Validation RMSE: 0.8134415847925469\n",
      "\n",
      "Training with params: \n",
      "{'colsample_bytree': 0.55, 'learning_rate': 0.05, 'max_depth': 12, 'metric': 'rmse', 'min_data_in_leaf': 20, 'objective': 'regression', 'seed': 1204, 'subsample': 0.55}\n",
      "6 fold results: [0.8433709758798887, 0.7704063016076794, 0.7019329441612484, 0.7672596386644015, 0.887157365645307, 0.9124196740434943]\n",
      "Mean Cross Validation RMSE: 0.8137578166670033\n",
      "\n",
      "Training with params: \n",
      "{'colsample_bytree': 0.5, 'learning_rate': 0.025, 'max_depth': 14, 'metric': 'rmse', 'min_data_in_leaf': 9, 'objective': 'regression', 'seed': 1204, 'subsample': 0.6000000000000001}\n",
      "6 fold results: [0.839646527827795, 0.7682785409388099, 0.7013579152432076, 0.7682110099237854, 0.8800629540453803, 0.912008974071293]\n",
      "Mean Cross Validation RMSE: 0.8115943203417118\n",
      "\n",
      "Training with params: \n",
      "{'colsample_bytree': 0.55, 'learning_rate': 0.025, 'max_depth': 10, 'metric': 'rmse', 'min_data_in_leaf': 22, 'objective': 'regression', 'seed': 1204, 'subsample': 0.5}\n",
      "6 fold results: [0.8424871050087228, 0.7666388449543783, 0.7003073668695572, 0.7714620776918749, 0.8846581240843009, 0.9195862301764395]\n",
      "Mean Cross Validation RMSE: 0.8141899581308789\n",
      "\n",
      "Training with params: \n",
      "{'colsample_bytree': 0.65, 'learning_rate': 0.025, 'max_depth': 5, 'metric': 'rmse', 'min_data_in_leaf': 20, 'objective': 'regression', 'seed': 1204, 'subsample': 0.5}\n",
      "6 fold results: [0.8574032589874677, 0.7749074780644108, 0.7053016169725732, 0.7729925503014089, 0.8925842131425095, 0.9442617582450127]\n",
      "Mean Cross Validation RMSE: 0.8245751459522305\n",
      "\n",
      "Training with params: \n",
      "{'colsample_bytree': 0.6000000000000001, 'learning_rate': 0.07500000000000001, 'max_depth': 12, 'metric': 'rmse', 'min_data_in_leaf': 6, 'objective': 'regression', 'seed': 1204, 'subsample': 0.55}\n",
      "6 fold results: [0.848681663750527, 0.770948766632844, 0.7033420236658217, 0.7710836249889259, 0.8907311274882979, 0.9175148044441936]\n",
      "Mean Cross Validation RMSE: 0.8170503351617683\n",
      "\n",
      "Training with params: \n",
      "{'colsample_bytree': 0.7000000000000001, 'learning_rate': 0.05, 'max_depth': 11, 'metric': 'rmse', 'min_data_in_leaf': 7, 'objective': 'regression', 'seed': 1204, 'subsample': 0.5}\n",
      "6 fold results: [0.830845779664139, 0.7699091932972296, 0.7031478080910092, 0.7686171664803856, 0.8814974495266169, 0.912312152812911]\n",
      "Mean Cross Validation RMSE: 0.8110549249787152\n",
      "\n",
      "Training with params: \n",
      "{'colsample_bytree': 0.7000000000000001, 'learning_rate': 0.05, 'max_depth': 11, 'metric': 'rmse', 'min_data_in_leaf': 5, 'objective': 'regression', 'seed': 1204, 'subsample': 0.5}\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "6 fold results: [0.8392639464317898, 0.7674707856503984, 0.7023065639229119, 0.7700615958392388, 0.8822220899396659, 0.9221151615803624]\n",
      "Mean Cross Validation RMSE: 0.813906690560728\n",
      "\n",
      "Training with params: \n",
      "{'colsample_bytree': 0.7000000000000001, 'learning_rate': 0.1, 'max_depth': 11, 'metric': 'rmse', 'min_data_in_leaf': 7, 'objective': 'regression', 'seed': 1204, 'subsample': 0.5}\n",
      "6 fold results: [0.8292549119664023, 0.7717429072155731, 0.7005297601102368, 0.768690583764772, 0.8939118754986324, 0.9186061005027829]\n",
      "Mean Cross Validation RMSE: 0.8137893565097333\n",
      "\n",
      "Training with params: \n",
      "{'colsample_bytree': 0.75, 'learning_rate': 0.125, 'max_depth': 11, 'metric': 'rmse', 'min_data_in_leaf': 7, 'objective': 'regression', 'seed': 1204, 'subsample': 0.55}\n",
      "6 fold results: [0.847545004395674, 0.7724318150735455, 0.707698406218051, 0.7790165802190014, 0.8876117647650685, 0.9289897310640147]\n",
      "Mean Cross Validation RMSE: 0.8205488836225593\n",
      "\n",
      "Training with params: \n",
      "{'colsample_bytree': 0.65, 'learning_rate': 0.07500000000000001, 'max_depth': 11, 'metric': 'rmse', 'min_data_in_leaf': 7, 'objective': 'regression', 'seed': 1204, 'subsample': 0.5}\n",
      "6 fold results: [0.8384269298331961, 0.7717296684156323, 0.6979695034155543, 0.7679572275346629, 0.8817474774529337, 0.9110130115983675]\n",
      "Mean Cross Validation RMSE: 0.8114739697083911\n",
      "\n",
      "Training with params: \n",
      "{'colsample_bytree': 0.75, 'learning_rate': 0.125, 'max_depth': 11, 'metric': 'rmse', 'min_data_in_leaf': 13, 'objective': 'regression', 'seed': 1204, 'subsample': 0.6000000000000001}\n",
      "6 fold results: [0.8431038863706091, 0.7731138347450754, 0.7010311653616954, 0.7689586585887386, 0.8898581089360814, 0.9289897310640147]\n",
      "Mean Cross Validation RMSE: 0.8175092308443691\n",
      "\n",
      "Training with params: \n",
      "{'colsample_bytree': 0.7000000000000001, 'learning_rate': 0.05, 'max_depth': 11, 'metric': 'rmse', 'min_data_in_leaf': 7, 'objective': 'regression', 'seed': 1204, 'subsample': 0.55}\n",
      "6 fold results: [0.830845779664139, 0.7699091932972296, 0.7031478080910092, 0.7686171664803856, 0.8814974495266169, 0.912312152812911]\n",
      "Mean Cross Validation RMSE: 0.8110549249787152\n",
      "\n",
      "Training with params: \n",
      "{'colsample_bytree': 0.7000000000000001, 'learning_rate': 0.025, 'max_depth': 11, 'metric': 'rmse', 'min_data_in_leaf': 7, 'objective': 'regression', 'seed': 1204, 'subsample': 0.55}\n"
     ]
    },
    {
     "ename": "KeyboardInterrupt",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-7-0707184175de>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m      9\u001b[0m     \u001b[0;34m'metric'\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m'rmse'\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     10\u001b[0m }\n\u001b[0;32m---> 11\u001b[0;31m \u001b[0mbest_hyperparams\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0moptimize\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mspace\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0mmax_evals\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m300\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     12\u001b[0m \u001b[0mprint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"The best hyperparameters are: \"\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     13\u001b[0m \u001b[0mprint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mbest_hyperparams\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m<ipython-input-6-f4d943850cb7>\u001b[0m in \u001b[0;36moptimize\u001b[0;34m(space, seed, max_evals)\u001b[0m\n\u001b[1;32m     45\u001b[0m     best = fmin(score, space, algo=tpe.suggest, \n\u001b[1;32m     46\u001b[0m         \u001b[0;31m# trials=trials,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 47\u001b[0;31m         max_evals=max_evals)\n\u001b[0m\u001b[1;32m     48\u001b[0m     \u001b[0;32mreturn\u001b[0m \u001b[0mbest\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/envs/xgb/lib/python3.6/site-packages/hyperopt/fmin.py\u001b[0m in \u001b[0;36mfmin\u001b[0;34m(fn, space, algo, max_evals, trials, rstate, allow_trials_fmin, pass_expr_memo_ctrl, catch_eval_exceptions, verbose, return_argmin)\u001b[0m\n\u001b[1;32m    318\u001b[0m                     verbose=verbose)\n\u001b[1;32m    319\u001b[0m     \u001b[0mrval\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcatch_eval_exceptions\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mcatch_eval_exceptions\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 320\u001b[0;31m     \u001b[0mrval\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mexhaust\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    321\u001b[0m     \u001b[0;32mif\u001b[0m \u001b[0mreturn_argmin\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    322\u001b[0m         \u001b[0;32mreturn\u001b[0m \u001b[0mtrials\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0margmin\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/envs/xgb/lib/python3.6/site-packages/hyperopt/fmin.py\u001b[0m in \u001b[0;36mexhaust\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m    197\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0mexhaust\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    198\u001b[0m         \u001b[0mn_done\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mlen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtrials\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 199\u001b[0;31m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mrun\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmax_evals\u001b[0m \u001b[0;34m-\u001b[0m \u001b[0mn_done\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mblock_until_done\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0masync\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    200\u001b[0m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtrials\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mrefresh\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    201\u001b[0m         \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/envs/xgb/lib/python3.6/site-packages/hyperopt/fmin.py\u001b[0m in \u001b[0;36mrun\u001b[0;34m(self, N, block_until_done)\u001b[0m\n\u001b[1;32m    171\u001b[0m             \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    172\u001b[0m                 \u001b[0;31m# -- loop over trials and do the jobs directly\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 173\u001b[0;31m                 \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mserial_evaluate\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    174\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    175\u001b[0m             \u001b[0;32mif\u001b[0m \u001b[0mstopped\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/envs/xgb/lib/python3.6/site-packages/hyperopt/fmin.py\u001b[0m in \u001b[0;36mserial_evaluate\u001b[0;34m(self, N)\u001b[0m\n\u001b[1;32m     90\u001b[0m                 \u001b[0mctrl\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mbase\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mCtrl\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtrials\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcurrent_trial\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mtrial\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     91\u001b[0m                 \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 92\u001b[0;31m                     \u001b[0mresult\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdomain\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mevaluate\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mspec\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mctrl\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     93\u001b[0m                 \u001b[0;32mexcept\u001b[0m \u001b[0mException\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0me\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     94\u001b[0m                     \u001b[0mlogger\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0minfo\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'job exception: %s'\u001b[0m \u001b[0;34m%\u001b[0m \u001b[0mstr\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0me\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/envs/xgb/lib/python3.6/site-packages/hyperopt/base.py\u001b[0m in \u001b[0;36mevaluate\u001b[0;34m(self, config, ctrl, attach_attachments)\u001b[0m\n\u001b[1;32m    838\u001b[0m                 \u001b[0mmemo\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mmemo\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    839\u001b[0m                 print_node_on_error=self.rec_eval_print_node_on_error)\n\u001b[0;32m--> 840\u001b[0;31m             \u001b[0mrval\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfn\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mpyll_rval\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    841\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    842\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0misinstance\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mrval\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m(\u001b[0m\u001b[0mfloat\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mint\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mnp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mnumber\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m<ipython-input-6-f4d943850cb7>\u001b[0m in \u001b[0;36mscore\u001b[0;34m(params)\u001b[0m\n\u001b[1;32m     27\u001b[0m         lgb_model = lgb.train(params, lgb.Dataset(cv_train, label=cv_y_train), 2000, \n\u001b[1;32m     28\u001b[0m                           \u001b[0mlgb\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mDataset\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mcv_val\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mlabel\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mcv_y_val\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mverbose_eval\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mFalse\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 29\u001b[0;31m                           early_stopping_rounds=50)\n\u001b[0m\u001b[1;32m     30\u001b[0m         \u001b[0mpred\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mlgb_model\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mpredict\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mcv_val\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0mlgb_model\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mbest_iteration\u001b[0m\u001b[0;34m+\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     31\u001b[0m         \u001b[0mloss\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mroot_mean_squared_error\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mcv_y_val\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0mpred\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/envs/xgb/lib/python3.6/site-packages/lightgbm/engine.py\u001b[0m in \u001b[0;36mtrain\u001b[0;34m(params, train_set, num_boost_round, valid_sets, valid_names, fobj, feval, init_model, feature_name, categorical_feature, early_stopping_rounds, evals_result, verbose_eval, learning_rates, keep_training_booster, callbacks)\u001b[0m\n\u001b[1;32m    199\u001b[0m                                     evaluation_result_list=None))\n\u001b[1;32m    200\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 201\u001b[0;31m         \u001b[0mbooster\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mupdate\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mfobj\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mfobj\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    202\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    203\u001b[0m         \u001b[0mevaluation_result_list\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/envs/xgb/lib/python3.6/site-packages/lightgbm/basic.py\u001b[0m in \u001b[0;36mupdate\u001b[0;34m(self, train_set, fobj)\u001b[0m\n\u001b[1;32m   1508\u001b[0m             _safe_call(_LIB.LGBM_BoosterUpdateOneIter(\n\u001b[1;32m   1509\u001b[0m                 \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mhandle\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1510\u001b[0;31m                 ctypes.byref(is_finished)))\n\u001b[0m\u001b[1;32m   1511\u001b[0m             \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m__is_predicted_cur_iter\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0;32mFalse\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0m_\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mrange_\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m__num_dataset\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1512\u001b[0m             \u001b[0;32mreturn\u001b[0m \u001b[0mis_finished\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mvalue\u001b[0m \u001b[0;34m==\u001b[0m \u001b[0;36m1\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m: "
     ]
    }
   ],
   "source": [
    "space = {\n",
    "#     'max_depth': hp.choice('max_depth', np.arange(3, 15, dtype=int)),\n",
    "    'subsample': hp.quniform('subsample', 0.5, 1, 0.05),\n",
    "    'colsample_bytree': hp.quniform('colsample_bytree', 0.5, 1, 0.05),\n",
    "    'min_data_in_leaf': hp.choice('min_data_in_leaf',np.arange(5, 30,1, dtype=int)),\n",
    "    'learning_rate': hp.quniform('learning_rate', 0.025, 0.5, 0.025),\n",
    "    'seed':seed,\n",
    "    'objective': 'regression',\n",
    "    'metric':'rmse',\n",
    "}\n",
    "best_hyperparams = optimize(space,max_evals=300)\n",
    "print(\"The best hyperparameters are: \")\n",
    "print(best_hyperparams)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Get oof prediction and test prediction"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "all_data = get_all_data(data_path,'new_sales_lag_after12.pickle')\n",
    "\n",
    "X,y = get_X_y(all_data,33)\n",
    "X.drop('date_block_num',axis=1,inplace=True)\n",
    "\n",
    "cv = get_cv_idxs(all_data,28,33)\n",
    "\n",
    "\n",
    "# np.mean([0.8489192532696636, 0.7787329689560278, 0.7065547479353921, 0.7692630166068186, 0.8873116487735562, 0.9227562743578496])\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "lgb_params = {\n",
    "               'colsample_bytree': 0.75,\n",
    "               'metric': 'rmse',\n",
    "               'min_data_in_leaf': 128, \n",
    "               'subsample': 0.75, \n",
    "               'learning_rate': 0.03, \n",
    "               'objective': 'regression', \n",
    "               'bagging_seed': 128, \n",
    "               'num_leaves': 128,\n",
    "               'bagging_freq':1,\n",
    "               'seed':1204\n",
    "              }"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[10]\tvalid_0's rmse: 1.05715\n",
      "[20]\tvalid_0's rmse: 0.972785\n",
      "[30]\tvalid_0's rmse: 0.919024\n",
      "[40]\tvalid_0's rmse: 0.882785\n",
      "[50]\tvalid_0's rmse: 0.858666\n",
      "[60]\tvalid_0's rmse: 0.841719\n",
      "[70]\tvalid_0's rmse: 0.828854\n",
      "[80]\tvalid_0's rmse: 0.819173\n",
      "[90]\tvalid_0's rmse: 0.811504\n",
      "[100]\tvalid_0's rmse: 0.805388\n",
      "[110]\tvalid_0's rmse: 0.800436\n",
      "[120]\tvalid_0's rmse: 0.79595\n",
      "[130]\tvalid_0's rmse: 0.791915\n",
      "[140]\tvalid_0's rmse: 0.788522\n",
      "[150]\tvalid_0's rmse: 0.785449\n",
      "[160]\tvalid_0's rmse: 0.782645\n",
      "[170]\tvalid_0's rmse: 0.780153\n",
      "[180]\tvalid_0's rmse: 0.777169\n",
      "[190]\tvalid_0's rmse: 0.77505\n",
      "[200]\tvalid_0's rmse: 0.772982\n",
      "[210]\tvalid_0's rmse: 0.77107\n",
      "[220]\tvalid_0's rmse: 0.769111\n",
      "[230]\tvalid_0's rmse: 0.767513\n",
      "[240]\tvalid_0's rmse: 0.765918\n",
      "[250]\tvalid_0's rmse: 0.764354\n",
      "[260]\tvalid_0's rmse: 0.762917\n",
      "[270]\tvalid_0's rmse: 0.761617\n",
      "[280]\tvalid_0's rmse: 0.760319\n",
      "[290]\tvalid_0's rmse: 0.759001\n",
      "[300]\tvalid_0's rmse: 0.757857\n",
      "[310]\tvalid_0's rmse: 0.756796\n",
      "[320]\tvalid_0's rmse: 0.755716\n",
      "[330]\tvalid_0's rmse: 0.754554\n",
      "[340]\tvalid_0's rmse: 0.75333\n",
      "[350]\tvalid_0's rmse: 0.752259\n",
      "[360]\tvalid_0's rmse: 0.751384\n",
      "[370]\tvalid_0's rmse: 0.75022\n",
      "[380]\tvalid_0's rmse: 0.74925\n",
      "[390]\tvalid_0's rmse: 0.748295\n",
      "[400]\tvalid_0's rmse: 0.747382\n",
      "[410]\tvalid_0's rmse: 0.746545\n",
      "[420]\tvalid_0's rmse: 0.74572\n",
      "[430]\tvalid_0's rmse: 0.744944\n",
      "[440]\tvalid_0's rmse: 0.744187\n",
      "[450]\tvalid_0's rmse: 0.743434\n",
      "[460]\tvalid_0's rmse: 0.742513\n",
      "[470]\tvalid_0's rmse: 0.741842\n",
      "[480]\tvalid_0's rmse: 0.74053\n",
      "[490]\tvalid_0's rmse: 0.739597\n",
      "[500]\tvalid_0's rmse: 0.738668\n",
      "[510]\tvalid_0's rmse: 0.737975\n",
      "[520]\tvalid_0's rmse: 0.737292\n",
      "[530]\tvalid_0's rmse: 0.736653\n",
      "[540]\tvalid_0's rmse: 0.735963\n",
      "[550]\tvalid_0's rmse: 0.735189\n",
      "[560]\tvalid_0's rmse: 0.73461\n",
      "[570]\tvalid_0's rmse: 0.733949\n",
      "[580]\tvalid_0's rmse: 0.733182\n",
      "[590]\tvalid_0's rmse: 0.732259\n",
      "[600]\tvalid_0's rmse: 0.731704\n",
      "[610]\tvalid_0's rmse: 0.731046\n",
      "[620]\tvalid_0's rmse: 0.730506\n",
      "[630]\tvalid_0's rmse: 0.729993\n",
      "[640]\tvalid_0's rmse: 0.72924\n",
      "[650]\tvalid_0's rmse: 0.728686\n",
      "CPU times: user 17min 20s, sys: 14.1 s, total: 17min 35s\n",
      "Wall time: 4min 36s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "\n",
    "\n",
    "lgb_model_full = lgb.train(lgb_params, lgb.Dataset(X, label=y), 708, \n",
    "                      lgb.Dataset(X, label=y), verbose_eval=10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "test = pd.read_csv(os.path.join(data_path, 'test_lag.csv'))\n",
    "test.drop(['ID','item_name','date_block_num'],axis=1,inplace=True)\n",
    "test_pred = lgb_model_full.predict(test,708)\n",
    "get_submission(test_pred,'coursera_tuned_lightgbm_basic_6folds');"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<matplotlib.axes._subplots.AxesSubplot at 0x7f52382d0630>"
      ]
     },
     "execution_count": 35,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAY0AAAD8CAYAAACLrvgBAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMS4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvNQv5yAAAEodJREFUeJzt3X+s3fV93/Hna7i0Li0BynKFbDbT1dpKg5YSC7xlqu7KBoZOM5VKRRQNEyF5ikibTkib0388JUVKpKU0TCmSFzxMlYYymg2rOPMskqNuUkKBJIpDvMhXhMEtHrQzoThRGzl974/zcTi1j+/9cM+Fc3L9fEhX9/t9fz/f7/nct4788vfHPTdVhSRJPf7WtCcgSfrhYWhIkroZGpKkboaGJKmboSFJ6mZoSJK6GRqSpG6GhiSpm6EhSeq2btoTWG2XXnppbdq0adWO953vfIcLLrhg1Y631tifpdmf5dmjpb1V/Xn66af/vKr+9nLj1lxobNq0iaeeemrVjjcYDJifn1+146019mdp9md59mhpb1V/kvyfnnFenpIkdTM0JEndDA1JUjdDQ5LUzdCQJHUzNCRJ3QwNSVI3Q0OS1M3QkCR1W3O/Eb5aNu16DIC7rjrJ/HSnIkkzwzMNSVI3Q0OS1M3QkCR1MzQkSd0MDUlSN0NDktTN0JAkdTM0JEndDA1JUjdDQ5LUzdCQJHUzNCRJ3QwNSVI3Q0OS1G3Z0EiyN8nLSb4+UrskyaEkR9v3i1s9Se5NspDka0muHtlnRxt/NMmOkfq7khxu+9ybJEu9hiRpenrONB4Atp1W2wU8XlWbgcfbOsCNwOb2tRO4D4YBAOwGrgWuAXaPhMB9beyp/bYt8xqSpClZNjSq6o+B46eVtwP72vI+4OaR+oM19CXgoiSXATcAh6rqeFW9AhwCtrVtF1bVF6uqgAdPO9a415AkTclK/3LfXFUdA6iqY0ne3uobgBdGxi222lL1xTH1pV7jDEl2MjxbYW5ujsFgsMIf63V3XXVyOIn1rMrx1qoTJ07YnyXYn+XZo6XNWn9W+8+9ZkytVlB/Q6pqD7AHYMuWLTU/P/9GD3GG20f+3OuvrsLx1qrBYMBq9Hutsj/Ls0dLm7X+rPTpqZfapSXa95dbfRG4fGTcRuDFZeobx9SXeg1J0pSsNDT2A6eegNoBPDpSv609RbUVeLVdYjoIXJ/k4nYD/HrgYNv2WpKt7amp20471rjXkCRNybKXp5J8BpgHLk2yyPApqI8CDye5A3geuKUNPwDcBCwA3wXeB1BVx5N8BHiyjftwVZ26uf5+hk9orQc+175Y4jUkSVOybGhU1XvOsum6MWMLuPMsx9kL7B1Tfwp4x5j6/xv3GpKk6fE3wiVJ3QwNSVI3Q0OS1M3QkCR1MzQkSd0MDUlSN0NDktTN0JAkdTM0JEndDA1JUjdDQ5LUzdCQJHUzNCRJ3QwNSVI3Q0OS1M3QkCR1MzQkSd0MDUlSN0NDktTN0JAkdTM0JEndDA1JUjdDQ5LUzdCQJHUzNCRJ3QwNSVI3Q0OS1M3QkCR1myg0kvybJM8k+XqSzyT5sSRXJHkiydEkf5Dk/Db2R9v6Qtu+aeQ4H2r1bya5YaS+rdUWkuyaZK6SpMmtODSSbAB+HdhSVe8AzgNuBT4G3FNVm4FXgDvaLncAr1TVzwD3tHEkubLt93PANuB3k5yX5Dzgk8CNwJXAe9pYSdKUTHp5ah2wPsk64MeBY8AvAo+07fuAm9vy9rZO235dkrT6Q1X1V1X1LWABuKZ9LVTVs1X1PeChNlaSNCUrDo2q+lPgPwDPMwyLV4GngW9X1ck2bBHY0JY3AC+0fU+28T81Wj9tn7PVJUlTsm6lOya5mOH//K8Avg38F4aXkk5Xp3Y5y7az1ccFWo2pkWQnsBNgbm6OwWCw1NS73HXVMPfm1rMqx1urTpw4YX+WYH+WZ4+WNmv9WXFoAP8M+FZV/RlAks8C/xi4KMm6djaxEXixjV8ELgcW2+WstwHHR+qnjO5ztvrfUFV7gD0AW7Zsqfn5+Ql+rKHbdz0GDMPjV1fheGvVYDBgNfq9Vtmf5dmjpc1afya5p/E8sDXJj7d7E9cB3wC+APxKG7MDeLQt72/rtO2fr6pq9Vvb01VXAJuBPwGeBDa3p7HOZ3izfP8E85UkTWjFZxpV9USSR4AvAyeBrzD83/5jwENJfqvV7m+73A/8XpIFhmcYt7bjPJPkYYaBcxK4s6q+D5DkA8BBhk9m7a2qZ1Y6X0nS5Ca5PEVV7QZ2n1Z+luGTT6eP/UvglrMc527g7jH1A8CBSeYoSVo9/ka4JKmboSFJ6mZoSJK6GRqSpG6GhiSpm6EhSepmaEiSuhkakqRuhoYkqZuhIUnqZmhIkroZGpKkboaGJKmboSFJ6mZoSJK6GRqSpG6GhiSpm6EhSepmaEiSuhkakqRuhoYkqZuhIUnqZmhIkroZGpKkboaGJKmboSFJ6mZoSJK6GRqSpG6GhiSp20ShkeSiJI8k+d9JjiT5R0kuSXIoydH2/eI2NknuTbKQ5GtJrh45zo42/miSHSP1dyU53Pa5N0kmma8kaTKTnml8AvjvVfUPgH8IHAF2AY9X1Wbg8bYOcCOwuX3tBO4DSHIJsBu4FrgG2H0qaNqYnSP7bZtwvpKkCaw4NJJcCPwCcD9AVX2vqr4NbAf2tWH7gJvb8nbgwRr6EnBRksuAG4BDVXW8ql4BDgHb2rYLq+qLVVXAgyPHkiRNwSRnGj8N/Bnwn5N8JcmnklwAzFXVMYD2/e1t/AbghZH9F1ttqfrimLokaUrWTbjv1cCvVdUTST7B65eixhl3P6JWUD/zwMlOhpexmJubYzAYLDGNPndddRKAufXwHz/96A/qV21428THXktOnDixKv1eq+zP8uzR0matP5OExiKwWFVPtPVHGIbGS0kuq6pj7RLTyyPjLx/ZfyPwYqvPn1YftPrGMePPUFV7gD0AW7Zsqfn5+XHD3pDbdz0GDMPj44dfb9Nz75382GvJYDBgNfq9Vtmf5dmjpc1af1Z8eaqq/i/wQpK/30rXAd8A9gOnnoDaAZz6b/p+4Lb2FNVW4NV2+eogcH2Si9sN8OuBg23ba0m2tqembhs5liRpCiY50wD4NeDTSc4HngXexzCIHk5yB/A8cEsbewC4CVgAvtvGUlXHk3wEeLKN+3BVHW/L7wceANYDn2tfkqQpmSg0quqrwJYxm64bM7aAO89ynL3A3jH1p4B3TDJHSdLq8TfCJUndDA1JUjdDQ5LUzdCQJHUzNCRJ3QwNSVI3Q0OS1M3QkCR1MzQkSd0MDUlSN0NDktTN0JAkdTM0JEndDA1JUjdDQ5LUzdCQJHUzNCRJ3QwNSVI3Q0OS1M3QkCR1MzQkSd0MDUlSN0NDktTN0JAkdTM0JEndDA1JUjdDQ5LUzdCQJHUzNCRJ3SYOjSTnJflKkj9q61ckeSLJ0SR/kOT8Vv/Rtr7Qtm8aOcaHWv2bSW4YqW9rtYUkuyadqyRpMqtxpvFB4MjI+seAe6pqM/AKcEer3wG8UlU/A9zTxpHkSuBW4OeAbcDvtiA6D/gkcCNwJfCeNlaSNCUThUaSjcAvAZ9q6wF+EXikDdkH3NyWt7d12vbr2vjtwENV9VdV9S1gAbimfS1U1bNV9T3goTZWkjQl6ybc/3eAfwv8ZFv/KeDbVXWyrS8CG9ryBuAFgKo6meTVNn4D8KWRY47u88Jp9WvHTSLJTmAnwNzcHIPBYOU/UXPXVcMfYW7968vAqhx7LTlx4oQ9WYL9WZ49Wtqs9WfFoZHkXwAvV9XTSeZPlccMrWW2na0+7iyoxtSoqj3AHoAtW7bU/Pz8uGFvyO27HgOGgfHxw6+36bn3Tn7stWQwGLAa/V6r7M/y7NHSZq0/k5xpvBv4l0luAn4MuJDhmcdFSda1s42NwItt/CJwObCYZB3wNuD4SP2U0X3OVpckTcGK72lU1YeqamNVbWJ4I/vzVfVe4AvAr7RhO4BH2/L+tk7b/vmqqla/tT1ddQWwGfgT4Elgc3sa6/z2GvtXOl9J0uQmvacxzr8DHkryW8BXgPtb/X7g95IsMDzDuBWgqp5J8jDwDeAkcGdVfR8gyQeAg8B5wN6qeuZNmK8kqdOqhEZVDYBBW36W4ZNPp4/5S+CWs+x/N3D3mPoB4MBqzFGSNDl/I1yS1M3QkCR1MzQkSd0MDUlSN0NDktTN0JAkdTM0JEndDA1JUjdDQ5LUzdCQJHUzNCRJ3QwNSVI3Q0OS1M3QkCR1MzQkSd0MDUlSN0NDktTN0JAkdTM0JEndDA1JUjdDQ5LUzdCQJHUzNCRJ3QwNSVI3Q0OS1M3QkCR1MzQkSd0MDUlStxWHRpLLk3whyZEkzyT5YKtfkuRQkqPt+8WtniT3JllI8rUkV48ca0cbfzTJjpH6u5IcbvvcmyST/LCSpMlMcqZxErirqn4W2ArcmeRKYBfweFVtBh5v6wA3Apvb107gPhiGDLAbuBa4Bth9KmjamJ0j+22bYL6SpAmtODSq6lhVfbktvwYcATYA24F9bdg+4Oa2vB14sIa+BFyU5DLgBuBQVR2vqleAQ8C2tu3CqvpiVRXw4MixJElTsCr3NJJsAn4eeAKYq6pjMAwW4O1t2AbghZHdFlttqfrimLokaUrWTXqAJD8B/CHwG1X1F0vcdhi3oVZQHzeHnQwvYzE3N8dgMFhm1su766qTAMytf30ZWJVjryUnTpywJ0uwP8uzR0ubtf5MFBpJfoRhYHy6qj7byi8luayqjrVLTC+3+iJw+cjuG4EXW33+tPqg1TeOGX+GqtoD7AHYsmVLzc/Pjxv2hty+6zFgGBgfP/x6m5577+THXksGgwGr0e+1yv4szx4tbdb6M8nTUwHuB45U1W+PbNoPnHoCagfw6Ej9tvYU1Vbg1Xb56iBwfZKL2w3w64GDbdtrSba217pt5FhTs2nXYz/4kqRzzSRnGu8G/hVwOMlXW+03gY8CDye5A3geuKVtOwDcBCwA3wXeB1BVx5N8BHiyjftwVR1vy+8HHgDWA59rX5KkKVlxaFTV/2L8fQeA68aML+DOsxxrL7B3TP0p4B0rnaMkaXX5G+GSpG6GhiSpm6EhSepmaEiSuhkakqRuhoYkqZuhIUnqZmhIkroZGpKkboaGJKmboSFJ6mZoSJK6GRqSpG6GhiSpm6EhSepmaEiSuhkakqRuk/y513Pe6N8Jf+6jvzTFmUjSW8MzDUlSN0NDktTN0JAkdTM0JEndDA1JUjdDQ5LUzUduV4mP30o6F3imIUnqZmhIkrp5eepN4KUqSWuVZxqSpG4zf6aRZBvwCeA84FNV9dEpT+kN8axD0loy06GR5Dzgk8A/BxaBJ5Psr6pvTHdmK2OASPphN9OhAVwDLFTVswBJHgK2Az+UoTFqNECWYrhImiWzHhobgBdG1heBa6c0l6noDZc3wiCStFKzHhoZU6szBiU7gZ1t9USSb67WBH4dLgX+fLWONwvysVU93JrrzyqzP8uzR0t7q/rzd3sGzXpoLAKXj6xvBF48fVBV7QH2vBkTSPJUVW15M469Ftifpdmf5dmjpc1af2b9kdsngc1JrkhyPnArsH/Kc5Kkc9ZMn2lU1ckkHwAOMnzkdm9VPTPlaUnSOWumQwOgqg4AB6Y4hTflstcaYn+WZn+WZ4+WNlP9SdUZ95UlSRpr1u9pSJJmiKFxFkm2JflmkoUku6Y9n1mU5Lkkh5N8NclT057PtCXZm+TlJF8fqV2S5FCSo+37xdOc4zSdpT//PsmftvfQV5PcNM05TlOSy5N8IcmRJM8k+WCrz9R7yNAYY+TjS24ErgTek+TK6c5qZv3TqnrnLD0SOEUPANtOq+0CHq+qzcDjbf1c9QBn9gfgnvYeeme7h3muOgncVVU/C2wF7mz/7szUe8jQGO8HH19SVd8DTn18iXRWVfXHwPHTytuBfW15H3DzWzqpGXKW/qipqmNV9eW2/BpwhOGnYszUe8jQGG/cx5dsmNJcZlkB/yPJ0+238nWmuao6BsN/FIC3T3k+s+gDSb7WLl+ds5fvRiXZBPw88AQz9h4yNMbr+vgS8e6quprhZbw7k/zCtCekHzr3AX8PeCdwDPj4dKczfUl+AvhD4Deq6i+mPZ/TGRrjdX18ybmuql5s318G/ivDy3r6m15KchlA+/7ylOczU6rqpar6flX9NfCfOMffQ0l+hGFgfLqqPtvKM/UeMjTG8+NLlpHkgiQ/eWoZuB74+tJ7nZP2Azva8g7g0SnOZeac+sew+WXO4fdQkgD3A0eq6rdHNs3Ue8hf7juL9ujf7/D6x5fcPeUpzZQkP83w7AKGnyzw++d6j5J8Bphn+KmkLwG7gf8GPAz8HeB54JaqOidvBp+lP/MML00V8Bzwr09dvz/XJPknwP8EDgN/3cq/yfC+xsy8hwwNSVI3L09JkroZGpKkboaGJKmboSFJ6mZoSJK6GRqSpG6GhiSpm6EhSer2/wEzcL3cXjg3LQAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x7f52382dd828>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "pd.Series(test_pred).hist(bins=100)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# get out of fold features"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training with params: \n",
      "{'colsample_bytree': 0.75, 'metric': 'rmse', 'min_data_in_leaf': 128, 'subsample': 0.75, 'learning_rate': 0.03, 'objective': 'regression', 'bagging_seed': 128, 'num_leaves': 128, 'bagging_freq': 1, 'seed': 1204, 'verbose': 1}\n",
      "Train RMSE: 0.8060203656198298. Val RMSE: 0.842604340869596\n",
      "Best iteration: 109\n",
      "Train RMSE: 0.7187770224588755. Val RMSE: 0.7607151772408083\n",
      "Best iteration: 845\n",
      "Train RMSE: 0.7280455507519825. Val RMSE: 0.6960586737223564\n",
      "Best iteration: 642\n",
      "Train RMSE: 0.6881746410827226. Val RMSE: 0.7624647527297275\n",
      "Best iteration: 1471\n",
      "Train RMSE: 0.713518541419691. Val RMSE: 0.8757157548862285\n",
      "Best iteration: 819\n",
      "Train RMSE: 0.7476038494661554. Val RMSE: 0.915767486370869\n",
      "Best iteration: 357\n",
      "6 fold results: [0.842604340869596, 0.7607151772408083, 0.6960586737223564, 0.7624647527297275, 0.8757157548862285, 0.915767486370869]\n",
      "Mean Cross Validation RMSE: 0.8088876976365976\n",
      "\n"
     ]
    }
   ],
   "source": [
    "oof_train,_ = timeseries_cv('lgb',X,y,lgb_params,cv,root_mean_squared_error,150,True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "oof_df = pd.Series(oof_train)\n",
    "oof_df.to_pickle(data_path+'oof/lgb_best.pickle')"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}