exhaustive_feature_selection_and_deployment.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Using matplotlib backend: Qt5Agg\n"
     ]
    }
   ],
   "source": [
    "# import necessary libraries\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "from tsfresh import select_features\n",
    "import xgboost\n",
    "import seaborn as sns\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "import math, datetime \n",
    "\n",
    "from sklearn.preprocessing import StandardScaler\n",
    "from sklearn.metrics import explained_variance_score\n",
    "from sklearn.metrics import mean_squared_error\n",
    "from sklearn.pipeline import Pipeline\n",
    "from sklearn.feature_selection import SelectFromModel\n",
    "\n",
    "from feature_selection import FeatureSelector\n",
    "\n",
    "from hyperopt import STATUS_OK, Trials, fmin, hp, tpe\n",
    "\n",
    "%matplotlib"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 87,
   "metadata": {},
   "outputs": [],
   "source": [
    "from matplotlib import pylab"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "# all extracted features from tsfresh\n",
    "# the extracted features are the same for all models\n",
    "full_features = pd.read_pickle('/Users/Rohil/Documents/iGEM/yemen/exhaustive_extracted_features_new.pkl')\n",
    "\n",
    "# test values for different time frames (in a nice format)\n",
    "y_df = pd.read_pickle('/Users/Rohil/Documents/iGEM/yemen/y_df_for_feature_selection_new.pkl')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "# adding governorates\n",
    "full_features = pd.concat([full_features, pd.get_dummies(full_features.gov_iso)], axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "# ignoring anything that happens before Jul 1, as there's not enough data for feature calculation\n",
    "full_features = full_features[full_features.date>='2017-07-01'].reset_index(drop=True)\n",
    "y_df = y_df[y_df.date>='2017-07-01'].reset_index(drop=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "ESTIMATORS = 750\n",
    "EARLY_STOPPING_ROUNDS = 50"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "# scale only continuos columns using standard scaler\n",
    "def scale_features(scaler, X):\n",
    "\n",
    "    continuous_cols = [col for col in X.columns if 'YE-' not in col]\n",
    "    \n",
    "    X_scaled = X\n",
    "    \n",
    "    X_scaled[continuous_cols] = scaler.transform(X[continuous_cols])\n",
    "    \n",
    "    return (X_scaled)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "# split data into train, cross-validation, and test based on date\n",
    "# note that the below function is NOT FOR ROLLING WINDOW CROSS VALIDATION\n",
    "# note that later on train and cv are usually combined (appended) when doing time series cross-validation\n",
    "# in those cases, the test set acts as the cross-validation and the train + cv set acts as the train set\n",
    "# the code below was not changed to retain its 3-split functionality\n",
    "\n",
    "def train_cv_test_split(data, cv_split_date='2017-09-17', test_split_date='2017-11-06'):\n",
    "    \n",
    "    data_train, data_cv, data_test = data[(data.date<cv_split_date)], data[(data.date<test_split_date) & (data.date>=cv_split_date)], data[ (data.date>=test_split_date) ]\n",
    "    \n",
    "    return (data_train.drop('date', axis=1), data_cv.drop('date', axis=1), data_test.drop('date', axis=1))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Start 2017-07-01 00:00:00 Stop 2017-11-10 00:00:00 No of days 132\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "8"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# functions to obtain start/stop, particularly the start of the train/cv and start of the holdout (which becomes the stop of the train/cv)\n",
    "#dates based on defined hold out and rolling window size\n",
    "\n",
    "HOLD_OUT_WINDOW = 100\n",
    "ROLLING_WINDOW_SIZE = 45\n",
    "WINDOW = round(ROLLING_WINDOW_SIZE/3)\n",
    "\n",
    "def getMaxDate():\n",
    "    return y_df.date.max() - datetime.timedelta(days=HOLD_OUT_WINDOW)\n",
    "\n",
    "def getMinDate():\n",
    "    return y_df.date.min()\n",
    "\n",
    "delta = getMaxDate() - getMinDate()\n",
    "print('Start {} Stop {} No of days {}'.format(getMinDate(), getMaxDate(), delta.days))\n",
    "\n",
    "number_rolling_windows = math.ceil(round(delta.days/WINDOW))  - 1\n",
    "number_rolling_windows\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "# builds off previous function to get hold-out dates\n",
    "def getHoldOutDate():\n",
    "    start = getMaxDate() + datetime.timedelta(days=1)\n",
    "    end   = y_df.date.max() \n",
    "    \n",
    "    return (start, end)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "# get dates for rolling window cross-validation\n",
    "\n",
    "def getRollingWindowDates(idx):\n",
    "    maxDate = getMaxDate()  \n",
    "    minDate = getMinDate() \n",
    "    \n",
    "    trainStart = minDate\n",
    "    trainStop  = minDate + datetime.timedelta(days=WINDOW*idx)\n",
    "    \n",
    "    validationStart  = trainStop + datetime.timedelta(days=1)\n",
    "    validationStop   = validationStart + datetime.timedelta(days=WINDOW)\n",
    "    \n",
    "    testStart  = validationStop + datetime.timedelta(days=1)\n",
    "    testStop   = testStart + datetime.timedelta(days=WINDOW)\n",
    "    \n",
    "    if (maxDate - testStop).days < WINDOW:\n",
    "        print('Rolling window to end date')\n",
    "        testStop = maxDate\n",
    "        \n",
    "    print('Train [{} {}] Val [{} {}] Test [{} {}]'.format(trainStart.date(), trainStop.date(), \n",
    "                                                       validationStart.date(), validationStop.date(), \n",
    "                                                       testStart.date(), testStop.date()))\n",
    "    \n",
    "    return (trainStart, trainStop, validationStart, validationStop, testStart, testStop)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "# using dates from getRollingWindowDates, split data into train, validation, test\n",
    "\n",
    "def getRollingWindow(data, trainStart, trainStop, validationStart, validationStop, testStart, testStop):\n",
    "    \n",
    "    train = data[(data.date >= trainStart) & (data.date <= trainStop)]\n",
    "    val   = data[(data.date >= validationStart) & (data.date <= validationStop)]\n",
    "    test  = data[(data.date >= testStart) & (data.date <= testStop)] \n",
    "    \n",
    "    print('Window Train/Val/Test shape {} {} {}'.format(train.shape, val.shape, test.shape))\n",
    "    \n",
    "    return (train.drop('date', axis=1), val.drop('date', axis=1), test.drop('date', axis=1))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "# splits data into holdout and train based on defined minimum date and hold-out date (see relevant functions above)\n",
    "\n",
    "def getHoldOutData(data):\n",
    "    minDate = getMinDate() \n",
    "    start, end = getHoldOutDate()\n",
    "\n",
    "    train = data[(data.date >= minDate) & (data.date < start)]\n",
    "    test =  data[(data.date >= start) & (data.date <= end)]\n",
    "    \n",
    "    return (train.drop('date', axis=1), test.drop('date', axis=1))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(8, 15, (Timestamp('2017-11-11 00:00:00'), Timestamp('2018-02-18 00:00:00')))"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "number_rolling_windows, round((ROLLING_WINDOW_SIZE)/3), getHoldOutDate()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "# uses scalable hypothesis tests to identify statistically relevant features\n",
    "class TsFresh():\n",
    "    def __init__(self):\n",
    "        pass\n",
    "    \n",
    "    def postProcessor(self, X, y, dateSeries):\n",
    "        self.selected_features = select_features(X, y, fdr_level=0.001)\n",
    "        print('Selected features {}'.format(self.selected_features.shape))      \n",
    "        self.selected_features = pd.concat([dateSeries, self.selected_features], axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "# removes features that are correlated with each other\n",
    "class OptimizeFeatures():\n",
    "    def __init__(self):\n",
    "        pass\n",
    "        \n",
    "    def selectFeatures(self, X, y):\n",
    "        self.X  = X\n",
    "        self.y  = y\n",
    "        self.fs = FeatureSelector(data = X, labels = y)        \n",
    "    \n",
    "    def identifyCollinearFeatures(self, correlation_threshold=0.975):\n",
    "        self.fs.identify_collinear(correlation_threshold)\n",
    "    \n",
    "    def collinerFeaturesColumnsToKeep(self):\n",
    "        return self.fs.ops['collinear']\n",
    "    \n",
    "    def removeCollinerFeatures(self):\n",
    "        self.cols_to_keep = set(self.X.columns) - set(self.fs.ops['collinear'])\n",
    "        self.corr_selected_features = self.X[list(self.cols_to_keep)]\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "# trains XGBoost model to rank features\n",
    "\n",
    "class ModelCustomRegressor():\n",
    "    \n",
    "    def __init__(self):\n",
    "        pass\n",
    "    \n",
    "    def extract(self, model_params, X_train, X_cv, X_test, y_train, y_cv, y_test):\n",
    "        print('Creating baseline model to extract features')\n",
    "        \n",
    "        X_train_cv = X_train.append(X_cv)\n",
    "        y_train_cv = y_train.append(y_cv)\n",
    "        \n",
    "        scaler = StandardScaler()\n",
    "        continuous_cols = [col for col in X_train_cv.columns if 'YE-' not in col]\n",
    "        scaler.fit(X_train_cv[continuous_cols])\n",
    "\n",
    "        X_train_cv, X_test = scale_features(scaler, X_train_cv), scale_features(scaler, X_test)\n",
    "        \n",
    "        print('all features {}'.format(X_train_cv.shape))\n",
    "            \n",
    "        eval_set = [(X_test, y_test)]\n",
    "    \n",
    "        self.feature_importance_df = pd.DataFrame(index = X_train.columns)\n",
    "        \n",
    "        self.regressor = xgboost.XGBRegressor(**model_params)                                      \n",
    "        \n",
    "        self.regressor.fit(X_train_cv, y_train_cv, eval_metric='rmse', \n",
    "                                  eval_set=eval_set, \n",
    "                                  early_stopping_rounds=EARLY_STOPPING_ROUNDS, verbose=False)\n",
    "\n",
    "        preds = self.regressor.predict((X_test))\n",
    "        \n",
    "        self.feature_importance_df['threshold'] = self.regressor.feature_importances_\n",
    "        \n",
    "        self.thresholds = np.unique(self.regressor.feature_importances_)\n",
    "        self.thresholds.sort();\n",
    "        print (len(self.thresholds))\n",
    "        self.thresholds = self.thresholds[::-1][:50]\n",
    "        \n",
    "        print('# features {} # thresholds {}; thresholds:{}'.format(len(self.regressor.feature_importances_), len(self.thresholds), self.thresholds))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "# performs rolling-window cross-validation\n",
    "\n",
    "class RollingWindowCrossValidation():\n",
    "    \n",
    "    def __init__(self, corr_selected_features_bi_week, corr_selected_features_bi_week_y, preselect_params):\n",
    "        self.corr_selected_features_bi_week = corr_selected_features_bi_week\n",
    "        self.corr_selected_features_bi_week_y   = corr_selected_features_bi_week_y\n",
    "        self.preselect_params = preselect_params\n",
    "        pass\n",
    "    \n",
    "    def extract(self, regressor, thresh):\n",
    "        \n",
    "        mse_list = []\n",
    "        \n",
    "        for idx in range(3, number_rolling_windows):\n",
    "            print('CV - Window {}'.format(idx))\n",
    "            trainStart, trainStop, validationStart, validationStop, testStart, testStop = getRollingWindowDates(idx)\n",
    "            \n",
    "            X_train, X_cv, X_test = getRollingWindow(self.corr_selected_features_bi_week, \n",
    "                                                     trainStart, trainStop, \n",
    "                                                     validationStart, validationStop, testStart, testStop)\n",
    "            y_train, y_cv, y_test = getRollingWindow(self.corr_selected_features_bi_week_y, \n",
    "                                                     trainStart, trainStop, \n",
    "                                                     validationStart, validationStop, testStart, testStop)\n",
    "            \n",
    "            X_train_cv = X_train.append(X_cv)\n",
    "            y_train_cv = y_train.append(y_cv)\n",
    "            \n",
    "            scaler = StandardScaler()\n",
    "            continuous_cols = [col for col in X_train_cv.columns if 'YE-' not in col]\n",
    "            scaler.fit(X_train_cv[continuous_cols])\n",
    "\n",
    "            X_train_cv, X_test = scale_features(scaler, X_train_cv), scale_features(scaler, X_test)\n",
    "        \n",
    "            print('X_train_cv  {}'.format(X_train_cv.shape))\n",
    "            \n",
    "            # select features using threshold\n",
    "            selection = SelectFromModel(regressor, threshold=thresh, prefit=True)\n",
    "            select_X_train_cv = selection.transform(X_train_cv)\n",
    "            \n",
    "            select_X_test = selection.transform(X_test)\n",
    "            \n",
    "            eval_set = [(select_X_test, y_test)]\n",
    "            \n",
    "            # train model\n",
    "            selection_model = xgboost.XGBRegressor(**self.preselect_params)\n",
    "            \n",
    "            selection_model.fit(select_X_train_cv, y_train_cv, eval_metric='rmse', \n",
    "                                      eval_set=eval_set, \n",
    "                                      early_stopping_rounds=50,\n",
    "                                      verbose=False)\n",
    "            # eval model\n",
    "            y_pred = selection_model.predict(select_X_test)\n",
    "           \n",
    "            mse = mean_squared_error(y_test, y_pred)\n",
    "\n",
    "            print(\"%d Thresh=%.5f, n=%d, mse: %.3f\" % (idx, thresh, select_X_train_cv.shape[1], mse))\n",
    "            \n",
    "            mse_list.append(mse)\n",
    "        \n",
    "        return_dict = {'threshold':thresh, 'num_features':select_X_train_cv.shape[1], 'mse_list':mse_list, 'mean_mse':np.mean(mse_list)}\n",
    "        print (return_dict)\n",
    "        return (return_dict)\n",
    "        "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "# takes feature ranking and recursively adds features\n",
    "# evaluates how each feature set performs using rolling window cross validation\n",
    "\n",
    "class RollingCustomFeatureExtractor():\n",
    "    \n",
    "    def __init__(self, corr_selected_features_bi_week, corr_selected_features_bi_week_y, preselect_params):\n",
    "        self.corr_selected_features_bi_week  = corr_selected_features_bi_week\n",
    "        self.corr_selected_features_bi_week_y   = corr_selected_features_bi_week_y\n",
    "        self.preselect_params = preselect_params\n",
    "        pass\n",
    "    \n",
    "    def extract(self):\n",
    "        X = self.corr_selected_features_bi_week \n",
    "        y = self.corr_selected_features_bi_week_y \n",
    "        \n",
    "        X_train, X_cv, X_test = train_cv_test_split(X)\n",
    "        y_train, y_cv, y_test = train_cv_test_split(y)\n",
    "        \n",
    "        self.mcr = ModelCustomRegressor()\n",
    "        self.mcr.extract(self.preselect_params, X_train, X_cv, X_test, y_train, y_cv, y_test) \n",
    "            \n",
    "        thresholds = self.mcr.thresholds        \n",
    "    \n",
    "        self.rwcv = RollingWindowCrossValidation(self.corr_selected_features_bi_week, \n",
    "                                                 self.corr_selected_features_bi_week_y,\n",
    "                                                 self.preselect_params)\n",
    "        \n",
    "        self.summary = pd.DataFrame(columns = ['threshold', 'num_features', 'mse_list', 'mean_mse'])\n",
    "        \n",
    "        for thresh in thresholds:\n",
    "            return_dict = self.rwcv.extract(self.mcr.regressor, thresh)\n",
    "            \n",
    "            self.summary = self.summary.append(return_dict, ignore_index = True)\n",
    "            \n",
    "            print(\"\\n\")\n",
    "            \n",
    "        print(self.summary.head(20))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "def set_style(color):\n",
    "    plt.style.use(['seaborn-' + color, 'seaborn-paper'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "# bayesian optimization with rolling window cross validation to select optimal hyperparameters\n",
    "\n",
    "class BayesianOptimizer():\n",
    "\n",
    "    def __init__(self, corr_selected_features_bi_week, corr_selected_features_bi_week_y, max_evals):\n",
    "        self.corr_selected_features_bi_week  = corr_selected_features_bi_week\n",
    "        self.corr_selected_features_bi_week_y   = corr_selected_features_bi_week_y\n",
    "        self.max_evals = max_evals\n",
    "    \n",
    "    def objective(self, space):\n",
    "    \n",
    "        mse_list = []\n",
    "\n",
    "        for idx in range(3, number_rolling_windows):\n",
    "\n",
    "            trainStart, trainStop, validationStart, validationStop, testStart, testStop = getRollingWindowDates(idx)\n",
    "\n",
    "            X_train, X_cv, X_test = getRollingWindow(self.corr_selected_features_bi_week, \n",
    "                                                     trainStart, trainStop, \n",
    "                                                     validationStart, validationStop, testStart, testStop)\n",
    "            y_train, y_cv, y_test = getRollingWindow(self.corr_selected_features_bi_week_y, \n",
    "                                                     trainStart, trainStop, \n",
    "                                                     validationStart, validationStop, testStart, testStop)\n",
    "\n",
    "            X_train_cv = X_train.append(X_cv)\n",
    "            y_train_cv = y_train.append(y_cv)\n",
    "\n",
    "            scaler = StandardScaler()\n",
    "            continuous_cols = [col for col in X_train_cv.columns if 'YE-' not in col]\n",
    "            scaler.fit(X_train_cv[continuous_cols])\n",
    "\n",
    "            X_train_cv, X_test = scale_features(scaler, X_train_cv), scale_features(scaler, X_test)\n",
    "\n",
    "            xgb = xgboost.XGBRegressor(n_estimators = int(space['n_estimators']),\n",
    "                           max_depth = int(space['max_depth']),\n",
    "                           min_child_weight = space['min_child_weight'],\n",
    "                           subsample = space['subsample'],\n",
    "                           learning_rate = space['learning_rate'],\n",
    "                           gamma = space['gamma'],\n",
    "                           colsample_bytree = space['colsample_bytree'],\n",
    "                           objective='reg:linear', n_jobs = -1\n",
    "                           )    \n",
    "\n",
    "            xgb.fit(X_train_cv ,y_train_cv, eval_metric = 'rmse')\n",
    "\n",
    "            # eval model\n",
    "            y_pred = xgb.predict(X_test)\n",
    "\n",
    "            mse = mean_squared_error(y_test, y_pred)    \n",
    "\n",
    "            mse_list.append(mse)\n",
    "            \n",
    "        print(mse_list)    \n",
    "\n",
    "        return_dict = {'mse_list':mse_list}\n",
    "\n",
    "        return_dict['loss'] = np.mean(mse_list)\n",
    "        return_dict['status'] = STATUS_OK\n",
    "\n",
    "        print (\"mean mse:\", return_dict['loss'])\n",
    "\n",
    "        return (return_dict)\n",
    "\n",
    "    def run(self):\n",
    "        \n",
    "        self.space ={'max_depth': hp.quniform('max_depth', 4, 12, 1),\n",
    "                'min_child_weight': hp.quniform('min_child_weight', 1, 10, 1),\n",
    "                'subsample':  hp.quniform('subsample', 0.5, 1, 0.05),\n",
    "                'n_estimators' : hp.quniform('n_estimators', 50, 750, 50),\n",
    "                'learning_rate' : hp.quniform('learning_rate', 0.01, 0.3, 0.025),\n",
    "                'gamma' : hp.quniform('gamma', 0, 5, 0.5),\n",
    "                'colsample_bytree' : hp.quniform('colsample_bytree', 0.3, 1, 0.05)}\n",
    "        \n",
    "        self.trials = Trials()\n",
    "        self.best = fmin(fn=self.objective,\n",
    "                    space=self.space,\n",
    "                    algo=tpe.suggest,\n",
    "                    max_evals=self.max_evals, \n",
    "                    trials=self.trials)\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 74,
   "metadata": {},
   "outputs": [],
   "source": [
    "ylabel_dict = {'week_1_to_2_cases':'0-2 week', 'week_2_to_4_cases':'2-4 week', 'week_4_to_6_cases':'4-6 week', 'week_6_to_8_cases':'6-8 week'}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 161,
   "metadata": {},
   "outputs": [],
   "source": [
    "def suplabel(axis,label,side,label_prop=None,labelpad=6.7,ha='center',va='center'):\n",
    "    ''' Add super ylabel or xlabel to the figure\n",
    "    Similar to matplotlib.suptitle\n",
    "    axis       - string: \"x\" or \"y\"\n",
    "    label      - string\n",
    "    label_prop - keyword dictionary for Text\n",
    "    labelpad   - padding from the axis (default: 5)\n",
    "    ha         - horizontal alignment (default: \"center\")\n",
    "    va         - vertical alignment (default: \"center\")\n",
    "    '''\n",
    "    fig = pylab.gcf()\n",
    "    xmin = []\n",
    "    ymin = []\n",
    "    xmax = []\n",
    "    ymax = []\n",
    "    for ax in fig.axes:\n",
    "        xmin.append(ax.get_position().xmin)\n",
    "        ymin.append(ax.get_position().ymin)\n",
    "        xmax.append(ax.get_position().xmax)\n",
    "        ymax.append(ax.get_position().ymax)\n",
    "        \n",
    "    xmin,ymin = min(xmin),min(ymin)\n",
    "    xmax,ymax = min(xmax),min(ymax)\n",
    "\n",
    "    dpi = fig.dpi\n",
    "    if axis.lower() == \"y\":\n",
    "        rotation=90.\n",
    "        \n",
    "        if side == 'left':\n",
    "            x = xmin-float(labelpad)/dpi\n",
    "            \n",
    "        if side == 'right':\n",
    "            x = xmax+float(labelpad)/dpi\n",
    "\n",
    "        y = 0.5\n",
    "    elif axis.lower() == 'x':\n",
    "        rotation = 0.\n",
    "        x = 0.5\n",
    "        y = ymin - float(labelpad)/dpi\n",
    "    else:\n",
    "        raise Exception(\"Unexpected axis: x or y\")\n",
    "    if label_prop is None: \n",
    "        label_prop = dict()\n",
    "    pylab.text(x,y,label,rotation=rotation,\n",
    "               transform=fig.transFigure,\n",
    "               ha=ha,va=va,\n",
    "               **label_prop)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 154,
   "metadata": {},
   "outputs": [],
   "source": [
    "# plotting function that visualizes predictions\n",
    "\n",
    "def plot_pred_against_actual(cv_pred_crosstab, test_pred_crosstab, true_crosstab, y_label, cv_mse, holdout_mse, sharey):\n",
    "    \n",
    "    set_style('white')\n",
    "    \n",
    "    fig, ax = plt.subplots(21,1,figsize = (6,15), sharex=True, sharey = sharey)\n",
    "    \n",
    "    cols = true_crosstab.columns\n",
    "        \n",
    "    for i in range(0,21):\n",
    "        \n",
    "        true_crosstab[cols[i]].plot(kind='line', ax = ax[i], label = 'true_val', legend = True, color = 'red')\n",
    "\n",
    "        ax[i].set_prop_cycle('color', ['seagreen', 'blue'])\n",
    "                            \n",
    "        #train_pred_crosstab[cols[i]].plot(kind='line', ax = ax[i], label= 'xgboost train-pred',linestyle= '-.', legend = True)\n",
    "        \n",
    "        cv_pred_crosstab[cols[i]].plot(kind='line', ax = ax[i], label= 'xgboost cv-pred;\\nmean error: %.3f'%(np.sqrt(cv_mse)) ,linestyle= '-.', legend = True)\n",
    "\n",
    "        test_pred_crosstab[cols[i]].plot(kind='line', ax = ax[i], label= 'xgboost holdout-pred;\\nmean error: %.3f'%(np.sqrt(holdout_mse)), linestyle= '-.', legend = True)   \n",
    "          \n",
    "        ax[i].legend().set_visible(False)\n",
    "        ax[i].set_ylabel(cols[i], rotation=0)\n",
    "        ax[i].yaxis.set_label_position('right')\n",
    "        ax[i].spines['right'].set_visible(False)\n",
    "        ax[i].spines['top'].set_visible(False)\n",
    "        ax[i].spines['bottom'].set_visible(True)\n",
    "        \n",
    "    ax[10].legend().set_visible(True)\n",
    "    ax[10].legend(fontsize=10, loc='center left', bbox_to_anchor=(1.05, 0.5))\n",
    " \n",
    "    fig.subplots_adjust(hspace = .2)    \n",
    "        \n",
    "    if sharey:\n",
    "    \n",
    "        fig.savefig('/Users/Rohil/Documents/iGEM/yemen/final_results/' + y_label + '_deployed_sharey.png', dpi = 500, bbox_inches = 'tight')\n",
    "    \n",
    "    else:\n",
    "        \n",
    "        fig.savefig('/Users/Rohil/Documents/iGEM/yemen/final_results/' + y_label + '_deployed.png', dpi = 500, bbox_inches = 'tight')\n",
    "\n",
    "    plt.close('all')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 155,
   "metadata": {},
   "outputs": [],
   "source": [
    "# plotting function that visualizes predictions, but only for a certain subset of governorates\n",
    "\n",
    "def plot_pred_against_actual_filtered(cv_pred_crosstab, test_pred_crosstab, true_crosstab, y_label, cv_mse, holdout_mse, sharey):\n",
    "    \n",
    "    set_style('white')\n",
    "        \n",
    "    #cols = list(set(true_crosstab.columns) - set(['YE-HD-AL', 'YE-MR', 'YE-SH', 'YE-TA', 'YE-MA', 'YE-SD']))\n",
    "    \n",
    "    cols = sorted(['YE-AM', 'YE-DA', 'YE-MW', 'YE-RA', 'YE-SN'])\n",
    "#sorted(['YE-AM', 'YE-DA', 'YE-TA', 'YE-RA', 'YE-SN'])\n",
    "    \n",
    "    fig, ax = plt.subplots(len(cols),1,figsize = (5,5), sharex=True, sharey = sharey)\n",
    "        \n",
    "    \n",
    "    for i in range(0,len(cols)):\n",
    "        \n",
    "        true_crosstab[cols[i]].plot(kind='line', ax = ax[i], label = 'true', legend = True, color = 'red')\n",
    "\n",
    "        ax[i].set_prop_cycle('color', ['seagreen', 'blue'])\n",
    "                            \n",
    "        #train_pred_crosstab[cols[i]].plot(kind='line', ax = ax[i], label= 'xgboost train-pred',linestyle= '-.', legend = True)\n",
    "        \n",
    "        cv_pred_crosstab[cols[i]].plot(kind='line', ax = ax[i], label= 'xgboost cross-validation prediction' ,linestyle= '-.', legend = True) #;\\nmean error: %.3f'%(np.sqrt(cv_mse))\n",
    "\n",
    "        test_pred_crosstab[cols[i]].plot(kind='line', ax = ax[i], label= 'xgboost holdout prediction', linestyle= '-.', legend = True) #;\\nmean error: %.3f'%(np.sqrt(holdout_mse))  \n",
    "          \n",
    "        ax[i].legend().set_visible(False)\n",
    "        ax[i].set_ylabel(cols[i], rotation=0)\n",
    "        ax[i].yaxis.set_label_position('right')\n",
    "        ax[i].spines['right'].set_visible(False)\n",
    "        ax[i].spines['top'].set_visible(False)\n",
    "        ax[i].spines['bottom'].set_visible(True)\n",
    "        \n",
    "    #ax[int(len(cols)/2)].legend().set_visible(True)\n",
    "    #ax[int(len(cols)/2)].legend(fontsize=10, loc='center left', bbox_to_anchor=(1.05, 0.5))\n",
    " \n",
    "    fig.subplots_adjust(hspace = .2)\n",
    "    \n",
    "    ax[len(cols)-1].set_xlabel('time (days)', rotation=0)\n",
    "\n",
    "    \n",
    "    fig.suptitle(ylabel_dict[y_label] + 's forecast')\n",
    "    suplabel('y', 'sliding-scale prediction of new cases per 10000 people', 'left')\n",
    "    suplabel('y', 'governorates', 'right')\n",
    "\n",
    "        \n",
    "    if sharey:\n",
    "    \n",
    "        fig.savefig('/Users/Rohil/Documents/iGEM/yemen/final_results/' + y_label + '_deployed_sharey_filtered.png', dpi = 500, bbox_inches = 'tight')\n",
    "    \n",
    "    else:\n",
    "        \n",
    "        fig.savefig('/Users/Rohil/Documents/iGEM/yemen/final_results/' + y_label + '_deployed_filtered.png', dpi = 500, bbox_inches = 'tight')\n",
    "\n",
    "    plt.close('all')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 123,
   "metadata": {},
   "outputs": [],
   "source": [
    "# used to deploy final iteration of model\n",
    "# uses rolling window cross-validation and evaluates on hold-out set\n",
    "# feeds into plotting function\n",
    "\n",
    "class DeployRegressor():\n",
    "    \n",
    "    def __init__(self):\n",
    "        pass\n",
    "    \n",
    "    def execute(self, model_params, X, y, y_to_plot, y_bi_week_label):\n",
    "        \n",
    "        X.drop(columns = 'days_from', inplace = True)\n",
    "        \n",
    "        X_train_all, X_hold_test = getHoldOutData(X)\n",
    "        y_train_all, y_hold_test = getHoldOutData(y)\n",
    "        \n",
    "        mse_list = []\n",
    "        y_cv_preds = []\n",
    "        y_cv_all = pd.DataFrame(columns = [y_bi_week_label])\n",
    "\n",
    "        for idx in range(3, number_rolling_windows):\n",
    "\n",
    "            trainStart, trainStop, validationStart, validationStop, testStart, testStop = getRollingWindowDates(idx)\n",
    "\n",
    "            X_train, X_cv, X_test = getRollingWindow(X, \n",
    "                                                     trainStart, trainStop, \n",
    "                                                     validationStart, validationStop, testStart, testStop)\n",
    "            y_train, y_cv, y_test = getRollingWindow(y, \n",
    "                                                     trainStart, trainStop, \n",
    "                                                     validationStart, validationStop, testStart, testStop)\n",
    "\n",
    "            X_train_cv = X_train.append(X_cv)\n",
    "            y_train_cv = y_train.append(y_cv)\n",
    "            \n",
    "#             if idx == 3:\n",
    "#                 X_base_train = X_train_cv\n",
    "#                 y_base_train = y_train_cv\n",
    "            \n",
    "            scaler = StandardScaler()\n",
    "            continuous_cols = [col for col in X_train_cv.columns if 'YE-' not in col]\n",
    "            scaler.fit(X_train_cv[continuous_cols])\n",
    "\n",
    "            X_train_cv, X_test = scale_features(scaler, X_train_cv), scale_features(scaler, X_test)\n",
    "\n",
    "            xgb = xgboost.XGBRegressor(**model_params)    \n",
    "\n",
    "            xgb.fit(X_train_cv ,y_train_cv, eval_metric = 'rmse')\n",
    "\n",
    "            # eval model\n",
    "            y_pred = xgb.predict(X_test)\n",
    "            \n",
    "            y_cv_preds.extend(list(y_pred))\n",
    "            y_cv_all = y_cv_all.append(y_test)\n",
    "\n",
    "        \n",
    "        end_xgb = xgboost.XGBRegressor(**model_params)    \n",
    "\n",
    "        end_xgb.fit(X_train_all, y_train_all, eval_metric = 'rmse')\n",
    "        \n",
    "        y_holdout_preds = end_xgb.predict(X_hold_test)\n",
    "        \n",
    "#         y_base_train_preds = end_xgb.predict(X_base_train)\n",
    "#         base_train_mse = mean_squared_error(y_base_train.values, y_base_train_preds)\n",
    "        \n",
    "        cv_mse = mean_squared_error(y_cv_all.values, y_cv_preds)\n",
    "        holdout_mse = mean_squared_error(y_hold_test.values, y_holdout_preds)\n",
    "                        \n",
    "        print('y-test mean {}, y-test std {}'.format(np.mean(y_test.values), np.std(y_test.values)))\n",
    "        print('cv mse {}, holdout mse{}'.format(cv_mse, holdout_mse))\n",
    "        \n",
    "#         y_base_train_pred_df = pd.DataFrame(y_base_train_preds, columns=[y_bi_week_label], index = y_base_train.index)\n",
    "#         y_base_train_pred_df = y_base_train_pred_df.merge(y_to_plot[['gov_iso', 'date']],  how = 'left',left_index = True, right_index = True)\n",
    "#         y_base_train_pred_crosstab = y_base_train_pred_df.pivot_table(index = 'date', columns = 'gov_iso', values = y_bi_week_label, aggfunc='sum')\n",
    "\n",
    "        \n",
    "        y_cv_pred_df = pd.DataFrame(y_cv_preds, columns=[y_bi_week_label], index = y_cv_all.index)\n",
    "        y_cv_pred_df = y_cv_pred_df.merge(y_to_plot[['gov_iso', 'date']],  how = 'left',left_index = True, right_index = True)\n",
    "        y_cv_pred_crosstab = y_cv_pred_df.pivot_table(index = 'date', columns = 'gov_iso', values = y_bi_week_label, aggfunc='mean')\n",
    "        \n",
    "        y_holdout_pred_df = pd.DataFrame(y_holdout_preds, columns=[y_bi_week_label], index = y_hold_test.index)\n",
    "        y_holdout_pred_df = y_holdout_pred_df.merge(y_to_plot[['gov_iso', 'date']], how = 'left', left_index = True, right_index = True)\n",
    "        y_holdout_pred_crosstab = y_holdout_pred_df.pivot_table(index = 'date', columns = 'gov_iso', values = y_bi_week_label, aggfunc='mean')\n",
    "                \n",
    "        true_val_pivot = y_to_plot.pivot_table(index = 'date', columns = 'gov_iso', values = y_bi_week_label, aggfunc='mean')\n",
    "        \n",
    "        #print (y_base_train_pred_df.gov_iso.nunique(), y_cv_pred_df, y_holdout_pred_df)\n",
    "\n",
    "        \n",
    "        plot_pred_against_actual(y_cv_pred_crosstab, y_holdout_pred_crosstab, true_val_pivot, y_bi_week_label, cv_mse, holdout_mse, sharey = True)\n",
    "        plot_pred_against_actual(y_cv_pred_crosstab, y_holdout_pred_crosstab, true_val_pivot, y_bi_week_label, cv_mse, holdout_mse, sharey = False)\n",
    "        \n",
    "        plot_pred_against_actual_filtered(y_cv_pred_crosstab, y_holdout_pred_crosstab, true_val_pivot, y_bi_week_label, cv_mse, holdout_mse, sharey = True)\n",
    "        plot_pred_against_actual_filtered(y_cv_pred_crosstab, y_holdout_pred_crosstab, true_val_pivot, y_bi_week_label, cv_mse, holdout_mse, sharey = False)\n",
    "        \n",
    "        y_cv_pred_crosstab.append(y_holdout_pred_crosstab).to_csv('/Users/Rohil/Documents/iGEM/yemen/' + y_bi_week_label + '_deployed_cv_holdout_preds.csv')\n",
    "        \n",
    "        fig1, ax1 = plt.subplots(figsize = (5,8))\n",
    "        xgboost.plot_importance(end_xgb, ax=ax1)\n",
    "        fig1.savefig('/Users/Rohil/Documents/iGEM/yemen/' + y_bi_week_label + '_deployed_feature_importances.png', dpi = 300, bbox_inches = 'tight')\n",
    "        plt.close('all')\n",
    "\n",
    "\n",
    "    \n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [],
   "source": [
    "# class from which all other processes are called (master class)\n",
    "# this is the class you initialize as the user\n",
    "\n",
    "class Orchestrator():\n",
    "    \n",
    "    def __init__(self, full_data_bi_week, y_bi_week_label):\n",
    "        self.full_data_bi_week = full_data_bi_week\n",
    "        self.y_bi_week_label   = y_bi_week_label\n",
    "        pass\n",
    "    \n",
    "    def runTsFresh(self):\n",
    "        print('Running TSFresh....')\n",
    "        \n",
    "        X_ts = self.full_data_bi_week.drop(columns=['date', 'gov_iso', y_bi_week_label])\n",
    "        y_ts = self.full_data_bi_week[self.y_bi_week_label]\n",
    "        dateSeries_ts = self.full_data_bi_week.date\n",
    "        \n",
    "        self.tf = TsFresh()\n",
    "        self.tf.postProcessor(X_ts, y_ts, dateSeries_ts)\n",
    "        \n",
    "        print('Finished running TSFresh....')\n",
    "        \n",
    "    def runOptimizeFeatures(self):\n",
    "        print('Running Feature Selection module ....')\n",
    "        \n",
    "        y = self.full_data_bi_week[self.y_bi_week_label]\n",
    "            \n",
    "        self.op = OptimizeFeatures()\n",
    "        self.op.selectFeatures(self.tf.selected_features, y)\n",
    "        self.op.identifyCollinearFeatures(0.975)\n",
    "        \n",
    "        self.op.removeCollinerFeatures()\n",
    "        print('Original {} and after {}'.format(self.op.X.shape, self.op.corr_selected_features.shape))\n",
    "        \n",
    "        print('Finished running Feature Selection ....')\n",
    "        \n",
    "    def performHyperparameterOptimization(self, X, max_evals):\n",
    "        self.bo = BayesianOptimizer(X, self.full_data_bi_week[['date', self.y_bi_week_label]], max_evals)\n",
    "        self.bo.run()\n",
    "    \n",
    "    def runRollingCustomFeatureExtractor(self, preselect_params, corr_selected_features):\n",
    "        print('Running custom feature selection module ....')\n",
    "        \n",
    "        # user can specify their corr_selected_features if they please (if the job has been batched)\n",
    "        if corr_selected_features is None:\n",
    "            self.corr_selected_features_bi_week = self.op.corr_selected_features\n",
    "        else:\n",
    "            self.corr_selected_features_bi_week = corr_selected_features\n",
    "        \n",
    "        self.corr_selected_features_bi_week_y = self.full_data_bi_week[['date', self.y_bi_week_label]]\n",
    "\n",
    "        #corr_selected_features_4_6 = pd.concat([full_data_4_6.date, corr_selected_features_4_6], axis=1)\n",
    "        #corr_selected_features_4_6.date.head()\n",
    "\n",
    "        \"\"\"\n",
    "        X = corr_selected_features_4_6 \n",
    "        y = corr_selected_features_4_6_y \n",
    "        \n",
    "        X_train, X_cv, X_test = train_cv_test_split(X)\n",
    "        y_train, y_cv, y_test = train_cv_test_split(y)\n",
    "        \"\"\"\n",
    "\n",
    "        self.rcfe = RollingCustomFeatureExtractor(self.corr_selected_features_bi_week, \n",
    "                                                  self.corr_selected_features_bi_week_y, preselect_params)\n",
    "        self.rcfe.extract()\n",
    "\n",
    "    def deploy(self, threshold, model_params, selected_x):\n",
    "        \n",
    "        if selected_x is None:\n",
    "            selected_features_from_threshold = list(self.rcfe.mcr.feature_importance_df[self.rcfe.mcr.feature_importance_df.threshold >= threshold].index)\n",
    "            selected_features_from_threshold.append('date')\n",
    "            self.X_final = self.corr_selected_features_bi_week[selected_features_from_threshold]\n",
    "            \n",
    "        else:\n",
    "            self.X_final = selected_x\n",
    "\n",
    "        y = self.corr_selected_features_bi_week_y \n",
    "        #self.X_final['days_from'] = self.full_data_bi_week.days_from\n",
    "        y_to_plot = self.corr_selected_features_bi_week_y.merge(full_features, on = 'date', left_index=True, right_index=True, how = 'left')[[self.y_bi_week_label, 'gov_iso', 'date']]\n",
    "                \n",
    "        self.dr = DeployRegressor()\n",
    "        self.dr.execute(model_params, self.X_final, y, y_to_plot, self.y_bi_week_label)\n",
    "        "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2017-07-01 00:00:00\n",
      "2018-02-18 00:00:00\n"
     ]
    }
   ],
   "source": [
    "print (y_df.date.min())\n",
    "print (y_df.date.max())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [],
   "source": [
    "y1_2 = y_df[['date', 'gov_iso', 'week_1_to_2_cases']]\n",
    "y2_4 = y_df[['date', 'gov_iso', 'week_2_to_4_cases']]\n",
    "y4_6 = y_df[['date', 'gov_iso', 'week_4_to_6_cases']]\n",
    "y6_8 = y_df[['date', 'gov_iso', 'week_6_to_8_cases']]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [],
   "source": [
    "# instead of creating copies here, will run these lines directly in the object instantiation\n",
    "# full_data_1_2 = y1_2.dropna().merge(full_features, how = 'left', on = ['gov_iso', 'date']).sort_values('date')\n",
    "# full_data_2_4 = y2_4.dropna().merge(full_features, how = 'left', on = ['gov_iso', 'date']).sort_values('date')\n",
    "# full_data_4_6 = y4_6.dropna().merge(full_features, how = 'left', on = ['gov_iso', 'date']).sort_values('date')\n",
    "# full_data_6_8 = y6_8.dropna().merge(full_features, how = 'left', on = ['gov_iso', 'date']).sort_values('date')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [],
   "source": [
    "# running feature selection for 1-2 week model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [],
   "source": [
    "orchestrator12 = Orchestrator(full_features, full_data_1_2, 'week_1_to_2_cases' )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Running TSFresh....\n",
      "Selected features (4599, 15252)\n",
      "Finished running TSFresh....\n"
     ]
    }
   ],
   "source": [
    "orchestrator12.runTsFresh()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Running Feature Selection module ....\n",
      "4825 features with a correlation magnitude greater than 0.97.\n",
      "\n",
      "Original (4599, 15253) and after (4599, 10428)\n",
      "Finished running Feature Selection ....\n"
     ]
    }
   ],
   "source": [
    "orchestrator12.runOptimizeFeatures()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "metadata": {},
   "outputs": [],
   "source": [
    "X_12_preselect = orchestrator12.op.corr_selected_features\n",
    "y_12_preselect = orchestrator12.full_data_bi_week[['date', orchestrator12.y_bi_week_label]]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "metadata": {},
   "outputs": [],
   "source": [
    "# X_12_preselect.to_csv('/Users/Rohil/Documents/iGEM/yemen/intermediary_files/X_12_preselect.csv')\n",
    "# y_12_preselect.to_csv('/Users/Rohil/Documents/iGEM/yemen/intermediary_files/y_12_preselect.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [],
   "source": [
    "orchestrator12 = Orchestrator(y1_2.dropna().merge(full_features, how = 'left', on = ['gov_iso', 'date']).sort_values('date'), 'week_1_to_2_cases' )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [],
   "source": [
    "# X_12_preselect = pd.read_csv('/Users/Rohil/Documents/iGEM/yemen/intermediary_files/X_12_preselect.csv', index_col = 0)\n",
    "# X_12_preselect.date = pd.to_datetime(X_12_preselect.date, format = '%Y-%m-%d')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 170,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train [2017-07-01 2017-08-15] Val [2017-08-16 2017-08-31] Test [2017-09-01 2017-09-16]\n",
      "Window Train/Val/Test shape (966, 10428) (336, 10428) (336, 10428)\n",
      "Window Train/Val/Test shape (966, 2) (336, 2) (336, 2)\n"
     ]
    },
    {
     "ename": "KeyboardInterrupt",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
      "\u001b[1;32m<ipython-input-170-7896f58f19ee>\u001b[0m in \u001b[0;36m<module>\u001b[1;34m()\u001b[0m\n\u001b[1;32m----> 1\u001b[1;33m \u001b[0morchestrator12\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mperformHyperparameterOptimization\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mX_12_preselect\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;36m25\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[1;32m<ipython-input-154-3560c0907952>\u001b[0m in \u001b[0;36mperformHyperparameterOptimization\u001b[1;34m(self, X, max_evals)\u001b[0m\n\u001b[0;32m     34\u001b[0m     \u001b[1;32mdef\u001b[0m \u001b[0mperformHyperparameterOptimization\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mX\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mmax_evals\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     35\u001b[0m         \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mbo\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mBayesianOptimizer\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mX\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mfull_data_bi_week\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;34m'date'\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0my_bi_week_label\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mmax_evals\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m---> 36\u001b[1;33m         \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mbo\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mrun\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m     37\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     38\u001b[0m     \u001b[1;32mdef\u001b[0m \u001b[0mrunRollingCustomFeatureExtractor\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mpreselect_params\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mcorr_selected_features\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m<ipython-input-48-535a0f59a4eb>\u001b[0m in \u001b[0;36mrun\u001b[1;34m(self)\u001b[0m\n\u001b[0;32m     75\u001b[0m                     \u001b[0malgo\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mtpe\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0msuggest\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     76\u001b[0m                     \u001b[0mmax_evals\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mmax_evals\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m---> 77\u001b[1;33m                     trials=self.trials)\n\u001b[0m",
      "\u001b[1;32m~\\Anaconda3\\lib\\site-packages\\hyperopt\\fmin.py\u001b[0m in \u001b[0;36mfmin\u001b[1;34m(fn, space, algo, max_evals, trials, rstate, allow_trials_fmin, pass_expr_memo_ctrl, catch_eval_exceptions, verbose, return_argmin, points_to_evaluate, max_queue_len)\u001b[0m\n\u001b[0;32m    365\u001b[0m             \u001b[0mverbose\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mverbose\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    366\u001b[0m             \u001b[0mcatch_eval_exceptions\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mcatch_eval_exceptions\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 367\u001b[1;33m             \u001b[0mreturn_argmin\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mreturn_argmin\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    368\u001b[0m         )\n\u001b[0;32m    369\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\Anaconda3\\lib\\site-packages\\hyperopt\\base.py\u001b[0m in \u001b[0;36mfmin\u001b[1;34m(self, fn, space, algo, max_evals, rstate, verbose, pass_expr_memo_ctrl, catch_eval_exceptions, return_argmin)\u001b[0m\n\u001b[0;32m    633\u001b[0m             \u001b[0mpass_expr_memo_ctrl\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mpass_expr_memo_ctrl\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    634\u001b[0m             \u001b[0mcatch_eval_exceptions\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mcatch_eval_exceptions\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 635\u001b[1;33m             return_argmin=return_argmin)\n\u001b[0m\u001b[0;32m    636\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    637\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\Anaconda3\\lib\\site-packages\\hyperopt\\fmin.py\u001b[0m in \u001b[0;36mfmin\u001b[1;34m(fn, space, algo, max_evals, trials, rstate, allow_trials_fmin, pass_expr_memo_ctrl, catch_eval_exceptions, verbose, return_argmin, points_to_evaluate, max_queue_len)\u001b[0m\n\u001b[0;32m    383\u001b[0m                     max_queue_len=max_queue_len)\n\u001b[0;32m    384\u001b[0m     \u001b[0mrval\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mcatch_eval_exceptions\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mcatch_eval_exceptions\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 385\u001b[1;33m     \u001b[0mrval\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mexhaust\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    386\u001b[0m     \u001b[1;32mif\u001b[0m \u001b[0mreturn_argmin\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    387\u001b[0m         \u001b[1;32mreturn\u001b[0m \u001b[0mtrials\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0margmin\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\Anaconda3\\lib\\site-packages\\hyperopt\\fmin.py\u001b[0m in \u001b[0;36mexhaust\u001b[1;34m(self)\u001b[0m\n\u001b[0;32m    242\u001b[0m     \u001b[1;32mdef\u001b[0m \u001b[0mexhaust\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    243\u001b[0m         \u001b[0mn_done\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mlen\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mtrials\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 244\u001b[1;33m         \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mrun\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mmax_evals\u001b[0m \u001b[1;33m-\u001b[0m \u001b[0mn_done\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mblock_until_done\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0masynchronous\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    245\u001b[0m         \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mtrials\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mrefresh\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    246\u001b[0m         \u001b[1;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\Anaconda3\\lib\\site-packages\\hyperopt\\fmin.py\u001b[0m in \u001b[0;36mrun\u001b[1;34m(self, N, block_until_done)\u001b[0m\n\u001b[0;32m    216\u001b[0m             \u001b[1;32melse\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    217\u001b[0m                 \u001b[1;31m# -- loop over trials and do the jobs directly\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 218\u001b[1;33m                 \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mserial_evaluate\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    219\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    220\u001b[0m             \u001b[1;32mif\u001b[0m \u001b[0mstopped\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\Anaconda3\\lib\\site-packages\\hyperopt\\fmin.py\u001b[0m in \u001b[0;36mserial_evaluate\u001b[1;34m(self, N)\u001b[0m\n\u001b[0;32m    135\u001b[0m                 \u001b[0mctrl\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mbase\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mCtrl\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mtrials\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mcurrent_trial\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mtrial\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    136\u001b[0m                 \u001b[1;32mtry\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 137\u001b[1;33m                     \u001b[0mresult\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mdomain\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mevaluate\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mspec\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mctrl\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    138\u001b[0m                 \u001b[1;32mexcept\u001b[0m \u001b[0mException\u001b[0m \u001b[1;32mas\u001b[0m \u001b[0me\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    139\u001b[0m                     \u001b[0mlogger\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0minfo\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;34m'job exception: %s'\u001b[0m \u001b[1;33m%\u001b[0m \u001b[0mstr\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0me\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\Anaconda3\\lib\\site-packages\\hyperopt\\base.py\u001b[0m in \u001b[0;36mevaluate\u001b[1;34m(self, config, ctrl, attach_attachments)\u001b[0m\n\u001b[0;32m    838\u001b[0m                 \u001b[0mmemo\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mmemo\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    839\u001b[0m                 print_node_on_error=self.rec_eval_print_node_on_error)\n\u001b[1;32m--> 840\u001b[1;33m             \u001b[0mrval\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mfn\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mpyll_rval\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    841\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    842\u001b[0m         \u001b[1;32mif\u001b[0m \u001b[0misinstance\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mrval\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;33m(\u001b[0m\u001b[0mfloat\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mint\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mnp\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mnumber\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m<ipython-input-48-535a0f59a4eb>\u001b[0m in \u001b[0;36mobjective\u001b[1;34m(self, space)\u001b[0m\n\u001b[0;32m     40\u001b[0m                            )    \n\u001b[0;32m     41\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m---> 42\u001b[1;33m             \u001b[0mxgb\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mfit\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mX_train_cv\u001b[0m \u001b[1;33m,\u001b[0m\u001b[0my_train_cv\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0meval_metric\u001b[0m \u001b[1;33m=\u001b[0m \u001b[1;34m'rmse'\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m     43\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     44\u001b[0m             \u001b[1;31m# eval model\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\Anaconda3\\lib\\site-packages\\xgboost\\sklearn.py\u001b[0m in \u001b[0;36mfit\u001b[1;34m(self, X, y, sample_weight, eval_set, eval_metric, early_stopping_rounds, verbose, xgb_model, sample_weight_eval_set)\u001b[0m\n\u001b[0;32m    322\u001b[0m                               \u001b[0mearly_stopping_rounds\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mearly_stopping_rounds\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    323\u001b[0m                               \u001b[0mevals_result\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mevals_result\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mobj\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mobj\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mfeval\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mfeval\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 324\u001b[1;33m                               verbose_eval=verbose, xgb_model=xgb_model)\n\u001b[0m\u001b[0;32m    325\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    326\u001b[0m         \u001b[1;32mif\u001b[0m \u001b[0mevals_result\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\Anaconda3\\lib\\site-packages\\xgboost\\training.py\u001b[0m in \u001b[0;36mtrain\u001b[1;34m(params, dtrain, num_boost_round, evals, obj, feval, maximize, early_stopping_rounds, evals_result, verbose_eval, xgb_model, callbacks, learning_rates)\u001b[0m\n\u001b[0;32m    202\u001b[0m                            \u001b[0mevals\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mevals\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    203\u001b[0m                            \u001b[0mobj\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mobj\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mfeval\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mfeval\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 204\u001b[1;33m                            xgb_model=xgb_model, callbacks=callbacks)\n\u001b[0m\u001b[0;32m    205\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    206\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\Anaconda3\\lib\\site-packages\\xgboost\\training.py\u001b[0m in \u001b[0;36m_train_internal\u001b[1;34m(params, dtrain, num_boost_round, evals, obj, feval, xgb_model, callbacks)\u001b[0m\n\u001b[0;32m     72\u001b[0m         \u001b[1;31m# Skip the first update if it is a recovery step.\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     73\u001b[0m         \u001b[1;32mif\u001b[0m \u001b[0mversion\u001b[0m \u001b[1;33m%\u001b[0m \u001b[1;36m2\u001b[0m \u001b[1;33m==\u001b[0m \u001b[1;36m0\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m---> 74\u001b[1;33m             \u001b[0mbst\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mupdate\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mdtrain\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mi\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mobj\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m     75\u001b[0m             \u001b[0mbst\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0msave_rabit_checkpoint\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     76\u001b[0m             \u001b[0mversion\u001b[0m \u001b[1;33m+=\u001b[0m \u001b[1;36m1\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\Anaconda3\\lib\\site-packages\\xgboost\\core.py\u001b[0m in \u001b[0;36mupdate\u001b[1;34m(self, dtrain, iteration, fobj)\u001b[0m\n\u001b[0;32m   1019\u001b[0m         \u001b[1;32mif\u001b[0m \u001b[0mfobj\u001b[0m \u001b[1;32mis\u001b[0m \u001b[1;32mNone\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   1020\u001b[0m             _check_call(_LIB.XGBoosterUpdateOneIter(self.handle, ctypes.c_int(iteration),\n\u001b[1;32m-> 1021\u001b[1;33m                                                     dtrain.handle))\n\u001b[0m\u001b[0;32m   1022\u001b[0m         \u001b[1;32melse\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   1023\u001b[0m             \u001b[0mpred\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mpredict\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mdtrain\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;31mKeyboardInterrupt\u001b[0m: "
     ]
    }
   ],
   "source": [
    "orchestrator12.performHyperparameterOptimization(X_12_preselect, 25)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "orchestrator12.bo.best"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [],
   "source": [
    "preselect_params_12 = {'colsample_bytree': 0.60,\n",
    "                        'gamma': 2.0,\n",
    "                        'learning_rate': 0.05,\n",
    "                        'max_depth': 11,\n",
    "                        'min_child_weight': 10.0,\n",
    "                        'n_estimators': 50,\n",
    "                        'subsample': 0.8}\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Running custom feature selection module ....\n",
      "Creating baseline model to extract features\n",
      "all features (2688, 10427)\n",
      "19\n",
      "# features 10427 # thresholds 19; thresholds:[0.01544021 0.01051248 0.00722733 0.00689882 0.00591327 0.00492773\n",
      " 0.0042707  0.00361367 0.00328515 0.00295664 0.00262812 0.00229961\n",
      " 0.00197109 0.00164258 0.00131406 0.00098555 0.00065703 0.00032852\n",
      " 0.        ]\n",
      "CV - Window 3\n",
      "Train [2017-07-01 2017-08-15] Val [2017-08-16 2017-08-31] Test [2017-09-01 2017-09-16]\n",
      "Window Train/Val/Test shape (966, 10428) (336, 10428) (336, 10428)\n",
      "Window Train/Val/Test shape (966, 2) (336, 2) (336, 2)\n",
      "X_train_cv  (1302, 10427)\n",
      "3 Thresh=0.01544, n=2, mse: 149.084\n",
      "CV - Window 4\n",
      "Train [2017-07-01 2017-08-30] Val [2017-08-31 2017-09-15] Test [2017-09-16 2017-10-01]\n",
      "Window Train/Val/Test shape (1281, 10428) (336, 10428) (336, 10428)\n",
      "Window Train/Val/Test shape (1281, 2) (336, 2) (336, 2)\n",
      "X_train_cv  (1617, 10427)\n",
      "4 Thresh=0.01544, n=2, mse: 124.931\n",
      "CV - Window 5\n",
      "Train [2017-07-01 2017-09-14] Val [2017-09-15 2017-09-30] Test [2017-10-01 2017-10-16]\n",
      "Window Train/Val/Test shape (1596, 10428) (336, 10428) (336, 10428)\n",
      "Window Train/Val/Test shape (1596, 2) (336, 2) (336, 2)\n",
      "X_train_cv  (1932, 10427)\n",
      "5 Thresh=0.01544, n=2, mse: 49.998\n",
      "CV - Window 6\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-09-29] Val [2017-09-30 2017-10-15] Test [2017-10-16 2017-11-10]\n",
      "Window Train/Val/Test shape (1911, 10428) (336, 10428) (546, 10428)\n",
      "Window Train/Val/Test shape (1911, 2) (336, 2) (546, 2)\n",
      "X_train_cv  (2247, 10427)\n",
      "6 Thresh=0.01544, n=2, mse: 18.819\n",
      "CV - Window 7\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-10-14] Val [2017-10-15 2017-10-30] Test [2017-10-31 2017-11-10]\n",
      "Window Train/Val/Test shape (2226, 10428) (336, 10428) (231, 10428)\n",
      "Window Train/Val/Test shape (2226, 2) (336, 2) (231, 2)\n",
      "X_train_cv  (2562, 10427)\n",
      "7 Thresh=0.01544, n=2, mse: 13.747\n",
      "{'threshold': 0.015440211, 'num_features': 2, 'mse_list': [149.083527043454, 124.93147620655877, 49.99802554664133, 18.819360306533035, 13.74666712906619], 'mean_mse': 71.31581124645066}\n",
      "\n",
      "\n",
      "CV - Window 3\n",
      "Train [2017-07-01 2017-08-15] Val [2017-08-16 2017-08-31] Test [2017-09-01 2017-09-16]\n",
      "Window Train/Val/Test shape (966, 10428) (336, 10428) (336, 10428)\n",
      "Window Train/Val/Test shape (966, 2) (336, 2) (336, 2)\n",
      "X_train_cv  (1302, 10427)\n",
      "3 Thresh=0.01051, n=3, mse: 149.521\n",
      "CV - Window 4\n",
      "Train [2017-07-01 2017-08-30] Val [2017-08-31 2017-09-15] Test [2017-09-16 2017-10-01]\n",
      "Window Train/Val/Test shape (1281, 10428) (336, 10428) (336, 10428)\n",
      "Window Train/Val/Test shape (1281, 2) (336, 2) (336, 2)\n",
      "X_train_cv  (1617, 10427)\n",
      "4 Thresh=0.01051, n=3, mse: 125.677\n",
      "CV - Window 5\n",
      "Train [2017-07-01 2017-09-14] Val [2017-09-15 2017-09-30] Test [2017-10-01 2017-10-16]\n",
      "Window Train/Val/Test shape (1596, 10428) (336, 10428) (336, 10428)\n",
      "Window Train/Val/Test shape (1596, 2) (336, 2) (336, 2)\n",
      "X_train_cv  (1932, 10427)\n",
      "5 Thresh=0.01051, n=3, mse: 45.855\n",
      "CV - Window 6\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-09-29] Val [2017-09-30 2017-10-15] Test [2017-10-16 2017-11-10]\n",
      "Window Train/Val/Test shape (1911, 10428) (336, 10428) (546, 10428)\n",
      "Window Train/Val/Test shape (1911, 2) (336, 2) (546, 2)\n",
      "X_train_cv  (2247, 10427)\n",
      "6 Thresh=0.01051, n=3, mse: 18.687\n",
      "CV - Window 7\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-10-14] Val [2017-10-15 2017-10-30] Test [2017-10-31 2017-11-10]\n",
      "Window Train/Val/Test shape (2226, 10428) (336, 10428) (231, 10428)\n",
      "Window Train/Val/Test shape (2226, 2) (336, 2) (231, 2)\n",
      "X_train_cv  (2562, 10427)\n",
      "7 Thresh=0.01051, n=3, mse: 12.878\n",
      "{'threshold': 0.010512483, 'num_features': 3, 'mse_list': [149.52148992973295, 125.67727159292792, 45.854568868790686, 18.687150538987765, 12.878068796221925], 'mean_mse': 70.52370994533226}\n",
      "\n",
      "\n",
      "CV - Window 3\n",
      "Train [2017-07-01 2017-08-15] Val [2017-08-16 2017-08-31] Test [2017-09-01 2017-09-16]\n",
      "Window Train/Val/Test shape (966, 10428) (336, 10428) (336, 10428)\n",
      "Window Train/Val/Test shape (966, 2) (336, 2) (336, 2)\n",
      "X_train_cv  (1302, 10427)\n",
      "3 Thresh=0.00723, n=4, mse: 148.988\n",
      "CV - Window 4\n",
      "Train [2017-07-01 2017-08-30] Val [2017-08-31 2017-09-15] Test [2017-09-16 2017-10-01]\n",
      "Window Train/Val/Test shape (1281, 10428) (336, 10428) (336, 10428)\n",
      "Window Train/Val/Test shape (1281, 2) (336, 2) (336, 2)\n",
      "X_train_cv  (1617, 10427)\n",
      "4 Thresh=0.00723, n=4, mse: 123.682\n",
      "CV - Window 5\n",
      "Train [2017-07-01 2017-09-14] Val [2017-09-15 2017-09-30] Test [2017-10-01 2017-10-16]\n",
      "Window Train/Val/Test shape (1596, 10428) (336, 10428) (336, 10428)\n",
      "Window Train/Val/Test shape (1596, 2) (336, 2) (336, 2)\n",
      "X_train_cv  (1932, 10427)\n",
      "5 Thresh=0.00723, n=4, mse: 41.577\n",
      "CV - Window 6\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-09-29] Val [2017-09-30 2017-10-15] Test [2017-10-16 2017-11-10]\n",
      "Window Train/Val/Test shape (1911, 10428) (336, 10428) (546, 10428)\n",
      "Window Train/Val/Test shape (1911, 2) (336, 2) (546, 2)\n",
      "X_train_cv  (2247, 10427)\n",
      "6 Thresh=0.00723, n=4, mse: 16.674\n",
      "CV - Window 7\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-10-14] Val [2017-10-15 2017-10-30] Test [2017-10-31 2017-11-10]\n",
      "Window Train/Val/Test shape (2226, 10428) (336, 10428) (231, 10428)\n",
      "Window Train/Val/Test shape (2226, 2) (336, 2) (231, 2)\n",
      "X_train_cv  (2562, 10427)\n",
      "7 Thresh=0.00723, n=4, mse: 12.152\n",
      "{'threshold': 0.0072273323, 'num_features': 4, 'mse_list': [148.98804438037072, 123.6824049157656, 41.576841051607275, 16.673878220307817, 12.151966913740285], 'mean_mse': 68.61462709635835}\n",
      "\n",
      "\n",
      "CV - Window 3\n",
      "Train [2017-07-01 2017-08-15] Val [2017-08-16 2017-08-31] Test [2017-09-01 2017-09-16]\n",
      "Window Train/Val/Test shape (966, 10428) (336, 10428) (336, 10428)\n",
      "Window Train/Val/Test shape (966, 2) (336, 2) (336, 2)\n",
      "X_train_cv  (1302, 10427)\n",
      "3 Thresh=0.00690, n=5, mse: 144.759\n",
      "CV - Window 4\n",
      "Train [2017-07-01 2017-08-30] Val [2017-08-31 2017-09-15] Test [2017-09-16 2017-10-01]\n",
      "Window Train/Val/Test shape (1281, 10428) (336, 10428) (336, 10428)\n",
      "Window Train/Val/Test shape (1281, 2) (336, 2) (336, 2)\n",
      "X_train_cv  (1617, 10427)\n",
      "4 Thresh=0.00690, n=5, mse: 124.575\n",
      "CV - Window 5\n",
      "Train [2017-07-01 2017-09-14] Val [2017-09-15 2017-09-30] Test [2017-10-01 2017-10-16]\n",
      "Window Train/Val/Test shape (1596, 10428) (336, 10428) (336, 10428)\n",
      "Window Train/Val/Test shape (1596, 2) (336, 2) (336, 2)\n",
      "X_train_cv  (1932, 10427)\n",
      "5 Thresh=0.00690, n=5, mse: 41.951\n",
      "CV - Window 6\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-09-29] Val [2017-09-30 2017-10-15] Test [2017-10-16 2017-11-10]\n",
      "Window Train/Val/Test shape (1911, 10428) (336, 10428) (546, 10428)\n",
      "Window Train/Val/Test shape (1911, 2) (336, 2) (546, 2)\n",
      "X_train_cv  (2247, 10427)\n",
      "6 Thresh=0.00690, n=5, mse: 16.498\n",
      "CV - Window 7\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-10-14] Val [2017-10-15 2017-10-30] Test [2017-10-31 2017-11-10]\n",
      "Window Train/Val/Test shape (2226, 10428) (336, 10428) (231, 10428)\n",
      "Window Train/Val/Test shape (2226, 2) (336, 2) (231, 2)\n",
      "X_train_cv  (2562, 10427)\n",
      "7 Thresh=0.00690, n=5, mse: 9.644\n",
      "{'threshold': 0.006898817, 'num_features': 5, 'mse_list': [144.75932550919464, 124.57481877299821, 41.95130959476892, 16.498498494792816, 9.644386174938479], 'mean_mse': 67.48566770933861}\n",
      "\n",
      "\n",
      "CV - Window 3\n",
      "Train [2017-07-01 2017-08-15] Val [2017-08-16 2017-08-31] Test [2017-09-01 2017-09-16]\n",
      "Window Train/Val/Test shape (966, 10428) (336, 10428) (336, 10428)\n",
      "Window Train/Val/Test shape (966, 2) (336, 2) (336, 2)\n",
      "X_train_cv  (1302, 10427)\n",
      "3 Thresh=0.00591, n=6, mse: 143.686\n",
      "CV - Window 4\n",
      "Train [2017-07-01 2017-08-30] Val [2017-08-31 2017-09-15] Test [2017-09-16 2017-10-01]\n",
      "Window Train/Val/Test shape (1281, 10428) (336, 10428) (336, 10428)\n",
      "Window Train/Val/Test shape (1281, 2) (336, 2) (336, 2)\n",
      "X_train_cv  (1617, 10427)\n",
      "4 Thresh=0.00591, n=6, mse: 124.363\n",
      "CV - Window 5\n",
      "Train [2017-07-01 2017-09-14] Val [2017-09-15 2017-09-30] Test [2017-10-01 2017-10-16]\n",
      "Window Train/Val/Test shape (1596, 10428) (336, 10428) (336, 10428)\n",
      "Window Train/Val/Test shape (1596, 2) (336, 2) (336, 2)\n",
      "X_train_cv  (1932, 10427)\n",
      "5 Thresh=0.00591, n=6, mse: 43.581\n",
      "CV - Window 6\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-09-29] Val [2017-09-30 2017-10-15] Test [2017-10-16 2017-11-10]\n",
      "Window Train/Val/Test shape (1911, 10428) (336, 10428) (546, 10428)\n",
      "Window Train/Val/Test shape (1911, 2) (336, 2) (546, 2)\n",
      "X_train_cv  (2247, 10427)\n",
      "6 Thresh=0.00591, n=6, mse: 14.961\n",
      "CV - Window 7\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-10-14] Val [2017-10-15 2017-10-30] Test [2017-10-31 2017-11-10]\n",
      "Window Train/Val/Test shape (2226, 10428) (336, 10428) (231, 10428)\n",
      "Window Train/Val/Test shape (2226, 2) (336, 2) (231, 2)\n",
      "X_train_cv  (2562, 10427)\n",
      "7 Thresh=0.00591, n=6, mse: 9.915\n",
      "{'threshold': 0.005913272, 'num_features': 6, 'mse_list': [143.68633116899045, 124.36296975381886, 43.580654344093254, 14.961303468884507, 9.914548359752263], 'mean_mse': 67.30116141910787}\n",
      "\n",
      "\n",
      "CV - Window 3\n",
      "Train [2017-07-01 2017-08-15] Val [2017-08-16 2017-08-31] Test [2017-09-01 2017-09-16]\n",
      "Window Train/Val/Test shape (966, 10428) (336, 10428) (336, 10428)\n",
      "Window Train/Val/Test shape (966, 2) (336, 2) (336, 2)\n",
      "X_train_cv  (1302, 10427)\n",
      "3 Thresh=0.00493, n=7, mse: 146.995\n",
      "CV - Window 4\n",
      "Train [2017-07-01 2017-08-30] Val [2017-08-31 2017-09-15] Test [2017-09-16 2017-10-01]\n",
      "Window Train/Val/Test shape (1281, 10428) (336, 10428) (336, 10428)\n",
      "Window Train/Val/Test shape (1281, 2) (336, 2) (336, 2)\n",
      "X_train_cv  (1617, 10427)\n",
      "4 Thresh=0.00493, n=7, mse: 126.947\n",
      "CV - Window 5\n",
      "Train [2017-07-01 2017-09-14] Val [2017-09-15 2017-09-30] Test [2017-10-01 2017-10-16]\n",
      "Window Train/Val/Test shape (1596, 10428) (336, 10428) (336, 10428)\n",
      "Window Train/Val/Test shape (1596, 2) (336, 2) (336, 2)\n",
      "X_train_cv  (1932, 10427)\n",
      "5 Thresh=0.00493, n=7, mse: 43.877\n",
      "CV - Window 6\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-09-29] Val [2017-09-30 2017-10-15] Test [2017-10-16 2017-11-10]\n",
      "Window Train/Val/Test shape (1911, 10428) (336, 10428) (546, 10428)\n",
      "Window Train/Val/Test shape (1911, 2) (336, 2) (546, 2)\n",
      "X_train_cv  (2247, 10427)\n",
      "6 Thresh=0.00493, n=7, mse: 16.463\n",
      "CV - Window 7\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-10-14] Val [2017-10-15 2017-10-30] Test [2017-10-31 2017-11-10]\n",
      "Window Train/Val/Test shape (2226, 10428) (336, 10428) (231, 10428)\n",
      "Window Train/Val/Test shape (2226, 2) (336, 2) (231, 2)\n",
      "X_train_cv  (2562, 10427)\n",
      "7 Thresh=0.00493, n=7, mse: 8.802\n",
      "{'threshold': 0.0049277265, 'num_features': 7, 'mse_list': [146.99511150122953, 126.94747737237012, 43.876945229578475, 16.462973102478557, 8.802276724423368], 'mean_mse': 68.61695678601602}\n",
      "\n",
      "\n",
      "CV - Window 3\n",
      "Train [2017-07-01 2017-08-15] Val [2017-08-16 2017-08-31] Test [2017-09-01 2017-09-16]\n",
      "Window Train/Val/Test shape (966, 10428) (336, 10428) (336, 10428)\n",
      "Window Train/Val/Test shape (966, 2) (336, 2) (336, 2)\n",
      "X_train_cv  (1302, 10427)\n",
      "3 Thresh=0.00427, n=9, mse: 151.316\n",
      "CV - Window 4\n",
      "Train [2017-07-01 2017-08-30] Val [2017-08-31 2017-09-15] Test [2017-09-16 2017-10-01]\n",
      "Window Train/Val/Test shape (1281, 10428) (336, 10428) (336, 10428)\n",
      "Window Train/Val/Test shape (1281, 2) (336, 2) (336, 2)\n",
      "X_train_cv  (1617, 10427)\n",
      "4 Thresh=0.00427, n=9, mse: 133.017\n",
      "CV - Window 5\n",
      "Train [2017-07-01 2017-09-14] Val [2017-09-15 2017-09-30] Test [2017-10-01 2017-10-16]\n",
      "Window Train/Val/Test shape (1596, 10428) (336, 10428) (336, 10428)\n",
      "Window Train/Val/Test shape (1596, 2) (336, 2) (336, 2)\n",
      "X_train_cv  (1932, 10427)\n",
      "5 Thresh=0.00427, n=9, mse: 42.336\n",
      "CV - Window 6\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-09-29] Val [2017-09-30 2017-10-15] Test [2017-10-16 2017-11-10]\n",
      "Window Train/Val/Test shape (1911, 10428) (336, 10428) (546, 10428)\n",
      "Window Train/Val/Test shape (1911, 2) (336, 2) (546, 2)\n",
      "X_train_cv  (2247, 10427)\n",
      "6 Thresh=0.00427, n=9, mse: 25.183\n",
      "CV - Window 7\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-10-14] Val [2017-10-15 2017-10-30] Test [2017-10-31 2017-11-10]\n",
      "Window Train/Val/Test shape (2226, 10428) (336, 10428) (231, 10428)\n",
      "Window Train/Val/Test shape (2226, 2) (336, 2) (231, 2)\n",
      "X_train_cv  (2562, 10427)\n",
      "7 Thresh=0.00427, n=9, mse: 26.864\n",
      "{'threshold': 0.0042706965, 'num_features': 9, 'mse_list': [151.31624305439107, 133.0165576389683, 42.33587494296738, 25.183151649853446, 26.864298093835593], 'mean_mse': 75.74322507600316}\n",
      "\n",
      "\n",
      "CV - Window 3\n",
      "Train [2017-07-01 2017-08-15] Val [2017-08-16 2017-08-31] Test [2017-09-01 2017-09-16]\n",
      "Window Train/Val/Test shape (966, 10428) (336, 10428) (336, 10428)\n",
      "Window Train/Val/Test shape (966, 2) (336, 2) (336, 2)\n",
      "X_train_cv  (1302, 10427)\n",
      "3 Thresh=0.00361, n=11, mse: 142.570\n",
      "CV - Window 4\n",
      "Train [2017-07-01 2017-08-30] Val [2017-08-31 2017-09-15] Test [2017-09-16 2017-10-01]\n",
      "Window Train/Val/Test shape (1281, 10428) (336, 10428) (336, 10428)\n",
      "Window Train/Val/Test shape (1281, 2) (336, 2) (336, 2)\n",
      "X_train_cv  (1617, 10427)\n",
      "4 Thresh=0.00361, n=11, mse: 134.792\n",
      "CV - Window 5\n",
      "Train [2017-07-01 2017-09-14] Val [2017-09-15 2017-09-30] Test [2017-10-01 2017-10-16]\n",
      "Window Train/Val/Test shape (1596, 10428) (336, 10428) (336, 10428)\n",
      "Window Train/Val/Test shape (1596, 2) (336, 2) (336, 2)\n",
      "X_train_cv  (1932, 10427)\n",
      "5 Thresh=0.00361, n=11, mse: 41.903\n",
      "CV - Window 6\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-09-29] Val [2017-09-30 2017-10-15] Test [2017-10-16 2017-11-10]\n",
      "Window Train/Val/Test shape (1911, 10428) (336, 10428) (546, 10428)\n",
      "Window Train/Val/Test shape (1911, 2) (336, 2) (546, 2)\n",
      "X_train_cv  (2247, 10427)\n",
      "6 Thresh=0.00361, n=11, mse: 25.070\n",
      "CV - Window 7\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-10-14] Val [2017-10-15 2017-10-30] Test [2017-10-31 2017-11-10]\n",
      "Window Train/Val/Test shape (2226, 10428) (336, 10428) (231, 10428)\n",
      "Window Train/Val/Test shape (2226, 2) (336, 2) (231, 2)\n",
      "X_train_cv  (2562, 10427)\n",
      "7 Thresh=0.00361, n=11, mse: 23.750\n",
      "{'threshold': 0.0036136662, 'num_features': 11, 'mse_list': [142.57007415876288, 134.79152742615184, 41.90297239280843, 25.06967531720101, 23.749630334321644], 'mean_mse': 73.61677592584917}\n",
      "\n",
      "\n",
      "CV - Window 3\n",
      "Train [2017-07-01 2017-08-15] Val [2017-08-16 2017-08-31] Test [2017-09-01 2017-09-16]\n",
      "Window Train/Val/Test shape (966, 10428) (336, 10428) (336, 10428)\n",
      "Window Train/Val/Test shape (966, 2) (336, 2) (336, 2)\n",
      "X_train_cv  (1302, 10427)\n",
      "3 Thresh=0.00329, n=13, mse: 147.862\n",
      "CV - Window 4\n",
      "Train [2017-07-01 2017-08-30] Val [2017-08-31 2017-09-15] Test [2017-09-16 2017-10-01]\n",
      "Window Train/Val/Test shape (1281, 10428) (336, 10428) (336, 10428)\n",
      "Window Train/Val/Test shape (1281, 2) (336, 2) (336, 2)\n",
      "X_train_cv  (1617, 10427)\n",
      "4 Thresh=0.00329, n=13, mse: 109.874\n",
      "CV - Window 5\n",
      "Train [2017-07-01 2017-09-14] Val [2017-09-15 2017-09-30] Test [2017-10-01 2017-10-16]\n",
      "Window Train/Val/Test shape (1596, 10428) (336, 10428) (336, 10428)\n",
      "Window Train/Val/Test shape (1596, 2) (336, 2) (336, 2)\n",
      "X_train_cv  (1932, 10427)\n",
      "5 Thresh=0.00329, n=13, mse: 32.583\n",
      "CV - Window 6\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-09-29] Val [2017-09-30 2017-10-15] Test [2017-10-16 2017-11-10]\n",
      "Window Train/Val/Test shape (1911, 10428) (336, 10428) (546, 10428)\n",
      "Window Train/Val/Test shape (1911, 2) (336, 2) (546, 2)\n",
      "X_train_cv  (2247, 10427)\n",
      "6 Thresh=0.00329, n=13, mse: 28.600\n",
      "CV - Window 7\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-10-14] Val [2017-10-15 2017-10-30] Test [2017-10-31 2017-11-10]\n",
      "Window Train/Val/Test shape (2226, 10428) (336, 10428) (231, 10428)\n",
      "Window Train/Val/Test shape (2226, 2) (336, 2) (231, 2)\n",
      "X_train_cv  (2562, 10427)\n",
      "7 Thresh=0.00329, n=13, mse: 26.292\n",
      "{'threshold': 0.0032851512, 'num_features': 13, 'mse_list': [147.86202167204647, 109.87446865163263, 32.58267055565556, 28.600462744029272, 26.291504206517914], 'mean_mse': 69.04222556597637}\n",
      "\n",
      "\n",
      "CV - Window 3\n",
      "Train [2017-07-01 2017-08-15] Val [2017-08-16 2017-08-31] Test [2017-09-01 2017-09-16]\n",
      "Window Train/Val/Test shape (966, 10428) (336, 10428) (336, 10428)\n",
      "Window Train/Val/Test shape (966, 2) (336, 2) (336, 2)\n",
      "X_train_cv  (1302, 10427)\n",
      "3 Thresh=0.00296, n=19, mse: 134.055\n",
      "CV - Window 4\n",
      "Train [2017-07-01 2017-08-30] Val [2017-08-31 2017-09-15] Test [2017-09-16 2017-10-01]\n",
      "Window Train/Val/Test shape (1281, 10428) (336, 10428) (336, 10428)\n",
      "Window Train/Val/Test shape (1281, 2) (336, 2) (336, 2)\n",
      "X_train_cv  (1617, 10427)\n",
      "4 Thresh=0.00296, n=19, mse: 84.787\n",
      "CV - Window 5\n",
      "Train [2017-07-01 2017-09-14] Val [2017-09-15 2017-09-30] Test [2017-10-01 2017-10-16]\n",
      "Window Train/Val/Test shape (1596, 10428) (336, 10428) (336, 10428)\n",
      "Window Train/Val/Test shape (1596, 2) (336, 2) (336, 2)\n",
      "X_train_cv  (1932, 10427)\n",
      "5 Thresh=0.00296, n=19, mse: 32.251\n",
      "CV - Window 6\n",
      "Rolling window to end date\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train [2017-07-01 2017-09-29] Val [2017-09-30 2017-10-15] Test [2017-10-16 2017-11-10]\n",
      "Window Train/Val/Test shape (1911, 10428) (336, 10428) (546, 10428)\n",
      "Window Train/Val/Test shape (1911, 2) (336, 2) (546, 2)\n",
      "X_train_cv  (2247, 10427)\n",
      "6 Thresh=0.00296, n=19, mse: 26.690\n",
      "CV - Window 7\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-10-14] Val [2017-10-15 2017-10-30] Test [2017-10-31 2017-11-10]\n",
      "Window Train/Val/Test shape (2226, 10428) (336, 10428) (231, 10428)\n",
      "Window Train/Val/Test shape (2226, 2) (336, 2) (231, 2)\n",
      "X_train_cv  (2562, 10427)\n",
      "7 Thresh=0.00296, n=19, mse: 32.084\n",
      "{'threshold': 0.002956636, 'num_features': 19, 'mse_list': [134.0550972030669, 84.78690099572086, 32.25120946837106, 26.68973445366726, 32.08396458228555], 'mean_mse': 61.973381340622325}\n",
      "\n",
      "\n",
      "CV - Window 3\n",
      "Train [2017-07-01 2017-08-15] Val [2017-08-16 2017-08-31] Test [2017-09-01 2017-09-16]\n",
      "Window Train/Val/Test shape (966, 10428) (336, 10428) (336, 10428)\n",
      "Window Train/Val/Test shape (966, 2) (336, 2) (336, 2)\n",
      "X_train_cv  (1302, 10427)\n",
      "3 Thresh=0.00263, n=28, mse: 135.655\n",
      "CV - Window 4\n",
      "Train [2017-07-01 2017-08-30] Val [2017-08-31 2017-09-15] Test [2017-09-16 2017-10-01]\n",
      "Window Train/Val/Test shape (1281, 10428) (336, 10428) (336, 10428)\n",
      "Window Train/Val/Test shape (1281, 2) (336, 2) (336, 2)\n",
      "X_train_cv  (1617, 10427)\n",
      "4 Thresh=0.00263, n=28, mse: 81.672\n",
      "CV - Window 5\n",
      "Train [2017-07-01 2017-09-14] Val [2017-09-15 2017-09-30] Test [2017-10-01 2017-10-16]\n",
      "Window Train/Val/Test shape (1596, 10428) (336, 10428) (336, 10428)\n",
      "Window Train/Val/Test shape (1596, 2) (336, 2) (336, 2)\n",
      "X_train_cv  (1932, 10427)\n",
      "5 Thresh=0.00263, n=28, mse: 41.965\n",
      "CV - Window 6\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-09-29] Val [2017-09-30 2017-10-15] Test [2017-10-16 2017-11-10]\n",
      "Window Train/Val/Test shape (1911, 10428) (336, 10428) (546, 10428)\n",
      "Window Train/Val/Test shape (1911, 2) (336, 2) (546, 2)\n",
      "X_train_cv  (2247, 10427)\n",
      "6 Thresh=0.00263, n=28, mse: 26.247\n",
      "CV - Window 7\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-10-14] Val [2017-10-15 2017-10-30] Test [2017-10-31 2017-11-10]\n",
      "Window Train/Val/Test shape (2226, 10428) (336, 10428) (231, 10428)\n",
      "Window Train/Val/Test shape (2226, 2) (336, 2) (231, 2)\n",
      "X_train_cv  (2562, 10427)\n",
      "7 Thresh=0.00263, n=28, mse: 23.037\n",
      "{'threshold': 0.0026281208, 'num_features': 28, 'mse_list': [135.65509680519378, 81.67240770210165, 41.96450069158665, 26.247416911172547, 23.037242682807346], 'mean_mse': 61.71533295857239}\n",
      "\n",
      "\n",
      "CV - Window 3\n",
      "Train [2017-07-01 2017-08-15] Val [2017-08-16 2017-08-31] Test [2017-09-01 2017-09-16]\n",
      "Window Train/Val/Test shape (966, 10428) (336, 10428) (336, 10428)\n",
      "Window Train/Val/Test shape (966, 2) (336, 2) (336, 2)\n",
      "X_train_cv  (1302, 10427)\n",
      "3 Thresh=0.00230, n=31, mse: 132.604\n",
      "CV - Window 4\n",
      "Train [2017-07-01 2017-08-30] Val [2017-08-31 2017-09-15] Test [2017-09-16 2017-10-01]\n",
      "Window Train/Val/Test shape (1281, 10428) (336, 10428) (336, 10428)\n",
      "Window Train/Val/Test shape (1281, 2) (336, 2) (336, 2)\n",
      "X_train_cv  (1617, 10427)\n",
      "4 Thresh=0.00230, n=31, mse: 80.114\n",
      "CV - Window 5\n",
      "Train [2017-07-01 2017-09-14] Val [2017-09-15 2017-09-30] Test [2017-10-01 2017-10-16]\n",
      "Window Train/Val/Test shape (1596, 10428) (336, 10428) (336, 10428)\n",
      "Window Train/Val/Test shape (1596, 2) (336, 2) (336, 2)\n",
      "X_train_cv  (1932, 10427)\n",
      "5 Thresh=0.00230, n=31, mse: 41.293\n",
      "CV - Window 6\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-09-29] Val [2017-09-30 2017-10-15] Test [2017-10-16 2017-11-10]\n",
      "Window Train/Val/Test shape (1911, 10428) (336, 10428) (546, 10428)\n",
      "Window Train/Val/Test shape (1911, 2) (336, 2) (546, 2)\n",
      "X_train_cv  (2247, 10427)\n",
      "6 Thresh=0.00230, n=31, mse: 20.410\n",
      "CV - Window 7\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-10-14] Val [2017-10-15 2017-10-30] Test [2017-10-31 2017-11-10]\n",
      "Window Train/Val/Test shape (2226, 10428) (336, 10428) (231, 10428)\n",
      "Window Train/Val/Test shape (2226, 2) (336, 2) (231, 2)\n",
      "X_train_cv  (2562, 10427)\n",
      "7 Thresh=0.00230, n=31, mse: 16.157\n",
      "{'threshold': 0.0022996059, 'num_features': 31, 'mse_list': [132.60385135116053, 80.11427651688743, 41.293024660936474, 20.410224457673046, 16.156668787984525], 'mean_mse': 58.11560915492841}\n",
      "\n",
      "\n",
      "CV - Window 3\n",
      "Train [2017-07-01 2017-08-15] Val [2017-08-16 2017-08-31] Test [2017-09-01 2017-09-16]\n",
      "Window Train/Val/Test shape (966, 10428) (336, 10428) (336, 10428)\n",
      "Window Train/Val/Test shape (966, 2) (336, 2) (336, 2)\n",
      "X_train_cv  (1302, 10427)\n",
      "3 Thresh=0.00197, n=38, mse: 130.890\n",
      "CV - Window 4\n",
      "Train [2017-07-01 2017-08-30] Val [2017-08-31 2017-09-15] Test [2017-09-16 2017-10-01]\n",
      "Window Train/Val/Test shape (1281, 10428) (336, 10428) (336, 10428)\n",
      "Window Train/Val/Test shape (1281, 2) (336, 2) (336, 2)\n",
      "X_train_cv  (1617, 10427)\n",
      "4 Thresh=0.00197, n=38, mse: 81.248\n",
      "CV - Window 5\n",
      "Train [2017-07-01 2017-09-14] Val [2017-09-15 2017-09-30] Test [2017-10-01 2017-10-16]\n",
      "Window Train/Val/Test shape (1596, 10428) (336, 10428) (336, 10428)\n",
      "Window Train/Val/Test shape (1596, 2) (336, 2) (336, 2)\n",
      "X_train_cv  (1932, 10427)\n",
      "5 Thresh=0.00197, n=38, mse: 33.328\n",
      "CV - Window 6\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-09-29] Val [2017-09-30 2017-10-15] Test [2017-10-16 2017-11-10]\n",
      "Window Train/Val/Test shape (1911, 10428) (336, 10428) (546, 10428)\n",
      "Window Train/Val/Test shape (1911, 2) (336, 2) (546, 2)\n",
      "X_train_cv  (2247, 10427)\n",
      "6 Thresh=0.00197, n=38, mse: 16.822\n",
      "CV - Window 7\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-10-14] Val [2017-10-15 2017-10-30] Test [2017-10-31 2017-11-10]\n",
      "Window Train/Val/Test shape (2226, 10428) (336, 10428) (231, 10428)\n",
      "Window Train/Val/Test shape (2226, 2) (336, 2) (231, 2)\n",
      "X_train_cv  (2562, 10427)\n",
      "7 Thresh=0.00197, n=38, mse: 11.453\n",
      "{'threshold': 0.0019710907, 'num_features': 38, 'mse_list': [130.88952906794887, 81.24836756760475, 33.32827027032671, 16.821887275906974, 11.453208601122393], 'mean_mse': 54.74825255658194}\n",
      "\n",
      "\n",
      "CV - Window 3\n",
      "Train [2017-07-01 2017-08-15] Val [2017-08-16 2017-08-31] Test [2017-09-01 2017-09-16]\n",
      "Window Train/Val/Test shape (966, 10428) (336, 10428) (336, 10428)\n",
      "Window Train/Val/Test shape (966, 2) (336, 2) (336, 2)\n",
      "X_train_cv  (1302, 10427)\n",
      "3 Thresh=0.00164, n=64, mse: 127.092\n",
      "CV - Window 4\n",
      "Train [2017-07-01 2017-08-30] Val [2017-08-31 2017-09-15] Test [2017-09-16 2017-10-01]\n",
      "Window Train/Val/Test shape (1281, 10428) (336, 10428) (336, 10428)\n",
      "Window Train/Val/Test shape (1281, 2) (336, 2) (336, 2)\n",
      "X_train_cv  (1617, 10427)\n",
      "4 Thresh=0.00164, n=64, mse: 69.956\n",
      "CV - Window 5\n",
      "Train [2017-07-01 2017-09-14] Val [2017-09-15 2017-09-30] Test [2017-10-01 2017-10-16]\n",
      "Window Train/Val/Test shape (1596, 10428) (336, 10428) (336, 10428)\n",
      "Window Train/Val/Test shape (1596, 2) (336, 2) (336, 2)\n",
      "X_train_cv  (1932, 10427)\n",
      "5 Thresh=0.00164, n=64, mse: 32.429\n",
      "CV - Window 6\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-09-29] Val [2017-09-30 2017-10-15] Test [2017-10-16 2017-11-10]\n",
      "Window Train/Val/Test shape (1911, 10428) (336, 10428) (546, 10428)\n",
      "Window Train/Val/Test shape (1911, 2) (336, 2) (546, 2)\n",
      "X_train_cv  (2247, 10427)\n",
      "6 Thresh=0.00164, n=64, mse: 21.001\n",
      "CV - Window 7\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-10-14] Val [2017-10-15 2017-10-30] Test [2017-10-31 2017-11-10]\n",
      "Window Train/Val/Test shape (2226, 10428) (336, 10428) (231, 10428)\n",
      "Window Train/Val/Test shape (2226, 2) (336, 2) (231, 2)\n",
      "X_train_cv  (2562, 10427)\n",
      "7 Thresh=0.00164, n=64, mse: 16.151\n",
      "{'threshold': 0.0016425756, 'num_features': 64, 'mse_list': [127.09150562024566, 69.9564526961938, 32.4289804108263, 21.00130819217266, 16.150621611338273], 'mean_mse': 53.32577370615534}\n",
      "\n",
      "\n",
      "CV - Window 3\n",
      "Train [2017-07-01 2017-08-15] Val [2017-08-16 2017-08-31] Test [2017-09-01 2017-09-16]\n",
      "Window Train/Val/Test shape (966, 10428) (336, 10428) (336, 10428)\n",
      "Window Train/Val/Test shape (966, 2) (336, 2) (336, 2)\n",
      "X_train_cv  (1302, 10427)\n",
      "3 Thresh=0.00131, n=97, mse: 134.373\n",
      "CV - Window 4\n",
      "Train [2017-07-01 2017-08-30] Val [2017-08-31 2017-09-15] Test [2017-09-16 2017-10-01]\n",
      "Window Train/Val/Test shape (1281, 10428) (336, 10428) (336, 10428)\n",
      "Window Train/Val/Test shape (1281, 2) (336, 2) (336, 2)\n",
      "X_train_cv  (1617, 10427)\n",
      "4 Thresh=0.00131, n=97, mse: 78.932\n",
      "CV - Window 5\n",
      "Train [2017-07-01 2017-09-14] Val [2017-09-15 2017-09-30] Test [2017-10-01 2017-10-16]\n",
      "Window Train/Val/Test shape (1596, 10428) (336, 10428) (336, 10428)\n",
      "Window Train/Val/Test shape (1596, 2) (336, 2) (336, 2)\n",
      "X_train_cv  (1932, 10427)\n",
      "5 Thresh=0.00131, n=97, mse: 34.498\n",
      "CV - Window 6\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-09-29] Val [2017-09-30 2017-10-15] Test [2017-10-16 2017-11-10]\n",
      "Window Train/Val/Test shape (1911, 10428) (336, 10428) (546, 10428)\n",
      "Window Train/Val/Test shape (1911, 2) (336, 2) (546, 2)\n",
      "X_train_cv  (2247, 10427)\n",
      "6 Thresh=0.00131, n=97, mse: 20.733\n",
      "CV - Window 7\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-10-14] Val [2017-10-15 2017-10-30] Test [2017-10-31 2017-11-10]\n",
      "Window Train/Val/Test shape (2226, 10428) (336, 10428) (231, 10428)\n",
      "Window Train/Val/Test shape (2226, 2) (336, 2) (231, 2)\n",
      "X_train_cv  (2562, 10427)\n",
      "7 Thresh=0.00131, n=97, mse: 13.617\n",
      "{'threshold': 0.0013140604, 'num_features': 97, 'mse_list': [134.37292251067038, 78.93178654972893, 34.49822091036245, 20.732943041742697, 13.617278377993225], 'mean_mse': 56.430630278099535}\n",
      "\n",
      "\n",
      "CV - Window 3\n",
      "Train [2017-07-01 2017-08-15] Val [2017-08-16 2017-08-31] Test [2017-09-01 2017-09-16]\n",
      "Window Train/Val/Test shape (966, 10428) (336, 10428) (336, 10428)\n",
      "Window Train/Val/Test shape (966, 2) (336, 2) (336, 2)\n",
      "X_train_cv  (1302, 10427)\n",
      "3 Thresh=0.00099, n=208, mse: 132.953\n",
      "CV - Window 4\n",
      "Train [2017-07-01 2017-08-30] Val [2017-08-31 2017-09-15] Test [2017-09-16 2017-10-01]\n",
      "Window Train/Val/Test shape (1281, 10428) (336, 10428) (336, 10428)\n",
      "Window Train/Val/Test shape (1281, 2) (336, 2) (336, 2)\n",
      "X_train_cv  (1617, 10427)\n",
      "4 Thresh=0.00099, n=208, mse: 74.553\n",
      "CV - Window 5\n",
      "Train [2017-07-01 2017-09-14] Val [2017-09-15 2017-09-30] Test [2017-10-01 2017-10-16]\n",
      "Window Train/Val/Test shape (1596, 10428) (336, 10428) (336, 10428)\n",
      "Window Train/Val/Test shape (1596, 2) (336, 2) (336, 2)\n",
      "X_train_cv  (1932, 10427)\n",
      "5 Thresh=0.00099, n=208, mse: 35.085\n",
      "CV - Window 6\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-09-29] Val [2017-09-30 2017-10-15] Test [2017-10-16 2017-11-10]\n",
      "Window Train/Val/Test shape (1911, 10428) (336, 10428) (546, 10428)\n",
      "Window Train/Val/Test shape (1911, 2) (336, 2) (546, 2)\n",
      "X_train_cv  (2247, 10427)\n",
      "6 Thresh=0.00099, n=208, mse: 21.597\n",
      "CV - Window 7\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-10-14] Val [2017-10-15 2017-10-30] Test [2017-10-31 2017-11-10]\n",
      "Window Train/Val/Test shape (2226, 10428) (336, 10428) (231, 10428)\n",
      "Window Train/Val/Test shape (2226, 2) (336, 2) (231, 2)\n",
      "X_train_cv  (2562, 10427)\n",
      "7 Thresh=0.00099, n=208, mse: 11.990\n",
      "{'threshold': 0.0009855453, 'num_features': 208, 'mse_list': [132.9527246761228, 74.55345164256457, 35.08544423228711, 21.596993312663074, 11.989880185693314], 'mean_mse': 55.23569880986618}\n",
      "\n",
      "\n",
      "CV - Window 3\n",
      "Train [2017-07-01 2017-08-15] Val [2017-08-16 2017-08-31] Test [2017-09-01 2017-09-16]\n",
      "Window Train/Val/Test shape (966, 10428) (336, 10428) (336, 10428)\n",
      "Window Train/Val/Test shape (966, 2) (336, 2) (336, 2)\n",
      "X_train_cv  (1302, 10427)\n",
      "3 Thresh=0.00066, n=522, mse: 140.149\n",
      "CV - Window 4\n",
      "Train [2017-07-01 2017-08-30] Val [2017-08-31 2017-09-15] Test [2017-09-16 2017-10-01]\n",
      "Window Train/Val/Test shape (1281, 10428) (336, 10428) (336, 10428)\n",
      "Window Train/Val/Test shape (1281, 2) (336, 2) (336, 2)\n",
      "X_train_cv  (1617, 10427)\n",
      "4 Thresh=0.00066, n=522, mse: 77.884\n",
      "CV - Window 5\n",
      "Train [2017-07-01 2017-09-14] Val [2017-09-15 2017-09-30] Test [2017-10-01 2017-10-16]\n",
      "Window Train/Val/Test shape (1596, 10428) (336, 10428) (336, 10428)\n",
      "Window Train/Val/Test shape (1596, 2) (336, 2) (336, 2)\n",
      "X_train_cv  (1932, 10427)\n",
      "5 Thresh=0.00066, n=522, mse: 33.391\n",
      "CV - Window 6\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-09-29] Val [2017-09-30 2017-10-15] Test [2017-10-16 2017-11-10]\n",
      "Window Train/Val/Test shape (1911, 10428) (336, 10428) (546, 10428)\n",
      "Window Train/Val/Test shape (1911, 2) (336, 2) (546, 2)\n",
      "X_train_cv  (2247, 10427)\n",
      "6 Thresh=0.00066, n=522, mse: 31.483\n",
      "CV - Window 7\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-10-14] Val [2017-10-15 2017-10-30] Test [2017-10-31 2017-11-10]\n",
      "Window Train/Val/Test shape (2226, 10428) (336, 10428) (231, 10428)\n",
      "Window Train/Val/Test shape (2226, 2) (336, 2) (231, 2)\n",
      "X_train_cv  (2562, 10427)\n",
      "7 Thresh=0.00066, n=522, mse: 12.786\n",
      "{'threshold': 0.0006570302, 'num_features': 522, 'mse_list': [140.14937535864692, 77.8839930693229, 33.39086049344875, 31.483020642982385, 12.786372075945781], 'mean_mse': 59.13872432806935}\n",
      "\n",
      "\n",
      "CV - Window 3\n",
      "Train [2017-07-01 2017-08-15] Val [2017-08-16 2017-08-31] Test [2017-09-01 2017-09-16]\n",
      "Window Train/Val/Test shape (966, 10428) (336, 10428) (336, 10428)\n",
      "Window Train/Val/Test shape (966, 2) (336, 2) (336, 2)\n",
      "X_train_cv  (1302, 10427)\n",
      "3 Thresh=0.00033, n=1884, mse: 143.040\n",
      "CV - Window 4\n",
      "Train [2017-07-01 2017-08-30] Val [2017-08-31 2017-09-15] Test [2017-09-16 2017-10-01]\n",
      "Window Train/Val/Test shape (1281, 10428) (336, 10428) (336, 10428)\n",
      "Window Train/Val/Test shape (1281, 2) (336, 2) (336, 2)\n",
      "X_train_cv  (1617, 10427)\n",
      "4 Thresh=0.00033, n=1884, mse: 72.389\n",
      "CV - Window 5\n",
      "Train [2017-07-01 2017-09-14] Val [2017-09-15 2017-09-30] Test [2017-10-01 2017-10-16]\n",
      "Window Train/Val/Test shape (1596, 10428) (336, 10428) (336, 10428)\n",
      "Window Train/Val/Test shape (1596, 2) (336, 2) (336, 2)\n",
      "X_train_cv  (1932, 10427)\n",
      "5 Thresh=0.00033, n=1884, mse: 36.088\n",
      "CV - Window 6\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-09-29] Val [2017-09-30 2017-10-15] Test [2017-10-16 2017-11-10]\n",
      "Window Train/Val/Test shape (1911, 10428) (336, 10428) (546, 10428)\n",
      "Window Train/Val/Test shape (1911, 2) (336, 2) (546, 2)\n",
      "X_train_cv  (2247, 10427)\n",
      "6 Thresh=0.00033, n=1884, mse: 34.473\n",
      "CV - Window 7\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-10-14] Val [2017-10-15 2017-10-30] Test [2017-10-31 2017-11-10]\n",
      "Window Train/Val/Test shape (2226, 10428) (336, 10428) (231, 10428)\n",
      "Window Train/Val/Test shape (2226, 2) (336, 2) (231, 2)\n",
      "X_train_cv  (2562, 10427)\n",
      "7 Thresh=0.00033, n=1884, mse: 12.922\n",
      "{'threshold': 0.0003285151, 'num_features': 1884, 'mse_list': [143.03955535953685, 72.38888339845897, 36.08781647118917, 34.47269578377974, 12.921861825668186], 'mean_mse': 59.78216256772659}\n",
      "\n",
      "\n",
      "CV - Window 3\n",
      "Train [2017-07-01 2017-08-15] Val [2017-08-16 2017-08-31] Test [2017-09-01 2017-09-16]\n",
      "Window Train/Val/Test shape (966, 10428) (336, 10428) (336, 10428)\n",
      "Window Train/Val/Test shape (966, 2) (336, 2) (336, 2)\n",
      "X_train_cv  (1302, 10427)\n",
      "3 Thresh=0.00000, n=10427, mse: 142.281\n",
      "CV - Window 4\n",
      "Train [2017-07-01 2017-08-30] Val [2017-08-31 2017-09-15] Test [2017-09-16 2017-10-01]\n",
      "Window Train/Val/Test shape (1281, 10428) (336, 10428) (336, 10428)\n",
      "Window Train/Val/Test shape (1281, 2) (336, 2) (336, 2)\n",
      "X_train_cv  (1617, 10427)\n",
      "4 Thresh=0.00000, n=10427, mse: 78.592\n",
      "CV - Window 5\n",
      "Train [2017-07-01 2017-09-14] Val [2017-09-15 2017-09-30] Test [2017-10-01 2017-10-16]\n",
      "Window Train/Val/Test shape (1596, 10428) (336, 10428) (336, 10428)\n",
      "Window Train/Val/Test shape (1596, 2) (336, 2) (336, 2)\n",
      "X_train_cv  (1932, 10427)\n",
      "5 Thresh=0.00000, n=10427, mse: 36.168\n",
      "CV - Window 6\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-09-29] Val [2017-09-30 2017-10-15] Test [2017-10-16 2017-11-10]\n",
      "Window Train/Val/Test shape (1911, 10428) (336, 10428) (546, 10428)\n",
      "Window Train/Val/Test shape (1911, 2) (336, 2) (546, 2)\n",
      "X_train_cv  (2247, 10427)\n",
      "6 Thresh=0.00000, n=10427, mse: 32.831\n",
      "CV - Window 7\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-10-14] Val [2017-10-15 2017-10-30] Test [2017-10-31 2017-11-10]\n",
      "Window Train/Val/Test shape (2226, 10428) (336, 10428) (231, 10428)\n",
      "Window Train/Val/Test shape (2226, 2) (336, 2) (231, 2)\n",
      "X_train_cv  (2562, 10427)\n",
      "7 Thresh=0.00000, n=10427, mse: 15.235\n",
      "{'threshold': 0.0, 'num_features': 10427, 'mse_list': [142.28148149361724, 78.59174176691447, 36.16768864203749, 32.830567274928846, 15.235275948510619], 'mean_mse': 61.021351025201724}\n",
      "\n",
      "\n",
      "    threshold num_features                                           mse_list  \\\n",
      "0    0.015440            2  [149.083527043454, 124.93147620655877, 49.9980...   \n",
      "1    0.010512            3  [149.52148992973295, 125.67727159292792, 45.85...   \n",
      "2    0.007227            4  [148.98804438037072, 123.6824049157656, 41.576...   \n",
      "3    0.006899            5  [144.75932550919464, 124.57481877299821, 41.95...   \n",
      "4    0.005913            6  [143.68633116899045, 124.36296975381886, 43.58...   \n",
      "5    0.004928            7  [146.99511150122953, 126.94747737237012, 43.87...   \n",
      "6    0.004271            9  [151.31624305439107, 133.0165576389683, 42.335...   \n",
      "7    0.003614           11  [142.57007415876288, 134.79152742615184, 41.90...   \n",
      "8    0.003285           13  [147.86202167204647, 109.87446865163263, 32.58...   \n",
      "9    0.002957           19  [134.0550972030669, 84.78690099572086, 32.2512...   \n",
      "10   0.002628           28  [135.65509680519378, 81.67240770210165, 41.964...   \n",
      "11   0.002300           31  [132.60385135116053, 80.11427651688743, 41.293...   \n",
      "12   0.001971           38  [130.88952906794887, 81.24836756760475, 33.328...   \n",
      "13   0.001643           64  [127.09150562024566, 69.9564526961938, 32.4289...   \n",
      "14   0.001314           97  [134.37292251067038, 78.93178654972893, 34.498...   \n",
      "15   0.000986          208  [132.9527246761228, 74.55345164256457, 35.0854...   \n",
      "16   0.000657          522  [140.14937535864692, 77.8839930693229, 33.3908...   \n",
      "17   0.000329         1884  [143.03955535953685, 72.38888339845897, 36.087...   \n",
      "18   0.000000        10427  [142.28148149361724, 78.59174176691447, 36.167...   \n",
      "\n",
      "     mean_mse  \n",
      "0   71.315811  \n",
      "1   70.523710  \n",
      "2   68.614627  \n",
      "3   67.485668  \n",
      "4   67.301161  \n",
      "5   68.616957  \n",
      "6   75.743225  \n",
      "7   73.616776  \n",
      "8   69.042226  \n",
      "9   61.973381  \n",
      "10  61.715333  \n",
      "11  58.115609  \n",
      "12  54.748253  \n",
      "13  53.325774  \n",
      "14  56.430630  \n",
      "15  55.235699  \n",
      "16  59.138724  \n",
      "17  59.782163  \n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "18  61.021351  \n"
     ]
    }
   ],
   "source": [
    "orchestrator12.runRollingCustomFeatureExtractor(preselect_params_12, X_12_preselect)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>threshold</th>\n",
       "      <th>num_features</th>\n",
       "      <th>mse_list</th>\n",
       "      <th>mean_mse</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0.015440</td>\n",
       "      <td>2</td>\n",
       "      <td>[149.083527043454, 124.93147620655877, 49.9980...</td>\n",
       "      <td>71.315811</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0.010512</td>\n",
       "      <td>3</td>\n",
       "      <td>[149.52148992973295, 125.67727159292792, 45.85...</td>\n",
       "      <td>70.523710</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0.007227</td>\n",
       "      <td>4</td>\n",
       "      <td>[148.98804438037072, 123.6824049157656, 41.576...</td>\n",
       "      <td>68.614627</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0.006899</td>\n",
       "      <td>5</td>\n",
       "      <td>[144.75932550919464, 124.57481877299821, 41.95...</td>\n",
       "      <td>67.485668</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0.005913</td>\n",
       "      <td>6</td>\n",
       "      <td>[143.68633116899045, 124.36296975381886, 43.58...</td>\n",
       "      <td>67.301161</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>0.004928</td>\n",
       "      <td>7</td>\n",
       "      <td>[146.99511150122953, 126.94747737237012, 43.87...</td>\n",
       "      <td>68.616957</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>0.004271</td>\n",
       "      <td>9</td>\n",
       "      <td>[151.31624305439107, 133.0165576389683, 42.335...</td>\n",
       "      <td>75.743225</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>0.003614</td>\n",
       "      <td>11</td>\n",
       "      <td>[142.57007415876288, 134.79152742615184, 41.90...</td>\n",
       "      <td>73.616776</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>0.003285</td>\n",
       "      <td>13</td>\n",
       "      <td>[147.86202167204647, 109.87446865163263, 32.58...</td>\n",
       "      <td>69.042226</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>0.002957</td>\n",
       "      <td>19</td>\n",
       "      <td>[134.0550972030669, 84.78690099572086, 32.2512...</td>\n",
       "      <td>61.973381</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>0.002628</td>\n",
       "      <td>28</td>\n",
       "      <td>[135.65509680519378, 81.67240770210165, 41.964...</td>\n",
       "      <td>61.715333</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>0.002300</td>\n",
       "      <td>31</td>\n",
       "      <td>[132.60385135116053, 80.11427651688743, 41.293...</td>\n",
       "      <td>58.115609</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>0.001971</td>\n",
       "      <td>38</td>\n",
       "      <td>[130.88952906794887, 81.24836756760475, 33.328...</td>\n",
       "      <td>54.748253</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>0.001643</td>\n",
       "      <td>64</td>\n",
       "      <td>[127.09150562024566, 69.9564526961938, 32.4289...</td>\n",
       "      <td>53.325774</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>0.001314</td>\n",
       "      <td>97</td>\n",
       "      <td>[134.37292251067038, 78.93178654972893, 34.498...</td>\n",
       "      <td>56.430630</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>0.000986</td>\n",
       "      <td>208</td>\n",
       "      <td>[132.9527246761228, 74.55345164256457, 35.0854...</td>\n",
       "      <td>55.235699</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16</th>\n",
       "      <td>0.000657</td>\n",
       "      <td>522</td>\n",
       "      <td>[140.14937535864692, 77.8839930693229, 33.3908...</td>\n",
       "      <td>59.138724</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17</th>\n",
       "      <td>0.000329</td>\n",
       "      <td>1884</td>\n",
       "      <td>[143.03955535953685, 72.38888339845897, 36.087...</td>\n",
       "      <td>59.782163</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18</th>\n",
       "      <td>0.000000</td>\n",
       "      <td>10427</td>\n",
       "      <td>[142.28148149361724, 78.59174176691447, 36.167...</td>\n",
       "      <td>61.021351</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "    threshold num_features                                           mse_list  \\\n",
       "0    0.015440            2  [149.083527043454, 124.93147620655877, 49.9980...   \n",
       "1    0.010512            3  [149.52148992973295, 125.67727159292792, 45.85...   \n",
       "2    0.007227            4  [148.98804438037072, 123.6824049157656, 41.576...   \n",
       "3    0.006899            5  [144.75932550919464, 124.57481877299821, 41.95...   \n",
       "4    0.005913            6  [143.68633116899045, 124.36296975381886, 43.58...   \n",
       "5    0.004928            7  [146.99511150122953, 126.94747737237012, 43.87...   \n",
       "6    0.004271            9  [151.31624305439107, 133.0165576389683, 42.335...   \n",
       "7    0.003614           11  [142.57007415876288, 134.79152742615184, 41.90...   \n",
       "8    0.003285           13  [147.86202167204647, 109.87446865163263, 32.58...   \n",
       "9    0.002957           19  [134.0550972030669, 84.78690099572086, 32.2512...   \n",
       "10   0.002628           28  [135.65509680519378, 81.67240770210165, 41.964...   \n",
       "11   0.002300           31  [132.60385135116053, 80.11427651688743, 41.293...   \n",
       "12   0.001971           38  [130.88952906794887, 81.24836756760475, 33.328...   \n",
       "13   0.001643           64  [127.09150562024566, 69.9564526961938, 32.4289...   \n",
       "14   0.001314           97  [134.37292251067038, 78.93178654972893, 34.498...   \n",
       "15   0.000986          208  [132.9527246761228, 74.55345164256457, 35.0854...   \n",
       "16   0.000657          522  [140.14937535864692, 77.8839930693229, 33.3908...   \n",
       "17   0.000329         1884  [143.03955535953685, 72.38888339845897, 36.087...   \n",
       "18   0.000000        10427  [142.28148149361724, 78.59174176691447, 36.167...   \n",
       "\n",
       "     mean_mse  \n",
       "0   71.315811  \n",
       "1   70.523710  \n",
       "2   68.614627  \n",
       "3   67.485668  \n",
       "4   67.301161  \n",
       "5   68.616957  \n",
       "6   75.743225  \n",
       "7   73.616776  \n",
       "8   69.042226  \n",
       "9   61.973381  \n",
       "10  61.715333  \n",
       "11  58.115609  \n",
       "12  54.748253  \n",
       "13  53.325774  \n",
       "14  56.430630  \n",
       "15  55.235699  \n",
       "16  59.138724  \n",
       "17  59.782163  \n",
       "18  61.021351  "
      ]
     },
     "execution_count": 34,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "orchestrator12.rcfe.summary"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {},
   "outputs": [],
   "source": [
    "selected_features12 = ['date'] + list(orchestrator12.rcfe.mcr.feature_importance_df[orchestrator12.rcfe.mcr.feature_importance_df.threshold >= 0.001643].index)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {},
   "outputs": [],
   "source": [
    "# orchestrator12.corr_selected_features_bi_week[selected_features12].to_csv('/Users/Rohil/Documents/iGEM/yemen/intermediary_files/X_1_2_postselect.csv')\n",
    "# orchestrator12.corr_selected_features_bi_week_y.to_csv('/Users/Rohil/Documents/iGEM/yemen/intermediary_files/y_1_2.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 143,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train [2017-07-01 2017-08-15] Val [2017-08-16 2017-08-31] Test [2017-09-01 2017-09-16]\n",
      "Window Train/Val/Test shape (966, 39) (336, 39) (336, 39)\n",
      "Window Train/Val/Test shape (966, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-08-30] Val [2017-08-31 2017-09-15] Test [2017-09-16 2017-10-01]\n",
      "Window Train/Val/Test shape (1281, 39) (336, 39) (336, 39)\n",
      "Window Train/Val/Test shape (1281, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-09-14] Val [2017-09-15 2017-09-30] Test [2017-10-01 2017-10-16]\n",
      "Window Train/Val/Test shape (1596, 39) (336, 39) (336, 39)\n",
      "Window Train/Val/Test shape (1596, 2) (336, 2) (336, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-09-29] Val [2017-09-30 2017-10-15] Test [2017-10-16 2017-11-10]\n",
      "Window Train/Val/Test shape (1911, 39) (336, 39) (546, 39)\n",
      "Window Train/Val/Test shape (1911, 2) (336, 2) (546, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-10-14] Val [2017-10-15 2017-10-30] Test [2017-10-31 2017-11-10]\n",
      "Window Train/Val/Test shape (2226, 39) (336, 39) (231, 39)\n",
      "Window Train/Val/Test shape (2226, 2) (336, 2) (231, 2)\n",
      "[112.67654225041538, 85.03936717361567, 34.05908656865604, 21.43872038418687, 16.557992238592874]\n",
      "mean mse: 53.954341723093364\n",
      "Train [2017-07-01 2017-08-15] Val [2017-08-16 2017-08-31] Test [2017-09-01 2017-09-16]\n",
      "Window Train/Val/Test shape (966, 39) (336, 39) (336, 39)\n",
      "Window Train/Val/Test shape (966, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-08-30] Val [2017-08-31 2017-09-15] Test [2017-09-16 2017-10-01]\n",
      "Window Train/Val/Test shape (1281, 39) (336, 39) (336, 39)\n",
      "Window Train/Val/Test shape (1281, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-09-14] Val [2017-09-15 2017-09-30] Test [2017-10-01 2017-10-16]\n",
      "Window Train/Val/Test shape (1596, 39) (336, 39) (336, 39)\n",
      "Window Train/Val/Test shape (1596, 2) (336, 2) (336, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-09-29] Val [2017-09-30 2017-10-15] Test [2017-10-16 2017-11-10]\n",
      "Window Train/Val/Test shape (1911, 39) (336, 39) (546, 39)\n",
      "Window Train/Val/Test shape (1911, 2) (336, 2) (546, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-10-14] Val [2017-10-15 2017-10-30] Test [2017-10-31 2017-11-10]\n",
      "Window Train/Val/Test shape (2226, 39) (336, 39) (231, 39)\n",
      "Window Train/Val/Test shape (2226, 2) (336, 2) (231, 2)\n",
      "[116.83659799555443, 75.86428643686602, 35.27586682167523, 15.395736163269945, 9.487372364266491]\n",
      "mean mse: 50.57197195632642\n",
      "Train [2017-07-01 2017-08-15] Val [2017-08-16 2017-08-31] Test [2017-09-01 2017-09-16]\n",
      "Window Train/Val/Test shape (966, 39) (336, 39) (336, 39)\n",
      "Window Train/Val/Test shape (966, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-08-30] Val [2017-08-31 2017-09-15] Test [2017-09-16 2017-10-01]\n",
      "Window Train/Val/Test shape (1281, 39) (336, 39) (336, 39)\n",
      "Window Train/Val/Test shape (1281, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-09-14] Val [2017-09-15 2017-09-30] Test [2017-10-01 2017-10-16]\n",
      "Window Train/Val/Test shape (1596, 39) (336, 39) (336, 39)\n",
      "Window Train/Val/Test shape (1596, 2) (336, 2) (336, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-09-29] Val [2017-09-30 2017-10-15] Test [2017-10-16 2017-11-10]\n",
      "Window Train/Val/Test shape (1911, 39) (336, 39) (546, 39)\n",
      "Window Train/Val/Test shape (1911, 2) (336, 2) (546, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-10-14] Val [2017-10-15 2017-10-30] Test [2017-10-31 2017-11-10]\n",
      "Window Train/Val/Test shape (2226, 39) (336, 39) (231, 39)\n",
      "Window Train/Val/Test shape (2226, 2) (336, 2) (231, 2)\n",
      "[110.39119605053243, 84.0829222769243, 55.98691404960114, 15.37052297710717, 5.799831631822532]\n",
      "mean mse: 54.32627739719751\n",
      "Train [2017-07-01 2017-08-15] Val [2017-08-16 2017-08-31] Test [2017-09-01 2017-09-16]\n",
      "Window Train/Val/Test shape (966, 39) (336, 39) (336, 39)\n",
      "Window Train/Val/Test shape (966, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-08-30] Val [2017-08-31 2017-09-15] Test [2017-09-16 2017-10-01]\n",
      "Window Train/Val/Test shape (1281, 39) (336, 39) (336, 39)\n",
      "Window Train/Val/Test shape (1281, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-09-14] Val [2017-09-15 2017-09-30] Test [2017-10-01 2017-10-16]\n",
      "Window Train/Val/Test shape (1596, 39) (336, 39) (336, 39)\n",
      "Window Train/Val/Test shape (1596, 2) (336, 2) (336, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-09-29] Val [2017-09-30 2017-10-15] Test [2017-10-16 2017-11-10]\n",
      "Window Train/Val/Test shape (1911, 39) (336, 39) (546, 39)\n",
      "Window Train/Val/Test shape (1911, 2) (336, 2) (546, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-10-14] Val [2017-10-15 2017-10-30] Test [2017-10-31 2017-11-10]\n",
      "Window Train/Val/Test shape (2226, 39) (336, 39) (231, 39)\n",
      "Window Train/Val/Test shape (2226, 2) (336, 2) (231, 2)\n",
      "[116.69758259530602, 75.83912686046978, 26.914912815769732, 21.078001317026445, 8.272429492771527]\n",
      "mean mse: 49.76041061626869\n",
      "Train [2017-07-01 2017-08-15] Val [2017-08-16 2017-08-31] Test [2017-09-01 2017-09-16]\n",
      "Window Train/Val/Test shape (966, 39) (336, 39) (336, 39)\n",
      "Window Train/Val/Test shape (966, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-08-30] Val [2017-08-31 2017-09-15] Test [2017-09-16 2017-10-01]\n",
      "Window Train/Val/Test shape (1281, 39) (336, 39) (336, 39)\n",
      "Window Train/Val/Test shape (1281, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-09-14] Val [2017-09-15 2017-09-30] Test [2017-10-01 2017-10-16]\n",
      "Window Train/Val/Test shape (1596, 39) (336, 39) (336, 39)\n",
      "Window Train/Val/Test shape (1596, 2) (336, 2) (336, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-09-29] Val [2017-09-30 2017-10-15] Test [2017-10-16 2017-11-10]\n",
      "Window Train/Val/Test shape (1911, 39) (336, 39) (546, 39)\n",
      "Window Train/Val/Test shape (1911, 2) (336, 2) (546, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-10-14] Val [2017-10-15 2017-10-30] Test [2017-10-31 2017-11-10]\n",
      "Window Train/Val/Test shape (2226, 39) (336, 39) (231, 39)\n",
      "Window Train/Val/Test shape (2226, 2) (336, 2) (231, 2)\n",
      "[113.38624038025695, 88.25314173122649, 30.3637428842967, 14.143099338547254, 5.026867606537084]\n",
      "mean mse: 50.23461838817289\n",
      "Train [2017-07-01 2017-08-15] Val [2017-08-16 2017-08-31] Test [2017-09-01 2017-09-16]\n",
      "Window Train/Val/Test shape (966, 39) (336, 39) (336, 39)\n",
      "Window Train/Val/Test shape (966, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-08-30] Val [2017-08-31 2017-09-15] Test [2017-09-16 2017-10-01]\n",
      "Window Train/Val/Test shape (1281, 39) (336, 39) (336, 39)\n",
      "Window Train/Val/Test shape (1281, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-09-14] Val [2017-09-15 2017-09-30] Test [2017-10-01 2017-10-16]\n",
      "Window Train/Val/Test shape (1596, 39) (336, 39) (336, 39)\n",
      "Window Train/Val/Test shape (1596, 2) (336, 2) (336, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-09-29] Val [2017-09-30 2017-10-15] Test [2017-10-16 2017-11-10]\n",
      "Window Train/Val/Test shape (1911, 39) (336, 39) (546, 39)\n",
      "Window Train/Val/Test shape (1911, 2) (336, 2) (546, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-10-14] Val [2017-10-15 2017-10-30] Test [2017-10-31 2017-11-10]\n",
      "Window Train/Val/Test shape (2226, 39) (336, 39) (231, 39)\n",
      "Window Train/Val/Test shape (2226, 2) (336, 2) (231, 2)\n",
      "[115.45977796289634, 116.72572767224652, 64.92084356242758, 22.125445999160444, 15.463642475698457]\n",
      "mean mse: 66.93908753448588\n",
      "Train [2017-07-01 2017-08-15] Val [2017-08-16 2017-08-31] Test [2017-09-01 2017-09-16]\n",
      "Window Train/Val/Test shape (966, 39) (336, 39) (336, 39)\n",
      "Window Train/Val/Test shape (966, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-08-30] Val [2017-08-31 2017-09-15] Test [2017-09-16 2017-10-01]\n",
      "Window Train/Val/Test shape (1281, 39) (336, 39) (336, 39)\n",
      "Window Train/Val/Test shape (1281, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-09-14] Val [2017-09-15 2017-09-30] Test [2017-10-01 2017-10-16]\n",
      "Window Train/Val/Test shape (1596, 39) (336, 39) (336, 39)\n",
      "Window Train/Val/Test shape (1596, 2) (336, 2) (336, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-09-29] Val [2017-09-30 2017-10-15] Test [2017-10-16 2017-11-10]\n",
      "Window Train/Val/Test shape (1911, 39) (336, 39) (546, 39)\n",
      "Window Train/Val/Test shape (1911, 2) (336, 2) (546, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-10-14] Val [2017-10-15 2017-10-30] Test [2017-10-31 2017-11-10]\n",
      "Window Train/Val/Test shape (2226, 39) (336, 39) (231, 39)\n",
      "Window Train/Val/Test shape (2226, 2) (336, 2) (231, 2)\n",
      "[115.41628028619428, 77.6317312306556, 35.27324056664621, 18.196547992435235, 8.02457815712044]\n",
      "mean mse: 50.908475646610356\n",
      "Train [2017-07-01 2017-08-15] Val [2017-08-16 2017-08-31] Test [2017-09-01 2017-09-16]\n",
      "Window Train/Val/Test shape (966, 39) (336, 39) (336, 39)\n",
      "Window Train/Val/Test shape (966, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-08-30] Val [2017-08-31 2017-09-15] Test [2017-09-16 2017-10-01]\n",
      "Window Train/Val/Test shape (1281, 39) (336, 39) (336, 39)\n",
      "Window Train/Val/Test shape (1281, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-09-14] Val [2017-09-15 2017-09-30] Test [2017-10-01 2017-10-16]\n",
      "Window Train/Val/Test shape (1596, 39) (336, 39) (336, 39)\n",
      "Window Train/Val/Test shape (1596, 2) (336, 2) (336, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-09-29] Val [2017-09-30 2017-10-15] Test [2017-10-16 2017-11-10]\n",
      "Window Train/Val/Test shape (1911, 39) (336, 39) (546, 39)\n",
      "Window Train/Val/Test shape (1911, 2) (336, 2) (546, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-10-14] Val [2017-10-15 2017-10-30] Test [2017-10-31 2017-11-10]\n",
      "Window Train/Val/Test shape (2226, 39) (336, 39) (231, 39)\n",
      "Window Train/Val/Test shape (2226, 2) (336, 2) (231, 2)\n",
      "[119.09389962636708, 79.7327188052345, 36.26559608366805, 15.270831151327176, 10.757428978024985]\n",
      "mean mse: 52.22409492892435\n",
      "Train [2017-07-01 2017-08-15] Val [2017-08-16 2017-08-31] Test [2017-09-01 2017-09-16]\n",
      "Window Train/Val/Test shape (966, 39) (336, 39) (336, 39)\n",
      "Window Train/Val/Test shape (966, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-08-30] Val [2017-08-31 2017-09-15] Test [2017-09-16 2017-10-01]\n",
      "Window Train/Val/Test shape (1281, 39) (336, 39) (336, 39)\n",
      "Window Train/Val/Test shape (1281, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-09-14] Val [2017-09-15 2017-09-30] Test [2017-10-01 2017-10-16]\n",
      "Window Train/Val/Test shape (1596, 39) (336, 39) (336, 39)\n",
      "Window Train/Val/Test shape (1596, 2) (336, 2) (336, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-09-29] Val [2017-09-30 2017-10-15] Test [2017-10-16 2017-11-10]\n",
      "Window Train/Val/Test shape (1911, 39) (336, 39) (546, 39)\n",
      "Window Train/Val/Test shape (1911, 2) (336, 2) (546, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-10-14] Val [2017-10-15 2017-10-30] Test [2017-10-31 2017-11-10]\n",
      "Window Train/Val/Test shape (2226, 39) (336, 39) (231, 39)\n",
      "Window Train/Val/Test shape (2226, 2) (336, 2) (231, 2)\n",
      "[116.48447647344793, 79.43942053279886, 35.011569743347124, 15.296340309217873, 8.772027662935695]\n",
      "mean mse: 51.00076694434949\n",
      "Train [2017-07-01 2017-08-15] Val [2017-08-16 2017-08-31] Test [2017-09-01 2017-09-16]\n",
      "Window Train/Val/Test shape (966, 39) (336, 39) (336, 39)\n",
      "Window Train/Val/Test shape (966, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-08-30] Val [2017-08-31 2017-09-15] Test [2017-09-16 2017-10-01]\n",
      "Window Train/Val/Test shape (1281, 39) (336, 39) (336, 39)\n",
      "Window Train/Val/Test shape (1281, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-09-14] Val [2017-09-15 2017-09-30] Test [2017-10-01 2017-10-16]\n",
      "Window Train/Val/Test shape (1596, 39) (336, 39) (336, 39)\n",
      "Window Train/Val/Test shape (1596, 2) (336, 2) (336, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-09-29] Val [2017-09-30 2017-10-15] Test [2017-10-16 2017-11-10]\n",
      "Window Train/Val/Test shape (1911, 39) (336, 39) (546, 39)\n",
      "Window Train/Val/Test shape (1911, 2) (336, 2) (546, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-10-14] Val [2017-10-15 2017-10-30] Test [2017-10-31 2017-11-10]\n",
      "Window Train/Val/Test shape (2226, 39) (336, 39) (231, 39)\n",
      "Window Train/Val/Test shape (2226, 2) (336, 2) (231, 2)\n",
      "[112.00797280605586, 73.5231079546835, 39.34246922066767, 15.273836424693677, 5.67270875037813]\n",
      "mean mse: 49.16401903129577\n",
      "Train [2017-07-01 2017-08-15] Val [2017-08-16 2017-08-31] Test [2017-09-01 2017-09-16]\n",
      "Window Train/Val/Test shape (966, 39) (336, 39) (336, 39)\n",
      "Window Train/Val/Test shape (966, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-08-30] Val [2017-08-31 2017-09-15] Test [2017-09-16 2017-10-01]\n",
      "Window Train/Val/Test shape (1281, 39) (336, 39) (336, 39)\n",
      "Window Train/Val/Test shape (1281, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-09-14] Val [2017-09-15 2017-09-30] Test [2017-10-01 2017-10-16]\n",
      "Window Train/Val/Test shape (1596, 39) (336, 39) (336, 39)\n",
      "Window Train/Val/Test shape (1596, 2) (336, 2) (336, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-09-29] Val [2017-09-30 2017-10-15] Test [2017-10-16 2017-11-10]\n",
      "Window Train/Val/Test shape (1911, 39) (336, 39) (546, 39)\n",
      "Window Train/Val/Test shape (1911, 2) (336, 2) (546, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-10-14] Val [2017-10-15 2017-10-30] Test [2017-10-31 2017-11-10]\n",
      "Window Train/Val/Test shape (2226, 39) (336, 39) (231, 39)\n",
      "Window Train/Val/Test shape (2226, 2) (336, 2) (231, 2)\n",
      "[114.99824013822551, 81.05071090776615, 40.267208869792334, 15.290768521904653, 5.966055454148288]\n",
      "mean mse: 51.51459677836739\n",
      "Train [2017-07-01 2017-08-15] Val [2017-08-16 2017-08-31] Test [2017-09-01 2017-09-16]\n",
      "Window Train/Val/Test shape (966, 39) (336, 39) (336, 39)\n",
      "Window Train/Val/Test shape (966, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-08-30] Val [2017-08-31 2017-09-15] Test [2017-09-16 2017-10-01]\n",
      "Window Train/Val/Test shape (1281, 39) (336, 39) (336, 39)\n",
      "Window Train/Val/Test shape (1281, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-09-14] Val [2017-09-15 2017-09-30] Test [2017-10-01 2017-10-16]\n",
      "Window Train/Val/Test shape (1596, 39) (336, 39) (336, 39)\n",
      "Window Train/Val/Test shape (1596, 2) (336, 2) (336, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-09-29] Val [2017-09-30 2017-10-15] Test [2017-10-16 2017-11-10]\n",
      "Window Train/Val/Test shape (1911, 39) (336, 39) (546, 39)\n",
      "Window Train/Val/Test shape (1911, 2) (336, 2) (546, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-10-14] Val [2017-10-15 2017-10-30] Test [2017-10-31 2017-11-10]\n",
      "Window Train/Val/Test shape (2226, 39) (336, 39) (231, 39)\n",
      "Window Train/Val/Test shape (2226, 2) (336, 2) (231, 2)\n",
      "[115.17752137886562, 81.58636227690799, 41.219806024914256, 19.022488926632764, 6.532176912846225]\n",
      "mean mse: 52.70767110403337\n",
      "Train [2017-07-01 2017-08-15] Val [2017-08-16 2017-08-31] Test [2017-09-01 2017-09-16]\n",
      "Window Train/Val/Test shape (966, 39) (336, 39) (336, 39)\n",
      "Window Train/Val/Test shape (966, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-08-30] Val [2017-08-31 2017-09-15] Test [2017-09-16 2017-10-01]\n",
      "Window Train/Val/Test shape (1281, 39) (336, 39) (336, 39)\n",
      "Window Train/Val/Test shape (1281, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-09-14] Val [2017-09-15 2017-09-30] Test [2017-10-01 2017-10-16]\n",
      "Window Train/Val/Test shape (1596, 39) (336, 39) (336, 39)\n",
      "Window Train/Val/Test shape (1596, 2) (336, 2) (336, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-09-29] Val [2017-09-30 2017-10-15] Test [2017-10-16 2017-11-10]\n",
      "Window Train/Val/Test shape (1911, 39) (336, 39) (546, 39)\n",
      "Window Train/Val/Test shape (1911, 2) (336, 2) (546, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-10-14] Val [2017-10-15 2017-10-30] Test [2017-10-31 2017-11-10]\n",
      "Window Train/Val/Test shape (2226, 39) (336, 39) (231, 39)\n",
      "Window Train/Val/Test shape (2226, 2) (336, 2) (231, 2)\n",
      "[124.90147170517974, 87.38173305192866, 41.70605749852052, 18.72425582049228, 14.144378536154779]\n",
      "mean mse: 57.37157932245519\n",
      "Train [2017-07-01 2017-08-15] Val [2017-08-16 2017-08-31] Test [2017-09-01 2017-09-16]\n",
      "Window Train/Val/Test shape (966, 39) (336, 39) (336, 39)\n",
      "Window Train/Val/Test shape (966, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-08-30] Val [2017-08-31 2017-09-15] Test [2017-09-16 2017-10-01]\n",
      "Window Train/Val/Test shape (1281, 39) (336, 39) (336, 39)\n",
      "Window Train/Val/Test shape (1281, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-09-14] Val [2017-09-15 2017-09-30] Test [2017-10-01 2017-10-16]\n",
      "Window Train/Val/Test shape (1596, 39) (336, 39) (336, 39)\n",
      "Window Train/Val/Test shape (1596, 2) (336, 2) (336, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-09-29] Val [2017-09-30 2017-10-15] Test [2017-10-16 2017-11-10]\n",
      "Window Train/Val/Test shape (1911, 39) (336, 39) (546, 39)\n",
      "Window Train/Val/Test shape (1911, 2) (336, 2) (546, 2)\n",
      "Rolling window to end date\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train [2017-07-01 2017-10-14] Val [2017-10-15 2017-10-30] Test [2017-10-31 2017-11-10]\n",
      "Window Train/Val/Test shape (2226, 39) (336, 39) (231, 39)\n",
      "Window Train/Val/Test shape (2226, 2) (336, 2) (231, 2)\n",
      "[120.2318300453442, 75.25740480503505, 29.631467060923892, 14.999769787628592, 14.666435397476373]\n",
      "mean mse: 50.95738141928162\n",
      "Train [2017-07-01 2017-08-15] Val [2017-08-16 2017-08-31] Test [2017-09-01 2017-09-16]\n",
      "Window Train/Val/Test shape (966, 39) (336, 39) (336, 39)\n",
      "Window Train/Val/Test shape (966, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-08-30] Val [2017-08-31 2017-09-15] Test [2017-09-16 2017-10-01]\n",
      "Window Train/Val/Test shape (1281, 39) (336, 39) (336, 39)\n",
      "Window Train/Val/Test shape (1281, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-09-14] Val [2017-09-15 2017-09-30] Test [2017-10-01 2017-10-16]\n",
      "Window Train/Val/Test shape (1596, 39) (336, 39) (336, 39)\n",
      "Window Train/Val/Test shape (1596, 2) (336, 2) (336, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-09-29] Val [2017-09-30 2017-10-15] Test [2017-10-16 2017-11-10]\n",
      "Window Train/Val/Test shape (1911, 39) (336, 39) (546, 39)\n",
      "Window Train/Val/Test shape (1911, 2) (336, 2) (546, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-10-14] Val [2017-10-15 2017-10-30] Test [2017-10-31 2017-11-10]\n",
      "Window Train/Val/Test shape (2226, 39) (336, 39) (231, 39)\n",
      "Window Train/Val/Test shape (2226, 2) (336, 2) (231, 2)\n",
      "[115.67022881726969, 88.36300310445812, 45.677553131190464, 23.547563167700954, 10.592440577941451]\n",
      "mean mse: 56.77015775971213\n",
      "Train [2017-07-01 2017-08-15] Val [2017-08-16 2017-08-31] Test [2017-09-01 2017-09-16]\n",
      "Window Train/Val/Test shape (966, 39) (336, 39) (336, 39)\n",
      "Window Train/Val/Test shape (966, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-08-30] Val [2017-08-31 2017-09-15] Test [2017-09-16 2017-10-01]\n",
      "Window Train/Val/Test shape (1281, 39) (336, 39) (336, 39)\n",
      "Window Train/Val/Test shape (1281, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-09-14] Val [2017-09-15 2017-09-30] Test [2017-10-01 2017-10-16]\n",
      "Window Train/Val/Test shape (1596, 39) (336, 39) (336, 39)\n",
      "Window Train/Val/Test shape (1596, 2) (336, 2) (336, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-09-29] Val [2017-09-30 2017-10-15] Test [2017-10-16 2017-11-10]\n",
      "Window Train/Val/Test shape (1911, 39) (336, 39) (546, 39)\n",
      "Window Train/Val/Test shape (1911, 2) (336, 2) (546, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-10-14] Val [2017-10-15 2017-10-30] Test [2017-10-31 2017-11-10]\n",
      "Window Train/Val/Test shape (2226, 39) (336, 39) (231, 39)\n",
      "Window Train/Val/Test shape (2226, 2) (336, 2) (231, 2)\n",
      "[108.58489438254067, 90.66902309930852, 31.445948471625645, 17.04343209052317, 9.649247483647043]\n",
      "mean mse: 51.47850910552901\n",
      "Train [2017-07-01 2017-08-15] Val [2017-08-16 2017-08-31] Test [2017-09-01 2017-09-16]\n",
      "Window Train/Val/Test shape (966, 39) (336, 39) (336, 39)\n",
      "Window Train/Val/Test shape (966, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-08-30] Val [2017-08-31 2017-09-15] Test [2017-09-16 2017-10-01]\n",
      "Window Train/Val/Test shape (1281, 39) (336, 39) (336, 39)\n",
      "Window Train/Val/Test shape (1281, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-09-14] Val [2017-09-15 2017-09-30] Test [2017-10-01 2017-10-16]\n",
      "Window Train/Val/Test shape (1596, 39) (336, 39) (336, 39)\n",
      "Window Train/Val/Test shape (1596, 2) (336, 2) (336, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-09-29] Val [2017-09-30 2017-10-15] Test [2017-10-16 2017-11-10]\n",
      "Window Train/Val/Test shape (1911, 39) (336, 39) (546, 39)\n",
      "Window Train/Val/Test shape (1911, 2) (336, 2) (546, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-10-14] Val [2017-10-15 2017-10-30] Test [2017-10-31 2017-11-10]\n",
      "Window Train/Val/Test shape (2226, 39) (336, 39) (231, 39)\n",
      "Window Train/Val/Test shape (2226, 2) (336, 2) (231, 2)\n",
      "[116.25177337893133, 95.90673390917588, 39.564863650731716, 18.266980387576954, 10.098358066699669]\n",
      "mean mse: 56.01774187862312\n",
      "Train [2017-07-01 2017-08-15] Val [2017-08-16 2017-08-31] Test [2017-09-01 2017-09-16]\n",
      "Window Train/Val/Test shape (966, 39) (336, 39) (336, 39)\n",
      "Window Train/Val/Test shape (966, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-08-30] Val [2017-08-31 2017-09-15] Test [2017-09-16 2017-10-01]\n",
      "Window Train/Val/Test shape (1281, 39) (336, 39) (336, 39)\n",
      "Window Train/Val/Test shape (1281, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-09-14] Val [2017-09-15 2017-09-30] Test [2017-10-01 2017-10-16]\n",
      "Window Train/Val/Test shape (1596, 39) (336, 39) (336, 39)\n",
      "Window Train/Val/Test shape (1596, 2) (336, 2) (336, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-09-29] Val [2017-09-30 2017-10-15] Test [2017-10-16 2017-11-10]\n",
      "Window Train/Val/Test shape (1911, 39) (336, 39) (546, 39)\n",
      "Window Train/Val/Test shape (1911, 2) (336, 2) (546, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-10-14] Val [2017-10-15 2017-10-30] Test [2017-10-31 2017-11-10]\n",
      "Window Train/Val/Test shape (2226, 39) (336, 39) (231, 39)\n",
      "Window Train/Val/Test shape (2226, 2) (336, 2) (231, 2)\n",
      "[115.0096501372739, 124.18029847957868, 45.511355045743976, 24.18278828095234, 18.235055505460984]\n",
      "mean mse: 65.42382948980199\n",
      "Train [2017-07-01 2017-08-15] Val [2017-08-16 2017-08-31] Test [2017-09-01 2017-09-16]\n",
      "Window Train/Val/Test shape (966, 39) (336, 39) (336, 39)\n",
      "Window Train/Val/Test shape (966, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-08-30] Val [2017-08-31 2017-09-15] Test [2017-09-16 2017-10-01]\n",
      "Window Train/Val/Test shape (1281, 39) (336, 39) (336, 39)\n",
      "Window Train/Val/Test shape (1281, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-09-14] Val [2017-09-15 2017-09-30] Test [2017-10-01 2017-10-16]\n",
      "Window Train/Val/Test shape (1596, 39) (336, 39) (336, 39)\n",
      "Window Train/Val/Test shape (1596, 2) (336, 2) (336, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-09-29] Val [2017-09-30 2017-10-15] Test [2017-10-16 2017-11-10]\n",
      "Window Train/Val/Test shape (1911, 39) (336, 39) (546, 39)\n",
      "Window Train/Val/Test shape (1911, 2) (336, 2) (546, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-10-14] Val [2017-10-15 2017-10-30] Test [2017-10-31 2017-11-10]\n",
      "Window Train/Val/Test shape (2226, 39) (336, 39) (231, 39)\n",
      "Window Train/Val/Test shape (2226, 2) (336, 2) (231, 2)\n",
      "[122.77063582697589, 86.1736705354737, 40.68980444577725, 17.89838473158563, 15.027229268380868]\n",
      "mean mse: 56.511944961638676\n",
      "Train [2017-07-01 2017-08-15] Val [2017-08-16 2017-08-31] Test [2017-09-01 2017-09-16]\n",
      "Window Train/Val/Test shape (966, 39) (336, 39) (336, 39)\n",
      "Window Train/Val/Test shape (966, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-08-30] Val [2017-08-31 2017-09-15] Test [2017-09-16 2017-10-01]\n",
      "Window Train/Val/Test shape (1281, 39) (336, 39) (336, 39)\n",
      "Window Train/Val/Test shape (1281, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-09-14] Val [2017-09-15 2017-09-30] Test [2017-10-01 2017-10-16]\n",
      "Window Train/Val/Test shape (1596, 39) (336, 39) (336, 39)\n",
      "Window Train/Val/Test shape (1596, 2) (336, 2) (336, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-09-29] Val [2017-09-30 2017-10-15] Test [2017-10-16 2017-11-10]\n",
      "Window Train/Val/Test shape (1911, 39) (336, 39) (546, 39)\n",
      "Window Train/Val/Test shape (1911, 2) (336, 2) (546, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-10-14] Val [2017-10-15 2017-10-30] Test [2017-10-31 2017-11-10]\n",
      "Window Train/Val/Test shape (2226, 39) (336, 39) (231, 39)\n",
      "Window Train/Val/Test shape (2226, 2) (336, 2) (231, 2)\n",
      "[122.00647601245153, 114.98156689253123, 29.94920266272363, 17.53250751234576, 15.793670636318488]\n",
      "mean mse: 60.05268474327413\n",
      "Train [2017-07-01 2017-08-15] Val [2017-08-16 2017-08-31] Test [2017-09-01 2017-09-16]\n",
      "Window Train/Val/Test shape (966, 39) (336, 39) (336, 39)\n",
      "Window Train/Val/Test shape (966, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-08-30] Val [2017-08-31 2017-09-15] Test [2017-09-16 2017-10-01]\n",
      "Window Train/Val/Test shape (1281, 39) (336, 39) (336, 39)\n",
      "Window Train/Val/Test shape (1281, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-09-14] Val [2017-09-15 2017-09-30] Test [2017-10-01 2017-10-16]\n",
      "Window Train/Val/Test shape (1596, 39) (336, 39) (336, 39)\n",
      "Window Train/Val/Test shape (1596, 2) (336, 2) (336, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-09-29] Val [2017-09-30 2017-10-15] Test [2017-10-16 2017-11-10]\n",
      "Window Train/Val/Test shape (1911, 39) (336, 39) (546, 39)\n",
      "Window Train/Val/Test shape (1911, 2) (336, 2) (546, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-10-14] Val [2017-10-15 2017-10-30] Test [2017-10-31 2017-11-10]\n",
      "Window Train/Val/Test shape (2226, 39) (336, 39) (231, 39)\n",
      "Window Train/Val/Test shape (2226, 2) (336, 2) (231, 2)\n",
      "[110.9613063105299, 89.56990794362079, 47.874833033274726, 17.82067309630497, 9.00668061938873]\n",
      "mean mse: 55.046680200623825\n",
      "Train [2017-07-01 2017-08-15] Val [2017-08-16 2017-08-31] Test [2017-09-01 2017-09-16]\n",
      "Window Train/Val/Test shape (966, 39) (336, 39) (336, 39)\n",
      "Window Train/Val/Test shape (966, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-08-30] Val [2017-08-31 2017-09-15] Test [2017-09-16 2017-10-01]\n",
      "Window Train/Val/Test shape (1281, 39) (336, 39) (336, 39)\n",
      "Window Train/Val/Test shape (1281, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-09-14] Val [2017-09-15 2017-09-30] Test [2017-10-01 2017-10-16]\n",
      "Window Train/Val/Test shape (1596, 39) (336, 39) (336, 39)\n",
      "Window Train/Val/Test shape (1596, 2) (336, 2) (336, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-09-29] Val [2017-09-30 2017-10-15] Test [2017-10-16 2017-11-10]\n",
      "Window Train/Val/Test shape (1911, 39) (336, 39) (546, 39)\n",
      "Window Train/Val/Test shape (1911, 2) (336, 2) (546, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-10-14] Val [2017-10-15 2017-10-30] Test [2017-10-31 2017-11-10]\n",
      "Window Train/Val/Test shape (2226, 39) (336, 39) (231, 39)\n",
      "Window Train/Val/Test shape (2226, 2) (336, 2) (231, 2)\n",
      "[112.74091037127941, 87.96909724687482, 41.19435839787861, 14.51269483060203, 4.906246123331014]\n",
      "mean mse: 52.26466139399317\n",
      "Train [2017-07-01 2017-08-15] Val [2017-08-16 2017-08-31] Test [2017-09-01 2017-09-16]\n",
      "Window Train/Val/Test shape (966, 39) (336, 39) (336, 39)\n",
      "Window Train/Val/Test shape (966, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-08-30] Val [2017-08-31 2017-09-15] Test [2017-09-16 2017-10-01]\n",
      "Window Train/Val/Test shape (1281, 39) (336, 39) (336, 39)\n",
      "Window Train/Val/Test shape (1281, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-09-14] Val [2017-09-15 2017-09-30] Test [2017-10-01 2017-10-16]\n",
      "Window Train/Val/Test shape (1596, 39) (336, 39) (336, 39)\n",
      "Window Train/Val/Test shape (1596, 2) (336, 2) (336, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-09-29] Val [2017-09-30 2017-10-15] Test [2017-10-16 2017-11-10]\n",
      "Window Train/Val/Test shape (1911, 39) (336, 39) (546, 39)\n",
      "Window Train/Val/Test shape (1911, 2) (336, 2) (546, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-10-14] Val [2017-10-15 2017-10-30] Test [2017-10-31 2017-11-10]\n",
      "Window Train/Val/Test shape (2226, 39) (336, 39) (231, 39)\n",
      "Window Train/Val/Test shape (2226, 2) (336, 2) (231, 2)\n",
      "[109.87611199662963, 87.2969853586551, 41.8915464097373, 17.43869035369562, 7.464656145831657]\n",
      "mean mse: 52.793598052909864\n",
      "Train [2017-07-01 2017-08-15] Val [2017-08-16 2017-08-31] Test [2017-09-01 2017-09-16]\n",
      "Window Train/Val/Test shape (966, 39) (336, 39) (336, 39)\n",
      "Window Train/Val/Test shape (966, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-08-30] Val [2017-08-31 2017-09-15] Test [2017-09-16 2017-10-01]\n",
      "Window Train/Val/Test shape (1281, 39) (336, 39) (336, 39)\n",
      "Window Train/Val/Test shape (1281, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-09-14] Val [2017-09-15 2017-09-30] Test [2017-10-01 2017-10-16]\n",
      "Window Train/Val/Test shape (1596, 39) (336, 39) (336, 39)\n",
      "Window Train/Val/Test shape (1596, 2) (336, 2) (336, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-09-29] Val [2017-09-30 2017-10-15] Test [2017-10-16 2017-11-10]\n",
      "Window Train/Val/Test shape (1911, 39) (336, 39) (546, 39)\n",
      "Window Train/Val/Test shape (1911, 2) (336, 2) (546, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-10-14] Val [2017-10-15 2017-10-30] Test [2017-10-31 2017-11-10]\n",
      "Window Train/Val/Test shape (2226, 39) (336, 39) (231, 39)\n",
      "Window Train/Val/Test shape (2226, 2) (336, 2) (231, 2)\n",
      "[113.66030201717352, 97.02052804313232, 32.49015965543542, 20.87310821943709, 9.242164543507554]\n",
      "mean mse: 54.65725249573718\n",
      "Train [2017-07-01 2017-08-15] Val [2017-08-16 2017-08-31] Test [2017-09-01 2017-09-16]\n",
      "Window Train/Val/Test shape (966, 39) (336, 39) (336, 39)\n",
      "Window Train/Val/Test shape (966, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-08-30] Val [2017-08-31 2017-09-15] Test [2017-09-16 2017-10-01]\n",
      "Window Train/Val/Test shape (1281, 39) (336, 39) (336, 39)\n",
      "Window Train/Val/Test shape (1281, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-09-14] Val [2017-09-15 2017-09-30] Test [2017-10-01 2017-10-16]\n",
      "Window Train/Val/Test shape (1596, 39) (336, 39) (336, 39)\n",
      "Window Train/Val/Test shape (1596, 2) (336, 2) (336, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-09-29] Val [2017-09-30 2017-10-15] Test [2017-10-16 2017-11-10]\n",
      "Window Train/Val/Test shape (1911, 39) (336, 39) (546, 39)\n",
      "Window Train/Val/Test shape (1911, 2) (336, 2) (546, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-10-14] Val [2017-10-15 2017-10-30] Test [2017-10-31 2017-11-10]\n",
      "Window Train/Val/Test shape (2226, 39) (336, 39) (231, 39)\n",
      "Window Train/Val/Test shape (2226, 2) (336, 2) (231, 2)\n",
      "[105.56583317768929, 87.06963312731315, 38.70382745701249, 18.11749371903667, 5.369274349242042]\n",
      "mean mse: 50.96521236605873\n",
      "Train [2017-07-01 2017-08-15] Val [2017-08-16 2017-08-31] Test [2017-09-01 2017-09-16]\n",
      "Window Train/Val/Test shape (966, 39) (336, 39) (336, 39)\n",
      "Window Train/Val/Test shape (966, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-08-30] Val [2017-08-31 2017-09-15] Test [2017-09-16 2017-10-01]\n",
      "Window Train/Val/Test shape (1281, 39) (336, 39) (336, 39)\n",
      "Window Train/Val/Test shape (1281, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-09-14] Val [2017-09-15 2017-09-30] Test [2017-10-01 2017-10-16]\n",
      "Window Train/Val/Test shape (1596, 39) (336, 39) (336, 39)\n",
      "Window Train/Val/Test shape (1596, 2) (336, 2) (336, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-09-29] Val [2017-09-30 2017-10-15] Test [2017-10-16 2017-11-10]\n",
      "Window Train/Val/Test shape (1911, 39) (336, 39) (546, 39)\n",
      "Window Train/Val/Test shape (1911, 2) (336, 2) (546, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-10-14] Val [2017-10-15 2017-10-30] Test [2017-10-31 2017-11-10]\n",
      "Window Train/Val/Test shape (2226, 39) (336, 39) (231, 39)\n",
      "Window Train/Val/Test shape (2226, 2) (336, 2) (231, 2)\n",
      "[107.32037504770572, 88.59762563967959, 44.58865742558758, 13.911480768393698, 5.5378412458957404]\n",
      "mean mse: 51.99119602545246\n",
      "Train [2017-07-01 2017-08-15] Val [2017-08-16 2017-08-31] Test [2017-09-01 2017-09-16]\n",
      "Window Train/Val/Test shape (966, 39) (336, 39) (336, 39)\n",
      "Window Train/Val/Test shape (966, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-08-30] Val [2017-08-31 2017-09-15] Test [2017-09-16 2017-10-01]\n",
      "Window Train/Val/Test shape (1281, 39) (336, 39) (336, 39)\n",
      "Window Train/Val/Test shape (1281, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-09-14] Val [2017-09-15 2017-09-30] Test [2017-10-01 2017-10-16]\n",
      "Window Train/Val/Test shape (1596, 39) (336, 39) (336, 39)\n",
      "Window Train/Val/Test shape (1596, 2) (336, 2) (336, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-09-29] Val [2017-09-30 2017-10-15] Test [2017-10-16 2017-11-10]\n",
      "Window Train/Val/Test shape (1911, 39) (336, 39) (546, 39)\n",
      "Window Train/Val/Test shape (1911, 2) (336, 2) (546, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-10-14] Val [2017-10-15 2017-10-30] Test [2017-10-31 2017-11-10]\n",
      "Window Train/Val/Test shape (2226, 39) (336, 39) (231, 39)\n",
      "Window Train/Val/Test shape (2226, 2) (336, 2) (231, 2)\n",
      "[1221.4747544990453, 946.0202819161093, 504.6565290411965, 324.13823796560195, 291.370343515604]\n",
      "mean mse: 657.5320293875113\n",
      "Train [2017-07-01 2017-08-15] Val [2017-08-16 2017-08-31] Test [2017-09-01 2017-09-16]\n",
      "Window Train/Val/Test shape (966, 39) (336, 39) (336, 39)\n",
      "Window Train/Val/Test shape (966, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-08-30] Val [2017-08-31 2017-09-15] Test [2017-09-16 2017-10-01]\n",
      "Window Train/Val/Test shape (1281, 39) (336, 39) (336, 39)\n",
      "Window Train/Val/Test shape (1281, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-09-14] Val [2017-09-15 2017-09-30] Test [2017-10-01 2017-10-16]\n",
      "Window Train/Val/Test shape (1596, 39) (336, 39) (336, 39)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Window Train/Val/Test shape (1596, 2) (336, 2) (336, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-09-29] Val [2017-09-30 2017-10-15] Test [2017-10-16 2017-11-10]\n",
      "Window Train/Val/Test shape (1911, 39) (336, 39) (546, 39)\n",
      "Window Train/Val/Test shape (1911, 2) (336, 2) (546, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-10-14] Val [2017-10-15 2017-10-30] Test [2017-10-31 2017-11-10]\n",
      "Window Train/Val/Test shape (2226, 39) (336, 39) (231, 39)\n",
      "Window Train/Val/Test shape (2226, 2) (336, 2) (231, 2)\n",
      "[116.89116065841442, 98.08878449465499, 34.082713252226355, 17.786512028511186, 7.794412211557411]\n",
      "mean mse: 54.92871652907288\n",
      "Train [2017-07-01 2017-08-15] Val [2017-08-16 2017-08-31] Test [2017-09-01 2017-09-16]\n",
      "Window Train/Val/Test shape (966, 39) (336, 39) (336, 39)\n",
      "Window Train/Val/Test shape (966, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-08-30] Val [2017-08-31 2017-09-15] Test [2017-09-16 2017-10-01]\n",
      "Window Train/Val/Test shape (1281, 39) (336, 39) (336, 39)\n",
      "Window Train/Val/Test shape (1281, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-09-14] Val [2017-09-15 2017-09-30] Test [2017-10-01 2017-10-16]\n",
      "Window Train/Val/Test shape (1596, 39) (336, 39) (336, 39)\n",
      "Window Train/Val/Test shape (1596, 2) (336, 2) (336, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-09-29] Val [2017-09-30 2017-10-15] Test [2017-10-16 2017-11-10]\n",
      "Window Train/Val/Test shape (1911, 39) (336, 39) (546, 39)\n",
      "Window Train/Val/Test shape (1911, 2) (336, 2) (546, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-10-14] Val [2017-10-15 2017-10-30] Test [2017-10-31 2017-11-10]\n",
      "Window Train/Val/Test shape (2226, 39) (336, 39) (231, 39)\n",
      "Window Train/Val/Test shape (2226, 2) (336, 2) (231, 2)\n",
      "[110.53323891041752, 96.5368610400816, 30.65904149945922, 18.94762045269714, 9.779680032998657]\n",
      "mean mse: 53.29128838713083\n",
      "Train [2017-07-01 2017-08-15] Val [2017-08-16 2017-08-31] Test [2017-09-01 2017-09-16]\n",
      "Window Train/Val/Test shape (966, 39) (336, 39) (336, 39)\n",
      "Window Train/Val/Test shape (966, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-08-30] Val [2017-08-31 2017-09-15] Test [2017-09-16 2017-10-01]\n",
      "Window Train/Val/Test shape (1281, 39) (336, 39) (336, 39)\n",
      "Window Train/Val/Test shape (1281, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-09-14] Val [2017-09-15 2017-09-30] Test [2017-10-01 2017-10-16]\n",
      "Window Train/Val/Test shape (1596, 39) (336, 39) (336, 39)\n",
      "Window Train/Val/Test shape (1596, 2) (336, 2) (336, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-09-29] Val [2017-09-30 2017-10-15] Test [2017-10-16 2017-11-10]\n",
      "Window Train/Val/Test shape (1911, 39) (336, 39) (546, 39)\n",
      "Window Train/Val/Test shape (1911, 2) (336, 2) (546, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-10-14] Val [2017-10-15 2017-10-30] Test [2017-10-31 2017-11-10]\n",
      "Window Train/Val/Test shape (2226, 39) (336, 39) (231, 39)\n",
      "Window Train/Val/Test shape (2226, 2) (336, 2) (231, 2)\n",
      "[120.13748608258746, 83.82157932536099, 36.00278233786237, 16.179100798781185, 9.60690911932422]\n",
      "mean mse: 53.14957153278324\n",
      "Train [2017-07-01 2017-08-15] Val [2017-08-16 2017-08-31] Test [2017-09-01 2017-09-16]\n",
      "Window Train/Val/Test shape (966, 39) (336, 39) (336, 39)\n",
      "Window Train/Val/Test shape (966, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-08-30] Val [2017-08-31 2017-09-15] Test [2017-09-16 2017-10-01]\n",
      "Window Train/Val/Test shape (1281, 39) (336, 39) (336, 39)\n",
      "Window Train/Val/Test shape (1281, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-09-14] Val [2017-09-15 2017-09-30] Test [2017-10-01 2017-10-16]\n",
      "Window Train/Val/Test shape (1596, 39) (336, 39) (336, 39)\n",
      "Window Train/Val/Test shape (1596, 2) (336, 2) (336, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-09-29] Val [2017-09-30 2017-10-15] Test [2017-10-16 2017-11-10]\n",
      "Window Train/Val/Test shape (1911, 39) (336, 39) (546, 39)\n",
      "Window Train/Val/Test shape (1911, 2) (336, 2) (546, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-10-14] Val [2017-10-15 2017-10-30] Test [2017-10-31 2017-11-10]\n",
      "Window Train/Val/Test shape (2226, 39) (336, 39) (231, 39)\n",
      "Window Train/Val/Test shape (2226, 2) (336, 2) (231, 2)\n",
      "[122.66813713863387, 78.54919069984838, 34.87296878089358, 19.561402274781432, 7.740336071561466]\n",
      "mean mse: 52.678406993143746\n",
      "Train [2017-07-01 2017-08-15] Val [2017-08-16 2017-08-31] Test [2017-09-01 2017-09-16]\n",
      "Window Train/Val/Test shape (966, 39) (336, 39) (336, 39)\n",
      "Window Train/Val/Test shape (966, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-08-30] Val [2017-08-31 2017-09-15] Test [2017-09-16 2017-10-01]\n",
      "Window Train/Val/Test shape (1281, 39) (336, 39) (336, 39)\n",
      "Window Train/Val/Test shape (1281, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-09-14] Val [2017-09-15 2017-09-30] Test [2017-10-01 2017-10-16]\n",
      "Window Train/Val/Test shape (1596, 39) (336, 39) (336, 39)\n",
      "Window Train/Val/Test shape (1596, 2) (336, 2) (336, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-09-29] Val [2017-09-30 2017-10-15] Test [2017-10-16 2017-11-10]\n",
      "Window Train/Val/Test shape (1911, 39) (336, 39) (546, 39)\n",
      "Window Train/Val/Test shape (1911, 2) (336, 2) (546, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-10-14] Val [2017-10-15 2017-10-30] Test [2017-10-31 2017-11-10]\n",
      "Window Train/Val/Test shape (2226, 39) (336, 39) (231, 39)\n",
      "Window Train/Val/Test shape (2226, 2) (336, 2) (231, 2)\n",
      "[113.43821074945193, 72.75625495556096, 35.62913739949095, 17.24291615766813, 8.817837169152929]\n",
      "mean mse: 49.57687128626498\n",
      "Train [2017-07-01 2017-08-15] Val [2017-08-16 2017-08-31] Test [2017-09-01 2017-09-16]\n",
      "Window Train/Val/Test shape (966, 39) (336, 39) (336, 39)\n",
      "Window Train/Val/Test shape (966, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-08-30] Val [2017-08-31 2017-09-15] Test [2017-09-16 2017-10-01]\n",
      "Window Train/Val/Test shape (1281, 39) (336, 39) (336, 39)\n",
      "Window Train/Val/Test shape (1281, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-09-14] Val [2017-09-15 2017-09-30] Test [2017-10-01 2017-10-16]\n",
      "Window Train/Val/Test shape (1596, 39) (336, 39) (336, 39)\n",
      "Window Train/Val/Test shape (1596, 2) (336, 2) (336, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-09-29] Val [2017-09-30 2017-10-15] Test [2017-10-16 2017-11-10]\n",
      "Window Train/Val/Test shape (1911, 39) (336, 39) (546, 39)\n",
      "Window Train/Val/Test shape (1911, 2) (336, 2) (546, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-10-14] Val [2017-10-15 2017-10-30] Test [2017-10-31 2017-11-10]\n",
      "Window Train/Val/Test shape (2226, 39) (336, 39) (231, 39)\n",
      "Window Train/Val/Test shape (2226, 2) (336, 2) (231, 2)\n",
      "[103.85012048243023, 77.84959321540462, 44.168458840538214, 15.27380417897277, 5.831120527926743]\n",
      "mean mse: 49.39461944905452\n",
      "Train [2017-07-01 2017-08-15] Val [2017-08-16 2017-08-31] Test [2017-09-01 2017-09-16]\n",
      "Window Train/Val/Test shape (966, 39) (336, 39) (336, 39)\n",
      "Window Train/Val/Test shape (966, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-08-30] Val [2017-08-31 2017-09-15] Test [2017-09-16 2017-10-01]\n",
      "Window Train/Val/Test shape (1281, 39) (336, 39) (336, 39)\n",
      "Window Train/Val/Test shape (1281, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-09-14] Val [2017-09-15 2017-09-30] Test [2017-10-01 2017-10-16]\n",
      "Window Train/Val/Test shape (1596, 39) (336, 39) (336, 39)\n",
      "Window Train/Val/Test shape (1596, 2) (336, 2) (336, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-09-29] Val [2017-09-30 2017-10-15] Test [2017-10-16 2017-11-10]\n",
      "Window Train/Val/Test shape (1911, 39) (336, 39) (546, 39)\n",
      "Window Train/Val/Test shape (1911, 2) (336, 2) (546, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-10-14] Val [2017-10-15 2017-10-30] Test [2017-10-31 2017-11-10]\n",
      "Window Train/Val/Test shape (2226, 39) (336, 39) (231, 39)\n",
      "Window Train/Val/Test shape (2226, 2) (336, 2) (231, 2)\n",
      "[104.34197430774933, 79.43816772813587, 45.01461635287602, 14.314598901356124, 4.910624172769896]\n",
      "mean mse: 49.603996292577456\n",
      "Train [2017-07-01 2017-08-15] Val [2017-08-16 2017-08-31] Test [2017-09-01 2017-09-16]\n",
      "Window Train/Val/Test shape (966, 39) (336, 39) (336, 39)\n",
      "Window Train/Val/Test shape (966, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-08-30] Val [2017-08-31 2017-09-15] Test [2017-09-16 2017-10-01]\n",
      "Window Train/Val/Test shape (1281, 39) (336, 39) (336, 39)\n",
      "Window Train/Val/Test shape (1281, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-09-14] Val [2017-09-15 2017-09-30] Test [2017-10-01 2017-10-16]\n",
      "Window Train/Val/Test shape (1596, 39) (336, 39) (336, 39)\n",
      "Window Train/Val/Test shape (1596, 2) (336, 2) (336, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-09-29] Val [2017-09-30 2017-10-15] Test [2017-10-16 2017-11-10]\n",
      "Window Train/Val/Test shape (1911, 39) (336, 39) (546, 39)\n",
      "Window Train/Val/Test shape (1911, 2) (336, 2) (546, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-10-14] Val [2017-10-15 2017-10-30] Test [2017-10-31 2017-11-10]\n",
      "Window Train/Val/Test shape (2226, 39) (336, 39) (231, 39)\n",
      "Window Train/Val/Test shape (2226, 2) (336, 2) (231, 2)\n",
      "[105.23958831616466, 82.95824060107991, 49.43812762911644, 17.224031501291744, 6.640506079342727]\n",
      "mean mse: 52.3000988253991\n",
      "Train [2017-07-01 2017-08-15] Val [2017-08-16 2017-08-31] Test [2017-09-01 2017-09-16]\n",
      "Window Train/Val/Test shape (966, 39) (336, 39) (336, 39)\n",
      "Window Train/Val/Test shape (966, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-08-30] Val [2017-08-31 2017-09-15] Test [2017-09-16 2017-10-01]\n",
      "Window Train/Val/Test shape (1281, 39) (336, 39) (336, 39)\n",
      "Window Train/Val/Test shape (1281, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-09-14] Val [2017-09-15 2017-09-30] Test [2017-10-01 2017-10-16]\n",
      "Window Train/Val/Test shape (1596, 39) (336, 39) (336, 39)\n",
      "Window Train/Val/Test shape (1596, 2) (336, 2) (336, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-09-29] Val [2017-09-30 2017-10-15] Test [2017-10-16 2017-11-10]\n",
      "Window Train/Val/Test shape (1911, 39) (336, 39) (546, 39)\n",
      "Window Train/Val/Test shape (1911, 2) (336, 2) (546, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-10-14] Val [2017-10-15 2017-10-30] Test [2017-10-31 2017-11-10]\n",
      "Window Train/Val/Test shape (2226, 39) (336, 39) (231, 39)\n",
      "Window Train/Val/Test shape (2226, 2) (336, 2) (231, 2)\n",
      "[1221.4747544990453, 946.0202819161093, 504.6565290411965, 324.13823796560195, 291.370343515604]\n",
      "mean mse: 657.5320293875113\n",
      "Train [2017-07-01 2017-08-15] Val [2017-08-16 2017-08-31] Test [2017-09-01 2017-09-16]\n",
      "Window Train/Val/Test shape (966, 39) (336, 39) (336, 39)\n",
      "Window Train/Val/Test shape (966, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-08-30] Val [2017-08-31 2017-09-15] Test [2017-09-16 2017-10-01]\n",
      "Window Train/Val/Test shape (1281, 39) (336, 39) (336, 39)\n",
      "Window Train/Val/Test shape (1281, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-09-14] Val [2017-09-15 2017-09-30] Test [2017-10-01 2017-10-16]\n",
      "Window Train/Val/Test shape (1596, 39) (336, 39) (336, 39)\n",
      "Window Train/Val/Test shape (1596, 2) (336, 2) (336, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-09-29] Val [2017-09-30 2017-10-15] Test [2017-10-16 2017-11-10]\n",
      "Window Train/Val/Test shape (1911, 39) (336, 39) (546, 39)\n",
      "Window Train/Val/Test shape (1911, 2) (336, 2) (546, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-10-14] Val [2017-10-15 2017-10-30] Test [2017-10-31 2017-11-10]\n",
      "Window Train/Val/Test shape (2226, 39) (336, 39) (231, 39)\n",
      "Window Train/Val/Test shape (2226, 2) (336, 2) (231, 2)\n",
      "[108.4849121388155, 86.15498451276433, 51.6546511573648, 17.379899858286215, 5.6991756888911835]\n",
      "mean mse: 53.8747246712244\n",
      "Train [2017-07-01 2017-08-15] Val [2017-08-16 2017-08-31] Test [2017-09-01 2017-09-16]\n",
      "Window Train/Val/Test shape (966, 39) (336, 39) (336, 39)\n",
      "Window Train/Val/Test shape (966, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-08-30] Val [2017-08-31 2017-09-15] Test [2017-09-16 2017-10-01]\n",
      "Window Train/Val/Test shape (1281, 39) (336, 39) (336, 39)\n",
      "Window Train/Val/Test shape (1281, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-09-14] Val [2017-09-15 2017-09-30] Test [2017-10-01 2017-10-16]\n",
      "Window Train/Val/Test shape (1596, 39) (336, 39) (336, 39)\n",
      "Window Train/Val/Test shape (1596, 2) (336, 2) (336, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-09-29] Val [2017-09-30 2017-10-15] Test [2017-10-16 2017-11-10]\n",
      "Window Train/Val/Test shape (1911, 39) (336, 39) (546, 39)\n",
      "Window Train/Val/Test shape (1911, 2) (336, 2) (546, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-10-14] Val [2017-10-15 2017-10-30] Test [2017-10-31 2017-11-10]\n",
      "Window Train/Val/Test shape (2226, 39) (336, 39) (231, 39)\n",
      "Window Train/Val/Test shape (2226, 2) (336, 2) (231, 2)\n",
      "[111.09496896436765, 82.7839839854332, 40.74151301069915, 17.415160946442672, 6.3546094079216235]\n",
      "mean mse: 51.67804726297286\n",
      "Train [2017-07-01 2017-08-15] Val [2017-08-16 2017-08-31] Test [2017-09-01 2017-09-16]\n",
      "Window Train/Val/Test shape (966, 39) (336, 39) (336, 39)\n",
      "Window Train/Val/Test shape (966, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-08-30] Val [2017-08-31 2017-09-15] Test [2017-09-16 2017-10-01]\n",
      "Window Train/Val/Test shape (1281, 39) (336, 39) (336, 39)\n",
      "Window Train/Val/Test shape (1281, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-09-14] Val [2017-09-15 2017-09-30] Test [2017-10-01 2017-10-16]\n",
      "Window Train/Val/Test shape (1596, 39) (336, 39) (336, 39)\n",
      "Window Train/Val/Test shape (1596, 2) (336, 2) (336, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-09-29] Val [2017-09-30 2017-10-15] Test [2017-10-16 2017-11-10]\n",
      "Window Train/Val/Test shape (1911, 39) (336, 39) (546, 39)\n",
      "Window Train/Val/Test shape (1911, 2) (336, 2) (546, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-10-14] Val [2017-10-15 2017-10-30] Test [2017-10-31 2017-11-10]\n",
      "Window Train/Val/Test shape (2226, 39) (336, 39) (231, 39)\n",
      "Window Train/Val/Test shape (2226, 2) (336, 2) (231, 2)\n",
      "[104.90509365776282, 79.81816650958238, 47.254172136343946, 16.413521792800143, 5.153870435320031]\n",
      "mean mse: 50.708964906361864\n",
      "Train [2017-07-01 2017-08-15] Val [2017-08-16 2017-08-31] Test [2017-09-01 2017-09-16]\n",
      "Window Train/Val/Test shape (966, 39) (336, 39) (336, 39)\n",
      "Window Train/Val/Test shape (966, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-08-30] Val [2017-08-31 2017-09-15] Test [2017-09-16 2017-10-01]\n",
      "Window Train/Val/Test shape (1281, 39) (336, 39) (336, 39)\n",
      "Window Train/Val/Test shape (1281, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-09-14] Val [2017-09-15 2017-09-30] Test [2017-10-01 2017-10-16]\n",
      "Window Train/Val/Test shape (1596, 39) (336, 39) (336, 39)\n",
      "Window Train/Val/Test shape (1596, 2) (336, 2) (336, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-09-29] Val [2017-09-30 2017-10-15] Test [2017-10-16 2017-11-10]\n",
      "Window Train/Val/Test shape (1911, 39) (336, 39) (546, 39)\n",
      "Window Train/Val/Test shape (1911, 2) (336, 2) (546, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-10-14] Val [2017-10-15 2017-10-30] Test [2017-10-31 2017-11-10]\n",
      "Window Train/Val/Test shape (2226, 39) (336, 39) (231, 39)\n",
      "Window Train/Val/Test shape (2226, 2) (336, 2) (231, 2)\n",
      "[113.28296093493373, 97.06159561775682, 51.860359216662864, 24.153581774836468, 8.962952485120502]\n",
      "mean mse: 59.064290005862084\n",
      "Train [2017-07-01 2017-08-15] Val [2017-08-16 2017-08-31] Test [2017-09-01 2017-09-16]\n",
      "Window Train/Val/Test shape (966, 39) (336, 39) (336, 39)\n",
      "Window Train/Val/Test shape (966, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-08-30] Val [2017-08-31 2017-09-15] Test [2017-09-16 2017-10-01]\n",
      "Window Train/Val/Test shape (1281, 39) (336, 39) (336, 39)\n",
      "Window Train/Val/Test shape (1281, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-09-14] Val [2017-09-15 2017-09-30] Test [2017-10-01 2017-10-16]\n",
      "Window Train/Val/Test shape (1596, 39) (336, 39) (336, 39)\n",
      "Window Train/Val/Test shape (1596, 2) (336, 2) (336, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-09-29] Val [2017-09-30 2017-10-15] Test [2017-10-16 2017-11-10]\n",
      "Window Train/Val/Test shape (1911, 39) (336, 39) (546, 39)\n",
      "Window Train/Val/Test shape (1911, 2) (336, 2) (546, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-10-14] Val [2017-10-15 2017-10-30] Test [2017-10-31 2017-11-10]\n",
      "Window Train/Val/Test shape (2226, 39) (336, 39) (231, 39)\n",
      "Window Train/Val/Test shape (2226, 2) (336, 2) (231, 2)\n",
      "[109.63635358613915, 90.84958652664407, 45.24950646420299, 18.61799216236493, 5.3601029231213735]\n",
      "mean mse: 53.9427083324945\n",
      "Train [2017-07-01 2017-08-15] Val [2017-08-16 2017-08-31] Test [2017-09-01 2017-09-16]\n",
      "Window Train/Val/Test shape (966, 39) (336, 39) (336, 39)\n",
      "Window Train/Val/Test shape (966, 2) (336, 2) (336, 2)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train [2017-07-01 2017-08-30] Val [2017-08-31 2017-09-15] Test [2017-09-16 2017-10-01]\n",
      "Window Train/Val/Test shape (1281, 39) (336, 39) (336, 39)\n",
      "Window Train/Val/Test shape (1281, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-09-14] Val [2017-09-15 2017-09-30] Test [2017-10-01 2017-10-16]\n",
      "Window Train/Val/Test shape (1596, 39) (336, 39) (336, 39)\n",
      "Window Train/Val/Test shape (1596, 2) (336, 2) (336, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-09-29] Val [2017-09-30 2017-10-15] Test [2017-10-16 2017-11-10]\n",
      "Window Train/Val/Test shape (1911, 39) (336, 39) (546, 39)\n",
      "Window Train/Val/Test shape (1911, 2) (336, 2) (546, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-10-14] Val [2017-10-15 2017-10-30] Test [2017-10-31 2017-11-10]\n",
      "Window Train/Val/Test shape (2226, 39) (336, 39) (231, 39)\n",
      "Window Train/Val/Test shape (2226, 2) (336, 2) (231, 2)\n",
      "[1221.4747544990453, 946.0202819161093, 504.6565290411965, 324.13823796560195, 291.370343515604]\n",
      "mean mse: 657.5320293875113\n",
      "Train [2017-07-01 2017-08-15] Val [2017-08-16 2017-08-31] Test [2017-09-01 2017-09-16]\n",
      "Window Train/Val/Test shape (966, 39) (336, 39) (336, 39)\n",
      "Window Train/Val/Test shape (966, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-08-30] Val [2017-08-31 2017-09-15] Test [2017-09-16 2017-10-01]\n",
      "Window Train/Val/Test shape (1281, 39) (336, 39) (336, 39)\n",
      "Window Train/Val/Test shape (1281, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-09-14] Val [2017-09-15 2017-09-30] Test [2017-10-01 2017-10-16]\n",
      "Window Train/Val/Test shape (1596, 39) (336, 39) (336, 39)\n",
      "Window Train/Val/Test shape (1596, 2) (336, 2) (336, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-09-29] Val [2017-09-30 2017-10-15] Test [2017-10-16 2017-11-10]\n",
      "Window Train/Val/Test shape (1911, 39) (336, 39) (546, 39)\n",
      "Window Train/Val/Test shape (1911, 2) (336, 2) (546, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-10-14] Val [2017-10-15 2017-10-30] Test [2017-10-31 2017-11-10]\n",
      "Window Train/Val/Test shape (2226, 39) (336, 39) (231, 39)\n",
      "Window Train/Val/Test shape (2226, 2) (336, 2) (231, 2)\n",
      "[108.04487184534, 77.79821824869948, 36.113879309127135, 12.478999144692969, 5.851316577371282]\n",
      "mean mse: 48.057457025046176\n",
      "Train [2017-07-01 2017-08-15] Val [2017-08-16 2017-08-31] Test [2017-09-01 2017-09-16]\n",
      "Window Train/Val/Test shape (966, 39) (336, 39) (336, 39)\n",
      "Window Train/Val/Test shape (966, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-08-30] Val [2017-08-31 2017-09-15] Test [2017-09-16 2017-10-01]\n",
      "Window Train/Val/Test shape (1281, 39) (336, 39) (336, 39)\n",
      "Window Train/Val/Test shape (1281, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-09-14] Val [2017-09-15 2017-09-30] Test [2017-10-01 2017-10-16]\n",
      "Window Train/Val/Test shape (1596, 39) (336, 39) (336, 39)\n",
      "Window Train/Val/Test shape (1596, 2) (336, 2) (336, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-09-29] Val [2017-09-30 2017-10-15] Test [2017-10-16 2017-11-10]\n",
      "Window Train/Val/Test shape (1911, 39) (336, 39) (546, 39)\n",
      "Window Train/Val/Test shape (1911, 2) (336, 2) (546, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-10-14] Val [2017-10-15 2017-10-30] Test [2017-10-31 2017-11-10]\n",
      "Window Train/Val/Test shape (2226, 39) (336, 39) (231, 39)\n",
      "Window Train/Val/Test shape (2226, 2) (336, 2) (231, 2)\n",
      "[112.0260288430181, 75.87587158549448, 33.35770364295005, 11.939272851834886, 5.697545193740237]\n",
      "mean mse: 47.77928442340755\n",
      "Train [2017-07-01 2017-08-15] Val [2017-08-16 2017-08-31] Test [2017-09-01 2017-09-16]\n",
      "Window Train/Val/Test shape (966, 39) (336, 39) (336, 39)\n",
      "Window Train/Val/Test shape (966, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-08-30] Val [2017-08-31 2017-09-15] Test [2017-09-16 2017-10-01]\n",
      "Window Train/Val/Test shape (1281, 39) (336, 39) (336, 39)\n",
      "Window Train/Val/Test shape (1281, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-09-14] Val [2017-09-15 2017-09-30] Test [2017-10-01 2017-10-16]\n",
      "Window Train/Val/Test shape (1596, 39) (336, 39) (336, 39)\n",
      "Window Train/Val/Test shape (1596, 2) (336, 2) (336, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-09-29] Val [2017-09-30 2017-10-15] Test [2017-10-16 2017-11-10]\n",
      "Window Train/Val/Test shape (1911, 39) (336, 39) (546, 39)\n",
      "Window Train/Val/Test shape (1911, 2) (336, 2) (546, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-10-14] Val [2017-10-15 2017-10-30] Test [2017-10-31 2017-11-10]\n",
      "Window Train/Val/Test shape (2226, 39) (336, 39) (231, 39)\n",
      "Window Train/Val/Test shape (2226, 2) (336, 2) (231, 2)\n",
      "[111.79151628957334, 75.48065902650777, 33.623029764034925, 11.902065941762629, 5.716369908330526]\n",
      "mean mse: 47.70272818604184\n",
      "Train [2017-07-01 2017-08-15] Val [2017-08-16 2017-08-31] Test [2017-09-01 2017-09-16]\n",
      "Window Train/Val/Test shape (966, 39) (336, 39) (336, 39)\n",
      "Window Train/Val/Test shape (966, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-08-30] Val [2017-08-31 2017-09-15] Test [2017-09-16 2017-10-01]\n",
      "Window Train/Val/Test shape (1281, 39) (336, 39) (336, 39)\n",
      "Window Train/Val/Test shape (1281, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-09-14] Val [2017-09-15 2017-09-30] Test [2017-10-01 2017-10-16]\n",
      "Window Train/Val/Test shape (1596, 39) (336, 39) (336, 39)\n",
      "Window Train/Val/Test shape (1596, 2) (336, 2) (336, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-09-29] Val [2017-09-30 2017-10-15] Test [2017-10-16 2017-11-10]\n",
      "Window Train/Val/Test shape (1911, 39) (336, 39) (546, 39)\n",
      "Window Train/Val/Test shape (1911, 2) (336, 2) (546, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-10-14] Val [2017-10-15 2017-10-30] Test [2017-10-31 2017-11-10]\n",
      "Window Train/Val/Test shape (2226, 39) (336, 39) (231, 39)\n",
      "Window Train/Val/Test shape (2226, 2) (336, 2) (231, 2)\n",
      "[107.20657930817242, 82.18568571349549, 31.76889269204424, 10.913262716889164, 5.6401982396448]\n",
      "mean mse: 47.542923734049225\n",
      "Train [2017-07-01 2017-08-15] Val [2017-08-16 2017-08-31] Test [2017-09-01 2017-09-16]\n",
      "Window Train/Val/Test shape (966, 39) (336, 39) (336, 39)\n",
      "Window Train/Val/Test shape (966, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-08-30] Val [2017-08-31 2017-09-15] Test [2017-09-16 2017-10-01]\n",
      "Window Train/Val/Test shape (1281, 39) (336, 39) (336, 39)\n",
      "Window Train/Val/Test shape (1281, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-09-14] Val [2017-09-15 2017-09-30] Test [2017-10-01 2017-10-16]\n",
      "Window Train/Val/Test shape (1596, 39) (336, 39) (336, 39)\n",
      "Window Train/Val/Test shape (1596, 2) (336, 2) (336, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-09-29] Val [2017-09-30 2017-10-15] Test [2017-10-16 2017-11-10]\n",
      "Window Train/Val/Test shape (1911, 39) (336, 39) (546, 39)\n",
      "Window Train/Val/Test shape (1911, 2) (336, 2) (546, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-10-14] Val [2017-10-15 2017-10-30] Test [2017-10-31 2017-11-10]\n",
      "Window Train/Val/Test shape (2226, 39) (336, 39) (231, 39)\n",
      "Window Train/Val/Test shape (2226, 2) (336, 2) (231, 2)\n",
      "[110.63637796899464, 79.15664617515822, 32.908188135314006, 15.05646459327327, 6.778522519034141]\n",
      "mean mse: 48.90723987835485\n",
      "Train [2017-07-01 2017-08-15] Val [2017-08-16 2017-08-31] Test [2017-09-01 2017-09-16]\n",
      "Window Train/Val/Test shape (966, 39) (336, 39) (336, 39)\n",
      "Window Train/Val/Test shape (966, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-08-30] Val [2017-08-31 2017-09-15] Test [2017-09-16 2017-10-01]\n",
      "Window Train/Val/Test shape (1281, 39) (336, 39) (336, 39)\n",
      "Window Train/Val/Test shape (1281, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-09-14] Val [2017-09-15 2017-09-30] Test [2017-10-01 2017-10-16]\n",
      "Window Train/Val/Test shape (1596, 39) (336, 39) (336, 39)\n",
      "Window Train/Val/Test shape (1596, 2) (336, 2) (336, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-09-29] Val [2017-09-30 2017-10-15] Test [2017-10-16 2017-11-10]\n",
      "Window Train/Val/Test shape (1911, 39) (336, 39) (546, 39)\n",
      "Window Train/Val/Test shape (1911, 2) (336, 2) (546, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-10-14] Val [2017-10-15 2017-10-30] Test [2017-10-31 2017-11-10]\n",
      "Window Train/Val/Test shape (2226, 39) (336, 39) (231, 39)\n",
      "Window Train/Val/Test shape (2226, 2) (336, 2) (231, 2)\n",
      "[121.88357815372872, 74.77515868052117, 33.619362178576246, 17.541472868560145, 10.996785503934511]\n",
      "mean mse: 51.76327147706415\n",
      "Train [2017-07-01 2017-08-15] Val [2017-08-16 2017-08-31] Test [2017-09-01 2017-09-16]\n",
      "Window Train/Val/Test shape (966, 39) (336, 39) (336, 39)\n",
      "Window Train/Val/Test shape (966, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-08-30] Val [2017-08-31 2017-09-15] Test [2017-09-16 2017-10-01]\n",
      "Window Train/Val/Test shape (1281, 39) (336, 39) (336, 39)\n",
      "Window Train/Val/Test shape (1281, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-09-14] Val [2017-09-15 2017-09-30] Test [2017-10-01 2017-10-16]\n",
      "Window Train/Val/Test shape (1596, 39) (336, 39) (336, 39)\n",
      "Window Train/Val/Test shape (1596, 2) (336, 2) (336, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-09-29] Val [2017-09-30 2017-10-15] Test [2017-10-16 2017-11-10]\n",
      "Window Train/Val/Test shape (1911, 39) (336, 39) (546, 39)\n",
      "Window Train/Val/Test shape (1911, 2) (336, 2) (546, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-10-14] Val [2017-10-15 2017-10-30] Test [2017-10-31 2017-11-10]\n",
      "Window Train/Val/Test shape (2226, 39) (336, 39) (231, 39)\n",
      "Window Train/Val/Test shape (2226, 2) (336, 2) (231, 2)\n",
      "[114.04822248911753, 80.16091912130594, 36.16584541994215, 12.9932742726917, 5.059727970966583]\n",
      "mean mse: 49.68559785480478\n",
      "Train [2017-07-01 2017-08-15] Val [2017-08-16 2017-08-31] Test [2017-09-01 2017-09-16]\n",
      "Window Train/Val/Test shape (966, 39) (336, 39) (336, 39)\n",
      "Window Train/Val/Test shape (966, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-08-30] Val [2017-08-31 2017-09-15] Test [2017-09-16 2017-10-01]\n",
      "Window Train/Val/Test shape (1281, 39) (336, 39) (336, 39)\n",
      "Window Train/Val/Test shape (1281, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-09-14] Val [2017-09-15 2017-09-30] Test [2017-10-01 2017-10-16]\n",
      "Window Train/Val/Test shape (1596, 39) (336, 39) (336, 39)\n",
      "Window Train/Val/Test shape (1596, 2) (336, 2) (336, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-09-29] Val [2017-09-30 2017-10-15] Test [2017-10-16 2017-11-10]\n",
      "Window Train/Val/Test shape (1911, 39) (336, 39) (546, 39)\n",
      "Window Train/Val/Test shape (1911, 2) (336, 2) (546, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-10-14] Val [2017-10-15 2017-10-30] Test [2017-10-31 2017-11-10]\n",
      "Window Train/Val/Test shape (2226, 39) (336, 39) (231, 39)\n",
      "Window Train/Val/Test shape (2226, 2) (336, 2) (231, 2)\n",
      "[112.60991836503597, 85.49507158316598, 44.47653969103034, 16.769845831936678, 6.27396853892742]\n",
      "mean mse: 53.12506880201927\n",
      "Train [2017-07-01 2017-08-15] Val [2017-08-16 2017-08-31] Test [2017-09-01 2017-09-16]\n",
      "Window Train/Val/Test shape (966, 39) (336, 39) (336, 39)\n",
      "Window Train/Val/Test shape (966, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-08-30] Val [2017-08-31 2017-09-15] Test [2017-09-16 2017-10-01]\n",
      "Window Train/Val/Test shape (1281, 39) (336, 39) (336, 39)\n",
      "Window Train/Val/Test shape (1281, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-09-14] Val [2017-09-15 2017-09-30] Test [2017-10-01 2017-10-16]\n",
      "Window Train/Val/Test shape (1596, 39) (336, 39) (336, 39)\n",
      "Window Train/Val/Test shape (1596, 2) (336, 2) (336, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-09-29] Val [2017-09-30 2017-10-15] Test [2017-10-16 2017-11-10]\n",
      "Window Train/Val/Test shape (1911, 39) (336, 39) (546, 39)\n",
      "Window Train/Val/Test shape (1911, 2) (336, 2) (546, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-10-14] Val [2017-10-15 2017-10-30] Test [2017-10-31 2017-11-10]\n",
      "Window Train/Val/Test shape (2226, 39) (336, 39) (231, 39)\n",
      "Window Train/Val/Test shape (2226, 2) (336, 2) (231, 2)\n",
      "[110.27637345799428, 72.5986680513411, 43.478193348113095, 13.910015240290438, 5.482951782415906]\n",
      "mean mse: 49.14924037603096\n",
      "Train [2017-07-01 2017-08-15] Val [2017-08-16 2017-08-31] Test [2017-09-01 2017-09-16]\n",
      "Window Train/Val/Test shape (966, 39) (336, 39) (336, 39)\n",
      "Window Train/Val/Test shape (966, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-08-30] Val [2017-08-31 2017-09-15] Test [2017-09-16 2017-10-01]\n",
      "Window Train/Val/Test shape (1281, 39) (336, 39) (336, 39)\n",
      "Window Train/Val/Test shape (1281, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-09-14] Val [2017-09-15 2017-09-30] Test [2017-10-01 2017-10-16]\n",
      "Window Train/Val/Test shape (1596, 39) (336, 39) (336, 39)\n",
      "Window Train/Val/Test shape (1596, 2) (336, 2) (336, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-09-29] Val [2017-09-30 2017-10-15] Test [2017-10-16 2017-11-10]\n",
      "Window Train/Val/Test shape (1911, 39) (336, 39) (546, 39)\n",
      "Window Train/Val/Test shape (1911, 2) (336, 2) (546, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-10-14] Val [2017-10-15 2017-10-30] Test [2017-10-31 2017-11-10]\n",
      "Window Train/Val/Test shape (2226, 39) (336, 39) (231, 39)\n",
      "Window Train/Val/Test shape (2226, 2) (336, 2) (231, 2)\n",
      "[114.88489199261288, 88.9696400700597, 41.49448563185195, 17.17631538845899, 11.35487323147015]\n",
      "mean mse: 54.77604126289073\n",
      "Train [2017-07-01 2017-08-15] Val [2017-08-16 2017-08-31] Test [2017-09-01 2017-09-16]\n",
      "Window Train/Val/Test shape (966, 39) (336, 39) (336, 39)\n",
      "Window Train/Val/Test shape (966, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-08-30] Val [2017-08-31 2017-09-15] Test [2017-09-16 2017-10-01]\n",
      "Window Train/Val/Test shape (1281, 39) (336, 39) (336, 39)\n",
      "Window Train/Val/Test shape (1281, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-09-14] Val [2017-09-15 2017-09-30] Test [2017-10-01 2017-10-16]\n",
      "Window Train/Val/Test shape (1596, 39) (336, 39) (336, 39)\n",
      "Window Train/Val/Test shape (1596, 2) (336, 2) (336, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-09-29] Val [2017-09-30 2017-10-15] Test [2017-10-16 2017-11-10]\n",
      "Window Train/Val/Test shape (1911, 39) (336, 39) (546, 39)\n",
      "Window Train/Val/Test shape (1911, 2) (336, 2) (546, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-10-14] Val [2017-10-15 2017-10-30] Test [2017-10-31 2017-11-10]\n",
      "Window Train/Val/Test shape (2226, 39) (336, 39) (231, 39)\n",
      "Window Train/Val/Test shape (2226, 2) (336, 2) (231, 2)\n",
      "[1221.4747544990453, 946.0202819161093, 504.6565290411965, 324.13823796560195, 291.370343515604]\n",
      "mean mse: 657.5320293875113\n",
      "Train [2017-07-01 2017-08-15] Val [2017-08-16 2017-08-31] Test [2017-09-01 2017-09-16]\n",
      "Window Train/Val/Test shape (966, 39) (336, 39) (336, 39)\n",
      "Window Train/Val/Test shape (966, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-08-30] Val [2017-08-31 2017-09-15] Test [2017-09-16 2017-10-01]\n",
      "Window Train/Val/Test shape (1281, 39) (336, 39) (336, 39)\n",
      "Window Train/Val/Test shape (1281, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-09-14] Val [2017-09-15 2017-09-30] Test [2017-10-01 2017-10-16]\n",
      "Window Train/Val/Test shape (1596, 39) (336, 39) (336, 39)\n",
      "Window Train/Val/Test shape (1596, 2) (336, 2) (336, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-09-29] Val [2017-09-30 2017-10-15] Test [2017-10-16 2017-11-10]\n",
      "Window Train/Val/Test shape (1911, 39) (336, 39) (546, 39)\n",
      "Window Train/Val/Test shape (1911, 2) (336, 2) (546, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-10-14] Val [2017-10-15 2017-10-30] Test [2017-10-31 2017-11-10]\n",
      "Window Train/Val/Test shape (2226, 39) (336, 39) (231, 39)\n",
      "Window Train/Val/Test shape (2226, 2) (336, 2) (231, 2)\n",
      "[118.48497924314142, 75.22473440792108, 35.6408560836639, 14.823657765687203, 7.833183755358402]\n",
      "mean mse: 50.4014822511544\n",
      "Train [2017-07-01 2017-08-15] Val [2017-08-16 2017-08-31] Test [2017-09-01 2017-09-16]\n",
      "Window Train/Val/Test shape (966, 39) (336, 39) (336, 39)\n",
      "Window Train/Val/Test shape (966, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-08-30] Val [2017-08-31 2017-09-15] Test [2017-09-16 2017-10-01]\n",
      "Window Train/Val/Test shape (1281, 39) (336, 39) (336, 39)\n",
      "Window Train/Val/Test shape (1281, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-09-14] Val [2017-09-15 2017-09-30] Test [2017-10-01 2017-10-16]\n",
      "Window Train/Val/Test shape (1596, 39) (336, 39) (336, 39)\n",
      "Window Train/Val/Test shape (1596, 2) (336, 2) (336, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-09-29] Val [2017-09-30 2017-10-15] Test [2017-10-16 2017-11-10]\n",
      "Window Train/Val/Test shape (1911, 39) (336, 39) (546, 39)\n",
      "Window Train/Val/Test shape (1911, 2) (336, 2) (546, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-10-14] Val [2017-10-15 2017-10-30] Test [2017-10-31 2017-11-10]\n",
      "Window Train/Val/Test shape (2226, 39) (336, 39) (231, 39)\n",
      "Window Train/Val/Test shape (2226, 2) (336, 2) (231, 2)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[1221.4747544990453, 946.0202819161093, 504.6565290411965, 324.13823796560195, 291.370343515604]\n",
      "mean mse: 657.5320293875113\n",
      "Train [2017-07-01 2017-08-15] Val [2017-08-16 2017-08-31] Test [2017-09-01 2017-09-16]\n",
      "Window Train/Val/Test shape (966, 39) (336, 39) (336, 39)\n",
      "Window Train/Val/Test shape (966, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-08-30] Val [2017-08-31 2017-09-15] Test [2017-09-16 2017-10-01]\n",
      "Window Train/Val/Test shape (1281, 39) (336, 39) (336, 39)\n",
      "Window Train/Val/Test shape (1281, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-09-14] Val [2017-09-15 2017-09-30] Test [2017-10-01 2017-10-16]\n",
      "Window Train/Val/Test shape (1596, 39) (336, 39) (336, 39)\n",
      "Window Train/Val/Test shape (1596, 2) (336, 2) (336, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-09-29] Val [2017-09-30 2017-10-15] Test [2017-10-16 2017-11-10]\n",
      "Window Train/Val/Test shape (1911, 39) (336, 39) (546, 39)\n",
      "Window Train/Val/Test shape (1911, 2) (336, 2) (546, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-10-14] Val [2017-10-15 2017-10-30] Test [2017-10-31 2017-11-10]\n",
      "Window Train/Val/Test shape (2226, 39) (336, 39) (231, 39)\n",
      "Window Train/Val/Test shape (2226, 2) (336, 2) (231, 2)\n",
      "[106.98747425562087, 83.65179223969699, 45.15351811095881, 18.726459908770085, 8.657813903614366]\n",
      "mean mse: 52.635411683732215\n",
      "Train [2017-07-01 2017-08-15] Val [2017-08-16 2017-08-31] Test [2017-09-01 2017-09-16]\n",
      "Window Train/Val/Test shape (966, 39) (336, 39) (336, 39)\n",
      "Window Train/Val/Test shape (966, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-08-30] Val [2017-08-31 2017-09-15] Test [2017-09-16 2017-10-01]\n",
      "Window Train/Val/Test shape (1281, 39) (336, 39) (336, 39)\n",
      "Window Train/Val/Test shape (1281, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-09-14] Val [2017-09-15 2017-09-30] Test [2017-10-01 2017-10-16]\n",
      "Window Train/Val/Test shape (1596, 39) (336, 39) (336, 39)\n",
      "Window Train/Val/Test shape (1596, 2) (336, 2) (336, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-09-29] Val [2017-09-30 2017-10-15] Test [2017-10-16 2017-11-10]\n",
      "Window Train/Val/Test shape (1911, 39) (336, 39) (546, 39)\n",
      "Window Train/Val/Test shape (1911, 2) (336, 2) (546, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-10-14] Val [2017-10-15 2017-10-30] Test [2017-10-31 2017-11-10]\n",
      "Window Train/Val/Test shape (2226, 39) (336, 39) (231, 39)\n",
      "Window Train/Val/Test shape (2226, 2) (336, 2) (231, 2)\n",
      "[111.62921165844115, 96.82029776604179, 35.55520165668746, 22.333408529911537, 6.654072361712255]\n",
      "mean mse: 54.59843839455884\n",
      "Train [2017-07-01 2017-08-15] Val [2017-08-16 2017-08-31] Test [2017-09-01 2017-09-16]\n",
      "Window Train/Val/Test shape (966, 39) (336, 39) (336, 39)\n",
      "Window Train/Val/Test shape (966, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-08-30] Val [2017-08-31 2017-09-15] Test [2017-09-16 2017-10-01]\n",
      "Window Train/Val/Test shape (1281, 39) (336, 39) (336, 39)\n",
      "Window Train/Val/Test shape (1281, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-09-14] Val [2017-09-15 2017-09-30] Test [2017-10-01 2017-10-16]\n",
      "Window Train/Val/Test shape (1596, 39) (336, 39) (336, 39)\n",
      "Window Train/Val/Test shape (1596, 2) (336, 2) (336, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-09-29] Val [2017-09-30 2017-10-15] Test [2017-10-16 2017-11-10]\n",
      "Window Train/Val/Test shape (1911, 39) (336, 39) (546, 39)\n",
      "Window Train/Val/Test shape (1911, 2) (336, 2) (546, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-10-14] Val [2017-10-15 2017-10-30] Test [2017-10-31 2017-11-10]\n",
      "Window Train/Val/Test shape (2226, 39) (336, 39) (231, 39)\n",
      "Window Train/Val/Test shape (2226, 2) (336, 2) (231, 2)\n",
      "[1221.4747544990453, 946.0202819161093, 504.6565290411965, 324.13823796560195, 291.370343515604]\n",
      "mean mse: 657.5320293875113\n",
      "Train [2017-07-01 2017-08-15] Val [2017-08-16 2017-08-31] Test [2017-09-01 2017-09-16]\n",
      "Window Train/Val/Test shape (966, 39) (336, 39) (336, 39)\n",
      "Window Train/Val/Test shape (966, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-08-30] Val [2017-08-31 2017-09-15] Test [2017-09-16 2017-10-01]\n",
      "Window Train/Val/Test shape (1281, 39) (336, 39) (336, 39)\n",
      "Window Train/Val/Test shape (1281, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-09-14] Val [2017-09-15 2017-09-30] Test [2017-10-01 2017-10-16]\n",
      "Window Train/Val/Test shape (1596, 39) (336, 39) (336, 39)\n",
      "Window Train/Val/Test shape (1596, 2) (336, 2) (336, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-09-29] Val [2017-09-30 2017-10-15] Test [2017-10-16 2017-11-10]\n",
      "Window Train/Val/Test shape (1911, 39) (336, 39) (546, 39)\n",
      "Window Train/Val/Test shape (1911, 2) (336, 2) (546, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-10-14] Val [2017-10-15 2017-10-30] Test [2017-10-31 2017-11-10]\n",
      "Window Train/Val/Test shape (2226, 39) (336, 39) (231, 39)\n",
      "Window Train/Val/Test shape (2226, 2) (336, 2) (231, 2)\n",
      "[1221.4747544990453, 946.0202819161093, 504.6565290411965, 324.13823796560195, 291.370343515604]\n",
      "mean mse: 657.5320293875113\n",
      "Train [2017-07-01 2017-08-15] Val [2017-08-16 2017-08-31] Test [2017-09-01 2017-09-16]\n",
      "Window Train/Val/Test shape (966, 39) (336, 39) (336, 39)\n",
      "Window Train/Val/Test shape (966, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-08-30] Val [2017-08-31 2017-09-15] Test [2017-09-16 2017-10-01]\n",
      "Window Train/Val/Test shape (1281, 39) (336, 39) (336, 39)\n",
      "Window Train/Val/Test shape (1281, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-09-14] Val [2017-09-15 2017-09-30] Test [2017-10-01 2017-10-16]\n",
      "Window Train/Val/Test shape (1596, 39) (336, 39) (336, 39)\n",
      "Window Train/Val/Test shape (1596, 2) (336, 2) (336, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-09-29] Val [2017-09-30 2017-10-15] Test [2017-10-16 2017-11-10]\n",
      "Window Train/Val/Test shape (1911, 39) (336, 39) (546, 39)\n",
      "Window Train/Val/Test shape (1911, 2) (336, 2) (546, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-10-14] Val [2017-10-15 2017-10-30] Test [2017-10-31 2017-11-10]\n",
      "Window Train/Val/Test shape (2226, 39) (336, 39) (231, 39)\n",
      "Window Train/Val/Test shape (2226, 2) (336, 2) (231, 2)\n",
      "[112.83877318230606, 81.14174603648327, 35.50506708889799, 18.41434532028463, 11.544194721770102]\n",
      "mean mse: 51.888825269948406\n",
      "Train [2017-07-01 2017-08-15] Val [2017-08-16 2017-08-31] Test [2017-09-01 2017-09-16]\n",
      "Window Train/Val/Test shape (966, 39) (336, 39) (336, 39)\n",
      "Window Train/Val/Test shape (966, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-08-30] Val [2017-08-31 2017-09-15] Test [2017-09-16 2017-10-01]\n",
      "Window Train/Val/Test shape (1281, 39) (336, 39) (336, 39)\n",
      "Window Train/Val/Test shape (1281, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-09-14] Val [2017-09-15 2017-09-30] Test [2017-10-01 2017-10-16]\n",
      "Window Train/Val/Test shape (1596, 39) (336, 39) (336, 39)\n",
      "Window Train/Val/Test shape (1596, 2) (336, 2) (336, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-09-29] Val [2017-09-30 2017-10-15] Test [2017-10-16 2017-11-10]\n",
      "Window Train/Val/Test shape (1911, 39) (336, 39) (546, 39)\n",
      "Window Train/Val/Test shape (1911, 2) (336, 2) (546, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-10-14] Val [2017-10-15 2017-10-30] Test [2017-10-31 2017-11-10]\n",
      "Window Train/Val/Test shape (2226, 39) (336, 39) (231, 39)\n",
      "Window Train/Val/Test shape (2226, 2) (336, 2) (231, 2)\n",
      "[1221.4747544990453, 946.0202819161093, 504.6565290411965, 324.13823796560195, 291.370343515604]\n",
      "mean mse: 657.5320293875113\n",
      "Train [2017-07-01 2017-08-15] Val [2017-08-16 2017-08-31] Test [2017-09-01 2017-09-16]\n",
      "Window Train/Val/Test shape (966, 39) (336, 39) (336, 39)\n",
      "Window Train/Val/Test shape (966, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-08-30] Val [2017-08-31 2017-09-15] Test [2017-09-16 2017-10-01]\n",
      "Window Train/Val/Test shape (1281, 39) (336, 39) (336, 39)\n",
      "Window Train/Val/Test shape (1281, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-09-14] Val [2017-09-15 2017-09-30] Test [2017-10-01 2017-10-16]\n",
      "Window Train/Val/Test shape (1596, 39) (336, 39) (336, 39)\n",
      "Window Train/Val/Test shape (1596, 2) (336, 2) (336, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-09-29] Val [2017-09-30 2017-10-15] Test [2017-10-16 2017-11-10]\n",
      "Window Train/Val/Test shape (1911, 39) (336, 39) (546, 39)\n",
      "Window Train/Val/Test shape (1911, 2) (336, 2) (546, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-10-14] Val [2017-10-15 2017-10-30] Test [2017-10-31 2017-11-10]\n",
      "Window Train/Val/Test shape (2226, 39) (336, 39) (231, 39)\n",
      "Window Train/Val/Test shape (2226, 2) (336, 2) (231, 2)\n",
      "[105.36169914790273, 84.28787518857354, 33.3214182435893, 14.323928138098617, 8.334673476252789]\n",
      "mean mse: 49.12591883888339\n",
      "Train [2017-07-01 2017-08-15] Val [2017-08-16 2017-08-31] Test [2017-09-01 2017-09-16]\n",
      "Window Train/Val/Test shape (966, 39) (336, 39) (336, 39)\n",
      "Window Train/Val/Test shape (966, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-08-30] Val [2017-08-31 2017-09-15] Test [2017-09-16 2017-10-01]\n",
      "Window Train/Val/Test shape (1281, 39) (336, 39) (336, 39)\n",
      "Window Train/Val/Test shape (1281, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-09-14] Val [2017-09-15 2017-09-30] Test [2017-10-01 2017-10-16]\n",
      "Window Train/Val/Test shape (1596, 39) (336, 39) (336, 39)\n",
      "Window Train/Val/Test shape (1596, 2) (336, 2) (336, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-09-29] Val [2017-09-30 2017-10-15] Test [2017-10-16 2017-11-10]\n",
      "Window Train/Val/Test shape (1911, 39) (336, 39) (546, 39)\n",
      "Window Train/Val/Test shape (1911, 2) (336, 2) (546, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-10-14] Val [2017-10-15 2017-10-30] Test [2017-10-31 2017-11-10]\n",
      "Window Train/Val/Test shape (2226, 39) (336, 39) (231, 39)\n",
      "Window Train/Val/Test shape (2226, 2) (336, 2) (231, 2)\n",
      "[107.02616995585699, 86.30577470410208, 55.93434575703066, 14.543967770174028, 5.628279442594858]\n",
      "mean mse: 53.887707525951726\n",
      "Train [2017-07-01 2017-08-15] Val [2017-08-16 2017-08-31] Test [2017-09-01 2017-09-16]\n",
      "Window Train/Val/Test shape (966, 39) (336, 39) (336, 39)\n",
      "Window Train/Val/Test shape (966, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-08-30] Val [2017-08-31 2017-09-15] Test [2017-09-16 2017-10-01]\n",
      "Window Train/Val/Test shape (1281, 39) (336, 39) (336, 39)\n",
      "Window Train/Val/Test shape (1281, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-09-14] Val [2017-09-15 2017-09-30] Test [2017-10-01 2017-10-16]\n",
      "Window Train/Val/Test shape (1596, 39) (336, 39) (336, 39)\n",
      "Window Train/Val/Test shape (1596, 2) (336, 2) (336, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-09-29] Val [2017-09-30 2017-10-15] Test [2017-10-16 2017-11-10]\n",
      "Window Train/Val/Test shape (1911, 39) (336, 39) (546, 39)\n",
      "Window Train/Val/Test shape (1911, 2) (336, 2) (546, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-10-14] Val [2017-10-15 2017-10-30] Test [2017-10-31 2017-11-10]\n",
      "Window Train/Val/Test shape (2226, 39) (336, 39) (231, 39)\n",
      "Window Train/Val/Test shape (2226, 2) (336, 2) (231, 2)\n",
      "[103.86781566005534, 79.14133659016217, 36.41456407769389, 12.846824394892549, 5.956708456272319]\n",
      "mean mse: 47.64544983581525\n",
      "Train [2017-07-01 2017-08-15] Val [2017-08-16 2017-08-31] Test [2017-09-01 2017-09-16]\n",
      "Window Train/Val/Test shape (966, 39) (336, 39) (336, 39)\n",
      "Window Train/Val/Test shape (966, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-08-30] Val [2017-08-31 2017-09-15] Test [2017-09-16 2017-10-01]\n",
      "Window Train/Val/Test shape (1281, 39) (336, 39) (336, 39)\n",
      "Window Train/Val/Test shape (1281, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-09-14] Val [2017-09-15 2017-09-30] Test [2017-10-01 2017-10-16]\n",
      "Window Train/Val/Test shape (1596, 39) (336, 39) (336, 39)\n",
      "Window Train/Val/Test shape (1596, 2) (336, 2) (336, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-09-29] Val [2017-09-30 2017-10-15] Test [2017-10-16 2017-11-10]\n",
      "Window Train/Val/Test shape (1911, 39) (336, 39) (546, 39)\n",
      "Window Train/Val/Test shape (1911, 2) (336, 2) (546, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-10-14] Val [2017-10-15 2017-10-30] Test [2017-10-31 2017-11-10]\n",
      "Window Train/Val/Test shape (2226, 39) (336, 39) (231, 39)\n",
      "Window Train/Val/Test shape (2226, 2) (336, 2) (231, 2)\n",
      "[105.17301586334592, 75.27867918199397, 36.918978855979425, 13.362189363464026, 4.824896325745462]\n",
      "mean mse: 47.111551918105754\n",
      "Train [2017-07-01 2017-08-15] Val [2017-08-16 2017-08-31] Test [2017-09-01 2017-09-16]\n",
      "Window Train/Val/Test shape (966, 39) (336, 39) (336, 39)\n",
      "Window Train/Val/Test shape (966, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-08-30] Val [2017-08-31 2017-09-15] Test [2017-09-16 2017-10-01]\n",
      "Window Train/Val/Test shape (1281, 39) (336, 39) (336, 39)\n",
      "Window Train/Val/Test shape (1281, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-09-14] Val [2017-09-15 2017-09-30] Test [2017-10-01 2017-10-16]\n",
      "Window Train/Val/Test shape (1596, 39) (336, 39) (336, 39)\n",
      "Window Train/Val/Test shape (1596, 2) (336, 2) (336, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-09-29] Val [2017-09-30 2017-10-15] Test [2017-10-16 2017-11-10]\n",
      "Window Train/Val/Test shape (1911, 39) (336, 39) (546, 39)\n",
      "Window Train/Val/Test shape (1911, 2) (336, 2) (546, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-10-14] Val [2017-10-15 2017-10-30] Test [2017-10-31 2017-11-10]\n",
      "Window Train/Val/Test shape (2226, 39) (336, 39) (231, 39)\n",
      "Window Train/Val/Test shape (2226, 2) (336, 2) (231, 2)\n",
      "[1221.4747544990453, 946.0202819161093, 504.6565290411965, 324.13823796560195, 291.370343515604]\n",
      "mean mse: 657.5320293875113\n",
      "Train [2017-07-01 2017-08-15] Val [2017-08-16 2017-08-31] Test [2017-09-01 2017-09-16]\n",
      "Window Train/Val/Test shape (966, 39) (336, 39) (336, 39)\n",
      "Window Train/Val/Test shape (966, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-08-30] Val [2017-08-31 2017-09-15] Test [2017-09-16 2017-10-01]\n",
      "Window Train/Val/Test shape (1281, 39) (336, 39) (336, 39)\n",
      "Window Train/Val/Test shape (1281, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-09-14] Val [2017-09-15 2017-09-30] Test [2017-10-01 2017-10-16]\n",
      "Window Train/Val/Test shape (1596, 39) (336, 39) (336, 39)\n",
      "Window Train/Val/Test shape (1596, 2) (336, 2) (336, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-09-29] Val [2017-09-30 2017-10-15] Test [2017-10-16 2017-11-10]\n",
      "Window Train/Val/Test shape (1911, 39) (336, 39) (546, 39)\n",
      "Window Train/Val/Test shape (1911, 2) (336, 2) (546, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-10-14] Val [2017-10-15 2017-10-30] Test [2017-10-31 2017-11-10]\n",
      "Window Train/Val/Test shape (2226, 39) (336, 39) (231, 39)\n",
      "Window Train/Val/Test shape (2226, 2) (336, 2) (231, 2)\n",
      "[104.01046772549799, 74.72911401849353, 32.5093559382855, 15.236632343293959, 4.365843037989793]\n",
      "mean mse: 46.17028261271216\n",
      "Train [2017-07-01 2017-08-15] Val [2017-08-16 2017-08-31] Test [2017-09-01 2017-09-16]\n",
      "Window Train/Val/Test shape (966, 39) (336, 39) (336, 39)\n",
      "Window Train/Val/Test shape (966, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-08-30] Val [2017-08-31 2017-09-15] Test [2017-09-16 2017-10-01]\n",
      "Window Train/Val/Test shape (1281, 39) (336, 39) (336, 39)\n",
      "Window Train/Val/Test shape (1281, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-09-14] Val [2017-09-15 2017-09-30] Test [2017-10-01 2017-10-16]\n",
      "Window Train/Val/Test shape (1596, 39) (336, 39) (336, 39)\n",
      "Window Train/Val/Test shape (1596, 2) (336, 2) (336, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-09-29] Val [2017-09-30 2017-10-15] Test [2017-10-16 2017-11-10]\n",
      "Window Train/Val/Test shape (1911, 39) (336, 39) (546, 39)\n",
      "Window Train/Val/Test shape (1911, 2) (336, 2) (546, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-10-14] Val [2017-10-15 2017-10-30] Test [2017-10-31 2017-11-10]\n",
      "Window Train/Val/Test shape (2226, 39) (336, 39) (231, 39)\n",
      "Window Train/Val/Test shape (2226, 2) (336, 2) (231, 2)\n",
      "[101.36496680283521, 77.60793123900922, 40.92406366030906, 13.14777140976514, 4.247426566687089]\n",
      "mean mse: 47.458431935721144\n",
      "Train [2017-07-01 2017-08-15] Val [2017-08-16 2017-08-31] Test [2017-09-01 2017-09-16]\n",
      "Window Train/Val/Test shape (966, 39) (336, 39) (336, 39)\n",
      "Window Train/Val/Test shape (966, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-08-30] Val [2017-08-31 2017-09-15] Test [2017-09-16 2017-10-01]\n",
      "Window Train/Val/Test shape (1281, 39) (336, 39) (336, 39)\n",
      "Window Train/Val/Test shape (1281, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-09-14] Val [2017-09-15 2017-09-30] Test [2017-10-01 2017-10-16]\n",
      "Window Train/Val/Test shape (1596, 39) (336, 39) (336, 39)\n",
      "Window Train/Val/Test shape (1596, 2) (336, 2) (336, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-09-29] Val [2017-09-30 2017-10-15] Test [2017-10-16 2017-11-10]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Window Train/Val/Test shape (1911, 39) (336, 39) (546, 39)\n",
      "Window Train/Val/Test shape (1911, 2) (336, 2) (546, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-10-14] Val [2017-10-15 2017-10-30] Test [2017-10-31 2017-11-10]\n",
      "Window Train/Val/Test shape (2226, 39) (336, 39) (231, 39)\n",
      "Window Train/Val/Test shape (2226, 2) (336, 2) (231, 2)\n",
      "[1221.4747544990453, 946.0202819161093, 504.6565290411965, 324.13823796560195, 291.370343515604]\n",
      "mean mse: 657.5320293875113\n",
      "Train [2017-07-01 2017-08-15] Val [2017-08-16 2017-08-31] Test [2017-09-01 2017-09-16]\n",
      "Window Train/Val/Test shape (966, 39) (336, 39) (336, 39)\n",
      "Window Train/Val/Test shape (966, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-08-30] Val [2017-08-31 2017-09-15] Test [2017-09-16 2017-10-01]\n",
      "Window Train/Val/Test shape (1281, 39) (336, 39) (336, 39)\n",
      "Window Train/Val/Test shape (1281, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-09-14] Val [2017-09-15 2017-09-30] Test [2017-10-01 2017-10-16]\n",
      "Window Train/Val/Test shape (1596, 39) (336, 39) (336, 39)\n",
      "Window Train/Val/Test shape (1596, 2) (336, 2) (336, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-09-29] Val [2017-09-30 2017-10-15] Test [2017-10-16 2017-11-10]\n",
      "Window Train/Val/Test shape (1911, 39) (336, 39) (546, 39)\n",
      "Window Train/Val/Test shape (1911, 2) (336, 2) (546, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-10-14] Val [2017-10-15 2017-10-30] Test [2017-10-31 2017-11-10]\n",
      "Window Train/Val/Test shape (2226, 39) (336, 39) (231, 39)\n",
      "Window Train/Val/Test shape (2226, 2) (336, 2) (231, 2)\n",
      "[110.1581410912798, 80.04090145518587, 39.24816046709163, 16.262840862669382, 5.268357024663731]\n",
      "mean mse: 50.19568018017809\n",
      "Train [2017-07-01 2017-08-15] Val [2017-08-16 2017-08-31] Test [2017-09-01 2017-09-16]\n",
      "Window Train/Val/Test shape (966, 39) (336, 39) (336, 39)\n",
      "Window Train/Val/Test shape (966, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-08-30] Val [2017-08-31 2017-09-15] Test [2017-09-16 2017-10-01]\n",
      "Window Train/Val/Test shape (1281, 39) (336, 39) (336, 39)\n",
      "Window Train/Val/Test shape (1281, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-09-14] Val [2017-09-15 2017-09-30] Test [2017-10-01 2017-10-16]\n",
      "Window Train/Val/Test shape (1596, 39) (336, 39) (336, 39)\n",
      "Window Train/Val/Test shape (1596, 2) (336, 2) (336, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-09-29] Val [2017-09-30 2017-10-15] Test [2017-10-16 2017-11-10]\n",
      "Window Train/Val/Test shape (1911, 39) (336, 39) (546, 39)\n",
      "Window Train/Val/Test shape (1911, 2) (336, 2) (546, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-10-14] Val [2017-10-15 2017-10-30] Test [2017-10-31 2017-11-10]\n",
      "Window Train/Val/Test shape (2226, 39) (336, 39) (231, 39)\n",
      "Window Train/Val/Test shape (2226, 2) (336, 2) (231, 2)\n",
      "[105.70308665851726, 97.4811629426254, 44.1438334366252, 17.32969598067759, 6.1411331816732]\n",
      "mean mse: 54.15978244002372\n",
      "Train [2017-07-01 2017-08-15] Val [2017-08-16 2017-08-31] Test [2017-09-01 2017-09-16]\n",
      "Window Train/Val/Test shape (966, 39) (336, 39) (336, 39)\n",
      "Window Train/Val/Test shape (966, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-08-30] Val [2017-08-31 2017-09-15] Test [2017-09-16 2017-10-01]\n",
      "Window Train/Val/Test shape (1281, 39) (336, 39) (336, 39)\n",
      "Window Train/Val/Test shape (1281, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-09-14] Val [2017-09-15 2017-09-30] Test [2017-10-01 2017-10-16]\n",
      "Window Train/Val/Test shape (1596, 39) (336, 39) (336, 39)\n",
      "Window Train/Val/Test shape (1596, 2) (336, 2) (336, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-09-29] Val [2017-09-30 2017-10-15] Test [2017-10-16 2017-11-10]\n",
      "Window Train/Val/Test shape (1911, 39) (336, 39) (546, 39)\n",
      "Window Train/Val/Test shape (1911, 2) (336, 2) (546, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-10-14] Val [2017-10-15 2017-10-30] Test [2017-10-31 2017-11-10]\n",
      "Window Train/Val/Test shape (2226, 39) (336, 39) (231, 39)\n",
      "Window Train/Val/Test shape (2226, 2) (336, 2) (231, 2)\n",
      "[1221.4747544990453, 946.0202819161093, 504.6565290411965, 324.13823796560195, 291.370343515604]\n",
      "mean mse: 657.5320293875113\n",
      "Train [2017-07-01 2017-08-15] Val [2017-08-16 2017-08-31] Test [2017-09-01 2017-09-16]\n",
      "Window Train/Val/Test shape (966, 39) (336, 39) (336, 39)\n",
      "Window Train/Val/Test shape (966, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-08-30] Val [2017-08-31 2017-09-15] Test [2017-09-16 2017-10-01]\n",
      "Window Train/Val/Test shape (1281, 39) (336, 39) (336, 39)\n",
      "Window Train/Val/Test shape (1281, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-09-14] Val [2017-09-15 2017-09-30] Test [2017-10-01 2017-10-16]\n",
      "Window Train/Val/Test shape (1596, 39) (336, 39) (336, 39)\n",
      "Window Train/Val/Test shape (1596, 2) (336, 2) (336, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-09-29] Val [2017-09-30 2017-10-15] Test [2017-10-16 2017-11-10]\n",
      "Window Train/Val/Test shape (1911, 39) (336, 39) (546, 39)\n",
      "Window Train/Val/Test shape (1911, 2) (336, 2) (546, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-10-14] Val [2017-10-15 2017-10-30] Test [2017-10-31 2017-11-10]\n",
      "Window Train/Val/Test shape (2226, 39) (336, 39) (231, 39)\n",
      "Window Train/Val/Test shape (2226, 2) (336, 2) (231, 2)\n",
      "[1221.4747544990453, 946.0202819161093, 504.6565290411965, 324.13823796560195, 291.370343515604]\n",
      "mean mse: 657.5320293875113\n",
      "Train [2017-07-01 2017-08-15] Val [2017-08-16 2017-08-31] Test [2017-09-01 2017-09-16]\n",
      "Window Train/Val/Test shape (966, 39) (336, 39) (336, 39)\n",
      "Window Train/Val/Test shape (966, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-08-30] Val [2017-08-31 2017-09-15] Test [2017-09-16 2017-10-01]\n",
      "Window Train/Val/Test shape (1281, 39) (336, 39) (336, 39)\n",
      "Window Train/Val/Test shape (1281, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-09-14] Val [2017-09-15 2017-09-30] Test [2017-10-01 2017-10-16]\n",
      "Window Train/Val/Test shape (1596, 39) (336, 39) (336, 39)\n",
      "Window Train/Val/Test shape (1596, 2) (336, 2) (336, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-09-29] Val [2017-09-30 2017-10-15] Test [2017-10-16 2017-11-10]\n",
      "Window Train/Val/Test shape (1911, 39) (336, 39) (546, 39)\n",
      "Window Train/Val/Test shape (1911, 2) (336, 2) (546, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-10-14] Val [2017-10-15 2017-10-30] Test [2017-10-31 2017-11-10]\n",
      "Window Train/Val/Test shape (2226, 39) (336, 39) (231, 39)\n",
      "Window Train/Val/Test shape (2226, 2) (336, 2) (231, 2)\n",
      "[107.77837219245951, 73.52084028132658, 30.556398138249932, 15.18300644670784, 7.032812181340929]\n",
      "mean mse: 46.81428584801696\n",
      "Train [2017-07-01 2017-08-15] Val [2017-08-16 2017-08-31] Test [2017-09-01 2017-09-16]\n",
      "Window Train/Val/Test shape (966, 39) (336, 39) (336, 39)\n",
      "Window Train/Val/Test shape (966, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-08-30] Val [2017-08-31 2017-09-15] Test [2017-09-16 2017-10-01]\n",
      "Window Train/Val/Test shape (1281, 39) (336, 39) (336, 39)\n",
      "Window Train/Val/Test shape (1281, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-09-14] Val [2017-09-15 2017-09-30] Test [2017-10-01 2017-10-16]\n",
      "Window Train/Val/Test shape (1596, 39) (336, 39) (336, 39)\n",
      "Window Train/Val/Test shape (1596, 2) (336, 2) (336, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-09-29] Val [2017-09-30 2017-10-15] Test [2017-10-16 2017-11-10]\n",
      "Window Train/Val/Test shape (1911, 39) (336, 39) (546, 39)\n",
      "Window Train/Val/Test shape (1911, 2) (336, 2) (546, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-10-14] Val [2017-10-15 2017-10-30] Test [2017-10-31 2017-11-10]\n",
      "Window Train/Val/Test shape (2226, 39) (336, 39) (231, 39)\n",
      "Window Train/Val/Test shape (2226, 2) (336, 2) (231, 2)\n",
      "[104.32857656631909, 76.04392642613354, 36.194766707230166, 12.660505311006304, 6.633855413340877]\n",
      "mean mse: 47.17232608480599\n",
      "Train [2017-07-01 2017-08-15] Val [2017-08-16 2017-08-31] Test [2017-09-01 2017-09-16]\n",
      "Window Train/Val/Test shape (966, 39) (336, 39) (336, 39)\n",
      "Window Train/Val/Test shape (966, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-08-30] Val [2017-08-31 2017-09-15] Test [2017-09-16 2017-10-01]\n",
      "Window Train/Val/Test shape (1281, 39) (336, 39) (336, 39)\n",
      "Window Train/Val/Test shape (1281, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-09-14] Val [2017-09-15 2017-09-30] Test [2017-10-01 2017-10-16]\n",
      "Window Train/Val/Test shape (1596, 39) (336, 39) (336, 39)\n",
      "Window Train/Val/Test shape (1596, 2) (336, 2) (336, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-09-29] Val [2017-09-30 2017-10-15] Test [2017-10-16 2017-11-10]\n",
      "Window Train/Val/Test shape (1911, 39) (336, 39) (546, 39)\n",
      "Window Train/Val/Test shape (1911, 2) (336, 2) (546, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-10-14] Val [2017-10-15 2017-10-30] Test [2017-10-31 2017-11-10]\n",
      "Window Train/Val/Test shape (2226, 39) (336, 39) (231, 39)\n",
      "Window Train/Val/Test shape (2226, 2) (336, 2) (231, 2)\n",
      "[1221.4747544990453, 946.0202819161093, 504.6565290411965, 324.13823796560195, 291.370343515604]\n",
      "mean mse: 657.5320293875113\n",
      "Train [2017-07-01 2017-08-15] Val [2017-08-16 2017-08-31] Test [2017-09-01 2017-09-16]\n",
      "Window Train/Val/Test shape (966, 39) (336, 39) (336, 39)\n",
      "Window Train/Val/Test shape (966, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-08-30] Val [2017-08-31 2017-09-15] Test [2017-09-16 2017-10-01]\n",
      "Window Train/Val/Test shape (1281, 39) (336, 39) (336, 39)\n",
      "Window Train/Val/Test shape (1281, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-09-14] Val [2017-09-15 2017-09-30] Test [2017-10-01 2017-10-16]\n",
      "Window Train/Val/Test shape (1596, 39) (336, 39) (336, 39)\n",
      "Window Train/Val/Test shape (1596, 2) (336, 2) (336, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-09-29] Val [2017-09-30 2017-10-15] Test [2017-10-16 2017-11-10]\n",
      "Window Train/Val/Test shape (1911, 39) (336, 39) (546, 39)\n",
      "Window Train/Val/Test shape (1911, 2) (336, 2) (546, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-10-14] Val [2017-10-15 2017-10-30] Test [2017-10-31 2017-11-10]\n",
      "Window Train/Val/Test shape (2226, 39) (336, 39) (231, 39)\n",
      "Window Train/Val/Test shape (2226, 2) (336, 2) (231, 2)\n",
      "[122.42885923588578, 72.87310627400382, 35.13503619689496, 24.39983257445655, 8.92538065698247]\n",
      "mean mse: 52.75244298764473\n",
      "Train [2017-07-01 2017-08-15] Val [2017-08-16 2017-08-31] Test [2017-09-01 2017-09-16]\n",
      "Window Train/Val/Test shape (966, 39) (336, 39) (336, 39)\n",
      "Window Train/Val/Test shape (966, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-08-30] Val [2017-08-31 2017-09-15] Test [2017-09-16 2017-10-01]\n",
      "Window Train/Val/Test shape (1281, 39) (336, 39) (336, 39)\n",
      "Window Train/Val/Test shape (1281, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-09-14] Val [2017-09-15 2017-09-30] Test [2017-10-01 2017-10-16]\n",
      "Window Train/Val/Test shape (1596, 39) (336, 39) (336, 39)\n",
      "Window Train/Val/Test shape (1596, 2) (336, 2) (336, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-09-29] Val [2017-09-30 2017-10-15] Test [2017-10-16 2017-11-10]\n",
      "Window Train/Val/Test shape (1911, 39) (336, 39) (546, 39)\n",
      "Window Train/Val/Test shape (1911, 2) (336, 2) (546, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-10-14] Val [2017-10-15 2017-10-30] Test [2017-10-31 2017-11-10]\n",
      "Window Train/Val/Test shape (2226, 39) (336, 39) (231, 39)\n",
      "Window Train/Val/Test shape (2226, 2) (336, 2) (231, 2)\n",
      "[112.0029013361428, 72.7245447348801, 36.22188252743594, 14.975825596738655, 13.794069045608536]\n",
      "mean mse: 49.943844648161196\n",
      "Train [2017-07-01 2017-08-15] Val [2017-08-16 2017-08-31] Test [2017-09-01 2017-09-16]\n",
      "Window Train/Val/Test shape (966, 39) (336, 39) (336, 39)\n",
      "Window Train/Val/Test shape (966, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-08-30] Val [2017-08-31 2017-09-15] Test [2017-09-16 2017-10-01]\n",
      "Window Train/Val/Test shape (1281, 39) (336, 39) (336, 39)\n",
      "Window Train/Val/Test shape (1281, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-09-14] Val [2017-09-15 2017-09-30] Test [2017-10-01 2017-10-16]\n",
      "Window Train/Val/Test shape (1596, 39) (336, 39) (336, 39)\n",
      "Window Train/Val/Test shape (1596, 2) (336, 2) (336, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-09-29] Val [2017-09-30 2017-10-15] Test [2017-10-16 2017-11-10]\n",
      "Window Train/Val/Test shape (1911, 39) (336, 39) (546, 39)\n",
      "Window Train/Val/Test shape (1911, 2) (336, 2) (546, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-10-14] Val [2017-10-15 2017-10-30] Test [2017-10-31 2017-11-10]\n",
      "Window Train/Val/Test shape (2226, 39) (336, 39) (231, 39)\n",
      "Window Train/Val/Test shape (2226, 2) (336, 2) (231, 2)\n",
      "[1221.4747544990453, 946.0202819161093, 504.6565290411965, 324.13823796560195, 291.370343515604]\n",
      "mean mse: 657.5320293875113\n",
      "Train [2017-07-01 2017-08-15] Val [2017-08-16 2017-08-31] Test [2017-09-01 2017-09-16]\n",
      "Window Train/Val/Test shape (966, 39) (336, 39) (336, 39)\n",
      "Window Train/Val/Test shape (966, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-08-30] Val [2017-08-31 2017-09-15] Test [2017-09-16 2017-10-01]\n",
      "Window Train/Val/Test shape (1281, 39) (336, 39) (336, 39)\n",
      "Window Train/Val/Test shape (1281, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-09-14] Val [2017-09-15 2017-09-30] Test [2017-10-01 2017-10-16]\n",
      "Window Train/Val/Test shape (1596, 39) (336, 39) (336, 39)\n",
      "Window Train/Val/Test shape (1596, 2) (336, 2) (336, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-09-29] Val [2017-09-30 2017-10-15] Test [2017-10-16 2017-11-10]\n",
      "Window Train/Val/Test shape (1911, 39) (336, 39) (546, 39)\n",
      "Window Train/Val/Test shape (1911, 2) (336, 2) (546, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-10-14] Val [2017-10-15 2017-10-30] Test [2017-10-31 2017-11-10]\n",
      "Window Train/Val/Test shape (2226, 39) (336, 39) (231, 39)\n",
      "Window Train/Val/Test shape (2226, 2) (336, 2) (231, 2)\n",
      "[1221.4747544990453, 946.0202819161093, 504.6565290411965, 324.13823796560195, 291.370343515604]\n",
      "mean mse: 657.5320293875113\n",
      "Train [2017-07-01 2017-08-15] Val [2017-08-16 2017-08-31] Test [2017-09-01 2017-09-16]\n",
      "Window Train/Val/Test shape (966, 39) (336, 39) (336, 39)\n",
      "Window Train/Val/Test shape (966, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-08-30] Val [2017-08-31 2017-09-15] Test [2017-09-16 2017-10-01]\n",
      "Window Train/Val/Test shape (1281, 39) (336, 39) (336, 39)\n",
      "Window Train/Val/Test shape (1281, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-09-14] Val [2017-09-15 2017-09-30] Test [2017-10-01 2017-10-16]\n",
      "Window Train/Val/Test shape (1596, 39) (336, 39) (336, 39)\n",
      "Window Train/Val/Test shape (1596, 2) (336, 2) (336, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-09-29] Val [2017-09-30 2017-10-15] Test [2017-10-16 2017-11-10]\n",
      "Window Train/Val/Test shape (1911, 39) (336, 39) (546, 39)\n",
      "Window Train/Val/Test shape (1911, 2) (336, 2) (546, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-10-14] Val [2017-10-15 2017-10-30] Test [2017-10-31 2017-11-10]\n",
      "Window Train/Val/Test shape (2226, 39) (336, 39) (231, 39)\n",
      "Window Train/Val/Test shape (2226, 2) (336, 2) (231, 2)\n",
      "[1221.4747544990453, 946.0202819161093, 504.6565290411965, 324.13823796560195, 291.370343515604]\n",
      "mean mse: 657.5320293875113\n",
      "Train [2017-07-01 2017-08-15] Val [2017-08-16 2017-08-31] Test [2017-09-01 2017-09-16]\n",
      "Window Train/Val/Test shape (966, 39) (336, 39) (336, 39)\n",
      "Window Train/Val/Test shape (966, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-08-30] Val [2017-08-31 2017-09-15] Test [2017-09-16 2017-10-01]\n",
      "Window Train/Val/Test shape (1281, 39) (336, 39) (336, 39)\n",
      "Window Train/Val/Test shape (1281, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-09-14] Val [2017-09-15 2017-09-30] Test [2017-10-01 2017-10-16]\n",
      "Window Train/Val/Test shape (1596, 39) (336, 39) (336, 39)\n",
      "Window Train/Val/Test shape (1596, 2) (336, 2) (336, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-09-29] Val [2017-09-30 2017-10-15] Test [2017-10-16 2017-11-10]\n",
      "Window Train/Val/Test shape (1911, 39) (336, 39) (546, 39)\n",
      "Window Train/Val/Test shape (1911, 2) (336, 2) (546, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-10-14] Val [2017-10-15 2017-10-30] Test [2017-10-31 2017-11-10]\n",
      "Window Train/Val/Test shape (2226, 39) (336, 39) (231, 39)\n",
      "Window Train/Val/Test shape (2226, 2) (336, 2) (231, 2)\n",
      "[115.47480605191248, 71.45862603846457, 29.952346636542373, 14.639859940153588, 5.249099716246155]\n",
      "mean mse: 47.354947676663826\n",
      "Train [2017-07-01 2017-08-15] Val [2017-08-16 2017-08-31] Test [2017-09-01 2017-09-16]\n",
      "Window Train/Val/Test shape (966, 39) (336, 39) (336, 39)\n",
      "Window Train/Val/Test shape (966, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-08-30] Val [2017-08-31 2017-09-15] Test [2017-09-16 2017-10-01]\n",
      "Window Train/Val/Test shape (1281, 39) (336, 39) (336, 39)\n",
      "Window Train/Val/Test shape (1281, 2) (336, 2) (336, 2)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train [2017-07-01 2017-09-14] Val [2017-09-15 2017-09-30] Test [2017-10-01 2017-10-16]\n",
      "Window Train/Val/Test shape (1596, 39) (336, 39) (336, 39)\n",
      "Window Train/Val/Test shape (1596, 2) (336, 2) (336, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-09-29] Val [2017-09-30 2017-10-15] Test [2017-10-16 2017-11-10]\n",
      "Window Train/Val/Test shape (1911, 39) (336, 39) (546, 39)\n",
      "Window Train/Val/Test shape (1911, 2) (336, 2) (546, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-10-14] Val [2017-10-15 2017-10-30] Test [2017-10-31 2017-11-10]\n",
      "Window Train/Val/Test shape (2226, 39) (336, 39) (231, 39)\n",
      "Window Train/Val/Test shape (2226, 2) (336, 2) (231, 2)\n",
      "[1221.4747544990453, 946.0202819161093, 504.6565290411965, 324.13823796560195, 291.370343515604]\n",
      "mean mse: 657.5320293875113\n",
      "Train [2017-07-01 2017-08-15] Val [2017-08-16 2017-08-31] Test [2017-09-01 2017-09-16]\n",
      "Window Train/Val/Test shape (966, 39) (336, 39) (336, 39)\n",
      "Window Train/Val/Test shape (966, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-08-30] Val [2017-08-31 2017-09-15] Test [2017-09-16 2017-10-01]\n",
      "Window Train/Val/Test shape (1281, 39) (336, 39) (336, 39)\n",
      "Window Train/Val/Test shape (1281, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-09-14] Val [2017-09-15 2017-09-30] Test [2017-10-01 2017-10-16]\n",
      "Window Train/Val/Test shape (1596, 39) (336, 39) (336, 39)\n",
      "Window Train/Val/Test shape (1596, 2) (336, 2) (336, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-09-29] Val [2017-09-30 2017-10-15] Test [2017-10-16 2017-11-10]\n",
      "Window Train/Val/Test shape (1911, 39) (336, 39) (546, 39)\n",
      "Window Train/Val/Test shape (1911, 2) (336, 2) (546, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-10-14] Val [2017-10-15 2017-10-30] Test [2017-10-31 2017-11-10]\n",
      "Window Train/Val/Test shape (2226, 39) (336, 39) (231, 39)\n",
      "Window Train/Val/Test shape (2226, 2) (336, 2) (231, 2)\n",
      "[110.27492341372788, 87.18080418032518, 34.280026756023005, 17.300016734970686, 6.936654290871981]\n",
      "mean mse: 51.19448507518375\n",
      "Train [2017-07-01 2017-08-15] Val [2017-08-16 2017-08-31] Test [2017-09-01 2017-09-16]\n",
      "Window Train/Val/Test shape (966, 39) (336, 39) (336, 39)\n",
      "Window Train/Val/Test shape (966, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-08-30] Val [2017-08-31 2017-09-15] Test [2017-09-16 2017-10-01]\n",
      "Window Train/Val/Test shape (1281, 39) (336, 39) (336, 39)\n",
      "Window Train/Val/Test shape (1281, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-09-14] Val [2017-09-15 2017-09-30] Test [2017-10-01 2017-10-16]\n",
      "Window Train/Val/Test shape (1596, 39) (336, 39) (336, 39)\n",
      "Window Train/Val/Test shape (1596, 2) (336, 2) (336, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-09-29] Val [2017-09-30 2017-10-15] Test [2017-10-16 2017-11-10]\n",
      "Window Train/Val/Test shape (1911, 39) (336, 39) (546, 39)\n",
      "Window Train/Val/Test shape (1911, 2) (336, 2) (546, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-10-14] Val [2017-10-15 2017-10-30] Test [2017-10-31 2017-11-10]\n",
      "Window Train/Val/Test shape (2226, 39) (336, 39) (231, 39)\n",
      "Window Train/Val/Test shape (2226, 2) (336, 2) (231, 2)\n",
      "[101.14412875170812, 88.08726490257138, 46.28569445038433, 14.571119116735995, 7.777499911580482]\n",
      "mean mse: 51.57314142659605\n",
      "Train [2017-07-01 2017-08-15] Val [2017-08-16 2017-08-31] Test [2017-09-01 2017-09-16]\n",
      "Window Train/Val/Test shape (966, 39) (336, 39) (336, 39)\n",
      "Window Train/Val/Test shape (966, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-08-30] Val [2017-08-31 2017-09-15] Test [2017-09-16 2017-10-01]\n",
      "Window Train/Val/Test shape (1281, 39) (336, 39) (336, 39)\n",
      "Window Train/Val/Test shape (1281, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-09-14] Val [2017-09-15 2017-09-30] Test [2017-10-01 2017-10-16]\n",
      "Window Train/Val/Test shape (1596, 39) (336, 39) (336, 39)\n",
      "Window Train/Val/Test shape (1596, 2) (336, 2) (336, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-09-29] Val [2017-09-30 2017-10-15] Test [2017-10-16 2017-11-10]\n",
      "Window Train/Val/Test shape (1911, 39) (336, 39) (546, 39)\n",
      "Window Train/Val/Test shape (1911, 2) (336, 2) (546, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-10-14] Val [2017-10-15 2017-10-30] Test [2017-10-31 2017-11-10]\n",
      "Window Train/Val/Test shape (2226, 39) (336, 39) (231, 39)\n",
      "Window Train/Val/Test shape (2226, 2) (336, 2) (231, 2)\n",
      "[118.54646246834338, 73.75451383027819, 39.95211782480166, 15.012353319190746, 10.277160671355395]\n",
      "mean mse: 51.508521622793864\n",
      "Train [2017-07-01 2017-08-15] Val [2017-08-16 2017-08-31] Test [2017-09-01 2017-09-16]\n",
      "Window Train/Val/Test shape (966, 39) (336, 39) (336, 39)\n",
      "Window Train/Val/Test shape (966, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-08-30] Val [2017-08-31 2017-09-15] Test [2017-09-16 2017-10-01]\n",
      "Window Train/Val/Test shape (1281, 39) (336, 39) (336, 39)\n",
      "Window Train/Val/Test shape (1281, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-09-14] Val [2017-09-15 2017-09-30] Test [2017-10-01 2017-10-16]\n",
      "Window Train/Val/Test shape (1596, 39) (336, 39) (336, 39)\n",
      "Window Train/Val/Test shape (1596, 2) (336, 2) (336, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-09-29] Val [2017-09-30 2017-10-15] Test [2017-10-16 2017-11-10]\n",
      "Window Train/Val/Test shape (1911, 39) (336, 39) (546, 39)\n",
      "Window Train/Val/Test shape (1911, 2) (336, 2) (546, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-10-14] Val [2017-10-15 2017-10-30] Test [2017-10-31 2017-11-10]\n",
      "Window Train/Val/Test shape (2226, 39) (336, 39) (231, 39)\n",
      "Window Train/Val/Test shape (2226, 2) (336, 2) (231, 2)\n",
      "[107.40451401534764, 87.97231726522897, 48.050497144058816, 16.206720824323284, 11.287689270761962]\n",
      "mean mse: 54.18434770394414\n",
      "Train [2017-07-01 2017-08-15] Val [2017-08-16 2017-08-31] Test [2017-09-01 2017-09-16]\n",
      "Window Train/Val/Test shape (966, 39) (336, 39) (336, 39)\n",
      "Window Train/Val/Test shape (966, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-08-30] Val [2017-08-31 2017-09-15] Test [2017-09-16 2017-10-01]\n",
      "Window Train/Val/Test shape (1281, 39) (336, 39) (336, 39)\n",
      "Window Train/Val/Test shape (1281, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-09-14] Val [2017-09-15 2017-09-30] Test [2017-10-01 2017-10-16]\n",
      "Window Train/Val/Test shape (1596, 39) (336, 39) (336, 39)\n",
      "Window Train/Val/Test shape (1596, 2) (336, 2) (336, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-09-29] Val [2017-09-30 2017-10-15] Test [2017-10-16 2017-11-10]\n",
      "Window Train/Val/Test shape (1911, 39) (336, 39) (546, 39)\n",
      "Window Train/Val/Test shape (1911, 2) (336, 2) (546, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-10-14] Val [2017-10-15 2017-10-30] Test [2017-10-31 2017-11-10]\n",
      "Window Train/Val/Test shape (2226, 39) (336, 39) (231, 39)\n",
      "Window Train/Val/Test shape (2226, 2) (336, 2) (231, 2)\n",
      "[111.33126313781428, 88.95548298691759, 42.52380241257813, 14.347108792892659, 8.429847460159339]\n",
      "mean mse: 53.1175009580724\n",
      "Train [2017-07-01 2017-08-15] Val [2017-08-16 2017-08-31] Test [2017-09-01 2017-09-16]\n",
      "Window Train/Val/Test shape (966, 39) (336, 39) (336, 39)\n",
      "Window Train/Val/Test shape (966, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-08-30] Val [2017-08-31 2017-09-15] Test [2017-09-16 2017-10-01]\n",
      "Window Train/Val/Test shape (1281, 39) (336, 39) (336, 39)\n",
      "Window Train/Val/Test shape (1281, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-09-14] Val [2017-09-15 2017-09-30] Test [2017-10-01 2017-10-16]\n",
      "Window Train/Val/Test shape (1596, 39) (336, 39) (336, 39)\n",
      "Window Train/Val/Test shape (1596, 2) (336, 2) (336, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-09-29] Val [2017-09-30 2017-10-15] Test [2017-10-16 2017-11-10]\n",
      "Window Train/Val/Test shape (1911, 39) (336, 39) (546, 39)\n",
      "Window Train/Val/Test shape (1911, 2) (336, 2) (546, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-10-14] Val [2017-10-15 2017-10-30] Test [2017-10-31 2017-11-10]\n",
      "Window Train/Val/Test shape (2226, 39) (336, 39) (231, 39)\n",
      "Window Train/Val/Test shape (2226, 2) (336, 2) (231, 2)\n",
      "[109.60225673666731, 74.67604305486218, 27.043217643766845, 13.890276734390552, 5.319280205731232]\n",
      "mean mse: 46.106214875083616\n",
      "Train [2017-07-01 2017-08-15] Val [2017-08-16 2017-08-31] Test [2017-09-01 2017-09-16]\n",
      "Window Train/Val/Test shape (966, 39) (336, 39) (336, 39)\n",
      "Window Train/Val/Test shape (966, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-08-30] Val [2017-08-31 2017-09-15] Test [2017-09-16 2017-10-01]\n",
      "Window Train/Val/Test shape (1281, 39) (336, 39) (336, 39)\n",
      "Window Train/Val/Test shape (1281, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-09-14] Val [2017-09-15 2017-09-30] Test [2017-10-01 2017-10-16]\n",
      "Window Train/Val/Test shape (1596, 39) (336, 39) (336, 39)\n",
      "Window Train/Val/Test shape (1596, 2) (336, 2) (336, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-09-29] Val [2017-09-30 2017-10-15] Test [2017-10-16 2017-11-10]\n",
      "Window Train/Val/Test shape (1911, 39) (336, 39) (546, 39)\n",
      "Window Train/Val/Test shape (1911, 2) (336, 2) (546, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-10-14] Val [2017-10-15 2017-10-30] Test [2017-10-31 2017-11-10]\n",
      "Window Train/Val/Test shape (2226, 39) (336, 39) (231, 39)\n",
      "Window Train/Val/Test shape (2226, 2) (336, 2) (231, 2)\n",
      "[106.4953395852816, 82.49688858367729, 33.24475832863729, 14.272343135610399, 12.218546350127424]\n",
      "mean mse: 49.74557519666679\n",
      "Train [2017-07-01 2017-08-15] Val [2017-08-16 2017-08-31] Test [2017-09-01 2017-09-16]\n",
      "Window Train/Val/Test shape (966, 39) (336, 39) (336, 39)\n",
      "Window Train/Val/Test shape (966, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-08-30] Val [2017-08-31 2017-09-15] Test [2017-09-16 2017-10-01]\n",
      "Window Train/Val/Test shape (1281, 39) (336, 39) (336, 39)\n",
      "Window Train/Val/Test shape (1281, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-09-14] Val [2017-09-15 2017-09-30] Test [2017-10-01 2017-10-16]\n",
      "Window Train/Val/Test shape (1596, 39) (336, 39) (336, 39)\n",
      "Window Train/Val/Test shape (1596, 2) (336, 2) (336, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-09-29] Val [2017-09-30 2017-10-15] Test [2017-10-16 2017-11-10]\n",
      "Window Train/Val/Test shape (1911, 39) (336, 39) (546, 39)\n",
      "Window Train/Val/Test shape (1911, 2) (336, 2) (546, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-10-14] Val [2017-10-15 2017-10-30] Test [2017-10-31 2017-11-10]\n",
      "Window Train/Val/Test shape (2226, 39) (336, 39) (231, 39)\n",
      "Window Train/Val/Test shape (2226, 2) (336, 2) (231, 2)\n",
      "[1221.4747544990453, 946.0202819161093, 504.6565290411965, 324.13823796560195, 291.370343515604]\n",
      "mean mse: 657.5320293875113\n",
      "Train [2017-07-01 2017-08-15] Val [2017-08-16 2017-08-31] Test [2017-09-01 2017-09-16]\n",
      "Window Train/Val/Test shape (966, 39) (336, 39) (336, 39)\n",
      "Window Train/Val/Test shape (966, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-08-30] Val [2017-08-31 2017-09-15] Test [2017-09-16 2017-10-01]\n",
      "Window Train/Val/Test shape (1281, 39) (336, 39) (336, 39)\n",
      "Window Train/Val/Test shape (1281, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-09-14] Val [2017-09-15 2017-09-30] Test [2017-10-01 2017-10-16]\n",
      "Window Train/Val/Test shape (1596, 39) (336, 39) (336, 39)\n",
      "Window Train/Val/Test shape (1596, 2) (336, 2) (336, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-09-29] Val [2017-09-30 2017-10-15] Test [2017-10-16 2017-11-10]\n",
      "Window Train/Val/Test shape (1911, 39) (336, 39) (546, 39)\n",
      "Window Train/Val/Test shape (1911, 2) (336, 2) (546, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-10-14] Val [2017-10-15 2017-10-30] Test [2017-10-31 2017-11-10]\n",
      "Window Train/Val/Test shape (2226, 39) (336, 39) (231, 39)\n",
      "Window Train/Val/Test shape (2226, 2) (336, 2) (231, 2)\n",
      "[1221.4747544990453, 946.0202819161093, 504.6565290411965, 324.13823796560195, 291.370343515604]\n",
      "mean mse: 657.5320293875113\n",
      "Train [2017-07-01 2017-08-15] Val [2017-08-16 2017-08-31] Test [2017-09-01 2017-09-16]\n",
      "Window Train/Val/Test shape (966, 39) (336, 39) (336, 39)\n",
      "Window Train/Val/Test shape (966, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-08-30] Val [2017-08-31 2017-09-15] Test [2017-09-16 2017-10-01]\n",
      "Window Train/Val/Test shape (1281, 39) (336, 39) (336, 39)\n",
      "Window Train/Val/Test shape (1281, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-09-14] Val [2017-09-15 2017-09-30] Test [2017-10-01 2017-10-16]\n",
      "Window Train/Val/Test shape (1596, 39) (336, 39) (336, 39)\n",
      "Window Train/Val/Test shape (1596, 2) (336, 2) (336, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-09-29] Val [2017-09-30 2017-10-15] Test [2017-10-16 2017-11-10]\n",
      "Window Train/Val/Test shape (1911, 39) (336, 39) (546, 39)\n",
      "Window Train/Val/Test shape (1911, 2) (336, 2) (546, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-10-14] Val [2017-10-15 2017-10-30] Test [2017-10-31 2017-11-10]\n",
      "Window Train/Val/Test shape (2226, 39) (336, 39) (231, 39)\n",
      "Window Train/Val/Test shape (2226, 2) (336, 2) (231, 2)\n",
      "[109.11220192637319, 72.77566464807067, 38.61900791686383, 13.417079275539379, 5.327264399571328]\n",
      "mean mse: 47.85024363328368\n",
      "Train [2017-07-01 2017-08-15] Val [2017-08-16 2017-08-31] Test [2017-09-01 2017-09-16]\n",
      "Window Train/Val/Test shape (966, 39) (336, 39) (336, 39)\n",
      "Window Train/Val/Test shape (966, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-08-30] Val [2017-08-31 2017-09-15] Test [2017-09-16 2017-10-01]\n",
      "Window Train/Val/Test shape (1281, 39) (336, 39) (336, 39)\n",
      "Window Train/Val/Test shape (1281, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-09-14] Val [2017-09-15 2017-09-30] Test [2017-10-01 2017-10-16]\n",
      "Window Train/Val/Test shape (1596, 39) (336, 39) (336, 39)\n",
      "Window Train/Val/Test shape (1596, 2) (336, 2) (336, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-09-29] Val [2017-09-30 2017-10-15] Test [2017-10-16 2017-11-10]\n",
      "Window Train/Val/Test shape (1911, 39) (336, 39) (546, 39)\n",
      "Window Train/Val/Test shape (1911, 2) (336, 2) (546, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-10-14] Val [2017-10-15 2017-10-30] Test [2017-10-31 2017-11-10]\n",
      "Window Train/Val/Test shape (2226, 39) (336, 39) (231, 39)\n",
      "Window Train/Val/Test shape (2226, 2) (336, 2) (231, 2)\n",
      "[119.84047904160057, 67.86068694358808, 35.347752747257466, 14.335149185254862, 10.4734060255549]\n",
      "mean mse: 49.57149478865118\n",
      "Train [2017-07-01 2017-08-15] Val [2017-08-16 2017-08-31] Test [2017-09-01 2017-09-16]\n",
      "Window Train/Val/Test shape (966, 39) (336, 39) (336, 39)\n",
      "Window Train/Val/Test shape (966, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-08-30] Val [2017-08-31 2017-09-15] Test [2017-09-16 2017-10-01]\n",
      "Window Train/Val/Test shape (1281, 39) (336, 39) (336, 39)\n",
      "Window Train/Val/Test shape (1281, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-09-14] Val [2017-09-15 2017-09-30] Test [2017-10-01 2017-10-16]\n",
      "Window Train/Val/Test shape (1596, 39) (336, 39) (336, 39)\n",
      "Window Train/Val/Test shape (1596, 2) (336, 2) (336, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-09-29] Val [2017-09-30 2017-10-15] Test [2017-10-16 2017-11-10]\n",
      "Window Train/Val/Test shape (1911, 39) (336, 39) (546, 39)\n",
      "Window Train/Val/Test shape (1911, 2) (336, 2) (546, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-10-14] Val [2017-10-15 2017-10-30] Test [2017-10-31 2017-11-10]\n",
      "Window Train/Val/Test shape (2226, 39) (336, 39) (231, 39)\n",
      "Window Train/Val/Test shape (2226, 2) (336, 2) (231, 2)\n",
      "[109.827730443537, 74.73848352254981, 30.890675787625476, 14.300058469861415, 7.122357772438253]\n",
      "mean mse: 47.37586119920239\n",
      "Train [2017-07-01 2017-08-15] Val [2017-08-16 2017-08-31] Test [2017-09-01 2017-09-16]\n",
      "Window Train/Val/Test shape (966, 39) (336, 39) (336, 39)\n",
      "Window Train/Val/Test shape (966, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-08-30] Val [2017-08-31 2017-09-15] Test [2017-09-16 2017-10-01]\n",
      "Window Train/Val/Test shape (1281, 39) (336, 39) (336, 39)\n",
      "Window Train/Val/Test shape (1281, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-09-14] Val [2017-09-15 2017-09-30] Test [2017-10-01 2017-10-16]\n",
      "Window Train/Val/Test shape (1596, 39) (336, 39) (336, 39)\n",
      "Window Train/Val/Test shape (1596, 2) (336, 2) (336, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-09-29] Val [2017-09-30 2017-10-15] Test [2017-10-16 2017-11-10]\n",
      "Window Train/Val/Test shape (1911, 39) (336, 39) (546, 39)\n",
      "Window Train/Val/Test shape (1911, 2) (336, 2) (546, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-10-14] Val [2017-10-15 2017-10-30] Test [2017-10-31 2017-11-10]\n",
      "Window Train/Val/Test shape (2226, 39) (336, 39) (231, 39)\n",
      "Window Train/Val/Test shape (2226, 2) (336, 2) (231, 2)\n",
      "[1221.4747544990453, 946.0202819161093, 504.6565290411965, 324.13823796560195, 291.370343515604]\n",
      "mean mse: 657.5320293875113\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train [2017-07-01 2017-08-15] Val [2017-08-16 2017-08-31] Test [2017-09-01 2017-09-16]\n",
      "Window Train/Val/Test shape (966, 39) (336, 39) (336, 39)\n",
      "Window Train/Val/Test shape (966, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-08-30] Val [2017-08-31 2017-09-15] Test [2017-09-16 2017-10-01]\n",
      "Window Train/Val/Test shape (1281, 39) (336, 39) (336, 39)\n",
      "Window Train/Val/Test shape (1281, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-09-14] Val [2017-09-15 2017-09-30] Test [2017-10-01 2017-10-16]\n",
      "Window Train/Val/Test shape (1596, 39) (336, 39) (336, 39)\n",
      "Window Train/Val/Test shape (1596, 2) (336, 2) (336, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-09-29] Val [2017-09-30 2017-10-15] Test [2017-10-16 2017-11-10]\n",
      "Window Train/Val/Test shape (1911, 39) (336, 39) (546, 39)\n",
      "Window Train/Val/Test shape (1911, 2) (336, 2) (546, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-10-14] Val [2017-10-15 2017-10-30] Test [2017-10-31 2017-11-10]\n",
      "Window Train/Val/Test shape (2226, 39) (336, 39) (231, 39)\n",
      "Window Train/Val/Test shape (2226, 2) (336, 2) (231, 2)\n",
      "[1221.4747544990453, 946.0202819161093, 504.6565290411965, 324.13823796560195, 291.370343515604]\n",
      "mean mse: 657.5320293875113\n",
      "Train [2017-07-01 2017-08-15] Val [2017-08-16 2017-08-31] Test [2017-09-01 2017-09-16]\n",
      "Window Train/Val/Test shape (966, 39) (336, 39) (336, 39)\n",
      "Window Train/Val/Test shape (966, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-08-30] Val [2017-08-31 2017-09-15] Test [2017-09-16 2017-10-01]\n",
      "Window Train/Val/Test shape (1281, 39) (336, 39) (336, 39)\n",
      "Window Train/Val/Test shape (1281, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-09-14] Val [2017-09-15 2017-09-30] Test [2017-10-01 2017-10-16]\n",
      "Window Train/Val/Test shape (1596, 39) (336, 39) (336, 39)\n",
      "Window Train/Val/Test shape (1596, 2) (336, 2) (336, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-09-29] Val [2017-09-30 2017-10-15] Test [2017-10-16 2017-11-10]\n",
      "Window Train/Val/Test shape (1911, 39) (336, 39) (546, 39)\n",
      "Window Train/Val/Test shape (1911, 2) (336, 2) (546, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-10-14] Val [2017-10-15 2017-10-30] Test [2017-10-31 2017-11-10]\n",
      "Window Train/Val/Test shape (2226, 39) (336, 39) (231, 39)\n",
      "Window Train/Val/Test shape (2226, 2) (336, 2) (231, 2)\n",
      "[120.03530316081348, 82.0803800451954, 26.08071687094012, 17.39674517464949, 8.436201852355758]\n",
      "mean mse: 50.80586942079085\n",
      "Train [2017-07-01 2017-08-15] Val [2017-08-16 2017-08-31] Test [2017-09-01 2017-09-16]\n",
      "Window Train/Val/Test shape (966, 39) (336, 39) (336, 39)\n",
      "Window Train/Val/Test shape (966, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-08-30] Val [2017-08-31 2017-09-15] Test [2017-09-16 2017-10-01]\n",
      "Window Train/Val/Test shape (1281, 39) (336, 39) (336, 39)\n",
      "Window Train/Val/Test shape (1281, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-09-14] Val [2017-09-15 2017-09-30] Test [2017-10-01 2017-10-16]\n",
      "Window Train/Val/Test shape (1596, 39) (336, 39) (336, 39)\n",
      "Window Train/Val/Test shape (1596, 2) (336, 2) (336, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-09-29] Val [2017-09-30 2017-10-15] Test [2017-10-16 2017-11-10]\n",
      "Window Train/Val/Test shape (1911, 39) (336, 39) (546, 39)\n",
      "Window Train/Val/Test shape (1911, 2) (336, 2) (546, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-10-14] Val [2017-10-15 2017-10-30] Test [2017-10-31 2017-11-10]\n",
      "Window Train/Val/Test shape (2226, 39) (336, 39) (231, 39)\n",
      "Window Train/Val/Test shape (2226, 2) (336, 2) (231, 2)\n",
      "[1221.4747544990453, 946.0202819161093, 504.6565290411965, 324.13823796560195, 291.370343515604]\n",
      "mean mse: 657.5320293875113\n",
      "Train [2017-07-01 2017-08-15] Val [2017-08-16 2017-08-31] Test [2017-09-01 2017-09-16]\n",
      "Window Train/Val/Test shape (966, 39) (336, 39) (336, 39)\n",
      "Window Train/Val/Test shape (966, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-08-30] Val [2017-08-31 2017-09-15] Test [2017-09-16 2017-10-01]\n",
      "Window Train/Val/Test shape (1281, 39) (336, 39) (336, 39)\n",
      "Window Train/Val/Test shape (1281, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-09-14] Val [2017-09-15 2017-09-30] Test [2017-10-01 2017-10-16]\n",
      "Window Train/Val/Test shape (1596, 39) (336, 39) (336, 39)\n",
      "Window Train/Val/Test shape (1596, 2) (336, 2) (336, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-09-29] Val [2017-09-30 2017-10-15] Test [2017-10-16 2017-11-10]\n",
      "Window Train/Val/Test shape (1911, 39) (336, 39) (546, 39)\n",
      "Window Train/Val/Test shape (1911, 2) (336, 2) (546, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-10-14] Val [2017-10-15 2017-10-30] Test [2017-10-31 2017-11-10]\n",
      "Window Train/Val/Test shape (2226, 39) (336, 39) (231, 39)\n",
      "Window Train/Val/Test shape (2226, 2) (336, 2) (231, 2)\n",
      "[115.10051641375031, 75.84003935352902, 34.55082793549444, 16.52864291967859, 9.864265839361732]\n",
      "mean mse: 50.376858492362814\n"
     ]
    }
   ],
   "source": [
    "orchestrator12.performHyperparameterOptimization(X_12_preselect[selected_features12], 100)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 144,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'colsample_bytree': 0.75,\n",
       " 'gamma': 1.0,\n",
       " 'learning_rate': 0.05,\n",
       " 'max_depth': 6.0,\n",
       " 'min_child_weight': 1.0,\n",
       " 'n_estimators': 450.0,\n",
       " 'subsample': 0.9}"
      ]
     },
     "execution_count": 144,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "orchestrator12.bo.best"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "metadata": {},
   "outputs": [],
   "source": [
    "postselect_params_12 = {'colsample_bytree': 0.75,\n",
    "                         'gamma': 1.0,\n",
    "                         'learning_rate': 0.05,\n",
    "                         'max_depth': 6,\n",
    "                         'min_child_weight': 1.0,\n",
    "                         'n_estimators': 450,\n",
    "                         'subsample': 0.9}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 162,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\Rohil\\Anaconda3\\lib\\site-packages\\ipykernel_launcher.py:78: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/indexing.html#indexing-view-versus-copy\n",
      "C:\\Users\\Rohil\\Anaconda3\\lib\\site-packages\\pandas\\core\\frame.py:3694: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame\n",
      "\n",
      "See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/indexing.html#indexing-view-versus-copy\n",
      "  errors=errors)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train [2017-07-01 2017-08-15] Val [2017-08-16 2017-08-31] Test [2017-09-01 2017-09-16]\n",
      "Window Train/Val/Test shape (966, 39) (336, 39) (336, 39)\n",
      "Window Train/Val/Test shape (966, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-08-30] Val [2017-08-31 2017-09-15] Test [2017-09-16 2017-10-01]\n",
      "Window Train/Val/Test shape (1281, 39) (336, 39) (336, 39)\n",
      "Window Train/Val/Test shape (1281, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-09-14] Val [2017-09-15 2017-09-30] Test [2017-10-01 2017-10-16]\n",
      "Window Train/Val/Test shape (1596, 39) (336, 39) (336, 39)\n",
      "Window Train/Val/Test shape (1596, 2) (336, 2) (336, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-09-29] Val [2017-09-30 2017-10-15] Test [2017-10-16 2017-11-10]\n",
      "Window Train/Val/Test shape (1911, 39) (336, 39) (546, 39)\n",
      "Window Train/Val/Test shape (1911, 2) (336, 2) (546, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-10-14] Val [2017-10-15 2017-10-30] Test [2017-10-31 2017-11-10]\n",
      "Window Train/Val/Test shape (2226, 39) (336, 39) (231, 39)\n",
      "Window Train/Val/Test shape (2226, 2) (336, 2) (231, 2)\n",
      "y-test mean 12.50826011533399, y-test std 12.131448080014314\n",
      "cv mse 44.71533595672869, holdout mse18.77693774828676\n"
     ]
    }
   ],
   "source": [
    "orchestrator12.deploy(0.001643, postselect_params_12, selected_x=None)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [],
   "source": [
    "# running feature selection for 2-4 week model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Running TSFresh....\n",
      "Selected features (4305, 15146)\n",
      "Finished running TSFresh....\n"
     ]
    }
   ],
   "source": [
    "orchestrator24.runTsFresh()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Running Feature Selection module ....\n",
      "4810 features with a correlation magnitude greater than 0.97.\n",
      "\n",
      "Original (4305, 15147) and after (4305, 10337)\n",
      "Finished running Feature Selection ....\n"
     ]
    }
   ],
   "source": [
    "orchestrator24.runOptimizeFeatures()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 64,
   "metadata": {},
   "outputs": [],
   "source": [
    "X_24_preselect = orchestrator24.op.corr_selected_features\n",
    "y_24_preselect = orchestrator24.full_data_bi_week[['date', orchestrator24.y_bi_week_label]]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 65,
   "metadata": {},
   "outputs": [],
   "source": [
    "# X_24_preselect.to_csv('/Users/Rohil/Documents/iGEM/yemen/intermediary_files/X_24_preselect.csv')\n",
    "# y_24_preselect.to_csv('/Users/Rohil/Documents/iGEM/yemen/intermediary_files/y_24_preselect.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [],
   "source": [
    "orchestrator24 = Orchestrator(y2_4.dropna().merge(full_features, how = 'left', on = ['gov_iso', 'date']).sort_values('date'), 'week_2_to_4_cases' )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [],
   "source": [
    "# X_24_preselect = pd.read_csv('/Users/Rohil/Documents/iGEM/yemen/intermediary_files/X_24_preselect.csv', index_col = 0)\n",
    "# X_24_preselect.date = pd.to_datetime(X_24_preselect.date, format = '%Y-%m-%d')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 83,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train [2017-07-01 2017-08-15] Val [2017-08-16 2017-08-31] Test [2017-09-01 2017-09-16]\n",
      "Window Train/Val/Test shape (966, 10337) (336, 10337) (336, 10337)\n",
      "Window Train/Val/Test shape (966, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-08-30] Val [2017-08-31 2017-09-15] Test [2017-09-16 2017-10-01]\n",
      "Window Train/Val/Test shape (1281, 10337) (336, 10337) (336, 10337)\n",
      "Window Train/Val/Test shape (1281, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-09-14] Val [2017-09-15 2017-09-30] Test [2017-10-01 2017-10-16]\n",
      "Window Train/Val/Test shape (1596, 10337) (336, 10337) (336, 10337)\n",
      "Window Train/Val/Test shape (1596, 2) (336, 2) (336, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-09-29] Val [2017-09-30 2017-10-15] Test [2017-10-16 2017-11-10]\n",
      "Window Train/Val/Test shape (1911, 10337) (336, 10337) (546, 10337)\n",
      "Window Train/Val/Test shape (1911, 2) (336, 2) (546, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-10-14] Val [2017-10-15 2017-10-30] Test [2017-10-31 2017-11-10]\n",
      "Window Train/Val/Test shape (2226, 10337) (336, 10337) (231, 10337)\n",
      "Window Train/Val/Test shape (2226, 2) (336, 2) (231, 2)\n",
      "[111.61804677229122, 108.63686685614721, 53.073593370926204, 28.869864997078118, 18.623377713928964]\n",
      "mean mse: 64.16434994207435\n",
      "Train [2017-07-01 2017-08-15] Val [2017-08-16 2017-08-31] Test [2017-09-01 2017-09-16]\n",
      "Window Train/Val/Test shape (966, 10337) (336, 10337) (336, 10337)\n",
      "Window Train/Val/Test shape (966, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-08-30] Val [2017-08-31 2017-09-15] Test [2017-09-16 2017-10-01]\n",
      "Window Train/Val/Test shape (1281, 10337) (336, 10337) (336, 10337)\n",
      "Window Train/Val/Test shape (1281, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-09-14] Val [2017-09-15 2017-09-30] Test [2017-10-01 2017-10-16]\n",
      "Window Train/Val/Test shape (1596, 10337) (336, 10337) (336, 10337)\n",
      "Window Train/Val/Test shape (1596, 2) (336, 2) (336, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-09-29] Val [2017-09-30 2017-10-15] Test [2017-10-16 2017-11-10]\n",
      "Window Train/Val/Test shape (1911, 10337) (336, 10337) (546, 10337)\n",
      "Window Train/Val/Test shape (1911, 2) (336, 2) (546, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-10-14] Val [2017-10-15 2017-10-30] Test [2017-10-31 2017-11-10]\n",
      "Window Train/Val/Test shape (2226, 10337) (336, 10337) (231, 10337)\n",
      "Window Train/Val/Test shape (2226, 2) (336, 2) (231, 2)\n",
      "[125.85589131565195, 103.88363074152076, 61.34991068150878, 23.852292118844478, 26.611452893365485]\n",
      "mean mse: 68.31063555017829\n",
      "Train [2017-07-01 2017-08-15] Val [2017-08-16 2017-08-31] Test [2017-09-01 2017-09-16]\n",
      "Window Train/Val/Test shape (966, 10337) (336, 10337) (336, 10337)\n",
      "Window Train/Val/Test shape (966, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-08-30] Val [2017-08-31 2017-09-15] Test [2017-09-16 2017-10-01]\n",
      "Window Train/Val/Test shape (1281, 10337) (336, 10337) (336, 10337)\n",
      "Window Train/Val/Test shape (1281, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-09-14] Val [2017-09-15 2017-09-30] Test [2017-10-01 2017-10-16]\n",
      "Window Train/Val/Test shape (1596, 10337) (336, 10337) (336, 10337)\n",
      "Window Train/Val/Test shape (1596, 2) (336, 2) (336, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-09-29] Val [2017-09-30 2017-10-15] Test [2017-10-16 2017-11-10]\n",
      "Window Train/Val/Test shape (1911, 10337) (336, 10337) (546, 10337)\n",
      "Window Train/Val/Test shape (1911, 2) (336, 2) (546, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-10-14] Val [2017-10-15 2017-10-30] Test [2017-10-31 2017-11-10]\n",
      "Window Train/Val/Test shape (2226, 10337) (336, 10337) (231, 10337)\n",
      "Window Train/Val/Test shape (2226, 2) (336, 2) (231, 2)\n",
      "[147.0163785979502, 91.15533527256345, 50.2186958380491, 29.128293281205863, 19.460440185701177]\n",
      "mean mse: 67.39582863509396\n",
      "Train [2017-07-01 2017-08-15] Val [2017-08-16 2017-08-31] Test [2017-09-01 2017-09-16]\n",
      "Window Train/Val/Test shape (966, 10337) (336, 10337) (336, 10337)\n",
      "Window Train/Val/Test shape (966, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-08-30] Val [2017-08-31 2017-09-15] Test [2017-09-16 2017-10-01]\n",
      "Window Train/Val/Test shape (1281, 10337) (336, 10337) (336, 10337)\n",
      "Window Train/Val/Test shape (1281, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-09-14] Val [2017-09-15 2017-09-30] Test [2017-10-01 2017-10-16]\n",
      "Window Train/Val/Test shape (1596, 10337) (336, 10337) (336, 10337)\n",
      "Window Train/Val/Test shape (1596, 2) (336, 2) (336, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-09-29] Val [2017-09-30 2017-10-15] Test [2017-10-16 2017-11-10]\n",
      "Window Train/Val/Test shape (1911, 10337) (336, 10337) (546, 10337)\n",
      "Window Train/Val/Test shape (1911, 2) (336, 2) (546, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-10-14] Val [2017-10-15 2017-10-30] Test [2017-10-31 2017-11-10]\n",
      "Window Train/Val/Test shape (2226, 10337) (336, 10337) (231, 10337)\n",
      "Window Train/Val/Test shape (2226, 2) (336, 2) (231, 2)\n",
      "[126.54086698285643, 99.23713390377426, 44.11120983600989, 29.088407374018836, 14.659380566847034]\n",
      "mean mse: 62.72739973270129\n",
      "Train [2017-07-01 2017-08-15] Val [2017-08-16 2017-08-31] Test [2017-09-01 2017-09-16]\n",
      "Window Train/Val/Test shape (966, 10337) (336, 10337) (336, 10337)\n",
      "Window Train/Val/Test shape (966, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-08-30] Val [2017-08-31 2017-09-15] Test [2017-09-16 2017-10-01]\n",
      "Window Train/Val/Test shape (1281, 10337) (336, 10337) (336, 10337)\n",
      "Window Train/Val/Test shape (1281, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-09-14] Val [2017-09-15 2017-09-30] Test [2017-10-01 2017-10-16]\n",
      "Window Train/Val/Test shape (1596, 10337) (336, 10337) (336, 10337)\n",
      "Window Train/Val/Test shape (1596, 2) (336, 2) (336, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-09-29] Val [2017-09-30 2017-10-15] Test [2017-10-16 2017-11-10]\n",
      "Window Train/Val/Test shape (1911, 10337) (336, 10337) (546, 10337)\n",
      "Window Train/Val/Test shape (1911, 2) (336, 2) (546, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-10-14] Val [2017-10-15 2017-10-30] Test [2017-10-31 2017-11-10]\n",
      "Window Train/Val/Test shape (2226, 10337) (336, 10337) (231, 10337)\n",
      "Window Train/Val/Test shape (2226, 2) (336, 2) (231, 2)\n",
      "[124.69332437490482, 96.17936221280804, 41.040066790648076, 23.289188752986323, 11.765182663206648]\n",
      "mean mse: 59.393424958910785\n",
      "Train [2017-07-01 2017-08-15] Val [2017-08-16 2017-08-31] Test [2017-09-01 2017-09-16]\n",
      "Window Train/Val/Test shape (966, 10337) (336, 10337) (336, 10337)\n",
      "Window Train/Val/Test shape (966, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-08-30] Val [2017-08-31 2017-09-15] Test [2017-09-16 2017-10-01]\n",
      "Window Train/Val/Test shape (1281, 10337) (336, 10337) (336, 10337)\n",
      "Window Train/Val/Test shape (1281, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-09-14] Val [2017-09-15 2017-09-30] Test [2017-10-01 2017-10-16]\n",
      "Window Train/Val/Test shape (1596, 10337) (336, 10337) (336, 10337)\n",
      "Window Train/Val/Test shape (1596, 2) (336, 2) (336, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-09-29] Val [2017-09-30 2017-10-15] Test [2017-10-16 2017-11-10]\n",
      "Window Train/Val/Test shape (1911, 10337) (336, 10337) (546, 10337)\n",
      "Window Train/Val/Test shape (1911, 2) (336, 2) (546, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-10-14] Val [2017-10-15 2017-10-30] Test [2017-10-31 2017-11-10]\n",
      "Window Train/Val/Test shape (2226, 10337) (336, 10337) (231, 10337)\n",
      "Window Train/Val/Test shape (2226, 2) (336, 2) (231, 2)\n",
      "[140.3683612131413, 109.52284409031196, 54.47561418704822, 28.498423832780215, 17.67778675768]\n",
      "mean mse: 70.10860601619234\n",
      "Train [2017-07-01 2017-08-15] Val [2017-08-16 2017-08-31] Test [2017-09-01 2017-09-16]\n",
      "Window Train/Val/Test shape (966, 10337) (336, 10337) (336, 10337)\n",
      "Window Train/Val/Test shape (966, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-08-30] Val [2017-08-31 2017-09-15] Test [2017-09-16 2017-10-01]\n",
      "Window Train/Val/Test shape (1281, 10337) (336, 10337) (336, 10337)\n",
      "Window Train/Val/Test shape (1281, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-09-14] Val [2017-09-15 2017-09-30] Test [2017-10-01 2017-10-16]\n",
      "Window Train/Val/Test shape (1596, 10337) (336, 10337) (336, 10337)\n",
      "Window Train/Val/Test shape (1596, 2) (336, 2) (336, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-09-29] Val [2017-09-30 2017-10-15] Test [2017-10-16 2017-11-10]\n",
      "Window Train/Val/Test shape (1911, 10337) (336, 10337) (546, 10337)\n",
      "Window Train/Val/Test shape (1911, 2) (336, 2) (546, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-10-14] Val [2017-10-15 2017-10-30] Test [2017-10-31 2017-11-10]\n",
      "Window Train/Val/Test shape (2226, 10337) (336, 10337) (231, 10337)\n",
      "Window Train/Val/Test shape (2226, 2) (336, 2) (231, 2)\n",
      "[120.1662462639382, 118.17704091344673, 36.10507768097646, 23.519708815021364, 16.024347785060304]\n",
      "mean mse: 62.79848429168861\n",
      "Train [2017-07-01 2017-08-15] Val [2017-08-16 2017-08-31] Test [2017-09-01 2017-09-16]\n",
      "Window Train/Val/Test shape (966, 10337) (336, 10337) (336, 10337)\n",
      "Window Train/Val/Test shape (966, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-08-30] Val [2017-08-31 2017-09-15] Test [2017-09-16 2017-10-01]\n",
      "Window Train/Val/Test shape (1281, 10337) (336, 10337) (336, 10337)\n",
      "Window Train/Val/Test shape (1281, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-09-14] Val [2017-09-15 2017-09-30] Test [2017-10-01 2017-10-16]\n",
      "Window Train/Val/Test shape (1596, 10337) (336, 10337) (336, 10337)\n",
      "Window Train/Val/Test shape (1596, 2) (336, 2) (336, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-09-29] Val [2017-09-30 2017-10-15] Test [2017-10-16 2017-11-10]\n",
      "Window Train/Val/Test shape (1911, 10337) (336, 10337) (546, 10337)\n",
      "Window Train/Val/Test shape (1911, 2) (336, 2) (546, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-10-14] Val [2017-10-15 2017-10-30] Test [2017-10-31 2017-11-10]\n",
      "Window Train/Val/Test shape (2226, 10337) (336, 10337) (231, 10337)\n",
      "Window Train/Val/Test shape (2226, 2) (336, 2) (231, 2)\n",
      "[130.53402003358192, 106.33507411506419, 57.427475556646726, 25.29277417884096, 13.889828237206467]\n",
      "mean mse: 66.69583442426806\n",
      "Train [2017-07-01 2017-08-15] Val [2017-08-16 2017-08-31] Test [2017-09-01 2017-09-16]\n",
      "Window Train/Val/Test shape (966, 10337) (336, 10337) (336, 10337)\n",
      "Window Train/Val/Test shape (966, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-08-30] Val [2017-08-31 2017-09-15] Test [2017-09-16 2017-10-01]\n",
      "Window Train/Val/Test shape (1281, 10337) (336, 10337) (336, 10337)\n",
      "Window Train/Val/Test shape (1281, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-09-14] Val [2017-09-15 2017-09-30] Test [2017-10-01 2017-10-16]\n",
      "Window Train/Val/Test shape (1596, 10337) (336, 10337) (336, 10337)\n",
      "Window Train/Val/Test shape (1596, 2) (336, 2) (336, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-09-29] Val [2017-09-30 2017-10-15] Test [2017-10-16 2017-11-10]\n",
      "Window Train/Val/Test shape (1911, 10337) (336, 10337) (546, 10337)\n",
      "Window Train/Val/Test shape (1911, 2) (336, 2) (546, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-10-14] Val [2017-10-15 2017-10-30] Test [2017-10-31 2017-11-10]\n",
      "Window Train/Val/Test shape (2226, 10337) (336, 10337) (231, 10337)\n",
      "Window Train/Val/Test shape (2226, 2) (336, 2) (231, 2)\n",
      "[91.31883810402685, 102.33065924382178, 38.70097279844833, 22.65845363202696, 14.542013567733338]\n",
      "mean mse: 53.91018746921145\n",
      "Train [2017-07-01 2017-08-15] Val [2017-08-16 2017-08-31] Test [2017-09-01 2017-09-16]\n",
      "Window Train/Val/Test shape (966, 10337) (336, 10337) (336, 10337)\n",
      "Window Train/Val/Test shape (966, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-08-30] Val [2017-08-31 2017-09-15] Test [2017-09-16 2017-10-01]\n",
      "Window Train/Val/Test shape (1281, 10337) (336, 10337) (336, 10337)\n",
      "Window Train/Val/Test shape (1281, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-09-14] Val [2017-09-15 2017-09-30] Test [2017-10-01 2017-10-16]\n",
      "Window Train/Val/Test shape (1596, 10337) (336, 10337) (336, 10337)\n",
      "Window Train/Val/Test shape (1596, 2) (336, 2) (336, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-09-29] Val [2017-09-30 2017-10-15] Test [2017-10-16 2017-11-10]\n",
      "Window Train/Val/Test shape (1911, 10337) (336, 10337) (546, 10337)\n",
      "Window Train/Val/Test shape (1911, 2) (336, 2) (546, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-10-14] Val [2017-10-15 2017-10-30] Test [2017-10-31 2017-11-10]\n",
      "Window Train/Val/Test shape (2226, 10337) (336, 10337) (231, 10337)\n",
      "Window Train/Val/Test shape (2226, 2) (336, 2) (231, 2)\n",
      "[108.08003607627117, 93.64695576789514, 36.819362246729234, 23.96326939164932, 15.433957879167865]\n",
      "mean mse: 55.58871627234255\n",
      "Train [2017-07-01 2017-08-15] Val [2017-08-16 2017-08-31] Test [2017-09-01 2017-09-16]\n",
      "Window Train/Val/Test shape (966, 10337) (336, 10337) (336, 10337)\n",
      "Window Train/Val/Test shape (966, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-08-30] Val [2017-08-31 2017-09-15] Test [2017-09-16 2017-10-01]\n",
      "Window Train/Val/Test shape (1281, 10337) (336, 10337) (336, 10337)\n",
      "Window Train/Val/Test shape (1281, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-09-14] Val [2017-09-15 2017-09-30] Test [2017-10-01 2017-10-16]\n",
      "Window Train/Val/Test shape (1596, 10337) (336, 10337) (336, 10337)\n",
      "Window Train/Val/Test shape (1596, 2) (336, 2) (336, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-09-29] Val [2017-09-30 2017-10-15] Test [2017-10-16 2017-11-10]\n",
      "Window Train/Val/Test shape (1911, 10337) (336, 10337) (546, 10337)\n",
      "Window Train/Val/Test shape (1911, 2) (336, 2) (546, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-10-14] Val [2017-10-15 2017-10-30] Test [2017-10-31 2017-11-10]\n",
      "Window Train/Val/Test shape (2226, 10337) (336, 10337) (231, 10337)\n",
      "Window Train/Val/Test shape (2226, 2) (336, 2) (231, 2)\n",
      "[118.6659095708028, 63.067792083583385, 41.052858203249016, 24.44213454381191, 14.121152314250711]\n",
      "mean mse: 52.269969343139564\n",
      "Train [2017-07-01 2017-08-15] Val [2017-08-16 2017-08-31] Test [2017-09-01 2017-09-16]\n",
      "Window Train/Val/Test shape (966, 10337) (336, 10337) (336, 10337)\n",
      "Window Train/Val/Test shape (966, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-08-30] Val [2017-08-31 2017-09-15] Test [2017-09-16 2017-10-01]\n",
      "Window Train/Val/Test shape (1281, 10337) (336, 10337) (336, 10337)\n",
      "Window Train/Val/Test shape (1281, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-09-14] Val [2017-09-15 2017-09-30] Test [2017-10-01 2017-10-16]\n",
      "Window Train/Val/Test shape (1596, 10337) (336, 10337) (336, 10337)\n",
      "Window Train/Val/Test shape (1596, 2) (336, 2) (336, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-09-29] Val [2017-09-30 2017-10-15] Test [2017-10-16 2017-11-10]\n",
      "Window Train/Val/Test shape (1911, 10337) (336, 10337) (546, 10337)\n",
      "Window Train/Val/Test shape (1911, 2) (336, 2) (546, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-10-14] Val [2017-10-15 2017-10-30] Test [2017-10-31 2017-11-10]\n",
      "Window Train/Val/Test shape (2226, 10337) (336, 10337) (231, 10337)\n",
      "Window Train/Val/Test shape (2226, 2) (336, 2) (231, 2)\n",
      "[109.84412684473652, 92.85172147143643, 54.908753960721185, 27.571638286818715, 15.543233320255137]\n",
      "mean mse: 60.143894776793594\n",
      "Train [2017-07-01 2017-08-15] Val [2017-08-16 2017-08-31] Test [2017-09-01 2017-09-16]\n",
      "Window Train/Val/Test shape (966, 10337) (336, 10337) (336, 10337)\n",
      "Window Train/Val/Test shape (966, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-08-30] Val [2017-08-31 2017-09-15] Test [2017-09-16 2017-10-01]\n",
      "Window Train/Val/Test shape (1281, 10337) (336, 10337) (336, 10337)\n",
      "Window Train/Val/Test shape (1281, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-09-14] Val [2017-09-15 2017-09-30] Test [2017-10-01 2017-10-16]\n",
      "Window Train/Val/Test shape (1596, 10337) (336, 10337) (336, 10337)\n",
      "Window Train/Val/Test shape (1596, 2) (336, 2) (336, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-09-29] Val [2017-09-30 2017-10-15] Test [2017-10-16 2017-11-10]\n",
      "Window Train/Val/Test shape (1911, 10337) (336, 10337) (546, 10337)\n",
      "Window Train/Val/Test shape (1911, 2) (336, 2) (546, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-10-14] Val [2017-10-15 2017-10-30] Test [2017-10-31 2017-11-10]\n",
      "Window Train/Val/Test shape (2226, 10337) (336, 10337) (231, 10337)\n",
      "Window Train/Val/Test shape (2226, 2) (336, 2) (231, 2)\n",
      "[128.2115193503601, 98.82665235758469, 49.503239707765154, 27.7815962707098, 21.589138982206656]\n",
      "mean mse: 65.18242933372528\n",
      "Train [2017-07-01 2017-08-15] Val [2017-08-16 2017-08-31] Test [2017-09-01 2017-09-16]\n",
      "Window Train/Val/Test shape (966, 10337) (336, 10337) (336, 10337)\n",
      "Window Train/Val/Test shape (966, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-08-30] Val [2017-08-31 2017-09-15] Test [2017-09-16 2017-10-01]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Window Train/Val/Test shape (1281, 10337) (336, 10337) (336, 10337)\n",
      "Window Train/Val/Test shape (1281, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-09-14] Val [2017-09-15 2017-09-30] Test [2017-10-01 2017-10-16]\n",
      "Window Train/Val/Test shape (1596, 10337) (336, 10337) (336, 10337)\n",
      "Window Train/Val/Test shape (1596, 2) (336, 2) (336, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-09-29] Val [2017-09-30 2017-10-15] Test [2017-10-16 2017-11-10]\n",
      "Window Train/Val/Test shape (1911, 10337) (336, 10337) (546, 10337)\n",
      "Window Train/Val/Test shape (1911, 2) (336, 2) (546, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-10-14] Val [2017-10-15 2017-10-30] Test [2017-10-31 2017-11-10]\n",
      "Window Train/Val/Test shape (2226, 10337) (336, 10337) (231, 10337)\n",
      "Window Train/Val/Test shape (2226, 2) (336, 2) (231, 2)\n",
      "[109.22358942027475, 93.45216619537848, 34.82277186778052, 20.092146486803387, 13.75400768485361]\n",
      "mean mse: 54.26893633101815\n",
      "Train [2017-07-01 2017-08-15] Val [2017-08-16 2017-08-31] Test [2017-09-01 2017-09-16]\n",
      "Window Train/Val/Test shape (966, 10337) (336, 10337) (336, 10337)\n",
      "Window Train/Val/Test shape (966, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-08-30] Val [2017-08-31 2017-09-15] Test [2017-09-16 2017-10-01]\n",
      "Window Train/Val/Test shape (1281, 10337) (336, 10337) (336, 10337)\n",
      "Window Train/Val/Test shape (1281, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-09-14] Val [2017-09-15 2017-09-30] Test [2017-10-01 2017-10-16]\n",
      "Window Train/Val/Test shape (1596, 10337) (336, 10337) (336, 10337)\n",
      "Window Train/Val/Test shape (1596, 2) (336, 2) (336, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-09-29] Val [2017-09-30 2017-10-15] Test [2017-10-16 2017-11-10]\n",
      "Window Train/Val/Test shape (1911, 10337) (336, 10337) (546, 10337)\n",
      "Window Train/Val/Test shape (1911, 2) (336, 2) (546, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-10-14] Val [2017-10-15 2017-10-30] Test [2017-10-31 2017-11-10]\n",
      "Window Train/Val/Test shape (2226, 10337) (336, 10337) (231, 10337)\n",
      "Window Train/Val/Test shape (2226, 2) (336, 2) (231, 2)\n",
      "[161.32288269444697, 106.94805945175892, 64.49994850444898, 26.611493527714988, 19.120234531636502]\n",
      "mean mse: 75.70052374200128\n",
      "Train [2017-07-01 2017-08-15] Val [2017-08-16 2017-08-31] Test [2017-09-01 2017-09-16]\n",
      "Window Train/Val/Test shape (966, 10337) (336, 10337) (336, 10337)\n",
      "Window Train/Val/Test shape (966, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-08-30] Val [2017-08-31 2017-09-15] Test [2017-09-16 2017-10-01]\n",
      "Window Train/Val/Test shape (1281, 10337) (336, 10337) (336, 10337)\n",
      "Window Train/Val/Test shape (1281, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-09-14] Val [2017-09-15 2017-09-30] Test [2017-10-01 2017-10-16]\n",
      "Window Train/Val/Test shape (1596, 10337) (336, 10337) (336, 10337)\n",
      "Window Train/Val/Test shape (1596, 2) (336, 2) (336, 2)\n"
     ]
    },
    {
     "ename": "KeyboardInterrupt",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
      "\u001b[1;32m<ipython-input-83-83c8e1f6350d>\u001b[0m in \u001b[0;36m<module>\u001b[1;34m()\u001b[0m\n\u001b[1;32m----> 1\u001b[1;33m \u001b[0morchestrator24\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mperformHyperparameterOptimization\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mX_24_preselect\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;36m30\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[1;32m<ipython-input-24-3560c0907952>\u001b[0m in \u001b[0;36mperformHyperparameterOptimization\u001b[1;34m(self, X, max_evals)\u001b[0m\n\u001b[0;32m     34\u001b[0m     \u001b[1;32mdef\u001b[0m \u001b[0mperformHyperparameterOptimization\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mX\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mmax_evals\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     35\u001b[0m         \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mbo\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mBayesianOptimizer\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mX\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mfull_data_bi_week\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;34m'date'\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0my_bi_week_label\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mmax_evals\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m---> 36\u001b[1;33m         \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mbo\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mrun\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m     37\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     38\u001b[0m     \u001b[1;32mdef\u001b[0m \u001b[0mrunRollingCustomFeatureExtractor\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mpreselect_params\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mcorr_selected_features\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m<ipython-input-68-535a0f59a4eb>\u001b[0m in \u001b[0;36mrun\u001b[1;34m(self)\u001b[0m\n\u001b[0;32m     75\u001b[0m                     \u001b[0malgo\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mtpe\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0msuggest\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     76\u001b[0m                     \u001b[0mmax_evals\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mmax_evals\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m---> 77\u001b[1;33m                     trials=self.trials)\n\u001b[0m",
      "\u001b[1;32m~\\Anaconda3\\lib\\site-packages\\hyperopt\\fmin.py\u001b[0m in \u001b[0;36mfmin\u001b[1;34m(fn, space, algo, max_evals, trials, rstate, allow_trials_fmin, pass_expr_memo_ctrl, catch_eval_exceptions, verbose, return_argmin, points_to_evaluate, max_queue_len)\u001b[0m\n\u001b[0;32m    365\u001b[0m             \u001b[0mverbose\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mverbose\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    366\u001b[0m             \u001b[0mcatch_eval_exceptions\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mcatch_eval_exceptions\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 367\u001b[1;33m             \u001b[0mreturn_argmin\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mreturn_argmin\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    368\u001b[0m         )\n\u001b[0;32m    369\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\Anaconda3\\lib\\site-packages\\hyperopt\\base.py\u001b[0m in \u001b[0;36mfmin\u001b[1;34m(self, fn, space, algo, max_evals, rstate, verbose, pass_expr_memo_ctrl, catch_eval_exceptions, return_argmin)\u001b[0m\n\u001b[0;32m    633\u001b[0m             \u001b[0mpass_expr_memo_ctrl\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mpass_expr_memo_ctrl\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    634\u001b[0m             \u001b[0mcatch_eval_exceptions\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mcatch_eval_exceptions\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 635\u001b[1;33m             return_argmin=return_argmin)\n\u001b[0m\u001b[0;32m    636\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    637\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\Anaconda3\\lib\\site-packages\\hyperopt\\fmin.py\u001b[0m in \u001b[0;36mfmin\u001b[1;34m(fn, space, algo, max_evals, trials, rstate, allow_trials_fmin, pass_expr_memo_ctrl, catch_eval_exceptions, verbose, return_argmin, points_to_evaluate, max_queue_len)\u001b[0m\n\u001b[0;32m    383\u001b[0m                     max_queue_len=max_queue_len)\n\u001b[0;32m    384\u001b[0m     \u001b[0mrval\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mcatch_eval_exceptions\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mcatch_eval_exceptions\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 385\u001b[1;33m     \u001b[0mrval\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mexhaust\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    386\u001b[0m     \u001b[1;32mif\u001b[0m \u001b[0mreturn_argmin\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    387\u001b[0m         \u001b[1;32mreturn\u001b[0m \u001b[0mtrials\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0margmin\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\Anaconda3\\lib\\site-packages\\hyperopt\\fmin.py\u001b[0m in \u001b[0;36mexhaust\u001b[1;34m(self)\u001b[0m\n\u001b[0;32m    242\u001b[0m     \u001b[1;32mdef\u001b[0m \u001b[0mexhaust\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    243\u001b[0m         \u001b[0mn_done\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mlen\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mtrials\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 244\u001b[1;33m         \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mrun\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mmax_evals\u001b[0m \u001b[1;33m-\u001b[0m \u001b[0mn_done\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mblock_until_done\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0masynchronous\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    245\u001b[0m         \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mtrials\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mrefresh\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    246\u001b[0m         \u001b[1;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\Anaconda3\\lib\\site-packages\\hyperopt\\fmin.py\u001b[0m in \u001b[0;36mrun\u001b[1;34m(self, N, block_until_done)\u001b[0m\n\u001b[0;32m    216\u001b[0m             \u001b[1;32melse\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    217\u001b[0m                 \u001b[1;31m# -- loop over trials and do the jobs directly\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 218\u001b[1;33m                 \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mserial_evaluate\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    219\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    220\u001b[0m             \u001b[1;32mif\u001b[0m \u001b[0mstopped\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\Anaconda3\\lib\\site-packages\\hyperopt\\fmin.py\u001b[0m in \u001b[0;36mserial_evaluate\u001b[1;34m(self, N)\u001b[0m\n\u001b[0;32m    135\u001b[0m                 \u001b[0mctrl\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mbase\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mCtrl\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mtrials\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mcurrent_trial\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mtrial\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    136\u001b[0m                 \u001b[1;32mtry\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 137\u001b[1;33m                     \u001b[0mresult\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mdomain\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mevaluate\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mspec\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mctrl\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    138\u001b[0m                 \u001b[1;32mexcept\u001b[0m \u001b[0mException\u001b[0m \u001b[1;32mas\u001b[0m \u001b[0me\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    139\u001b[0m                     \u001b[0mlogger\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0minfo\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;34m'job exception: %s'\u001b[0m \u001b[1;33m%\u001b[0m \u001b[0mstr\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0me\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\Anaconda3\\lib\\site-packages\\hyperopt\\base.py\u001b[0m in \u001b[0;36mevaluate\u001b[1;34m(self, config, ctrl, attach_attachments)\u001b[0m\n\u001b[0;32m    838\u001b[0m                 \u001b[0mmemo\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mmemo\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    839\u001b[0m                 print_node_on_error=self.rec_eval_print_node_on_error)\n\u001b[1;32m--> 840\u001b[1;33m             \u001b[0mrval\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mfn\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mpyll_rval\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    841\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    842\u001b[0m         \u001b[1;32mif\u001b[0m \u001b[0misinstance\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mrval\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;33m(\u001b[0m\u001b[0mfloat\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mint\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mnp\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mnumber\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m<ipython-input-68-535a0f59a4eb>\u001b[0m in \u001b[0;36mobjective\u001b[1;34m(self, space)\u001b[0m\n\u001b[0;32m     40\u001b[0m                            )    \n\u001b[0;32m     41\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m---> 42\u001b[1;33m             \u001b[0mxgb\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mfit\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mX_train_cv\u001b[0m \u001b[1;33m,\u001b[0m\u001b[0my_train_cv\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0meval_metric\u001b[0m \u001b[1;33m=\u001b[0m \u001b[1;34m'rmse'\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m     43\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     44\u001b[0m             \u001b[1;31m# eval model\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\Anaconda3\\lib\\site-packages\\xgboost\\sklearn.py\u001b[0m in \u001b[0;36mfit\u001b[1;34m(self, X, y, sample_weight, eval_set, eval_metric, early_stopping_rounds, verbose, xgb_model, sample_weight_eval_set)\u001b[0m\n\u001b[0;32m    322\u001b[0m                               \u001b[0mearly_stopping_rounds\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mearly_stopping_rounds\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    323\u001b[0m                               \u001b[0mevals_result\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mevals_result\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mobj\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mobj\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mfeval\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mfeval\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 324\u001b[1;33m                               verbose_eval=verbose, xgb_model=xgb_model)\n\u001b[0m\u001b[0;32m    325\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    326\u001b[0m         \u001b[1;32mif\u001b[0m \u001b[0mevals_result\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\Anaconda3\\lib\\site-packages\\xgboost\\training.py\u001b[0m in \u001b[0;36mtrain\u001b[1;34m(params, dtrain, num_boost_round, evals, obj, feval, maximize, early_stopping_rounds, evals_result, verbose_eval, xgb_model, callbacks, learning_rates)\u001b[0m\n\u001b[0;32m    202\u001b[0m                            \u001b[0mevals\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mevals\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    203\u001b[0m                            \u001b[0mobj\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mobj\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mfeval\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mfeval\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 204\u001b[1;33m                            xgb_model=xgb_model, callbacks=callbacks)\n\u001b[0m\u001b[0;32m    205\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    206\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\Anaconda3\\lib\\site-packages\\xgboost\\training.py\u001b[0m in \u001b[0;36m_train_internal\u001b[1;34m(params, dtrain, num_boost_round, evals, obj, feval, xgb_model, callbacks)\u001b[0m\n\u001b[0;32m     72\u001b[0m         \u001b[1;31m# Skip the first update if it is a recovery step.\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     73\u001b[0m         \u001b[1;32mif\u001b[0m \u001b[0mversion\u001b[0m \u001b[1;33m%\u001b[0m \u001b[1;36m2\u001b[0m \u001b[1;33m==\u001b[0m \u001b[1;36m0\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m---> 74\u001b[1;33m             \u001b[0mbst\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mupdate\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mdtrain\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mi\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mobj\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m     75\u001b[0m             \u001b[0mbst\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0msave_rabit_checkpoint\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     76\u001b[0m             \u001b[0mversion\u001b[0m \u001b[1;33m+=\u001b[0m \u001b[1;36m1\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\Anaconda3\\lib\\site-packages\\xgboost\\core.py\u001b[0m in \u001b[0;36mupdate\u001b[1;34m(self, dtrain, iteration, fobj)\u001b[0m\n\u001b[0;32m   1019\u001b[0m         \u001b[1;32mif\u001b[0m \u001b[0mfobj\u001b[0m \u001b[1;32mis\u001b[0m \u001b[1;32mNone\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   1020\u001b[0m             _check_call(_LIB.XGBoosterUpdateOneIter(self.handle, ctypes.c_int(iteration),\n\u001b[1;32m-> 1021\u001b[1;33m                                                     dtrain.handle))\n\u001b[0m\u001b[0;32m   1022\u001b[0m         \u001b[1;32melse\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   1023\u001b[0m             \u001b[0mpred\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mpredict\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mdtrain\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;31mKeyboardInterrupt\u001b[0m: "
     ]
    }
   ],
   "source": [
    "orchestrator24.performHyperparameterOptimization(X_24_preselect, 30)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "orchestrator24.bo.best"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [],
   "source": [
    "preselect_params_24 = {'colsample_bytree': 0.9,\n",
    "                      'gamma': 3.5,\n",
    "                      'learning_rate': 0.15,\n",
    "                      'max_depth': 7,\n",
    "                      'min_child_weight': 4.0,\n",
    "                      'n_estimators': 150,\n",
    "                      'subsample': 1.0}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Running custom feature selection module ....\n",
      "Creating baseline model to extract features\n",
      "all features (2688, 10336)\n",
      "12\n",
      "# features 10336 # thresholds 12; thresholds:[0.01227679 0.00892857 0.00669643 0.00613839 0.00446429 0.00390625\n",
      " 0.00279018 0.00223214 0.00167411 0.00111607 0.00055804 0.        ]\n",
      "CV - Window 3\n",
      "Train [2017-07-01 2017-08-15] Val [2017-08-16 2017-08-31] Test [2017-09-01 2017-09-16]\n",
      "Window Train/Val/Test shape (966, 10337) (336, 10337) (336, 10337)\n",
      "Window Train/Val/Test shape (966, 2) (336, 2) (336, 2)\n",
      "X_train_cv  (1302, 10336)\n",
      "3 Thresh=0.01228, n=1, mse: 251.105\n",
      "CV - Window 4\n",
      "Train [2017-07-01 2017-08-30] Val [2017-08-31 2017-09-15] Test [2017-09-16 2017-10-01]\n",
      "Window Train/Val/Test shape (1281, 10337) (336, 10337) (336, 10337)\n",
      "Window Train/Val/Test shape (1281, 2) (336, 2) (336, 2)\n",
      "X_train_cv  (1617, 10336)\n",
      "4 Thresh=0.01228, n=1, mse: 81.223\n",
      "CV - Window 5\n",
      "Train [2017-07-01 2017-09-14] Val [2017-09-15 2017-09-30] Test [2017-10-01 2017-10-16]\n",
      "Window Train/Val/Test shape (1596, 10337) (336, 10337) (336, 10337)\n",
      "Window Train/Val/Test shape (1596, 2) (336, 2) (336, 2)\n",
      "X_train_cv  (1932, 10336)\n",
      "5 Thresh=0.01228, n=1, mse: 58.966\n",
      "CV - Window 6\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-09-29] Val [2017-09-30 2017-10-15] Test [2017-10-16 2017-11-10]\n",
      "Window Train/Val/Test shape (1911, 10337) (336, 10337) (546, 10337)\n",
      "Window Train/Val/Test shape (1911, 2) (336, 2) (546, 2)\n",
      "X_train_cv  (2247, 10336)\n",
      "6 Thresh=0.01228, n=1, mse: 37.126\n",
      "CV - Window 7\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-10-14] Val [2017-10-15 2017-10-30] Test [2017-10-31 2017-11-10]\n",
      "Window Train/Val/Test shape (2226, 10337) (336, 10337) (231, 10337)\n",
      "Window Train/Val/Test shape (2226, 2) (336, 2) (231, 2)\n",
      "X_train_cv  (2562, 10336)\n",
      "7 Thresh=0.01228, n=1, mse: 12.663\n",
      "{'threshold': 0.012276785, 'num_features': 1, 'mse_list': [251.10475750511748, 81.22296992591856, 58.965688718396116, 37.12566554888933, 12.6629638414197], 'mean_mse': 88.21640910794824}\n",
      "\n",
      "\n",
      "CV - Window 3\n",
      "Train [2017-07-01 2017-08-15] Val [2017-08-16 2017-08-31] Test [2017-09-01 2017-09-16]\n",
      "Window Train/Val/Test shape (966, 10337) (336, 10337) (336, 10337)\n",
      "Window Train/Val/Test shape (966, 2) (336, 2) (336, 2)\n",
      "X_train_cv  (1302, 10336)\n",
      "3 Thresh=0.00893, n=2, mse: 228.428\n",
      "CV - Window 4\n",
      "Train [2017-07-01 2017-08-30] Val [2017-08-31 2017-09-15] Test [2017-09-16 2017-10-01]\n",
      "Window Train/Val/Test shape (1281, 10337) (336, 10337) (336, 10337)\n",
      "Window Train/Val/Test shape (1281, 2) (336, 2) (336, 2)\n",
      "X_train_cv  (1617, 10336)\n",
      "4 Thresh=0.00893, n=2, mse: 75.170\n",
      "CV - Window 5\n",
      "Train [2017-07-01 2017-09-14] Val [2017-09-15 2017-09-30] Test [2017-10-01 2017-10-16]\n",
      "Window Train/Val/Test shape (1596, 10337) (336, 10337) (336, 10337)\n",
      "Window Train/Val/Test shape (1596, 2) (336, 2) (336, 2)\n",
      "X_train_cv  (1932, 10336)\n",
      "5 Thresh=0.00893, n=2, mse: 64.329\n",
      "CV - Window 6\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-09-29] Val [2017-09-30 2017-10-15] Test [2017-10-16 2017-11-10]\n",
      "Window Train/Val/Test shape (1911, 10337) (336, 10337) (546, 10337)\n",
      "Window Train/Val/Test shape (1911, 2) (336, 2) (546, 2)\n",
      "X_train_cv  (2247, 10336)\n",
      "6 Thresh=0.00893, n=2, mse: 35.706\n",
      "CV - Window 7\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-10-14] Val [2017-10-15 2017-10-30] Test [2017-10-31 2017-11-10]\n",
      "Window Train/Val/Test shape (2226, 10337) (336, 10337) (231, 10337)\n",
      "Window Train/Val/Test shape (2226, 2) (336, 2) (231, 2)\n",
      "X_train_cv  (2562, 10336)\n",
      "7 Thresh=0.00893, n=2, mse: 14.436\n",
      "{'threshold': 0.008928572, 'num_features': 2, 'mse_list': [228.42751571715976, 75.16984463184995, 64.32870873031905, 35.70585139515065, 14.435844083018235], 'mean_mse': 83.61355291149953}\n",
      "\n",
      "\n",
      "CV - Window 3\n",
      "Train [2017-07-01 2017-08-15] Val [2017-08-16 2017-08-31] Test [2017-09-01 2017-09-16]\n",
      "Window Train/Val/Test shape (966, 10337) (336, 10337) (336, 10337)\n",
      "Window Train/Val/Test shape (966, 2) (336, 2) (336, 2)\n",
      "X_train_cv  (1302, 10336)\n",
      "3 Thresh=0.00670, n=4, mse: 223.286\n",
      "CV - Window 4\n",
      "Train [2017-07-01 2017-08-30] Val [2017-08-31 2017-09-15] Test [2017-09-16 2017-10-01]\n",
      "Window Train/Val/Test shape (1281, 10337) (336, 10337) (336, 10337)\n",
      "Window Train/Val/Test shape (1281, 2) (336, 2) (336, 2)\n",
      "X_train_cv  (1617, 10336)\n",
      "4 Thresh=0.00670, n=4, mse: 74.933\n",
      "CV - Window 5\n",
      "Train [2017-07-01 2017-09-14] Val [2017-09-15 2017-09-30] Test [2017-10-01 2017-10-16]\n",
      "Window Train/Val/Test shape (1596, 10337) (336, 10337) (336, 10337)\n",
      "Window Train/Val/Test shape (1596, 2) (336, 2) (336, 2)\n",
      "X_train_cv  (1932, 10336)\n",
      "5 Thresh=0.00670, n=4, mse: 81.655\n",
      "CV - Window 6\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-09-29] Val [2017-09-30 2017-10-15] Test [2017-10-16 2017-11-10]\n",
      "Window Train/Val/Test shape (1911, 10337) (336, 10337) (546, 10337)\n",
      "Window Train/Val/Test shape (1911, 2) (336, 2) (546, 2)\n",
      "X_train_cv  (2247, 10336)\n",
      "6 Thresh=0.00670, n=4, mse: 38.175\n",
      "CV - Window 7\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-10-14] Val [2017-10-15 2017-10-30] Test [2017-10-31 2017-11-10]\n",
      "Window Train/Val/Test shape (2226, 10337) (336, 10337) (231, 10337)\n",
      "Window Train/Val/Test shape (2226, 2) (336, 2) (231, 2)\n",
      "X_train_cv  (2562, 10336)\n",
      "7 Thresh=0.00670, n=4, mse: 15.901\n",
      "{'threshold': 0.0066964286, 'num_features': 4, 'mse_list': [223.28589980212664, 74.9333470974012, 81.65467875668777, 38.17462292288409, 15.900932932509109], 'mean_mse': 86.78989630232176}\n",
      "\n",
      "\n",
      "CV - Window 3\n",
      "Train [2017-07-01 2017-08-15] Val [2017-08-16 2017-08-31] Test [2017-09-01 2017-09-16]\n",
      "Window Train/Val/Test shape (966, 10337) (336, 10337) (336, 10337)\n",
      "Window Train/Val/Test shape (966, 2) (336, 2) (336, 2)\n",
      "X_train_cv  (1302, 10336)\n",
      "3 Thresh=0.00614, n=6, mse: 231.335\n",
      "CV - Window 4\n",
      "Train [2017-07-01 2017-08-30] Val [2017-08-31 2017-09-15] Test [2017-09-16 2017-10-01]\n",
      "Window Train/Val/Test shape (1281, 10337) (336, 10337) (336, 10337)\n",
      "Window Train/Val/Test shape (1281, 2) (336, 2) (336, 2)\n",
      "X_train_cv  (1617, 10336)\n",
      "4 Thresh=0.00614, n=6, mse: 82.293\n",
      "CV - Window 5\n",
      "Train [2017-07-01 2017-09-14] Val [2017-09-15 2017-09-30] Test [2017-10-01 2017-10-16]\n",
      "Window Train/Val/Test shape (1596, 10337) (336, 10337) (336, 10337)\n",
      "Window Train/Val/Test shape (1596, 2) (336, 2) (336, 2)\n",
      "X_train_cv  (1932, 10336)\n",
      "5 Thresh=0.00614, n=6, mse: 82.776\n",
      "CV - Window 6\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-09-29] Val [2017-09-30 2017-10-15] Test [2017-10-16 2017-11-10]\n",
      "Window Train/Val/Test shape (1911, 10337) (336, 10337) (546, 10337)\n",
      "Window Train/Val/Test shape (1911, 2) (336, 2) (546, 2)\n",
      "X_train_cv  (2247, 10336)\n",
      "6 Thresh=0.00614, n=6, mse: 53.565\n",
      "CV - Window 7\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-10-14] Val [2017-10-15 2017-10-30] Test [2017-10-31 2017-11-10]\n",
      "Window Train/Val/Test shape (2226, 10337) (336, 10337) (231, 10337)\n",
      "Window Train/Val/Test shape (2226, 2) (336, 2) (231, 2)\n",
      "X_train_cv  (2562, 10336)\n",
      "7 Thresh=0.00614, n=6, mse: 14.185\n",
      "{'threshold': 0.0061383927, 'num_features': 6, 'mse_list': [231.3345894234506, 82.29263470739556, 82.7760042272423, 53.56517738989143, 14.18510721275739], 'mean_mse': 92.83070259214745}\n",
      "\n",
      "\n",
      "CV - Window 3\n",
      "Train [2017-07-01 2017-08-15] Val [2017-08-16 2017-08-31] Test [2017-09-01 2017-09-16]\n",
      "Window Train/Val/Test shape (966, 10337) (336, 10337) (336, 10337)\n",
      "Window Train/Val/Test shape (966, 2) (336, 2) (336, 2)\n",
      "X_train_cv  (1302, 10336)\n",
      "3 Thresh=0.00446, n=8, mse: 136.942\n",
      "CV - Window 4\n",
      "Train [2017-07-01 2017-08-30] Val [2017-08-31 2017-09-15] Test [2017-09-16 2017-10-01]\n",
      "Window Train/Val/Test shape (1281, 10337) (336, 10337) (336, 10337)\n",
      "Window Train/Val/Test shape (1281, 2) (336, 2) (336, 2)\n",
      "X_train_cv  (1617, 10336)\n",
      "4 Thresh=0.00446, n=8, mse: 86.117\n",
      "CV - Window 5\n",
      "Train [2017-07-01 2017-09-14] Val [2017-09-15 2017-09-30] Test [2017-10-01 2017-10-16]\n",
      "Window Train/Val/Test shape (1596, 10337) (336, 10337) (336, 10337)\n",
      "Window Train/Val/Test shape (1596, 2) (336, 2) (336, 2)\n",
      "X_train_cv  (1932, 10336)\n",
      "5 Thresh=0.00446, n=8, mse: 33.412\n",
      "CV - Window 6\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-09-29] Val [2017-09-30 2017-10-15] Test [2017-10-16 2017-11-10]\n",
      "Window Train/Val/Test shape (1911, 10337) (336, 10337) (546, 10337)\n",
      "Window Train/Val/Test shape (1911, 2) (336, 2) (546, 2)\n",
      "X_train_cv  (2247, 10336)\n",
      "6 Thresh=0.00446, n=8, mse: 22.572\n",
      "CV - Window 7\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-10-14] Val [2017-10-15 2017-10-30] Test [2017-10-31 2017-11-10]\n",
      "Window Train/Val/Test shape (2226, 10337) (336, 10337) (231, 10337)\n",
      "Window Train/Val/Test shape (2226, 2) (336, 2) (231, 2)\n",
      "X_train_cv  (2562, 10336)\n",
      "7 Thresh=0.00446, n=8, mse: 14.675\n",
      "{'threshold': 0.004464286, 'num_features': 8, 'mse_list': [136.94249832492858, 86.116811940815, 33.41235906489256, 22.572414984495314, 14.67454968324519], 'mean_mse': 58.74372679967532}\n",
      "\n",
      "\n",
      "CV - Window 3\n",
      "Train [2017-07-01 2017-08-15] Val [2017-08-16 2017-08-31] Test [2017-09-01 2017-09-16]\n",
      "Window Train/Val/Test shape (966, 10337) (336, 10337) (336, 10337)\n",
      "Window Train/Val/Test shape (966, 2) (336, 2) (336, 2)\n",
      "X_train_cv  (1302, 10336)\n",
      "3 Thresh=0.00391, n=12, mse: 104.745\n",
      "CV - Window 4\n",
      "Train [2017-07-01 2017-08-30] Val [2017-08-31 2017-09-15] Test [2017-09-16 2017-10-01]\n",
      "Window Train/Val/Test shape (1281, 10337) (336, 10337) (336, 10337)\n",
      "Window Train/Val/Test shape (1281, 2) (336, 2) (336, 2)\n",
      "X_train_cv  (1617, 10336)\n",
      "4 Thresh=0.00391, n=12, mse: 54.756\n",
      "CV - Window 5\n",
      "Train [2017-07-01 2017-09-14] Val [2017-09-15 2017-09-30] Test [2017-10-01 2017-10-16]\n",
      "Window Train/Val/Test shape (1596, 10337) (336, 10337) (336, 10337)\n",
      "Window Train/Val/Test shape (1596, 2) (336, 2) (336, 2)\n",
      "X_train_cv  (1932, 10336)\n",
      "5 Thresh=0.00391, n=12, mse: 27.441\n",
      "CV - Window 6\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-09-29] Val [2017-09-30 2017-10-15] Test [2017-10-16 2017-11-10]\n",
      "Window Train/Val/Test shape (1911, 10337) (336, 10337) (546, 10337)\n",
      "Window Train/Val/Test shape (1911, 2) (336, 2) (546, 2)\n",
      "X_train_cv  (2247, 10336)\n",
      "6 Thresh=0.00391, n=12, mse: 26.166\n",
      "CV - Window 7\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-10-14] Val [2017-10-15 2017-10-30] Test [2017-10-31 2017-11-10]\n",
      "Window Train/Val/Test shape (2226, 10337) (336, 10337) (231, 10337)\n",
      "Window Train/Val/Test shape (2226, 2) (336, 2) (231, 2)\n",
      "X_train_cv  (2562, 10336)\n",
      "7 Thresh=0.00391, n=12, mse: 11.711\n",
      "{'threshold': 0.00390625, 'num_features': 12, 'mse_list': [104.74463328628194, 54.755929280578975, 27.441081574084976, 26.16645344964324, 11.711474041574887], 'mean_mse': 44.9639143264328}\n",
      "\n",
      "\n",
      "CV - Window 3\n",
      "Train [2017-07-01 2017-08-15] Val [2017-08-16 2017-08-31] Test [2017-09-01 2017-09-16]\n",
      "Window Train/Val/Test shape (966, 10337) (336, 10337) (336, 10337)\n",
      "Window Train/Val/Test shape (966, 2) (336, 2) (336, 2)\n",
      "X_train_cv  (1302, 10336)\n",
      "3 Thresh=0.00279, n=14, mse: 71.067\n",
      "CV - Window 4\n",
      "Train [2017-07-01 2017-08-30] Val [2017-08-31 2017-09-15] Test [2017-09-16 2017-10-01]\n",
      "Window Train/Val/Test shape (1281, 10337) (336, 10337) (336, 10337)\n",
      "Window Train/Val/Test shape (1281, 2) (336, 2) (336, 2)\n",
      "X_train_cv  (1617, 10336)\n",
      "4 Thresh=0.00279, n=14, mse: 50.637\n",
      "CV - Window 5\n",
      "Train [2017-07-01 2017-09-14] Val [2017-09-15 2017-09-30] Test [2017-10-01 2017-10-16]\n",
      "Window Train/Val/Test shape (1596, 10337) (336, 10337) (336, 10337)\n",
      "Window Train/Val/Test shape (1596, 2) (336, 2) (336, 2)\n",
      "X_train_cv  (1932, 10336)\n",
      "5 Thresh=0.00279, n=14, mse: 26.106\n",
      "CV - Window 6\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-09-29] Val [2017-09-30 2017-10-15] Test [2017-10-16 2017-11-10]\n",
      "Window Train/Val/Test shape (1911, 10337) (336, 10337) (546, 10337)\n",
      "Window Train/Val/Test shape (1911, 2) (336, 2) (546, 2)\n",
      "X_train_cv  (2247, 10336)\n",
      "6 Thresh=0.00279, n=14, mse: 20.215\n",
      "CV - Window 7\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-10-14] Val [2017-10-15 2017-10-30] Test [2017-10-31 2017-11-10]\n",
      "Window Train/Val/Test shape (2226, 10337) (336, 10337) (231, 10337)\n",
      "Window Train/Val/Test shape (2226, 2) (336, 2) (231, 2)\n",
      "X_train_cv  (2562, 10336)\n",
      "7 Thresh=0.00279, n=14, mse: 7.740\n",
      "{'threshold': 0.0027901786, 'num_features': 14, 'mse_list': [71.06707665541056, 50.63736819605383, 26.106073833185206, 20.215377896049347, 7.739583087391219], 'mean_mse': 35.15309593361803}\n",
      "\n",
      "\n",
      "CV - Window 3\n",
      "Train [2017-07-01 2017-08-15] Val [2017-08-16 2017-08-31] Test [2017-09-01 2017-09-16]\n",
      "Window Train/Val/Test shape (966, 10337) (336, 10337) (336, 10337)\n",
      "Window Train/Val/Test shape (966, 2) (336, 2) (336, 2)\n",
      "X_train_cv  (1302, 10336)\n",
      "3 Thresh=0.00223, n=28, mse: 57.108\n",
      "CV - Window 4\n",
      "Train [2017-07-01 2017-08-30] Val [2017-08-31 2017-09-15] Test [2017-09-16 2017-10-01]\n",
      "Window Train/Val/Test shape (1281, 10337) (336, 10337) (336, 10337)\n",
      "Window Train/Val/Test shape (1281, 2) (336, 2) (336, 2)\n",
      "X_train_cv  (1617, 10336)\n",
      "4 Thresh=0.00223, n=28, mse: 41.301\n",
      "CV - Window 5\n",
      "Train [2017-07-01 2017-09-14] Val [2017-09-15 2017-09-30] Test [2017-10-01 2017-10-16]\n",
      "Window Train/Val/Test shape (1596, 10337) (336, 10337) (336, 10337)\n",
      "Window Train/Val/Test shape (1596, 2) (336, 2) (336, 2)\n",
      "X_train_cv  (1932, 10336)\n",
      "5 Thresh=0.00223, n=28, mse: 20.896\n",
      "CV - Window 6\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-09-29] Val [2017-09-30 2017-10-15] Test [2017-10-16 2017-11-10]\n",
      "Window Train/Val/Test shape (1911, 10337) (336, 10337) (546, 10337)\n",
      "Window Train/Val/Test shape (1911, 2) (336, 2) (546, 2)\n",
      "X_train_cv  (2247, 10336)\n",
      "6 Thresh=0.00223, n=28, mse: 20.603\n",
      "CV - Window 7\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-10-14] Val [2017-10-15 2017-10-30] Test [2017-10-31 2017-11-10]\n",
      "Window Train/Val/Test shape (2226, 10337) (336, 10337) (231, 10337)\n",
      "Window Train/Val/Test shape (2226, 2) (336, 2) (231, 2)\n",
      "X_train_cv  (2562, 10336)\n",
      "7 Thresh=0.00223, n=28, mse: 6.324\n",
      "{'threshold': 0.002232143, 'num_features': 28, 'mse_list': [57.1084370262155, 41.3008093922616, 20.896051014937328, 20.603155127989684, 6.324188236539581], 'mean_mse': 29.246528159588742}\n",
      "\n",
      "\n",
      "CV - Window 3\n",
      "Train [2017-07-01 2017-08-15] Val [2017-08-16 2017-08-31] Test [2017-09-01 2017-09-16]\n",
      "Window Train/Val/Test shape (966, 10337) (336, 10337) (336, 10337)\n",
      "Window Train/Val/Test shape (966, 2) (336, 2) (336, 2)\n",
      "X_train_cv  (1302, 10336)\n",
      "3 Thresh=0.00167, n=76, mse: 58.410\n",
      "CV - Window 4\n",
      "Train [2017-07-01 2017-08-30] Val [2017-08-31 2017-09-15] Test [2017-09-16 2017-10-01]\n",
      "Window Train/Val/Test shape (1281, 10337) (336, 10337) (336, 10337)\n",
      "Window Train/Val/Test shape (1281, 2) (336, 2) (336, 2)\n",
      "X_train_cv  (1617, 10336)\n",
      "4 Thresh=0.00167, n=76, mse: 49.472\n",
      "CV - Window 5\n",
      "Train [2017-07-01 2017-09-14] Val [2017-09-15 2017-09-30] Test [2017-10-01 2017-10-16]\n",
      "Window Train/Val/Test shape (1596, 10337) (336, 10337) (336, 10337)\n",
      "Window Train/Val/Test shape (1596, 2) (336, 2) (336, 2)\n",
      "X_train_cv  (1932, 10336)\n",
      "5 Thresh=0.00167, n=76, mse: 30.999\n",
      "CV - Window 6\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-09-29] Val [2017-09-30 2017-10-15] Test [2017-10-16 2017-11-10]\n",
      "Window Train/Val/Test shape (1911, 10337) (336, 10337) (546, 10337)\n",
      "Window Train/Val/Test shape (1911, 2) (336, 2) (546, 2)\n",
      "X_train_cv  (2247, 10336)\n",
      "6 Thresh=0.00167, n=76, mse: 16.866\n",
      "CV - Window 7\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-10-14] Val [2017-10-15 2017-10-30] Test [2017-10-31 2017-11-10]\n",
      "Window Train/Val/Test shape (2226, 10337) (336, 10337) (231, 10337)\n",
      "Window Train/Val/Test shape (2226, 2) (336, 2) (231, 2)\n",
      "X_train_cv  (2562, 10336)\n",
      "7 Thresh=0.00167, n=76, mse: 5.295\n",
      "{'threshold': 0.0016741072, 'num_features': 76, 'mse_list': [58.40954728673179, 49.47227697274227, 30.998777913793795, 16.865685637620324, 5.294782747955665], 'mean_mse': 32.20821411176877}\n",
      "\n",
      "\n",
      "CV - Window 3\n",
      "Train [2017-07-01 2017-08-15] Val [2017-08-16 2017-08-31] Test [2017-09-01 2017-09-16]\n",
      "Window Train/Val/Test shape (966, 10337) (336, 10337) (336, 10337)\n",
      "Window Train/Val/Test shape (966, 2) (336, 2) (336, 2)\n",
      "X_train_cv  (1302, 10336)\n",
      "3 Thresh=0.00112, n=272, mse: 54.863\n",
      "CV - Window 4\n",
      "Train [2017-07-01 2017-08-30] Val [2017-08-31 2017-09-15] Test [2017-09-16 2017-10-01]\n",
      "Window Train/Val/Test shape (1281, 10337) (336, 10337) (336, 10337)\n",
      "Window Train/Val/Test shape (1281, 2) (336, 2) (336, 2)\n",
      "X_train_cv  (1617, 10336)\n",
      "4 Thresh=0.00112, n=272, mse: 48.702\n",
      "CV - Window 5\n",
      "Train [2017-07-01 2017-09-14] Val [2017-09-15 2017-09-30] Test [2017-10-01 2017-10-16]\n",
      "Window Train/Val/Test shape (1596, 10337) (336, 10337) (336, 10337)\n",
      "Window Train/Val/Test shape (1596, 2) (336, 2) (336, 2)\n",
      "X_train_cv  (1932, 10336)\n",
      "5 Thresh=0.00112, n=272, mse: 44.601\n",
      "CV - Window 6\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-09-29] Val [2017-09-30 2017-10-15] Test [2017-10-16 2017-11-10]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Window Train/Val/Test shape (1911, 10337) (336, 10337) (546, 10337)\n",
      "Window Train/Val/Test shape (1911, 2) (336, 2) (546, 2)\n",
      "X_train_cv  (2247, 10336)\n",
      "6 Thresh=0.00112, n=272, mse: 17.593\n",
      "CV - Window 7\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-10-14] Val [2017-10-15 2017-10-30] Test [2017-10-31 2017-11-10]\n",
      "Window Train/Val/Test shape (2226, 10337) (336, 10337) (231, 10337)\n",
      "Window Train/Val/Test shape (2226, 2) (336, 2) (231, 2)\n",
      "X_train_cv  (2562, 10336)\n",
      "7 Thresh=0.00112, n=272, mse: 7.078\n",
      "{'threshold': 0.0011160715, 'num_features': 272, 'mse_list': [54.86282586406053, 48.70211355769078, 44.600514801959186, 17.59252020592548, 7.077724802636396], 'mean_mse': 34.567139846454474}\n",
      "\n",
      "\n",
      "CV - Window 3\n",
      "Train [2017-07-01 2017-08-15] Val [2017-08-16 2017-08-31] Test [2017-09-01 2017-09-16]\n",
      "Window Train/Val/Test shape (966, 10337) (336, 10337) (336, 10337)\n",
      "Window Train/Val/Test shape (966, 2) (336, 2) (336, 2)\n",
      "X_train_cv  (1302, 10336)\n",
      "3 Thresh=0.00056, n=1334, mse: 59.520\n",
      "CV - Window 4\n",
      "Train [2017-07-01 2017-08-30] Val [2017-08-31 2017-09-15] Test [2017-09-16 2017-10-01]\n",
      "Window Train/Val/Test shape (1281, 10337) (336, 10337) (336, 10337)\n",
      "Window Train/Val/Test shape (1281, 2) (336, 2) (336, 2)\n",
      "X_train_cv  (1617, 10336)\n",
      "4 Thresh=0.00056, n=1334, mse: 57.199\n",
      "CV - Window 5\n",
      "Train [2017-07-01 2017-09-14] Val [2017-09-15 2017-09-30] Test [2017-10-01 2017-10-16]\n",
      "Window Train/Val/Test shape (1596, 10337) (336, 10337) (336, 10337)\n",
      "Window Train/Val/Test shape (1596, 2) (336, 2) (336, 2)\n",
      "X_train_cv  (1932, 10336)\n",
      "5 Thresh=0.00056, n=1334, mse: 32.874\n",
      "CV - Window 6\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-09-29] Val [2017-09-30 2017-10-15] Test [2017-10-16 2017-11-10]\n",
      "Window Train/Val/Test shape (1911, 10337) (336, 10337) (546, 10337)\n",
      "Window Train/Val/Test shape (1911, 2) (336, 2) (546, 2)\n",
      "X_train_cv  (2247, 10336)\n",
      "6 Thresh=0.00056, n=1334, mse: 22.217\n",
      "CV - Window 7\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-10-14] Val [2017-10-15 2017-10-30] Test [2017-10-31 2017-11-10]\n",
      "Window Train/Val/Test shape (2226, 10337) (336, 10337) (231, 10337)\n",
      "Window Train/Val/Test shape (2226, 2) (336, 2) (231, 2)\n",
      "X_train_cv  (2562, 10336)\n",
      "7 Thresh=0.00056, n=1334, mse: 7.958\n",
      "{'threshold': 0.00055803574, 'num_features': 1334, 'mse_list': [59.520397111937676, 57.1985209548142, 32.87400825300657, 22.217228809253626, 7.958328125519546], 'mean_mse': 35.95369665090632}\n",
      "\n",
      "\n",
      "CV - Window 3\n",
      "Train [2017-07-01 2017-08-15] Val [2017-08-16 2017-08-31] Test [2017-09-01 2017-09-16]\n",
      "Window Train/Val/Test shape (966, 10337) (336, 10337) (336, 10337)\n",
      "Window Train/Val/Test shape (966, 2) (336, 2) (336, 2)\n",
      "X_train_cv  (1302, 10336)\n",
      "3 Thresh=0.00000, n=10336, mse: 98.676\n",
      "CV - Window 4\n",
      "Train [2017-07-01 2017-08-30] Val [2017-08-31 2017-09-15] Test [2017-09-16 2017-10-01]\n",
      "Window Train/Val/Test shape (1281, 10337) (336, 10337) (336, 10337)\n",
      "Window Train/Val/Test shape (1281, 2) (336, 2) (336, 2)\n",
      "X_train_cv  (1617, 10336)\n",
      "4 Thresh=0.00000, n=10336, mse: 59.255\n",
      "CV - Window 5\n",
      "Train [2017-07-01 2017-09-14] Val [2017-09-15 2017-09-30] Test [2017-10-01 2017-10-16]\n",
      "Window Train/Val/Test shape (1596, 10337) (336, 10337) (336, 10337)\n",
      "Window Train/Val/Test shape (1596, 2) (336, 2) (336, 2)\n",
      "X_train_cv  (1932, 10336)\n",
      "5 Thresh=0.00000, n=10336, mse: 47.263\n",
      "CV - Window 6\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-09-29] Val [2017-09-30 2017-10-15] Test [2017-10-16 2017-11-10]\n",
      "Window Train/Val/Test shape (1911, 10337) (336, 10337) (546, 10337)\n",
      "Window Train/Val/Test shape (1911, 2) (336, 2) (546, 2)\n",
      "X_train_cv  (2247, 10336)\n",
      "6 Thresh=0.00000, n=10336, mse: 23.615\n",
      "CV - Window 7\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-10-14] Val [2017-10-15 2017-10-30] Test [2017-10-31 2017-11-10]\n",
      "Window Train/Val/Test shape (2226, 10337) (336, 10337) (231, 10337)\n",
      "Window Train/Val/Test shape (2226, 2) (336, 2) (231, 2)\n",
      "X_train_cv  (2562, 10336)\n",
      "7 Thresh=0.00000, n=10336, mse: 13.735\n",
      "{'threshold': 0.0, 'num_features': 10336, 'mse_list': [98.67630574516781, 59.25521188643485, 47.26303805567107, 23.614762120549116, 13.73492731130917], 'mean_mse': 48.508849023826414}\n",
      "\n",
      "\n",
      "    threshold num_features                                           mse_list  \\\n",
      "0    0.012277            1  [251.10475750511748, 81.22296992591856, 58.965...   \n",
      "1    0.008929            2  [228.42751571715976, 75.16984463184995, 64.328...   \n",
      "2    0.006696            4  [223.28589980212664, 74.9333470974012, 81.6546...   \n",
      "3    0.006138            6  [231.3345894234506, 82.29263470739556, 82.7760...   \n",
      "4    0.004464            8  [136.94249832492858, 86.116811940815, 33.41235...   \n",
      "5    0.003906           12  [104.74463328628194, 54.755929280578975, 27.44...   \n",
      "6    0.002790           14  [71.06707665541056, 50.63736819605383, 26.1060...   \n",
      "7    0.002232           28  [57.1084370262155, 41.3008093922616, 20.896051...   \n",
      "8    0.001674           76  [58.40954728673179, 49.47227697274227, 30.9987...   \n",
      "9    0.001116          272  [54.86282586406053, 48.70211355769078, 44.6005...   \n",
      "10   0.000558         1334  [59.520397111937676, 57.1985209548142, 32.8740...   \n",
      "11   0.000000        10336  [98.67630574516781, 59.25521188643485, 47.2630...   \n",
      "\n",
      "     mean_mse  \n",
      "0   88.216409  \n",
      "1   83.613553  \n",
      "2   86.789896  \n",
      "3   92.830703  \n",
      "4   58.743727  \n",
      "5   44.963914  \n",
      "6   35.153096  \n",
      "7   29.246528  \n",
      "8   32.208214  \n",
      "9   34.567140  \n",
      "10  35.953697  \n",
      "11  48.508849  \n"
     ]
    }
   ],
   "source": [
    "orchestrator24.runRollingCustomFeatureExtractor(preselect_params_24, X_24_preselect)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 135,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>threshold</th>\n",
       "      <th>num_features</th>\n",
       "      <th>mse_list</th>\n",
       "      <th>mean_mse</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0.012277</td>\n",
       "      <td>1</td>\n",
       "      <td>[251.10475750511748, 81.22296992591856, 58.965...</td>\n",
       "      <td>88.216409</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0.008929</td>\n",
       "      <td>2</td>\n",
       "      <td>[228.42751571715976, 75.16984463184995, 64.328...</td>\n",
       "      <td>83.613553</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0.006696</td>\n",
       "      <td>4</td>\n",
       "      <td>[223.28589980212664, 74.9333470974012, 81.6546...</td>\n",
       "      <td>86.789896</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0.006138</td>\n",
       "      <td>6</td>\n",
       "      <td>[231.3345894234506, 82.29263470739556, 82.7760...</td>\n",
       "      <td>92.830703</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0.004464</td>\n",
       "      <td>8</td>\n",
       "      <td>[136.94249832492858, 86.116811940815, 33.41235...</td>\n",
       "      <td>58.743727</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>0.003906</td>\n",
       "      <td>12</td>\n",
       "      <td>[104.74463328628194, 54.755929280578975, 27.44...</td>\n",
       "      <td>44.963914</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>0.002790</td>\n",
       "      <td>14</td>\n",
       "      <td>[71.06707665541056, 50.63736819605383, 26.1060...</td>\n",
       "      <td>35.153096</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>0.002232</td>\n",
       "      <td>28</td>\n",
       "      <td>[57.1084370262155, 41.3008093922616, 20.896051...</td>\n",
       "      <td>29.246528</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>0.001674</td>\n",
       "      <td>76</td>\n",
       "      <td>[58.40954728673179, 49.47227697274227, 30.9987...</td>\n",
       "      <td>32.208214</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>0.001116</td>\n",
       "      <td>272</td>\n",
       "      <td>[54.86282586406053, 48.70211355769078, 44.6005...</td>\n",
       "      <td>34.567140</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>0.000558</td>\n",
       "      <td>1334</td>\n",
       "      <td>[59.520397111937676, 57.1985209548142, 32.8740...</td>\n",
       "      <td>35.953697</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>0.000000</td>\n",
       "      <td>10336</td>\n",
       "      <td>[98.67630574516781, 59.25521188643485, 47.2630...</td>\n",
       "      <td>48.508849</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "    threshold num_features                                           mse_list  \\\n",
       "0    0.012277            1  [251.10475750511748, 81.22296992591856, 58.965...   \n",
       "1    0.008929            2  [228.42751571715976, 75.16984463184995, 64.328...   \n",
       "2    0.006696            4  [223.28589980212664, 74.9333470974012, 81.6546...   \n",
       "3    0.006138            6  [231.3345894234506, 82.29263470739556, 82.7760...   \n",
       "4    0.004464            8  [136.94249832492858, 86.116811940815, 33.41235...   \n",
       "5    0.003906           12  [104.74463328628194, 54.755929280578975, 27.44...   \n",
       "6    0.002790           14  [71.06707665541056, 50.63736819605383, 26.1060...   \n",
       "7    0.002232           28  [57.1084370262155, 41.3008093922616, 20.896051...   \n",
       "8    0.001674           76  [58.40954728673179, 49.47227697274227, 30.9987...   \n",
       "9    0.001116          272  [54.86282586406053, 48.70211355769078, 44.6005...   \n",
       "10   0.000558         1334  [59.520397111937676, 57.1985209548142, 32.8740...   \n",
       "11   0.000000        10336  [98.67630574516781, 59.25521188643485, 47.2630...   \n",
       "\n",
       "     mean_mse  \n",
       "0   88.216409  \n",
       "1   83.613553  \n",
       "2   86.789896  \n",
       "3   92.830703  \n",
       "4   58.743727  \n",
       "5   44.963914  \n",
       "6   35.153096  \n",
       "7   29.246528  \n",
       "8   32.208214  \n",
       "9   34.567140  \n",
       "10  35.953697  \n",
       "11  48.508849  "
      ]
     },
     "execution_count": 135,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "orchestrator24.rcfe.summary"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {},
   "outputs": [],
   "source": [
    "selected_features24 = ['date'] + list(orchestrator24.rcfe.mcr.feature_importance_df[orchestrator24.rcfe.mcr.feature_importance_df.threshold >= 0.002232].index)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 137,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train [2017-07-01 2017-08-15] Val [2017-08-16 2017-08-31] Test [2017-09-01 2017-09-16]\n",
      "Window Train/Val/Test shape (966, 29) (336, 29) (336, 29)\n",
      "Window Train/Val/Test shape (966, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-08-30] Val [2017-08-31 2017-09-15] Test [2017-09-16 2017-10-01]\n",
      "Window Train/Val/Test shape (1281, 29) (336, 29) (336, 29)\n",
      "Window Train/Val/Test shape (1281, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-09-14] Val [2017-09-15 2017-09-30] Test [2017-10-01 2017-10-16]\n",
      "Window Train/Val/Test shape (1596, 29) (336, 29) (336, 29)\n",
      "Window Train/Val/Test shape (1596, 2) (336, 2) (336, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-09-29] Val [2017-09-30 2017-10-15] Test [2017-10-16 2017-11-10]\n",
      "Window Train/Val/Test shape (1911, 29) (336, 29) (546, 29)\n",
      "Window Train/Val/Test shape (1911, 2) (336, 2) (546, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-10-14] Val [2017-10-15 2017-10-30] Test [2017-10-31 2017-11-10]\n",
      "Window Train/Val/Test shape (2226, 29) (336, 29) (231, 29)\n",
      "Window Train/Val/Test shape (2226, 2) (336, 2) (231, 2)\n",
      "[59.274614110344935, 59.34926224410577, 18.107103725569885, 20.420373128414443, 19.199188351216506]\n",
      "mean mse: 35.270108311930315\n",
      "Train [2017-07-01 2017-08-15] Val [2017-08-16 2017-08-31] Test [2017-09-01 2017-09-16]\n",
      "Window Train/Val/Test shape (966, 29) (336, 29) (336, 29)\n",
      "Window Train/Val/Test shape (966, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-08-30] Val [2017-08-31 2017-09-15] Test [2017-09-16 2017-10-01]\n",
      "Window Train/Val/Test shape (1281, 29) (336, 29) (336, 29)\n",
      "Window Train/Val/Test shape (1281, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-09-14] Val [2017-09-15 2017-09-30] Test [2017-10-01 2017-10-16]\n",
      "Window Train/Val/Test shape (1596, 29) (336, 29) (336, 29)\n",
      "Window Train/Val/Test shape (1596, 2) (336, 2) (336, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-09-29] Val [2017-09-30 2017-10-15] Test [2017-10-16 2017-11-10]\n",
      "Window Train/Val/Test shape (1911, 29) (336, 29) (546, 29)\n",
      "Window Train/Val/Test shape (1911, 2) (336, 2) (546, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-10-14] Val [2017-10-15 2017-10-30] Test [2017-10-31 2017-11-10]\n",
      "Window Train/Val/Test shape (2226, 29) (336, 29) (231, 29)\n",
      "Window Train/Val/Test shape (2226, 2) (336, 2) (231, 2)\n",
      "[54.4568295764272, 50.18105154374875, 18.660674771257234, 19.893508172996835, 18.560128739671523]\n",
      "mean mse: 32.35043856082031\n",
      "Train [2017-07-01 2017-08-15] Val [2017-08-16 2017-08-31] Test [2017-09-01 2017-09-16]\n",
      "Window Train/Val/Test shape (966, 29) (336, 29) (336, 29)\n",
      "Window Train/Val/Test shape (966, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-08-30] Val [2017-08-31 2017-09-15] Test [2017-09-16 2017-10-01]\n",
      "Window Train/Val/Test shape (1281, 29) (336, 29) (336, 29)\n",
      "Window Train/Val/Test shape (1281, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-09-14] Val [2017-09-15 2017-09-30] Test [2017-10-01 2017-10-16]\n",
      "Window Train/Val/Test shape (1596, 29) (336, 29) (336, 29)\n",
      "Window Train/Val/Test shape (1596, 2) (336, 2) (336, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-09-29] Val [2017-09-30 2017-10-15] Test [2017-10-16 2017-11-10]\n",
      "Window Train/Val/Test shape (1911, 29) (336, 29) (546, 29)\n",
      "Window Train/Val/Test shape (1911, 2) (336, 2) (546, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-10-14] Val [2017-10-15 2017-10-30] Test [2017-10-31 2017-11-10]\n",
      "Window Train/Val/Test shape (2226, 29) (336, 29) (231, 29)\n",
      "Window Train/Val/Test shape (2226, 2) (336, 2) (231, 2)\n",
      "[70.02402494561031, 72.84348414010043, 25.79017279765673, 21.779250393077305, 24.271210080743415]\n",
      "mean mse: 42.94162847143764\n",
      "Train [2017-07-01 2017-08-15] Val [2017-08-16 2017-08-31] Test [2017-09-01 2017-09-16]\n",
      "Window Train/Val/Test shape (966, 29) (336, 29) (336, 29)\n",
      "Window Train/Val/Test shape (966, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-08-30] Val [2017-08-31 2017-09-15] Test [2017-09-16 2017-10-01]\n",
      "Window Train/Val/Test shape (1281, 29) (336, 29) (336, 29)\n",
      "Window Train/Val/Test shape (1281, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-09-14] Val [2017-09-15 2017-09-30] Test [2017-10-01 2017-10-16]\n",
      "Window Train/Val/Test shape (1596, 29) (336, 29) (336, 29)\n",
      "Window Train/Val/Test shape (1596, 2) (336, 2) (336, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-09-29] Val [2017-09-30 2017-10-15] Test [2017-10-16 2017-11-10]\n",
      "Window Train/Val/Test shape (1911, 29) (336, 29) (546, 29)\n",
      "Window Train/Val/Test shape (1911, 2) (336, 2) (546, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-10-14] Val [2017-10-15 2017-10-30] Test [2017-10-31 2017-11-10]\n",
      "Window Train/Val/Test shape (2226, 29) (336, 29) (231, 29)\n",
      "Window Train/Val/Test shape (2226, 2) (336, 2) (231, 2)\n",
      "[56.661515413030386, 61.65335733514318, 17.81509310686601, 21.532546109277785, 18.426795638046894]\n",
      "mean mse: 35.21786152047285\n",
      "Train [2017-07-01 2017-08-15] Val [2017-08-16 2017-08-31] Test [2017-09-01 2017-09-16]\n",
      "Window Train/Val/Test shape (966, 29) (336, 29) (336, 29)\n",
      "Window Train/Val/Test shape (966, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-08-30] Val [2017-08-31 2017-09-15] Test [2017-09-16 2017-10-01]\n",
      "Window Train/Val/Test shape (1281, 29) (336, 29) (336, 29)\n",
      "Window Train/Val/Test shape (1281, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-09-14] Val [2017-09-15 2017-09-30] Test [2017-10-01 2017-10-16]\n",
      "Window Train/Val/Test shape (1596, 29) (336, 29) (336, 29)\n",
      "Window Train/Val/Test shape (1596, 2) (336, 2) (336, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-09-29] Val [2017-09-30 2017-10-15] Test [2017-10-16 2017-11-10]\n",
      "Window Train/Val/Test shape (1911, 29) (336, 29) (546, 29)\n",
      "Window Train/Val/Test shape (1911, 2) (336, 2) (546, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-10-14] Val [2017-10-15 2017-10-30] Test [2017-10-31 2017-11-10]\n",
      "Window Train/Val/Test shape (2226, 29) (336, 29) (231, 29)\n",
      "Window Train/Val/Test shape (2226, 2) (336, 2) (231, 2)\n",
      "[62.4631477968555, 59.26598399404443, 23.91967851386323, 23.63400875337788, 22.590593011464843]\n",
      "mean mse: 38.37468241392118\n",
      "Train [2017-07-01 2017-08-15] Val [2017-08-16 2017-08-31] Test [2017-09-01 2017-09-16]\n",
      "Window Train/Val/Test shape (966, 29) (336, 29) (336, 29)\n",
      "Window Train/Val/Test shape (966, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-08-30] Val [2017-08-31 2017-09-15] Test [2017-09-16 2017-10-01]\n",
      "Window Train/Val/Test shape (1281, 29) (336, 29) (336, 29)\n",
      "Window Train/Val/Test shape (1281, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-09-14] Val [2017-09-15 2017-09-30] Test [2017-10-01 2017-10-16]\n",
      "Window Train/Val/Test shape (1596, 29) (336, 29) (336, 29)\n",
      "Window Train/Val/Test shape (1596, 2) (336, 2) (336, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-09-29] Val [2017-09-30 2017-10-15] Test [2017-10-16 2017-11-10]\n",
      "Window Train/Val/Test shape (1911, 29) (336, 29) (546, 29)\n",
      "Window Train/Val/Test shape (1911, 2) (336, 2) (546, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-10-14] Val [2017-10-15 2017-10-30] Test [2017-10-31 2017-11-10]\n",
      "Window Train/Val/Test shape (2226, 29) (336, 29) (231, 29)\n",
      "Window Train/Val/Test shape (2226, 2) (336, 2) (231, 2)\n",
      "[56.63005876736728, 60.06639721290871, 19.025637022408628, 20.242708615655072, 19.254124516340305]\n",
      "mean mse: 35.043785226936\n",
      "Train [2017-07-01 2017-08-15] Val [2017-08-16 2017-08-31] Test [2017-09-01 2017-09-16]\n",
      "Window Train/Val/Test shape (966, 29) (336, 29) (336, 29)\n",
      "Window Train/Val/Test shape (966, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-08-30] Val [2017-08-31 2017-09-15] Test [2017-09-16 2017-10-01]\n",
      "Window Train/Val/Test shape (1281, 29) (336, 29) (336, 29)\n",
      "Window Train/Val/Test shape (1281, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-09-14] Val [2017-09-15 2017-09-30] Test [2017-10-01 2017-10-16]\n",
      "Window Train/Val/Test shape (1596, 29) (336, 29) (336, 29)\n",
      "Window Train/Val/Test shape (1596, 2) (336, 2) (336, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-09-29] Val [2017-09-30 2017-10-15] Test [2017-10-16 2017-11-10]\n",
      "Window Train/Val/Test shape (1911, 29) (336, 29) (546, 29)\n",
      "Window Train/Val/Test shape (1911, 2) (336, 2) (546, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-10-14] Val [2017-10-15 2017-10-30] Test [2017-10-31 2017-11-10]\n",
      "Window Train/Val/Test shape (2226, 29) (336, 29) (231, 29)\n",
      "Window Train/Val/Test shape (2226, 2) (336, 2) (231, 2)\n",
      "[56.142475202391424, 58.54837675084619, 20.47056851539754, 19.361070664672823, 19.101768384263533]\n",
      "mean mse: 34.724851903514306\n",
      "Train [2017-07-01 2017-08-15] Val [2017-08-16 2017-08-31] Test [2017-09-01 2017-09-16]\n",
      "Window Train/Val/Test shape (966, 29) (336, 29) (336, 29)\n",
      "Window Train/Val/Test shape (966, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-08-30] Val [2017-08-31 2017-09-15] Test [2017-09-16 2017-10-01]\n",
      "Window Train/Val/Test shape (1281, 29) (336, 29) (336, 29)\n",
      "Window Train/Val/Test shape (1281, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-09-14] Val [2017-09-15 2017-09-30] Test [2017-10-01 2017-10-16]\n",
      "Window Train/Val/Test shape (1596, 29) (336, 29) (336, 29)\n",
      "Window Train/Val/Test shape (1596, 2) (336, 2) (336, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-09-29] Val [2017-09-30 2017-10-15] Test [2017-10-16 2017-11-10]\n",
      "Window Train/Val/Test shape (1911, 29) (336, 29) (546, 29)\n",
      "Window Train/Val/Test shape (1911, 2) (336, 2) (546, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-10-14] Val [2017-10-15 2017-10-30] Test [2017-10-31 2017-11-10]\n",
      "Window Train/Val/Test shape (2226, 29) (336, 29) (231, 29)\n",
      "Window Train/Val/Test shape (2226, 2) (336, 2) (231, 2)\n",
      "[57.941894464310266, 62.80748414551501, 17.516293573730234, 18.53387287399215, 18.466821759606635]\n",
      "mean mse: 35.053273363430854\n",
      "Train [2017-07-01 2017-08-15] Val [2017-08-16 2017-08-31] Test [2017-09-01 2017-09-16]\n",
      "Window Train/Val/Test shape (966, 29) (336, 29) (336, 29)\n",
      "Window Train/Val/Test shape (966, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-08-30] Val [2017-08-31 2017-09-15] Test [2017-09-16 2017-10-01]\n",
      "Window Train/Val/Test shape (1281, 29) (336, 29) (336, 29)\n",
      "Window Train/Val/Test shape (1281, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-09-14] Val [2017-09-15 2017-09-30] Test [2017-10-01 2017-10-16]\n",
      "Window Train/Val/Test shape (1596, 29) (336, 29) (336, 29)\n",
      "Window Train/Val/Test shape (1596, 2) (336, 2) (336, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-09-29] Val [2017-09-30 2017-10-15] Test [2017-10-16 2017-11-10]\n",
      "Window Train/Val/Test shape (1911, 29) (336, 29) (546, 29)\n",
      "Window Train/Val/Test shape (1911, 2) (336, 2) (546, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-10-14] Val [2017-10-15 2017-10-30] Test [2017-10-31 2017-11-10]\n",
      "Window Train/Val/Test shape (2226, 29) (336, 29) (231, 29)\n",
      "Window Train/Val/Test shape (2226, 2) (336, 2) (231, 2)\n",
      "[58.565379659507556, 65.38044174863089, 22.175967569820177, 18.690058260143182, 20.42378807136344]\n",
      "mean mse: 37.04712706189305\n",
      "Train [2017-07-01 2017-08-15] Val [2017-08-16 2017-08-31] Test [2017-09-01 2017-09-16]\n",
      "Window Train/Val/Test shape (966, 29) (336, 29) (336, 29)\n",
      "Window Train/Val/Test shape (966, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-08-30] Val [2017-08-31 2017-09-15] Test [2017-09-16 2017-10-01]\n",
      "Window Train/Val/Test shape (1281, 29) (336, 29) (336, 29)\n",
      "Window Train/Val/Test shape (1281, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-09-14] Val [2017-09-15 2017-09-30] Test [2017-10-01 2017-10-16]\n",
      "Window Train/Val/Test shape (1596, 29) (336, 29) (336, 29)\n",
      "Window Train/Val/Test shape (1596, 2) (336, 2) (336, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-09-29] Val [2017-09-30 2017-10-15] Test [2017-10-16 2017-11-10]\n",
      "Window Train/Val/Test shape (1911, 29) (336, 29) (546, 29)\n",
      "Window Train/Val/Test shape (1911, 2) (336, 2) (546, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-10-14] Val [2017-10-15 2017-10-30] Test [2017-10-31 2017-11-10]\n",
      "Window Train/Val/Test shape (2226, 29) (336, 29) (231, 29)\n",
      "Window Train/Val/Test shape (2226, 2) (336, 2) (231, 2)\n",
      "[61.45638677167515, 59.40102398533881, 17.290329589831966, 18.582215118308323, 20.20973600945043]\n",
      "mean mse: 35.387938294920936\n",
      "Train [2017-07-01 2017-08-15] Val [2017-08-16 2017-08-31] Test [2017-09-01 2017-09-16]\n",
      "Window Train/Val/Test shape (966, 29) (336, 29) (336, 29)\n",
      "Window Train/Val/Test shape (966, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-08-30] Val [2017-08-31 2017-09-15] Test [2017-09-16 2017-10-01]\n",
      "Window Train/Val/Test shape (1281, 29) (336, 29) (336, 29)\n",
      "Window Train/Val/Test shape (1281, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-09-14] Val [2017-09-15 2017-09-30] Test [2017-10-01 2017-10-16]\n",
      "Window Train/Val/Test shape (1596, 29) (336, 29) (336, 29)\n",
      "Window Train/Val/Test shape (1596, 2) (336, 2) (336, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-09-29] Val [2017-09-30 2017-10-15] Test [2017-10-16 2017-11-10]\n",
      "Window Train/Val/Test shape (1911, 29) (336, 29) (546, 29)\n",
      "Window Train/Val/Test shape (1911, 2) (336, 2) (546, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-10-14] Val [2017-10-15 2017-10-30] Test [2017-10-31 2017-11-10]\n",
      "Window Train/Val/Test shape (2226, 29) (336, 29) (231, 29)\n",
      "Window Train/Val/Test shape (2226, 2) (336, 2) (231, 2)\n",
      "[55.13786051020742, 63.0797690315084, 18.603685888369096, 22.215948531465482, 16.231031977699146]\n",
      "mean mse: 35.0536591878499\n",
      "Train [2017-07-01 2017-08-15] Val [2017-08-16 2017-08-31] Test [2017-09-01 2017-09-16]\n",
      "Window Train/Val/Test shape (966, 29) (336, 29) (336, 29)\n",
      "Window Train/Val/Test shape (966, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-08-30] Val [2017-08-31 2017-09-15] Test [2017-09-16 2017-10-01]\n",
      "Window Train/Val/Test shape (1281, 29) (336, 29) (336, 29)\n",
      "Window Train/Val/Test shape (1281, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-09-14] Val [2017-09-15 2017-09-30] Test [2017-10-01 2017-10-16]\n",
      "Window Train/Val/Test shape (1596, 29) (336, 29) (336, 29)\n",
      "Window Train/Val/Test shape (1596, 2) (336, 2) (336, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-09-29] Val [2017-09-30 2017-10-15] Test [2017-10-16 2017-11-10]\n",
      "Window Train/Val/Test shape (1911, 29) (336, 29) (546, 29)\n",
      "Window Train/Val/Test shape (1911, 2) (336, 2) (546, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-10-14] Val [2017-10-15 2017-10-30] Test [2017-10-31 2017-11-10]\n",
      "Window Train/Val/Test shape (2226, 29) (336, 29) (231, 29)\n",
      "Window Train/Val/Test shape (2226, 2) (336, 2) (231, 2)\n",
      "[43.72850855973647, 69.19516946543197, 23.024948088676222, 27.287211008435623, 17.303930041313368]\n",
      "mean mse: 36.10795343271873\n",
      "Train [2017-07-01 2017-08-15] Val [2017-08-16 2017-08-31] Test [2017-09-01 2017-09-16]\n",
      "Window Train/Val/Test shape (966, 29) (336, 29) (336, 29)\n",
      "Window Train/Val/Test shape (966, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-08-30] Val [2017-08-31 2017-09-15] Test [2017-09-16 2017-10-01]\n",
      "Window Train/Val/Test shape (1281, 29) (336, 29) (336, 29)\n",
      "Window Train/Val/Test shape (1281, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-09-14] Val [2017-09-15 2017-09-30] Test [2017-10-01 2017-10-16]\n",
      "Window Train/Val/Test shape (1596, 29) (336, 29) (336, 29)\n",
      "Window Train/Val/Test shape (1596, 2) (336, 2) (336, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-09-29] Val [2017-09-30 2017-10-15] Test [2017-10-16 2017-11-10]\n",
      "Window Train/Val/Test shape (1911, 29) (336, 29) (546, 29)\n",
      "Window Train/Val/Test shape (1911, 2) (336, 2) (546, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-10-14] Val [2017-10-15 2017-10-30] Test [2017-10-31 2017-11-10]\n",
      "Window Train/Val/Test shape (2226, 29) (336, 29) (231, 29)\n",
      "Window Train/Val/Test shape (2226, 2) (336, 2) (231, 2)\n",
      "[67.96405652308484, 59.33540949264558, 24.572338119697456, 24.55424552762494, 25.685179726339488]\n",
      "mean mse: 40.42224587787847\n",
      "Train [2017-07-01 2017-08-15] Val [2017-08-16 2017-08-31] Test [2017-09-01 2017-09-16]\n",
      "Window Train/Val/Test shape (966, 29) (336, 29) (336, 29)\n",
      "Window Train/Val/Test shape (966, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-08-30] Val [2017-08-31 2017-09-15] Test [2017-09-16 2017-10-01]\n",
      "Window Train/Val/Test shape (1281, 29) (336, 29) (336, 29)\n",
      "Window Train/Val/Test shape (1281, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-09-14] Val [2017-09-15 2017-09-30] Test [2017-10-01 2017-10-16]\n",
      "Window Train/Val/Test shape (1596, 29) (336, 29) (336, 29)\n",
      "Window Train/Val/Test shape (1596, 2) (336, 2) (336, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-09-29] Val [2017-09-30 2017-10-15] Test [2017-10-16 2017-11-10]\n",
      "Window Train/Val/Test shape (1911, 29) (336, 29) (546, 29)\n",
      "Window Train/Val/Test shape (1911, 2) (336, 2) (546, 2)\n",
      "Rolling window to end date\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train [2017-07-01 2017-10-14] Val [2017-10-15 2017-10-30] Test [2017-10-31 2017-11-10]\n",
      "Window Train/Val/Test shape (2226, 29) (336, 29) (231, 29)\n",
      "Window Train/Val/Test shape (2226, 2) (336, 2) (231, 2)\n",
      "[52.58794517784207, 57.98000231005614, 16.746808622878852, 19.435915762141438, 20.925747278521094]\n",
      "mean mse: 33.53528383028792\n",
      "Train [2017-07-01 2017-08-15] Val [2017-08-16 2017-08-31] Test [2017-09-01 2017-09-16]\n",
      "Window Train/Val/Test shape (966, 29) (336, 29) (336, 29)\n",
      "Window Train/Val/Test shape (966, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-08-30] Val [2017-08-31 2017-09-15] Test [2017-09-16 2017-10-01]\n",
      "Window Train/Val/Test shape (1281, 29) (336, 29) (336, 29)\n",
      "Window Train/Val/Test shape (1281, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-09-14] Val [2017-09-15 2017-09-30] Test [2017-10-01 2017-10-16]\n",
      "Window Train/Val/Test shape (1596, 29) (336, 29) (336, 29)\n",
      "Window Train/Val/Test shape (1596, 2) (336, 2) (336, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-09-29] Val [2017-09-30 2017-10-15] Test [2017-10-16 2017-11-10]\n",
      "Window Train/Val/Test shape (1911, 29) (336, 29) (546, 29)\n",
      "Window Train/Val/Test shape (1911, 2) (336, 2) (546, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-10-14] Val [2017-10-15 2017-10-30] Test [2017-10-31 2017-11-10]\n",
      "Window Train/Val/Test shape (2226, 29) (336, 29) (231, 29)\n",
      "Window Train/Val/Test shape (2226, 2) (336, 2) (231, 2)\n",
      "[57.20306744376715, 59.10412282104091, 21.987456578055724, 24.978639540172416, 22.28238611300619]\n",
      "mean mse: 37.11113449920848\n",
      "Train [2017-07-01 2017-08-15] Val [2017-08-16 2017-08-31] Test [2017-09-01 2017-09-16]\n",
      "Window Train/Val/Test shape (966, 29) (336, 29) (336, 29)\n",
      "Window Train/Val/Test shape (966, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-08-30] Val [2017-08-31 2017-09-15] Test [2017-09-16 2017-10-01]\n",
      "Window Train/Val/Test shape (1281, 29) (336, 29) (336, 29)\n",
      "Window Train/Val/Test shape (1281, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-09-14] Val [2017-09-15 2017-09-30] Test [2017-10-01 2017-10-16]\n",
      "Window Train/Val/Test shape (1596, 29) (336, 29) (336, 29)\n",
      "Window Train/Val/Test shape (1596, 2) (336, 2) (336, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-09-29] Val [2017-09-30 2017-10-15] Test [2017-10-16 2017-11-10]\n",
      "Window Train/Val/Test shape (1911, 29) (336, 29) (546, 29)\n",
      "Window Train/Val/Test shape (1911, 2) (336, 2) (546, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-10-14] Val [2017-10-15 2017-10-30] Test [2017-10-31 2017-11-10]\n",
      "Window Train/Val/Test shape (2226, 29) (336, 29) (231, 29)\n",
      "Window Train/Val/Test shape (2226, 2) (336, 2) (231, 2)\n",
      "[79.7462989376511, 56.99942460901683, 24.374655989952306, 20.577064602584763, 19.334373750123856]\n",
      "mean mse: 40.20636357786577\n",
      "Train [2017-07-01 2017-08-15] Val [2017-08-16 2017-08-31] Test [2017-09-01 2017-09-16]\n",
      "Window Train/Val/Test shape (966, 29) (336, 29) (336, 29)\n",
      "Window Train/Val/Test shape (966, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-08-30] Val [2017-08-31 2017-09-15] Test [2017-09-16 2017-10-01]\n",
      "Window Train/Val/Test shape (1281, 29) (336, 29) (336, 29)\n",
      "Window Train/Val/Test shape (1281, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-09-14] Val [2017-09-15 2017-09-30] Test [2017-10-01 2017-10-16]\n",
      "Window Train/Val/Test shape (1596, 29) (336, 29) (336, 29)\n",
      "Window Train/Val/Test shape (1596, 2) (336, 2) (336, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-09-29] Val [2017-09-30 2017-10-15] Test [2017-10-16 2017-11-10]\n",
      "Window Train/Val/Test shape (1911, 29) (336, 29) (546, 29)\n",
      "Window Train/Val/Test shape (1911, 2) (336, 2) (546, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-10-14] Val [2017-10-15 2017-10-30] Test [2017-10-31 2017-11-10]\n",
      "Window Train/Val/Test shape (2226, 29) (336, 29) (231, 29)\n",
      "Window Train/Val/Test shape (2226, 2) (336, 2) (231, 2)\n",
      "[65.95410318501776, 65.05691817008318, 20.15893402004991, 19.419346260630313, 23.24560790869186]\n",
      "mean mse: 38.7669819088946\n",
      "Train [2017-07-01 2017-08-15] Val [2017-08-16 2017-08-31] Test [2017-09-01 2017-09-16]\n",
      "Window Train/Val/Test shape (966, 29) (336, 29) (336, 29)\n",
      "Window Train/Val/Test shape (966, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-08-30] Val [2017-08-31 2017-09-15] Test [2017-09-16 2017-10-01]\n",
      "Window Train/Val/Test shape (1281, 29) (336, 29) (336, 29)\n",
      "Window Train/Val/Test shape (1281, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-09-14] Val [2017-09-15 2017-09-30] Test [2017-10-01 2017-10-16]\n",
      "Window Train/Val/Test shape (1596, 29) (336, 29) (336, 29)\n",
      "Window Train/Val/Test shape (1596, 2) (336, 2) (336, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-09-29] Val [2017-09-30 2017-10-15] Test [2017-10-16 2017-11-10]\n",
      "Window Train/Val/Test shape (1911, 29) (336, 29) (546, 29)\n",
      "Window Train/Val/Test shape (1911, 2) (336, 2) (546, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-10-14] Val [2017-10-15 2017-10-30] Test [2017-10-31 2017-11-10]\n",
      "Window Train/Val/Test shape (2226, 29) (336, 29) (231, 29)\n",
      "Window Train/Val/Test shape (2226, 2) (336, 2) (231, 2)\n",
      "[69.75510445695855, 63.420466229355355, 22.124935324818836, 17.416689201819224, 20.20885319462705]\n",
      "mean mse: 38.5852096815158\n",
      "Train [2017-07-01 2017-08-15] Val [2017-08-16 2017-08-31] Test [2017-09-01 2017-09-16]\n",
      "Window Train/Val/Test shape (966, 29) (336, 29) (336, 29)\n",
      "Window Train/Val/Test shape (966, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-08-30] Val [2017-08-31 2017-09-15] Test [2017-09-16 2017-10-01]\n",
      "Window Train/Val/Test shape (1281, 29) (336, 29) (336, 29)\n",
      "Window Train/Val/Test shape (1281, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-09-14] Val [2017-09-15 2017-09-30] Test [2017-10-01 2017-10-16]\n",
      "Window Train/Val/Test shape (1596, 29) (336, 29) (336, 29)\n",
      "Window Train/Val/Test shape (1596, 2) (336, 2) (336, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-09-29] Val [2017-09-30 2017-10-15] Test [2017-10-16 2017-11-10]\n",
      "Window Train/Val/Test shape (1911, 29) (336, 29) (546, 29)\n",
      "Window Train/Val/Test shape (1911, 2) (336, 2) (546, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-10-14] Val [2017-10-15 2017-10-30] Test [2017-10-31 2017-11-10]\n",
      "Window Train/Val/Test shape (2226, 29) (336, 29) (231, 29)\n",
      "Window Train/Val/Test shape (2226, 2) (336, 2) (231, 2)\n",
      "[70.4445606720781, 51.47101193407473, 15.159094298289082, 18.04606267367414, 30.449793409967015]\n",
      "mean mse: 37.11410459761662\n",
      "Train [2017-07-01 2017-08-15] Val [2017-08-16 2017-08-31] Test [2017-09-01 2017-09-16]\n",
      "Window Train/Val/Test shape (966, 29) (336, 29) (336, 29)\n",
      "Window Train/Val/Test shape (966, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-08-30] Val [2017-08-31 2017-09-15] Test [2017-09-16 2017-10-01]\n",
      "Window Train/Val/Test shape (1281, 29) (336, 29) (336, 29)\n",
      "Window Train/Val/Test shape (1281, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-09-14] Val [2017-09-15 2017-09-30] Test [2017-10-01 2017-10-16]\n",
      "Window Train/Val/Test shape (1596, 29) (336, 29) (336, 29)\n",
      "Window Train/Val/Test shape (1596, 2) (336, 2) (336, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-09-29] Val [2017-09-30 2017-10-15] Test [2017-10-16 2017-11-10]\n",
      "Window Train/Val/Test shape (1911, 29) (336, 29) (546, 29)\n",
      "Window Train/Val/Test shape (1911, 2) (336, 2) (546, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-10-14] Val [2017-10-15 2017-10-30] Test [2017-10-31 2017-11-10]\n",
      "Window Train/Val/Test shape (2226, 29) (336, 29) (231, 29)\n",
      "Window Train/Val/Test shape (2226, 2) (336, 2) (231, 2)\n",
      "[66.25922316449223, 61.66756726427997, 16.99919130485249, 20.074654475460488, 18.733331613469197]\n",
      "mean mse: 36.746793564510874\n",
      "Train [2017-07-01 2017-08-15] Val [2017-08-16 2017-08-31] Test [2017-09-01 2017-09-16]\n",
      "Window Train/Val/Test shape (966, 29) (336, 29) (336, 29)\n",
      "Window Train/Val/Test shape (966, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-08-30] Val [2017-08-31 2017-09-15] Test [2017-09-16 2017-10-01]\n",
      "Window Train/Val/Test shape (1281, 29) (336, 29) (336, 29)\n",
      "Window Train/Val/Test shape (1281, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-09-14] Val [2017-09-15 2017-09-30] Test [2017-10-01 2017-10-16]\n",
      "Window Train/Val/Test shape (1596, 29) (336, 29) (336, 29)\n",
      "Window Train/Val/Test shape (1596, 2) (336, 2) (336, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-09-29] Val [2017-09-30 2017-10-15] Test [2017-10-16 2017-11-10]\n",
      "Window Train/Val/Test shape (1911, 29) (336, 29) (546, 29)\n",
      "Window Train/Val/Test shape (1911, 2) (336, 2) (546, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-10-14] Val [2017-10-15 2017-10-30] Test [2017-10-31 2017-11-10]\n",
      "Window Train/Val/Test shape (2226, 29) (336, 29) (231, 29)\n",
      "Window Train/Val/Test shape (2226, 2) (336, 2) (231, 2)\n",
      "[69.97900187573657, 48.66361775666506, 34.177243503089066, 24.380514575932068, 19.51320447092034]\n",
      "mean mse: 39.34271643646862\n",
      "Train [2017-07-01 2017-08-15] Val [2017-08-16 2017-08-31] Test [2017-09-01 2017-09-16]\n",
      "Window Train/Val/Test shape (966, 29) (336, 29) (336, 29)\n",
      "Window Train/Val/Test shape (966, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-08-30] Val [2017-08-31 2017-09-15] Test [2017-09-16 2017-10-01]\n",
      "Window Train/Val/Test shape (1281, 29) (336, 29) (336, 29)\n",
      "Window Train/Val/Test shape (1281, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-09-14] Val [2017-09-15 2017-09-30] Test [2017-10-01 2017-10-16]\n",
      "Window Train/Val/Test shape (1596, 29) (336, 29) (336, 29)\n",
      "Window Train/Val/Test shape (1596, 2) (336, 2) (336, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-09-29] Val [2017-09-30 2017-10-15] Test [2017-10-16 2017-11-10]\n",
      "Window Train/Val/Test shape (1911, 29) (336, 29) (546, 29)\n",
      "Window Train/Val/Test shape (1911, 2) (336, 2) (546, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-10-14] Val [2017-10-15 2017-10-30] Test [2017-10-31 2017-11-10]\n",
      "Window Train/Val/Test shape (2226, 29) (336, 29) (231, 29)\n",
      "Window Train/Val/Test shape (2226, 2) (336, 2) (231, 2)\n",
      "[58.43799561142647, 52.42803184182261, 21.75225810306286, 26.752771825366896, 24.26683872640671]\n",
      "mean mse: 36.72757922161711\n",
      "Train [2017-07-01 2017-08-15] Val [2017-08-16 2017-08-31] Test [2017-09-01 2017-09-16]\n",
      "Window Train/Val/Test shape (966, 29) (336, 29) (336, 29)\n",
      "Window Train/Val/Test shape (966, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-08-30] Val [2017-08-31 2017-09-15] Test [2017-09-16 2017-10-01]\n",
      "Window Train/Val/Test shape (1281, 29) (336, 29) (336, 29)\n",
      "Window Train/Val/Test shape (1281, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-09-14] Val [2017-09-15 2017-09-30] Test [2017-10-01 2017-10-16]\n",
      "Window Train/Val/Test shape (1596, 29) (336, 29) (336, 29)\n",
      "Window Train/Val/Test shape (1596, 2) (336, 2) (336, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-09-29] Val [2017-09-30 2017-10-15] Test [2017-10-16 2017-11-10]\n",
      "Window Train/Val/Test shape (1911, 29) (336, 29) (546, 29)\n",
      "Window Train/Val/Test shape (1911, 2) (336, 2) (546, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-10-14] Val [2017-10-15 2017-10-30] Test [2017-10-31 2017-11-10]\n",
      "Window Train/Val/Test shape (2226, 29) (336, 29) (231, 29)\n",
      "Window Train/Val/Test shape (2226, 2) (336, 2) (231, 2)\n",
      "[61.62897122652081, 56.415017336973655, 22.184701936924945, 17.03379749315874, 29.67344145719147]\n",
      "mean mse: 37.38718589015393\n",
      "Train [2017-07-01 2017-08-15] Val [2017-08-16 2017-08-31] Test [2017-09-01 2017-09-16]\n",
      "Window Train/Val/Test shape (966, 29) (336, 29) (336, 29)\n",
      "Window Train/Val/Test shape (966, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-08-30] Val [2017-08-31 2017-09-15] Test [2017-09-16 2017-10-01]\n",
      "Window Train/Val/Test shape (1281, 29) (336, 29) (336, 29)\n",
      "Window Train/Val/Test shape (1281, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-09-14] Val [2017-09-15 2017-09-30] Test [2017-10-01 2017-10-16]\n",
      "Window Train/Val/Test shape (1596, 29) (336, 29) (336, 29)\n",
      "Window Train/Val/Test shape (1596, 2) (336, 2) (336, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-09-29] Val [2017-09-30 2017-10-15] Test [2017-10-16 2017-11-10]\n",
      "Window Train/Val/Test shape (1911, 29) (336, 29) (546, 29)\n",
      "Window Train/Val/Test shape (1911, 2) (336, 2) (546, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-10-14] Val [2017-10-15 2017-10-30] Test [2017-10-31 2017-11-10]\n",
      "Window Train/Val/Test shape (2226, 29) (336, 29) (231, 29)\n",
      "Window Train/Val/Test shape (2226, 2) (336, 2) (231, 2)\n",
      "[55.383929562904086, 55.664669210847045, 22.669500850486862, 21.39110540092216, 20.333949109800233]\n",
      "mean mse: 35.08863082699208\n",
      "Train [2017-07-01 2017-08-15] Val [2017-08-16 2017-08-31] Test [2017-09-01 2017-09-16]\n",
      "Window Train/Val/Test shape (966, 29) (336, 29) (336, 29)\n",
      "Window Train/Val/Test shape (966, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-08-30] Val [2017-08-31 2017-09-15] Test [2017-09-16 2017-10-01]\n",
      "Window Train/Val/Test shape (1281, 29) (336, 29) (336, 29)\n",
      "Window Train/Val/Test shape (1281, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-09-14] Val [2017-09-15 2017-09-30] Test [2017-10-01 2017-10-16]\n",
      "Window Train/Val/Test shape (1596, 29) (336, 29) (336, 29)\n",
      "Window Train/Val/Test shape (1596, 2) (336, 2) (336, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-09-29] Val [2017-09-30 2017-10-15] Test [2017-10-16 2017-11-10]\n",
      "Window Train/Val/Test shape (1911, 29) (336, 29) (546, 29)\n",
      "Window Train/Val/Test shape (1911, 2) (336, 2) (546, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-10-14] Val [2017-10-15 2017-10-30] Test [2017-10-31 2017-11-10]\n",
      "Window Train/Val/Test shape (2226, 29) (336, 29) (231, 29)\n",
      "Window Train/Val/Test shape (2226, 2) (336, 2) (231, 2)\n",
      "[72.01960565440416, 55.39539809541803, 26.11227177664286, 22.134487770732214, 20.734210876622456]\n",
      "mean mse: 39.27919483476394\n",
      "Train [2017-07-01 2017-08-15] Val [2017-08-16 2017-08-31] Test [2017-09-01 2017-09-16]\n",
      "Window Train/Val/Test shape (966, 29) (336, 29) (336, 29)\n",
      "Window Train/Val/Test shape (966, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-08-30] Val [2017-08-31 2017-09-15] Test [2017-09-16 2017-10-01]\n",
      "Window Train/Val/Test shape (1281, 29) (336, 29) (336, 29)\n",
      "Window Train/Val/Test shape (1281, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-09-14] Val [2017-09-15 2017-09-30] Test [2017-10-01 2017-10-16]\n",
      "Window Train/Val/Test shape (1596, 29) (336, 29) (336, 29)\n",
      "Window Train/Val/Test shape (1596, 2) (336, 2) (336, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-09-29] Val [2017-09-30 2017-10-15] Test [2017-10-16 2017-11-10]\n",
      "Window Train/Val/Test shape (1911, 29) (336, 29) (546, 29)\n",
      "Window Train/Val/Test shape (1911, 2) (336, 2) (546, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-10-14] Val [2017-10-15 2017-10-30] Test [2017-10-31 2017-11-10]\n",
      "Window Train/Val/Test shape (2226, 29) (336, 29) (231, 29)\n",
      "Window Train/Val/Test shape (2226, 2) (336, 2) (231, 2)\n",
      "[54.44255256209912, 60.29787348971941, 19.539709404494353, 20.08551084845182, 14.52231013113022]\n",
      "mean mse: 33.777591287178986\n",
      "Train [2017-07-01 2017-08-15] Val [2017-08-16 2017-08-31] Test [2017-09-01 2017-09-16]\n",
      "Window Train/Val/Test shape (966, 29) (336, 29) (336, 29)\n",
      "Window Train/Val/Test shape (966, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-08-30] Val [2017-08-31 2017-09-15] Test [2017-09-16 2017-10-01]\n",
      "Window Train/Val/Test shape (1281, 29) (336, 29) (336, 29)\n",
      "Window Train/Val/Test shape (1281, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-09-14] Val [2017-09-15 2017-09-30] Test [2017-10-01 2017-10-16]\n",
      "Window Train/Val/Test shape (1596, 29) (336, 29) (336, 29)\n",
      "Window Train/Val/Test shape (1596, 2) (336, 2) (336, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-09-29] Val [2017-09-30 2017-10-15] Test [2017-10-16 2017-11-10]\n",
      "Window Train/Val/Test shape (1911, 29) (336, 29) (546, 29)\n",
      "Window Train/Val/Test shape (1911, 2) (336, 2) (546, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-10-14] Val [2017-10-15 2017-10-30] Test [2017-10-31 2017-11-10]\n",
      "Window Train/Val/Test shape (2226, 29) (336, 29) (231, 29)\n",
      "Window Train/Val/Test shape (2226, 2) (336, 2) (231, 2)\n",
      "[47.98524973631404, 49.027811641168455, 22.29621887068087, 20.60557808838247, 17.8189230185926]\n",
      "mean mse: 31.546756271027686\n",
      "Train [2017-07-01 2017-08-15] Val [2017-08-16 2017-08-31] Test [2017-09-01 2017-09-16]\n",
      "Window Train/Val/Test shape (966, 29) (336, 29) (336, 29)\n",
      "Window Train/Val/Test shape (966, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-08-30] Val [2017-08-31 2017-09-15] Test [2017-09-16 2017-10-01]\n",
      "Window Train/Val/Test shape (1281, 29) (336, 29) (336, 29)\n",
      "Window Train/Val/Test shape (1281, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-09-14] Val [2017-09-15 2017-09-30] Test [2017-10-01 2017-10-16]\n",
      "Window Train/Val/Test shape (1596, 29) (336, 29) (336, 29)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Window Train/Val/Test shape (1596, 2) (336, 2) (336, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-09-29] Val [2017-09-30 2017-10-15] Test [2017-10-16 2017-11-10]\n",
      "Window Train/Val/Test shape (1911, 29) (336, 29) (546, 29)\n",
      "Window Train/Val/Test shape (1911, 2) (336, 2) (546, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-10-14] Val [2017-10-15 2017-10-30] Test [2017-10-31 2017-11-10]\n",
      "Window Train/Val/Test shape (2226, 29) (336, 29) (231, 29)\n",
      "Window Train/Val/Test shape (2226, 2) (336, 2) (231, 2)\n",
      "[71.9279968603582, 55.00529555976099, 20.13116437735047, 19.129944910019894, 23.67441472401715]\n",
      "mean mse: 37.97376328630134\n",
      "Train [2017-07-01 2017-08-15] Val [2017-08-16 2017-08-31] Test [2017-09-01 2017-09-16]\n",
      "Window Train/Val/Test shape (966, 29) (336, 29) (336, 29)\n",
      "Window Train/Val/Test shape (966, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-08-30] Val [2017-08-31 2017-09-15] Test [2017-09-16 2017-10-01]\n",
      "Window Train/Val/Test shape (1281, 29) (336, 29) (336, 29)\n",
      "Window Train/Val/Test shape (1281, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-09-14] Val [2017-09-15 2017-09-30] Test [2017-10-01 2017-10-16]\n",
      "Window Train/Val/Test shape (1596, 29) (336, 29) (336, 29)\n",
      "Window Train/Val/Test shape (1596, 2) (336, 2) (336, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-09-29] Val [2017-09-30 2017-10-15] Test [2017-10-16 2017-11-10]\n",
      "Window Train/Val/Test shape (1911, 29) (336, 29) (546, 29)\n",
      "Window Train/Val/Test shape (1911, 2) (336, 2) (546, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-10-14] Val [2017-10-15 2017-10-30] Test [2017-10-31 2017-11-10]\n",
      "Window Train/Val/Test shape (2226, 29) (336, 29) (231, 29)\n",
      "Window Train/Val/Test shape (2226, 2) (336, 2) (231, 2)\n",
      "[72.21032891264464, 62.09461983837588, 18.865599343067068, 21.70463927210425, 24.094320807565285]\n",
      "mean mse: 39.79390163475142\n",
      "Train [2017-07-01 2017-08-15] Val [2017-08-16 2017-08-31] Test [2017-09-01 2017-09-16]\n",
      "Window Train/Val/Test shape (966, 29) (336, 29) (336, 29)\n",
      "Window Train/Val/Test shape (966, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-08-30] Val [2017-08-31 2017-09-15] Test [2017-09-16 2017-10-01]\n",
      "Window Train/Val/Test shape (1281, 29) (336, 29) (336, 29)\n",
      "Window Train/Val/Test shape (1281, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-09-14] Val [2017-09-15 2017-09-30] Test [2017-10-01 2017-10-16]\n",
      "Window Train/Val/Test shape (1596, 29) (336, 29) (336, 29)\n",
      "Window Train/Val/Test shape (1596, 2) (336, 2) (336, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-09-29] Val [2017-09-30 2017-10-15] Test [2017-10-16 2017-11-10]\n",
      "Window Train/Val/Test shape (1911, 29) (336, 29) (546, 29)\n",
      "Window Train/Val/Test shape (1911, 2) (336, 2) (546, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-10-14] Val [2017-10-15 2017-10-30] Test [2017-10-31 2017-11-10]\n",
      "Window Train/Val/Test shape (2226, 29) (336, 29) (231, 29)\n",
      "Window Train/Val/Test shape (2226, 2) (336, 2) (231, 2)\n",
      "[60.06407746174252, 57.86982019068331, 14.464212841247544, 20.492452551570587, 19.343933600734807]\n",
      "mean mse: 34.446899329195745\n",
      "Train [2017-07-01 2017-08-15] Val [2017-08-16 2017-08-31] Test [2017-09-01 2017-09-16]\n",
      "Window Train/Val/Test shape (966, 29) (336, 29) (336, 29)\n",
      "Window Train/Val/Test shape (966, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-08-30] Val [2017-08-31 2017-09-15] Test [2017-09-16 2017-10-01]\n",
      "Window Train/Val/Test shape (1281, 29) (336, 29) (336, 29)\n",
      "Window Train/Val/Test shape (1281, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-09-14] Val [2017-09-15 2017-09-30] Test [2017-10-01 2017-10-16]\n",
      "Window Train/Val/Test shape (1596, 29) (336, 29) (336, 29)\n",
      "Window Train/Val/Test shape (1596, 2) (336, 2) (336, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-09-29] Val [2017-09-30 2017-10-15] Test [2017-10-16 2017-11-10]\n",
      "Window Train/Val/Test shape (1911, 29) (336, 29) (546, 29)\n",
      "Window Train/Val/Test shape (1911, 2) (336, 2) (546, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-10-14] Val [2017-10-15 2017-10-30] Test [2017-10-31 2017-11-10]\n",
      "Window Train/Val/Test shape (2226, 29) (336, 29) (231, 29)\n",
      "Window Train/Val/Test shape (2226, 2) (336, 2) (231, 2)\n",
      "[50.426005606148834, 61.50446900162987, 25.606437907866628, 17.59869366638203, 21.10574908204612]\n",
      "mean mse: 35.248271052814694\n",
      "Train [2017-07-01 2017-08-15] Val [2017-08-16 2017-08-31] Test [2017-09-01 2017-09-16]\n",
      "Window Train/Val/Test shape (966, 29) (336, 29) (336, 29)\n",
      "Window Train/Val/Test shape (966, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-08-30] Val [2017-08-31 2017-09-15] Test [2017-09-16 2017-10-01]\n",
      "Window Train/Val/Test shape (1281, 29) (336, 29) (336, 29)\n",
      "Window Train/Val/Test shape (1281, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-09-14] Val [2017-09-15 2017-09-30] Test [2017-10-01 2017-10-16]\n",
      "Window Train/Val/Test shape (1596, 29) (336, 29) (336, 29)\n",
      "Window Train/Val/Test shape (1596, 2) (336, 2) (336, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-09-29] Val [2017-09-30 2017-10-15] Test [2017-10-16 2017-11-10]\n",
      "Window Train/Val/Test shape (1911, 29) (336, 29) (546, 29)\n",
      "Window Train/Val/Test shape (1911, 2) (336, 2) (546, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-10-14] Val [2017-10-15 2017-10-30] Test [2017-10-31 2017-11-10]\n",
      "Window Train/Val/Test shape (2226, 29) (336, 29) (231, 29)\n",
      "Window Train/Val/Test shape (2226, 2) (336, 2) (231, 2)\n",
      "[74.51398789625433, 60.1921680198696, 22.025242337086834, 19.129197736472296, 19.577982644788285]\n",
      "mean mse: 39.08771572689427\n",
      "Train [2017-07-01 2017-08-15] Val [2017-08-16 2017-08-31] Test [2017-09-01 2017-09-16]\n",
      "Window Train/Val/Test shape (966, 29) (336, 29) (336, 29)\n",
      "Window Train/Val/Test shape (966, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-08-30] Val [2017-08-31 2017-09-15] Test [2017-09-16 2017-10-01]\n",
      "Window Train/Val/Test shape (1281, 29) (336, 29) (336, 29)\n",
      "Window Train/Val/Test shape (1281, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-09-14] Val [2017-09-15 2017-09-30] Test [2017-10-01 2017-10-16]\n",
      "Window Train/Val/Test shape (1596, 29) (336, 29) (336, 29)\n",
      "Window Train/Val/Test shape (1596, 2) (336, 2) (336, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-09-29] Val [2017-09-30 2017-10-15] Test [2017-10-16 2017-11-10]\n",
      "Window Train/Val/Test shape (1911, 29) (336, 29) (546, 29)\n",
      "Window Train/Val/Test shape (1911, 2) (336, 2) (546, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-10-14] Val [2017-10-15 2017-10-30] Test [2017-10-31 2017-11-10]\n",
      "Window Train/Val/Test shape (2226, 29) (336, 29) (231, 29)\n",
      "Window Train/Val/Test shape (2226, 2) (336, 2) (231, 2)\n",
      "[55.03940249352157, 55.525287944529175, 19.368573912185497, 24.21191930440658, 13.228404943984529]\n",
      "mean mse: 33.47471771972547\n",
      "Train [2017-07-01 2017-08-15] Val [2017-08-16 2017-08-31] Test [2017-09-01 2017-09-16]\n",
      "Window Train/Val/Test shape (966, 29) (336, 29) (336, 29)\n",
      "Window Train/Val/Test shape (966, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-08-30] Val [2017-08-31 2017-09-15] Test [2017-09-16 2017-10-01]\n",
      "Window Train/Val/Test shape (1281, 29) (336, 29) (336, 29)\n",
      "Window Train/Val/Test shape (1281, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-09-14] Val [2017-09-15 2017-09-30] Test [2017-10-01 2017-10-16]\n",
      "Window Train/Val/Test shape (1596, 29) (336, 29) (336, 29)\n",
      "Window Train/Val/Test shape (1596, 2) (336, 2) (336, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-09-29] Val [2017-09-30 2017-10-15] Test [2017-10-16 2017-11-10]\n",
      "Window Train/Val/Test shape (1911, 29) (336, 29) (546, 29)\n",
      "Window Train/Val/Test shape (1911, 2) (336, 2) (546, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-10-14] Val [2017-10-15 2017-10-30] Test [2017-10-31 2017-11-10]\n",
      "Window Train/Val/Test shape (2226, 29) (336, 29) (231, 29)\n",
      "Window Train/Val/Test shape (2226, 2) (336, 2) (231, 2)\n",
      "[62.01260774405924, 60.32248796606776, 21.420031663305487, 17.666524003067376, 23.08683544012242]\n",
      "mean mse: 36.90169736332446\n",
      "Train [2017-07-01 2017-08-15] Val [2017-08-16 2017-08-31] Test [2017-09-01 2017-09-16]\n",
      "Window Train/Val/Test shape (966, 29) (336, 29) (336, 29)\n",
      "Window Train/Val/Test shape (966, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-08-30] Val [2017-08-31 2017-09-15] Test [2017-09-16 2017-10-01]\n",
      "Window Train/Val/Test shape (1281, 29) (336, 29) (336, 29)\n",
      "Window Train/Val/Test shape (1281, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-09-14] Val [2017-09-15 2017-09-30] Test [2017-10-01 2017-10-16]\n",
      "Window Train/Val/Test shape (1596, 29) (336, 29) (336, 29)\n",
      "Window Train/Val/Test shape (1596, 2) (336, 2) (336, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-09-29] Val [2017-09-30 2017-10-15] Test [2017-10-16 2017-11-10]\n",
      "Window Train/Val/Test shape (1911, 29) (336, 29) (546, 29)\n",
      "Window Train/Val/Test shape (1911, 2) (336, 2) (546, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-10-14] Val [2017-10-15 2017-10-30] Test [2017-10-31 2017-11-10]\n",
      "Window Train/Val/Test shape (2226, 29) (336, 29) (231, 29)\n",
      "Window Train/Val/Test shape (2226, 2) (336, 2) (231, 2)\n",
      "[62.20841785092426, 53.33983351534971, 20.328939782488586, 22.528209859072597, 23.22630164838344]\n",
      "mean mse: 36.32634053124372\n",
      "Train [2017-07-01 2017-08-15] Val [2017-08-16 2017-08-31] Test [2017-09-01 2017-09-16]\n",
      "Window Train/Val/Test shape (966, 29) (336, 29) (336, 29)\n",
      "Window Train/Val/Test shape (966, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-08-30] Val [2017-08-31 2017-09-15] Test [2017-09-16 2017-10-01]\n",
      "Window Train/Val/Test shape (1281, 29) (336, 29) (336, 29)\n",
      "Window Train/Val/Test shape (1281, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-09-14] Val [2017-09-15 2017-09-30] Test [2017-10-01 2017-10-16]\n",
      "Window Train/Val/Test shape (1596, 29) (336, 29) (336, 29)\n",
      "Window Train/Val/Test shape (1596, 2) (336, 2) (336, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-09-29] Val [2017-09-30 2017-10-15] Test [2017-10-16 2017-11-10]\n",
      "Window Train/Val/Test shape (1911, 29) (336, 29) (546, 29)\n",
      "Window Train/Val/Test shape (1911, 2) (336, 2) (546, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-10-14] Val [2017-10-15 2017-10-30] Test [2017-10-31 2017-11-10]\n",
      "Window Train/Val/Test shape (2226, 29) (336, 29) (231, 29)\n",
      "Window Train/Val/Test shape (2226, 2) (336, 2) (231, 2)\n",
      "[51.6247705566342, 59.91297106505933, 28.35745288476961, 20.837999670047022, 20.840251057665796]\n",
      "mean mse: 36.31468904683519\n",
      "Train [2017-07-01 2017-08-15] Val [2017-08-16 2017-08-31] Test [2017-09-01 2017-09-16]\n",
      "Window Train/Val/Test shape (966, 29) (336, 29) (336, 29)\n",
      "Window Train/Val/Test shape (966, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-08-30] Val [2017-08-31 2017-09-15] Test [2017-09-16 2017-10-01]\n",
      "Window Train/Val/Test shape (1281, 29) (336, 29) (336, 29)\n",
      "Window Train/Val/Test shape (1281, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-09-14] Val [2017-09-15 2017-09-30] Test [2017-10-01 2017-10-16]\n",
      "Window Train/Val/Test shape (1596, 29) (336, 29) (336, 29)\n",
      "Window Train/Val/Test shape (1596, 2) (336, 2) (336, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-09-29] Val [2017-09-30 2017-10-15] Test [2017-10-16 2017-11-10]\n",
      "Window Train/Val/Test shape (1911, 29) (336, 29) (546, 29)\n",
      "Window Train/Val/Test shape (1911, 2) (336, 2) (546, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-10-14] Val [2017-10-15 2017-10-30] Test [2017-10-31 2017-11-10]\n",
      "Window Train/Val/Test shape (2226, 29) (336, 29) (231, 29)\n",
      "Window Train/Val/Test shape (2226, 2) (336, 2) (231, 2)\n",
      "[47.93918608201125, 69.77917639987172, 19.933181708395303, 23.06576327527488, 16.37062537149963]\n",
      "mean mse: 35.41758656741056\n",
      "Train [2017-07-01 2017-08-15] Val [2017-08-16 2017-08-31] Test [2017-09-01 2017-09-16]\n",
      "Window Train/Val/Test shape (966, 29) (336, 29) (336, 29)\n",
      "Window Train/Val/Test shape (966, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-08-30] Val [2017-08-31 2017-09-15] Test [2017-09-16 2017-10-01]\n",
      "Window Train/Val/Test shape (1281, 29) (336, 29) (336, 29)\n",
      "Window Train/Val/Test shape (1281, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-09-14] Val [2017-09-15 2017-09-30] Test [2017-10-01 2017-10-16]\n",
      "Window Train/Val/Test shape (1596, 29) (336, 29) (336, 29)\n",
      "Window Train/Val/Test shape (1596, 2) (336, 2) (336, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-09-29] Val [2017-09-30 2017-10-15] Test [2017-10-16 2017-11-10]\n",
      "Window Train/Val/Test shape (1911, 29) (336, 29) (546, 29)\n",
      "Window Train/Val/Test shape (1911, 2) (336, 2) (546, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-10-14] Val [2017-10-15 2017-10-30] Test [2017-10-31 2017-11-10]\n",
      "Window Train/Val/Test shape (2226, 29) (336, 29) (231, 29)\n",
      "Window Train/Val/Test shape (2226, 2) (336, 2) (231, 2)\n",
      "[59.73884398846409, 51.405640926043795, 19.90900901026275, 20.38375356106002, 19.79074991782386]\n",
      "mean mse: 34.2455994807309\n",
      "Train [2017-07-01 2017-08-15] Val [2017-08-16 2017-08-31] Test [2017-09-01 2017-09-16]\n",
      "Window Train/Val/Test shape (966, 29) (336, 29) (336, 29)\n",
      "Window Train/Val/Test shape (966, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-08-30] Val [2017-08-31 2017-09-15] Test [2017-09-16 2017-10-01]\n",
      "Window Train/Val/Test shape (1281, 29) (336, 29) (336, 29)\n",
      "Window Train/Val/Test shape (1281, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-09-14] Val [2017-09-15 2017-09-30] Test [2017-10-01 2017-10-16]\n",
      "Window Train/Val/Test shape (1596, 29) (336, 29) (336, 29)\n",
      "Window Train/Val/Test shape (1596, 2) (336, 2) (336, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-09-29] Val [2017-09-30 2017-10-15] Test [2017-10-16 2017-11-10]\n",
      "Window Train/Val/Test shape (1911, 29) (336, 29) (546, 29)\n",
      "Window Train/Val/Test shape (1911, 2) (336, 2) (546, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-10-14] Val [2017-10-15 2017-10-30] Test [2017-10-31 2017-11-10]\n",
      "Window Train/Val/Test shape (2226, 29) (336, 29) (231, 29)\n",
      "Window Train/Val/Test shape (2226, 2) (336, 2) (231, 2)\n",
      "[65.38698863546814, 58.45379214941958, 24.20425786989071, 19.352806751661227, 21.538474822021023]\n",
      "mean mse: 37.78726404569214\n",
      "Train [2017-07-01 2017-08-15] Val [2017-08-16 2017-08-31] Test [2017-09-01 2017-09-16]\n",
      "Window Train/Val/Test shape (966, 29) (336, 29) (336, 29)\n",
      "Window Train/Val/Test shape (966, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-08-30] Val [2017-08-31 2017-09-15] Test [2017-09-16 2017-10-01]\n",
      "Window Train/Val/Test shape (1281, 29) (336, 29) (336, 29)\n",
      "Window Train/Val/Test shape (1281, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-09-14] Val [2017-09-15 2017-09-30] Test [2017-10-01 2017-10-16]\n",
      "Window Train/Val/Test shape (1596, 29) (336, 29) (336, 29)\n",
      "Window Train/Val/Test shape (1596, 2) (336, 2) (336, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-09-29] Val [2017-09-30 2017-10-15] Test [2017-10-16 2017-11-10]\n",
      "Window Train/Val/Test shape (1911, 29) (336, 29) (546, 29)\n",
      "Window Train/Val/Test shape (1911, 2) (336, 2) (546, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-10-14] Val [2017-10-15 2017-10-30] Test [2017-10-31 2017-11-10]\n",
      "Window Train/Val/Test shape (2226, 29) (336, 29) (231, 29)\n",
      "Window Train/Val/Test shape (2226, 2) (336, 2) (231, 2)\n",
      "[54.397165005936, 59.81883375877381, 22.436626943407525, 19.928672945805545, 17.031464435963805]\n",
      "mean mse: 34.72255261797734\n",
      "Train [2017-07-01 2017-08-15] Val [2017-08-16 2017-08-31] Test [2017-09-01 2017-09-16]\n",
      "Window Train/Val/Test shape (966, 29) (336, 29) (336, 29)\n",
      "Window Train/Val/Test shape (966, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-08-30] Val [2017-08-31 2017-09-15] Test [2017-09-16 2017-10-01]\n",
      "Window Train/Val/Test shape (1281, 29) (336, 29) (336, 29)\n",
      "Window Train/Val/Test shape (1281, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-09-14] Val [2017-09-15 2017-09-30] Test [2017-10-01 2017-10-16]\n",
      "Window Train/Val/Test shape (1596, 29) (336, 29) (336, 29)\n",
      "Window Train/Val/Test shape (1596, 2) (336, 2) (336, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-09-29] Val [2017-09-30 2017-10-15] Test [2017-10-16 2017-11-10]\n",
      "Window Train/Val/Test shape (1911, 29) (336, 29) (546, 29)\n",
      "Window Train/Val/Test shape (1911, 2) (336, 2) (546, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-10-14] Val [2017-10-15 2017-10-30] Test [2017-10-31 2017-11-10]\n",
      "Window Train/Val/Test shape (2226, 29) (336, 29) (231, 29)\n",
      "Window Train/Val/Test shape (2226, 2) (336, 2) (231, 2)\n",
      "[70.40672683739439, 73.14173270118762, 29.936397484300443, 23.385013193642372, 23.575020974401408]\n",
      "mean mse: 44.08897823818525\n",
      "Train [2017-07-01 2017-08-15] Val [2017-08-16 2017-08-31] Test [2017-09-01 2017-09-16]\n",
      "Window Train/Val/Test shape (966, 29) (336, 29) (336, 29)\n",
      "Window Train/Val/Test shape (966, 2) (336, 2) (336, 2)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train [2017-07-01 2017-08-30] Val [2017-08-31 2017-09-15] Test [2017-09-16 2017-10-01]\n",
      "Window Train/Val/Test shape (1281, 29) (336, 29) (336, 29)\n",
      "Window Train/Val/Test shape (1281, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-09-14] Val [2017-09-15 2017-09-30] Test [2017-10-01 2017-10-16]\n",
      "Window Train/Val/Test shape (1596, 29) (336, 29) (336, 29)\n",
      "Window Train/Val/Test shape (1596, 2) (336, 2) (336, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-09-29] Val [2017-09-30 2017-10-15] Test [2017-10-16 2017-11-10]\n",
      "Window Train/Val/Test shape (1911, 29) (336, 29) (546, 29)\n",
      "Window Train/Val/Test shape (1911, 2) (336, 2) (546, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-10-14] Val [2017-10-15 2017-10-30] Test [2017-10-31 2017-11-10]\n",
      "Window Train/Val/Test shape (2226, 29) (336, 29) (231, 29)\n",
      "Window Train/Val/Test shape (2226, 2) (336, 2) (231, 2)\n",
      "[65.72708814520028, 60.449223876282176, 22.69457452856961, 20.235627202759737, 17.031850664851277]\n",
      "mean mse: 37.22767288353261\n",
      "Train [2017-07-01 2017-08-15] Val [2017-08-16 2017-08-31] Test [2017-09-01 2017-09-16]\n",
      "Window Train/Val/Test shape (966, 29) (336, 29) (336, 29)\n",
      "Window Train/Val/Test shape (966, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-08-30] Val [2017-08-31 2017-09-15] Test [2017-09-16 2017-10-01]\n",
      "Window Train/Val/Test shape (1281, 29) (336, 29) (336, 29)\n",
      "Window Train/Val/Test shape (1281, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-09-14] Val [2017-09-15 2017-09-30] Test [2017-10-01 2017-10-16]\n",
      "Window Train/Val/Test shape (1596, 29) (336, 29) (336, 29)\n",
      "Window Train/Val/Test shape (1596, 2) (336, 2) (336, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-09-29] Val [2017-09-30 2017-10-15] Test [2017-10-16 2017-11-10]\n",
      "Window Train/Val/Test shape (1911, 29) (336, 29) (546, 29)\n",
      "Window Train/Val/Test shape (1911, 2) (336, 2) (546, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-10-14] Val [2017-10-15 2017-10-30] Test [2017-10-31 2017-11-10]\n",
      "Window Train/Val/Test shape (2226, 29) (336, 29) (231, 29)\n",
      "Window Train/Val/Test shape (2226, 2) (336, 2) (231, 2)\n",
      "[64.79747178242675, 61.35228959582084, 19.97742803142611, 20.006960514922486, 21.14540761895382]\n",
      "mean mse: 37.45591150871\n",
      "Train [2017-07-01 2017-08-15] Val [2017-08-16 2017-08-31] Test [2017-09-01 2017-09-16]\n",
      "Window Train/Val/Test shape (966, 29) (336, 29) (336, 29)\n",
      "Window Train/Val/Test shape (966, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-08-30] Val [2017-08-31 2017-09-15] Test [2017-09-16 2017-10-01]\n",
      "Window Train/Val/Test shape (1281, 29) (336, 29) (336, 29)\n",
      "Window Train/Val/Test shape (1281, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-09-14] Val [2017-09-15 2017-09-30] Test [2017-10-01 2017-10-16]\n",
      "Window Train/Val/Test shape (1596, 29) (336, 29) (336, 29)\n",
      "Window Train/Val/Test shape (1596, 2) (336, 2) (336, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-09-29] Val [2017-09-30 2017-10-15] Test [2017-10-16 2017-11-10]\n",
      "Window Train/Val/Test shape (1911, 29) (336, 29) (546, 29)\n",
      "Window Train/Val/Test shape (1911, 2) (336, 2) (546, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-10-14] Val [2017-10-15 2017-10-30] Test [2017-10-31 2017-11-10]\n",
      "Window Train/Val/Test shape (2226, 29) (336, 29) (231, 29)\n",
      "Window Train/Val/Test shape (2226, 2) (336, 2) (231, 2)\n",
      "[74.73570053277102, 71.79352859726012, 21.141338031376932, 20.330930646696647, 21.552449091911097]\n",
      "mean mse: 41.91078938000316\n",
      "Train [2017-07-01 2017-08-15] Val [2017-08-16 2017-08-31] Test [2017-09-01 2017-09-16]\n",
      "Window Train/Val/Test shape (966, 29) (336, 29) (336, 29)\n",
      "Window Train/Val/Test shape (966, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-08-30] Val [2017-08-31 2017-09-15] Test [2017-09-16 2017-10-01]\n",
      "Window Train/Val/Test shape (1281, 29) (336, 29) (336, 29)\n",
      "Window Train/Val/Test shape (1281, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-09-14] Val [2017-09-15 2017-09-30] Test [2017-10-01 2017-10-16]\n",
      "Window Train/Val/Test shape (1596, 29) (336, 29) (336, 29)\n",
      "Window Train/Val/Test shape (1596, 2) (336, 2) (336, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-09-29] Val [2017-09-30 2017-10-15] Test [2017-10-16 2017-11-10]\n",
      "Window Train/Val/Test shape (1911, 29) (336, 29) (546, 29)\n",
      "Window Train/Val/Test shape (1911, 2) (336, 2) (546, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-10-14] Val [2017-10-15 2017-10-30] Test [2017-10-31 2017-11-10]\n",
      "Window Train/Val/Test shape (2226, 29) (336, 29) (231, 29)\n",
      "Window Train/Val/Test shape (2226, 2) (336, 2) (231, 2)\n",
      "[60.54623791340646, 64.23996097614152, 24.82602271712214, 19.411639883257948, 18.857753860488334]\n",
      "mean mse: 37.57632307008328\n",
      "Train [2017-07-01 2017-08-15] Val [2017-08-16 2017-08-31] Test [2017-09-01 2017-09-16]\n",
      "Window Train/Val/Test shape (966, 29) (336, 29) (336, 29)\n",
      "Window Train/Val/Test shape (966, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-08-30] Val [2017-08-31 2017-09-15] Test [2017-09-16 2017-10-01]\n",
      "Window Train/Val/Test shape (1281, 29) (336, 29) (336, 29)\n",
      "Window Train/Val/Test shape (1281, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-09-14] Val [2017-09-15 2017-09-30] Test [2017-10-01 2017-10-16]\n",
      "Window Train/Val/Test shape (1596, 29) (336, 29) (336, 29)\n",
      "Window Train/Val/Test shape (1596, 2) (336, 2) (336, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-09-29] Val [2017-09-30 2017-10-15] Test [2017-10-16 2017-11-10]\n",
      "Window Train/Val/Test shape (1911, 29) (336, 29) (546, 29)\n",
      "Window Train/Val/Test shape (1911, 2) (336, 2) (546, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-10-14] Val [2017-10-15 2017-10-30] Test [2017-10-31 2017-11-10]\n",
      "Window Train/Val/Test shape (2226, 29) (336, 29) (231, 29)\n",
      "Window Train/Val/Test shape (2226, 2) (336, 2) (231, 2)\n",
      "[52.62312666750073, 62.582953653398384, 15.06697999625607, 19.31982377295391, 21.27904129579008]\n",
      "mean mse: 34.17438507717983\n",
      "Train [2017-07-01 2017-08-15] Val [2017-08-16 2017-08-31] Test [2017-09-01 2017-09-16]\n",
      "Window Train/Val/Test shape (966, 29) (336, 29) (336, 29)\n",
      "Window Train/Val/Test shape (966, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-08-30] Val [2017-08-31 2017-09-15] Test [2017-09-16 2017-10-01]\n",
      "Window Train/Val/Test shape (1281, 29) (336, 29) (336, 29)\n",
      "Window Train/Val/Test shape (1281, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-09-14] Val [2017-09-15 2017-09-30] Test [2017-10-01 2017-10-16]\n",
      "Window Train/Val/Test shape (1596, 29) (336, 29) (336, 29)\n",
      "Window Train/Val/Test shape (1596, 2) (336, 2) (336, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-09-29] Val [2017-09-30 2017-10-15] Test [2017-10-16 2017-11-10]\n",
      "Window Train/Val/Test shape (1911, 29) (336, 29) (546, 29)\n",
      "Window Train/Val/Test shape (1911, 2) (336, 2) (546, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-10-14] Val [2017-10-15 2017-10-30] Test [2017-10-31 2017-11-10]\n",
      "Window Train/Val/Test shape (2226, 29) (336, 29) (231, 29)\n",
      "Window Train/Val/Test shape (2226, 2) (336, 2) (231, 2)\n",
      "[61.60728562024486, 54.431776574629815, 15.820922476978202, 22.45815269121555, 17.83580579798036]\n",
      "mean mse: 34.43078863220976\n",
      "Train [2017-07-01 2017-08-15] Val [2017-08-16 2017-08-31] Test [2017-09-01 2017-09-16]\n",
      "Window Train/Val/Test shape (966, 29) (336, 29) (336, 29)\n",
      "Window Train/Val/Test shape (966, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-08-30] Val [2017-08-31 2017-09-15] Test [2017-09-16 2017-10-01]\n",
      "Window Train/Val/Test shape (1281, 29) (336, 29) (336, 29)\n",
      "Window Train/Val/Test shape (1281, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-09-14] Val [2017-09-15 2017-09-30] Test [2017-10-01 2017-10-16]\n",
      "Window Train/Val/Test shape (1596, 29) (336, 29) (336, 29)\n",
      "Window Train/Val/Test shape (1596, 2) (336, 2) (336, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-09-29] Val [2017-09-30 2017-10-15] Test [2017-10-16 2017-11-10]\n",
      "Window Train/Val/Test shape (1911, 29) (336, 29) (546, 29)\n",
      "Window Train/Val/Test shape (1911, 2) (336, 2) (546, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-10-14] Val [2017-10-15 2017-10-30] Test [2017-10-31 2017-11-10]\n",
      "Window Train/Val/Test shape (2226, 29) (336, 29) (231, 29)\n",
      "Window Train/Val/Test shape (2226, 2) (336, 2) (231, 2)\n",
      "[987.931295689933, 521.0934633649847, 350.8162566999191, 222.68899875926647, 143.22159405120408]\n",
      "mean mse: 445.1503217130615\n",
      "Train [2017-07-01 2017-08-15] Val [2017-08-16 2017-08-31] Test [2017-09-01 2017-09-16]\n",
      "Window Train/Val/Test shape (966, 29) (336, 29) (336, 29)\n",
      "Window Train/Val/Test shape (966, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-08-30] Val [2017-08-31 2017-09-15] Test [2017-09-16 2017-10-01]\n",
      "Window Train/Val/Test shape (1281, 29) (336, 29) (336, 29)\n",
      "Window Train/Val/Test shape (1281, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-09-14] Val [2017-09-15 2017-09-30] Test [2017-10-01 2017-10-16]\n",
      "Window Train/Val/Test shape (1596, 29) (336, 29) (336, 29)\n",
      "Window Train/Val/Test shape (1596, 2) (336, 2) (336, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-09-29] Val [2017-09-30 2017-10-15] Test [2017-10-16 2017-11-10]\n",
      "Window Train/Val/Test shape (1911, 29) (336, 29) (546, 29)\n",
      "Window Train/Val/Test shape (1911, 2) (336, 2) (546, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-10-14] Val [2017-10-15 2017-10-30] Test [2017-10-31 2017-11-10]\n",
      "Window Train/Val/Test shape (2226, 29) (336, 29) (231, 29)\n",
      "Window Train/Val/Test shape (2226, 2) (336, 2) (231, 2)\n",
      "[58.514920947549115, 54.87863028465127, 16.921767841970347, 17.43071965702312, 17.850471013811134]\n",
      "mean mse: 33.11930194900099\n",
      "Train [2017-07-01 2017-08-15] Val [2017-08-16 2017-08-31] Test [2017-09-01 2017-09-16]\n",
      "Window Train/Val/Test shape (966, 29) (336, 29) (336, 29)\n",
      "Window Train/Val/Test shape (966, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-08-30] Val [2017-08-31 2017-09-15] Test [2017-09-16 2017-10-01]\n",
      "Window Train/Val/Test shape (1281, 29) (336, 29) (336, 29)\n",
      "Window Train/Val/Test shape (1281, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-09-14] Val [2017-09-15 2017-09-30] Test [2017-10-01 2017-10-16]\n",
      "Window Train/Val/Test shape (1596, 29) (336, 29) (336, 29)\n",
      "Window Train/Val/Test shape (1596, 2) (336, 2) (336, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-09-29] Val [2017-09-30 2017-10-15] Test [2017-10-16 2017-11-10]\n",
      "Window Train/Val/Test shape (1911, 29) (336, 29) (546, 29)\n",
      "Window Train/Val/Test shape (1911, 2) (336, 2) (546, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-10-14] Val [2017-10-15 2017-10-30] Test [2017-10-31 2017-11-10]\n",
      "Window Train/Val/Test shape (2226, 29) (336, 29) (231, 29)\n",
      "Window Train/Val/Test shape (2226, 2) (336, 2) (231, 2)\n",
      "[76.08420237214747, 51.05133672852302, 28.9242288301099, 23.116457952275795, 22.864304154657738]\n",
      "mean mse: 40.408106007542784\n",
      "Train [2017-07-01 2017-08-15] Val [2017-08-16 2017-08-31] Test [2017-09-01 2017-09-16]\n",
      "Window Train/Val/Test shape (966, 29) (336, 29) (336, 29)\n",
      "Window Train/Val/Test shape (966, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-08-30] Val [2017-08-31 2017-09-15] Test [2017-09-16 2017-10-01]\n",
      "Window Train/Val/Test shape (1281, 29) (336, 29) (336, 29)\n",
      "Window Train/Val/Test shape (1281, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-09-14] Val [2017-09-15 2017-09-30] Test [2017-10-01 2017-10-16]\n",
      "Window Train/Val/Test shape (1596, 29) (336, 29) (336, 29)\n",
      "Window Train/Val/Test shape (1596, 2) (336, 2) (336, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-09-29] Val [2017-09-30 2017-10-15] Test [2017-10-16 2017-11-10]\n",
      "Window Train/Val/Test shape (1911, 29) (336, 29) (546, 29)\n",
      "Window Train/Val/Test shape (1911, 2) (336, 2) (546, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-10-14] Val [2017-10-15 2017-10-30] Test [2017-10-31 2017-11-10]\n",
      "Window Train/Val/Test shape (2226, 29) (336, 29) (231, 29)\n",
      "Window Train/Val/Test shape (2226, 2) (336, 2) (231, 2)\n",
      "[48.5765397292641, 63.28887443905848, 25.741851365412312, 20.437004427674783, 20.43583091725064]\n",
      "mean mse: 35.696020175732066\n",
      "Train [2017-07-01 2017-08-15] Val [2017-08-16 2017-08-31] Test [2017-09-01 2017-09-16]\n",
      "Window Train/Val/Test shape (966, 29) (336, 29) (336, 29)\n",
      "Window Train/Val/Test shape (966, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-08-30] Val [2017-08-31 2017-09-15] Test [2017-09-16 2017-10-01]\n",
      "Window Train/Val/Test shape (1281, 29) (336, 29) (336, 29)\n",
      "Window Train/Val/Test shape (1281, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-09-14] Val [2017-09-15 2017-09-30] Test [2017-10-01 2017-10-16]\n",
      "Window Train/Val/Test shape (1596, 29) (336, 29) (336, 29)\n",
      "Window Train/Val/Test shape (1596, 2) (336, 2) (336, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-09-29] Val [2017-09-30 2017-10-15] Test [2017-10-16 2017-11-10]\n",
      "Window Train/Val/Test shape (1911, 29) (336, 29) (546, 29)\n",
      "Window Train/Val/Test shape (1911, 2) (336, 2) (546, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-10-14] Val [2017-10-15 2017-10-30] Test [2017-10-31 2017-11-10]\n",
      "Window Train/Val/Test shape (2226, 29) (336, 29) (231, 29)\n",
      "Window Train/Val/Test shape (2226, 2) (336, 2) (231, 2)\n",
      "[51.387739729853614, 52.21562961755368, 22.63067301167223, 23.617774816243102, 19.399277659251737]\n",
      "mean mse: 33.85021896691487\n",
      "Train [2017-07-01 2017-08-15] Val [2017-08-16 2017-08-31] Test [2017-09-01 2017-09-16]\n",
      "Window Train/Val/Test shape (966, 29) (336, 29) (336, 29)\n",
      "Window Train/Val/Test shape (966, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-08-30] Val [2017-08-31 2017-09-15] Test [2017-09-16 2017-10-01]\n",
      "Window Train/Val/Test shape (1281, 29) (336, 29) (336, 29)\n",
      "Window Train/Val/Test shape (1281, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-09-14] Val [2017-09-15 2017-09-30] Test [2017-10-01 2017-10-16]\n",
      "Window Train/Val/Test shape (1596, 29) (336, 29) (336, 29)\n",
      "Window Train/Val/Test shape (1596, 2) (336, 2) (336, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-09-29] Val [2017-09-30 2017-10-15] Test [2017-10-16 2017-11-10]\n",
      "Window Train/Val/Test shape (1911, 29) (336, 29) (546, 29)\n",
      "Window Train/Val/Test shape (1911, 2) (336, 2) (546, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-10-14] Val [2017-10-15 2017-10-30] Test [2017-10-31 2017-11-10]\n",
      "Window Train/Val/Test shape (2226, 29) (336, 29) (231, 29)\n",
      "Window Train/Val/Test shape (2226, 2) (336, 2) (231, 2)\n",
      "[73.65855459990341, 70.15063325140802, 21.428330774593807, 21.95178052622809, 22.435720763219564]\n",
      "mean mse: 41.92500398307058\n",
      "Train [2017-07-01 2017-08-15] Val [2017-08-16 2017-08-31] Test [2017-09-01 2017-09-16]\n",
      "Window Train/Val/Test shape (966, 29) (336, 29) (336, 29)\n",
      "Window Train/Val/Test shape (966, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-08-30] Val [2017-08-31 2017-09-15] Test [2017-09-16 2017-10-01]\n",
      "Window Train/Val/Test shape (1281, 29) (336, 29) (336, 29)\n",
      "Window Train/Val/Test shape (1281, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-09-14] Val [2017-09-15 2017-09-30] Test [2017-10-01 2017-10-16]\n",
      "Window Train/Val/Test shape (1596, 29) (336, 29) (336, 29)\n",
      "Window Train/Val/Test shape (1596, 2) (336, 2) (336, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-09-29] Val [2017-09-30 2017-10-15] Test [2017-10-16 2017-11-10]\n",
      "Window Train/Val/Test shape (1911, 29) (336, 29) (546, 29)\n",
      "Window Train/Val/Test shape (1911, 2) (336, 2) (546, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-10-14] Val [2017-10-15 2017-10-30] Test [2017-10-31 2017-11-10]\n",
      "Window Train/Val/Test shape (2226, 29) (336, 29) (231, 29)\n",
      "Window Train/Val/Test shape (2226, 2) (336, 2) (231, 2)\n",
      "[57.35350944451836, 56.11176599892094, 19.834603573703962, 22.27601928026665, 18.618928688313204]\n",
      "mean mse: 34.83896539714463\n",
      "Train [2017-07-01 2017-08-15] Val [2017-08-16 2017-08-31] Test [2017-09-01 2017-09-16]\n",
      "Window Train/Val/Test shape (966, 29) (336, 29) (336, 29)\n",
      "Window Train/Val/Test shape (966, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-08-30] Val [2017-08-31 2017-09-15] Test [2017-09-16 2017-10-01]\n",
      "Window Train/Val/Test shape (1281, 29) (336, 29) (336, 29)\n",
      "Window Train/Val/Test shape (1281, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-09-14] Val [2017-09-15 2017-09-30] Test [2017-10-01 2017-10-16]\n",
      "Window Train/Val/Test shape (1596, 29) (336, 29) (336, 29)\n",
      "Window Train/Val/Test shape (1596, 2) (336, 2) (336, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-09-29] Val [2017-09-30 2017-10-15] Test [2017-10-16 2017-11-10]\n",
      "Window Train/Val/Test shape (1911, 29) (336, 29) (546, 29)\n",
      "Window Train/Val/Test shape (1911, 2) (336, 2) (546, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-10-14] Val [2017-10-15 2017-10-30] Test [2017-10-31 2017-11-10]\n",
      "Window Train/Val/Test shape (2226, 29) (336, 29) (231, 29)\n",
      "Window Train/Val/Test shape (2226, 2) (336, 2) (231, 2)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[62.49790774744759, 62.253096335004514, 19.674337295759784, 19.681664441229106, 27.85834397178694]\n",
      "mean mse: 38.39306995824559\n",
      "Train [2017-07-01 2017-08-15] Val [2017-08-16 2017-08-31] Test [2017-09-01 2017-09-16]\n",
      "Window Train/Val/Test shape (966, 29) (336, 29) (336, 29)\n",
      "Window Train/Val/Test shape (966, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-08-30] Val [2017-08-31 2017-09-15] Test [2017-09-16 2017-10-01]\n",
      "Window Train/Val/Test shape (1281, 29) (336, 29) (336, 29)\n",
      "Window Train/Val/Test shape (1281, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-09-14] Val [2017-09-15 2017-09-30] Test [2017-10-01 2017-10-16]\n",
      "Window Train/Val/Test shape (1596, 29) (336, 29) (336, 29)\n",
      "Window Train/Val/Test shape (1596, 2) (336, 2) (336, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-09-29] Val [2017-09-30 2017-10-15] Test [2017-10-16 2017-11-10]\n",
      "Window Train/Val/Test shape (1911, 29) (336, 29) (546, 29)\n",
      "Window Train/Val/Test shape (1911, 2) (336, 2) (546, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-10-14] Val [2017-10-15 2017-10-30] Test [2017-10-31 2017-11-10]\n",
      "Window Train/Val/Test shape (2226, 29) (336, 29) (231, 29)\n",
      "Window Train/Val/Test shape (2226, 2) (336, 2) (231, 2)\n",
      "[78.12379581591199, 63.76805446550571, 22.82052876761586, 20.533015134554624, 24.45916962584562]\n",
      "mean mse: 41.940912761886764\n",
      "Train [2017-07-01 2017-08-15] Val [2017-08-16 2017-08-31] Test [2017-09-01 2017-09-16]\n",
      "Window Train/Val/Test shape (966, 29) (336, 29) (336, 29)\n",
      "Window Train/Val/Test shape (966, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-08-30] Val [2017-08-31 2017-09-15] Test [2017-09-16 2017-10-01]\n",
      "Window Train/Val/Test shape (1281, 29) (336, 29) (336, 29)\n",
      "Window Train/Val/Test shape (1281, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-09-14] Val [2017-09-15 2017-09-30] Test [2017-10-01 2017-10-16]\n",
      "Window Train/Val/Test shape (1596, 29) (336, 29) (336, 29)\n",
      "Window Train/Val/Test shape (1596, 2) (336, 2) (336, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-09-29] Val [2017-09-30 2017-10-15] Test [2017-10-16 2017-11-10]\n",
      "Window Train/Val/Test shape (1911, 29) (336, 29) (546, 29)\n",
      "Window Train/Val/Test shape (1911, 2) (336, 2) (546, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-10-14] Val [2017-10-15 2017-10-30] Test [2017-10-31 2017-11-10]\n",
      "Window Train/Val/Test shape (2226, 29) (336, 29) (231, 29)\n",
      "Window Train/Val/Test shape (2226, 2) (336, 2) (231, 2)\n",
      "[68.78437749639198, 42.45393126918724, 43.494342946245254, 27.92953866817119, 25.49300994528127]\n",
      "mean mse: 41.63104006505539\n",
      "Train [2017-07-01 2017-08-15] Val [2017-08-16 2017-08-31] Test [2017-09-01 2017-09-16]\n",
      "Window Train/Val/Test shape (966, 29) (336, 29) (336, 29)\n",
      "Window Train/Val/Test shape (966, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-08-30] Val [2017-08-31 2017-09-15] Test [2017-09-16 2017-10-01]\n",
      "Window Train/Val/Test shape (1281, 29) (336, 29) (336, 29)\n",
      "Window Train/Val/Test shape (1281, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-09-14] Val [2017-09-15 2017-09-30] Test [2017-10-01 2017-10-16]\n",
      "Window Train/Val/Test shape (1596, 29) (336, 29) (336, 29)\n",
      "Window Train/Val/Test shape (1596, 2) (336, 2) (336, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-09-29] Val [2017-09-30 2017-10-15] Test [2017-10-16 2017-11-10]\n",
      "Window Train/Val/Test shape (1911, 29) (336, 29) (546, 29)\n",
      "Window Train/Val/Test shape (1911, 2) (336, 2) (546, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-10-14] Val [2017-10-15 2017-10-30] Test [2017-10-31 2017-11-10]\n",
      "Window Train/Val/Test shape (2226, 29) (336, 29) (231, 29)\n",
      "Window Train/Val/Test shape (2226, 2) (336, 2) (231, 2)\n",
      "[61.642389050533424, 63.6903356122064, 20.973574389381458, 22.43519914665898, 22.85801446904687]\n",
      "mean mse: 38.31990253356543\n",
      "Train [2017-07-01 2017-08-15] Val [2017-08-16 2017-08-31] Test [2017-09-01 2017-09-16]\n",
      "Window Train/Val/Test shape (966, 29) (336, 29) (336, 29)\n",
      "Window Train/Val/Test shape (966, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-08-30] Val [2017-08-31 2017-09-15] Test [2017-09-16 2017-10-01]\n",
      "Window Train/Val/Test shape (1281, 29) (336, 29) (336, 29)\n",
      "Window Train/Val/Test shape (1281, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-09-14] Val [2017-09-15 2017-09-30] Test [2017-10-01 2017-10-16]\n",
      "Window Train/Val/Test shape (1596, 29) (336, 29) (336, 29)\n",
      "Window Train/Val/Test shape (1596, 2) (336, 2) (336, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-09-29] Val [2017-09-30 2017-10-15] Test [2017-10-16 2017-11-10]\n",
      "Window Train/Val/Test shape (1911, 29) (336, 29) (546, 29)\n",
      "Window Train/Val/Test shape (1911, 2) (336, 2) (546, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-10-14] Val [2017-10-15 2017-10-30] Test [2017-10-31 2017-11-10]\n",
      "Window Train/Val/Test shape (2226, 29) (336, 29) (231, 29)\n",
      "Window Train/Val/Test shape (2226, 2) (336, 2) (231, 2)\n",
      "[69.47113434713093, 64.4016953989442, 19.443559721740264, 20.389698538673482, 18.238612304826628]\n",
      "mean mse: 38.3889400622631\n",
      "Train [2017-07-01 2017-08-15] Val [2017-08-16 2017-08-31] Test [2017-09-01 2017-09-16]\n",
      "Window Train/Val/Test shape (966, 29) (336, 29) (336, 29)\n",
      "Window Train/Val/Test shape (966, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-08-30] Val [2017-08-31 2017-09-15] Test [2017-09-16 2017-10-01]\n",
      "Window Train/Val/Test shape (1281, 29) (336, 29) (336, 29)\n",
      "Window Train/Val/Test shape (1281, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-09-14] Val [2017-09-15 2017-09-30] Test [2017-10-01 2017-10-16]\n",
      "Window Train/Val/Test shape (1596, 29) (336, 29) (336, 29)\n",
      "Window Train/Val/Test shape (1596, 2) (336, 2) (336, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-09-29] Val [2017-09-30 2017-10-15] Test [2017-10-16 2017-11-10]\n",
      "Window Train/Val/Test shape (1911, 29) (336, 29) (546, 29)\n",
      "Window Train/Val/Test shape (1911, 2) (336, 2) (546, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-10-14] Val [2017-10-15 2017-10-30] Test [2017-10-31 2017-11-10]\n",
      "Window Train/Val/Test shape (2226, 29) (336, 29) (231, 29)\n",
      "Window Train/Val/Test shape (2226, 2) (336, 2) (231, 2)\n",
      "[65.52077779084541, 59.51580625247092, 22.45077906195007, 20.999396936374616, 16.4995504316387]\n",
      "mean mse: 36.997262094655945\n",
      "Train [2017-07-01 2017-08-15] Val [2017-08-16 2017-08-31] Test [2017-09-01 2017-09-16]\n",
      "Window Train/Val/Test shape (966, 29) (336, 29) (336, 29)\n",
      "Window Train/Val/Test shape (966, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-08-30] Val [2017-08-31 2017-09-15] Test [2017-09-16 2017-10-01]\n",
      "Window Train/Val/Test shape (1281, 29) (336, 29) (336, 29)\n",
      "Window Train/Val/Test shape (1281, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-09-14] Val [2017-09-15 2017-09-30] Test [2017-10-01 2017-10-16]\n",
      "Window Train/Val/Test shape (1596, 29) (336, 29) (336, 29)\n",
      "Window Train/Val/Test shape (1596, 2) (336, 2) (336, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-09-29] Val [2017-09-30 2017-10-15] Test [2017-10-16 2017-11-10]\n",
      "Window Train/Val/Test shape (1911, 29) (336, 29) (546, 29)\n",
      "Window Train/Val/Test shape (1911, 2) (336, 2) (546, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-10-14] Val [2017-10-15 2017-10-30] Test [2017-10-31 2017-11-10]\n",
      "Window Train/Val/Test shape (2226, 29) (336, 29) (231, 29)\n",
      "Window Train/Val/Test shape (2226, 2) (336, 2) (231, 2)\n",
      "[54.45861427032384, 57.65006313177984, 19.06628156370511, 21.268597192975232, 20.50929559784665]\n",
      "mean mse: 34.590570351326136\n",
      "Train [2017-07-01 2017-08-15] Val [2017-08-16 2017-08-31] Test [2017-09-01 2017-09-16]\n",
      "Window Train/Val/Test shape (966, 29) (336, 29) (336, 29)\n",
      "Window Train/Val/Test shape (966, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-08-30] Val [2017-08-31 2017-09-15] Test [2017-09-16 2017-10-01]\n",
      "Window Train/Val/Test shape (1281, 29) (336, 29) (336, 29)\n",
      "Window Train/Val/Test shape (1281, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-09-14] Val [2017-09-15 2017-09-30] Test [2017-10-01 2017-10-16]\n",
      "Window Train/Val/Test shape (1596, 29) (336, 29) (336, 29)\n",
      "Window Train/Val/Test shape (1596, 2) (336, 2) (336, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-09-29] Val [2017-09-30 2017-10-15] Test [2017-10-16 2017-11-10]\n",
      "Window Train/Val/Test shape (1911, 29) (336, 29) (546, 29)\n",
      "Window Train/Val/Test shape (1911, 2) (336, 2) (546, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-10-14] Val [2017-10-15 2017-10-30] Test [2017-10-31 2017-11-10]\n",
      "Window Train/Val/Test shape (2226, 29) (336, 29) (231, 29)\n",
      "Window Train/Val/Test shape (2226, 2) (336, 2) (231, 2)\n",
      "[64.6344236066613, 50.73089607128618, 23.97519325122897, 16.037535785259276, 22.323004824361814]\n",
      "mean mse: 35.540210707759506\n",
      "Train [2017-07-01 2017-08-15] Val [2017-08-16 2017-08-31] Test [2017-09-01 2017-09-16]\n",
      "Window Train/Val/Test shape (966, 29) (336, 29) (336, 29)\n",
      "Window Train/Val/Test shape (966, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-08-30] Val [2017-08-31 2017-09-15] Test [2017-09-16 2017-10-01]\n",
      "Window Train/Val/Test shape (1281, 29) (336, 29) (336, 29)\n",
      "Window Train/Val/Test shape (1281, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-09-14] Val [2017-09-15 2017-09-30] Test [2017-10-01 2017-10-16]\n",
      "Window Train/Val/Test shape (1596, 29) (336, 29) (336, 29)\n",
      "Window Train/Val/Test shape (1596, 2) (336, 2) (336, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-09-29] Val [2017-09-30 2017-10-15] Test [2017-10-16 2017-11-10]\n",
      "Window Train/Val/Test shape (1911, 29) (336, 29) (546, 29)\n",
      "Window Train/Val/Test shape (1911, 2) (336, 2) (546, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-10-14] Val [2017-10-15 2017-10-30] Test [2017-10-31 2017-11-10]\n",
      "Window Train/Val/Test shape (2226, 29) (336, 29) (231, 29)\n",
      "Window Train/Val/Test shape (2226, 2) (336, 2) (231, 2)\n",
      "[61.16525168240359, 70.12515653109867, 20.062233185226347, 21.25827258764657, 21.311122202299916]\n",
      "mean mse: 38.78440723773501\n",
      "Train [2017-07-01 2017-08-15] Val [2017-08-16 2017-08-31] Test [2017-09-01 2017-09-16]\n",
      "Window Train/Val/Test shape (966, 29) (336, 29) (336, 29)\n",
      "Window Train/Val/Test shape (966, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-08-30] Val [2017-08-31 2017-09-15] Test [2017-09-16 2017-10-01]\n",
      "Window Train/Val/Test shape (1281, 29) (336, 29) (336, 29)\n",
      "Window Train/Val/Test shape (1281, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-09-14] Val [2017-09-15 2017-09-30] Test [2017-10-01 2017-10-16]\n",
      "Window Train/Val/Test shape (1596, 29) (336, 29) (336, 29)\n",
      "Window Train/Val/Test shape (1596, 2) (336, 2) (336, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-09-29] Val [2017-09-30 2017-10-15] Test [2017-10-16 2017-11-10]\n",
      "Window Train/Val/Test shape (1911, 29) (336, 29) (546, 29)\n",
      "Window Train/Val/Test shape (1911, 2) (336, 2) (546, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-10-14] Val [2017-10-15 2017-10-30] Test [2017-10-31 2017-11-10]\n",
      "Window Train/Val/Test shape (2226, 29) (336, 29) (231, 29)\n",
      "Window Train/Val/Test shape (2226, 2) (336, 2) (231, 2)\n",
      "[65.76266976203694, 73.32641527961925, 24.34856989457144, 21.990754270589186, 21.14887957655157]\n",
      "mean mse: 41.315457756673666\n",
      "Train [2017-07-01 2017-08-15] Val [2017-08-16 2017-08-31] Test [2017-09-01 2017-09-16]\n",
      "Window Train/Val/Test shape (966, 29) (336, 29) (336, 29)\n",
      "Window Train/Val/Test shape (966, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-08-30] Val [2017-08-31 2017-09-15] Test [2017-09-16 2017-10-01]\n",
      "Window Train/Val/Test shape (1281, 29) (336, 29) (336, 29)\n",
      "Window Train/Val/Test shape (1281, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-09-14] Val [2017-09-15 2017-09-30] Test [2017-10-01 2017-10-16]\n",
      "Window Train/Val/Test shape (1596, 29) (336, 29) (336, 29)\n",
      "Window Train/Val/Test shape (1596, 2) (336, 2) (336, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-09-29] Val [2017-09-30 2017-10-15] Test [2017-10-16 2017-11-10]\n",
      "Window Train/Val/Test shape (1911, 29) (336, 29) (546, 29)\n",
      "Window Train/Val/Test shape (1911, 2) (336, 2) (546, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-10-14] Val [2017-10-15 2017-10-30] Test [2017-10-31 2017-11-10]\n",
      "Window Train/Val/Test shape (2226, 29) (336, 29) (231, 29)\n",
      "Window Train/Val/Test shape (2226, 2) (336, 2) (231, 2)\n",
      "[59.135948290512445, 61.91672198948812, 20.732461268750736, 20.62257921077896, 25.950760064632693]\n",
      "mean mse: 37.671694164832594\n",
      "Train [2017-07-01 2017-08-15] Val [2017-08-16 2017-08-31] Test [2017-09-01 2017-09-16]\n",
      "Window Train/Val/Test shape (966, 29) (336, 29) (336, 29)\n",
      "Window Train/Val/Test shape (966, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-08-30] Val [2017-08-31 2017-09-15] Test [2017-09-16 2017-10-01]\n",
      "Window Train/Val/Test shape (1281, 29) (336, 29) (336, 29)\n",
      "Window Train/Val/Test shape (1281, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-09-14] Val [2017-09-15 2017-09-30] Test [2017-10-01 2017-10-16]\n",
      "Window Train/Val/Test shape (1596, 29) (336, 29) (336, 29)\n",
      "Window Train/Val/Test shape (1596, 2) (336, 2) (336, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-09-29] Val [2017-09-30 2017-10-15] Test [2017-10-16 2017-11-10]\n",
      "Window Train/Val/Test shape (1911, 29) (336, 29) (546, 29)\n",
      "Window Train/Val/Test shape (1911, 2) (336, 2) (546, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-10-14] Val [2017-10-15 2017-10-30] Test [2017-10-31 2017-11-10]\n",
      "Window Train/Val/Test shape (2226, 29) (336, 29) (231, 29)\n",
      "Window Train/Val/Test shape (2226, 2) (336, 2) (231, 2)\n",
      "[58.18009142298326, 51.552905679631856, 15.955975054657115, 21.676435386708814, 20.546818955749945]\n",
      "mean mse: 33.5824452999462\n",
      "Train [2017-07-01 2017-08-15] Val [2017-08-16 2017-08-31] Test [2017-09-01 2017-09-16]\n",
      "Window Train/Val/Test shape (966, 29) (336, 29) (336, 29)\n",
      "Window Train/Val/Test shape (966, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-08-30] Val [2017-08-31 2017-09-15] Test [2017-09-16 2017-10-01]\n",
      "Window Train/Val/Test shape (1281, 29) (336, 29) (336, 29)\n",
      "Window Train/Val/Test shape (1281, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-09-14] Val [2017-09-15 2017-09-30] Test [2017-10-01 2017-10-16]\n",
      "Window Train/Val/Test shape (1596, 29) (336, 29) (336, 29)\n",
      "Window Train/Val/Test shape (1596, 2) (336, 2) (336, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-09-29] Val [2017-09-30 2017-10-15] Test [2017-10-16 2017-11-10]\n",
      "Window Train/Val/Test shape (1911, 29) (336, 29) (546, 29)\n",
      "Window Train/Val/Test shape (1911, 2) (336, 2) (546, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-10-14] Val [2017-10-15 2017-10-30] Test [2017-10-31 2017-11-10]\n",
      "Window Train/Val/Test shape (2226, 29) (336, 29) (231, 29)\n",
      "Window Train/Val/Test shape (2226, 2) (336, 2) (231, 2)\n",
      "[71.50980595228316, 71.5430756830068, 32.911798223151195, 18.104541877330657, 23.264859719040903]\n",
      "mean mse: 43.46681629096255\n",
      "Train [2017-07-01 2017-08-15] Val [2017-08-16 2017-08-31] Test [2017-09-01 2017-09-16]\n",
      "Window Train/Val/Test shape (966, 29) (336, 29) (336, 29)\n",
      "Window Train/Val/Test shape (966, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-08-30] Val [2017-08-31 2017-09-15] Test [2017-09-16 2017-10-01]\n",
      "Window Train/Val/Test shape (1281, 29) (336, 29) (336, 29)\n",
      "Window Train/Val/Test shape (1281, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-09-14] Val [2017-09-15 2017-09-30] Test [2017-10-01 2017-10-16]\n",
      "Window Train/Val/Test shape (1596, 29) (336, 29) (336, 29)\n",
      "Window Train/Val/Test shape (1596, 2) (336, 2) (336, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-09-29] Val [2017-09-30 2017-10-15] Test [2017-10-16 2017-11-10]\n",
      "Window Train/Val/Test shape (1911, 29) (336, 29) (546, 29)\n",
      "Window Train/Val/Test shape (1911, 2) (336, 2) (546, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-10-14] Val [2017-10-15 2017-10-30] Test [2017-10-31 2017-11-10]\n",
      "Window Train/Val/Test shape (2226, 29) (336, 29) (231, 29)\n",
      "Window Train/Val/Test shape (2226, 2) (336, 2) (231, 2)\n",
      "[62.45410840712633, 65.35008356341847, 16.201764960961338, 22.685675144408194, 20.2078181172127]\n",
      "mean mse: 37.3798900386254\n",
      "Train [2017-07-01 2017-08-15] Val [2017-08-16 2017-08-31] Test [2017-09-01 2017-09-16]\n",
      "Window Train/Val/Test shape (966, 29) (336, 29) (336, 29)\n",
      "Window Train/Val/Test shape (966, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-08-30] Val [2017-08-31 2017-09-15] Test [2017-09-16 2017-10-01]\n",
      "Window Train/Val/Test shape (1281, 29) (336, 29) (336, 29)\n",
      "Window Train/Val/Test shape (1281, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-09-14] Val [2017-09-15 2017-09-30] Test [2017-10-01 2017-10-16]\n",
      "Window Train/Val/Test shape (1596, 29) (336, 29) (336, 29)\n",
      "Window Train/Val/Test shape (1596, 2) (336, 2) (336, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-09-29] Val [2017-09-30 2017-10-15] Test [2017-10-16 2017-11-10]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Window Train/Val/Test shape (1911, 29) (336, 29) (546, 29)\n",
      "Window Train/Val/Test shape (1911, 2) (336, 2) (546, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-10-14] Val [2017-10-15 2017-10-30] Test [2017-10-31 2017-11-10]\n",
      "Window Train/Val/Test shape (2226, 29) (336, 29) (231, 29)\n",
      "Window Train/Val/Test shape (2226, 2) (336, 2) (231, 2)\n",
      "[73.05799310875769, 56.33807331722249, 25.878767707871216, 28.64052717726264, 27.252653920368136]\n",
      "mean mse: 42.23360304629644\n",
      "Train [2017-07-01 2017-08-15] Val [2017-08-16 2017-08-31] Test [2017-09-01 2017-09-16]\n",
      "Window Train/Val/Test shape (966, 29) (336, 29) (336, 29)\n",
      "Window Train/Val/Test shape (966, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-08-30] Val [2017-08-31 2017-09-15] Test [2017-09-16 2017-10-01]\n",
      "Window Train/Val/Test shape (1281, 29) (336, 29) (336, 29)\n",
      "Window Train/Val/Test shape (1281, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-09-14] Val [2017-09-15 2017-09-30] Test [2017-10-01 2017-10-16]\n",
      "Window Train/Val/Test shape (1596, 29) (336, 29) (336, 29)\n",
      "Window Train/Val/Test shape (1596, 2) (336, 2) (336, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-09-29] Val [2017-09-30 2017-10-15] Test [2017-10-16 2017-11-10]\n",
      "Window Train/Val/Test shape (1911, 29) (336, 29) (546, 29)\n",
      "Window Train/Val/Test shape (1911, 2) (336, 2) (546, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-10-14] Val [2017-10-15 2017-10-30] Test [2017-10-31 2017-11-10]\n",
      "Window Train/Val/Test shape (2226, 29) (336, 29) (231, 29)\n",
      "Window Train/Val/Test shape (2226, 2) (336, 2) (231, 2)\n",
      "[67.89853315700864, 58.26802698077, 18.77597280646437, 21.273107783824756, 37.22475061207878]\n",
      "mean mse: 40.688078268029315\n",
      "Train [2017-07-01 2017-08-15] Val [2017-08-16 2017-08-31] Test [2017-09-01 2017-09-16]\n",
      "Window Train/Val/Test shape (966, 29) (336, 29) (336, 29)\n",
      "Window Train/Val/Test shape (966, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-08-30] Val [2017-08-31 2017-09-15] Test [2017-09-16 2017-10-01]\n",
      "Window Train/Val/Test shape (1281, 29) (336, 29) (336, 29)\n",
      "Window Train/Val/Test shape (1281, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-09-14] Val [2017-09-15 2017-09-30] Test [2017-10-01 2017-10-16]\n",
      "Window Train/Val/Test shape (1596, 29) (336, 29) (336, 29)\n",
      "Window Train/Val/Test shape (1596, 2) (336, 2) (336, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-09-29] Val [2017-09-30 2017-10-15] Test [2017-10-16 2017-11-10]\n",
      "Window Train/Val/Test shape (1911, 29) (336, 29) (546, 29)\n",
      "Window Train/Val/Test shape (1911, 2) (336, 2) (546, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-10-14] Val [2017-10-15 2017-10-30] Test [2017-10-31 2017-11-10]\n",
      "Window Train/Val/Test shape (2226, 29) (336, 29) (231, 29)\n",
      "Window Train/Val/Test shape (2226, 2) (336, 2) (231, 2)\n",
      "[90.94631527356644, 70.84109196139738, 25.012561474082055, 26.885213750338767, 20.580728870304434]\n",
      "mean mse: 46.85318226593782\n",
      "Train [2017-07-01 2017-08-15] Val [2017-08-16 2017-08-31] Test [2017-09-01 2017-09-16]\n",
      "Window Train/Val/Test shape (966, 29) (336, 29) (336, 29)\n",
      "Window Train/Val/Test shape (966, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-08-30] Val [2017-08-31 2017-09-15] Test [2017-09-16 2017-10-01]\n",
      "Window Train/Val/Test shape (1281, 29) (336, 29) (336, 29)\n",
      "Window Train/Val/Test shape (1281, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-09-14] Val [2017-09-15 2017-09-30] Test [2017-10-01 2017-10-16]\n",
      "Window Train/Val/Test shape (1596, 29) (336, 29) (336, 29)\n",
      "Window Train/Val/Test shape (1596, 2) (336, 2) (336, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-09-29] Val [2017-09-30 2017-10-15] Test [2017-10-16 2017-11-10]\n",
      "Window Train/Val/Test shape (1911, 29) (336, 29) (546, 29)\n",
      "Window Train/Val/Test shape (1911, 2) (336, 2) (546, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-10-14] Val [2017-10-15 2017-10-30] Test [2017-10-31 2017-11-10]\n",
      "Window Train/Val/Test shape (2226, 29) (336, 29) (231, 29)\n",
      "Window Train/Val/Test shape (2226, 2) (336, 2) (231, 2)\n",
      "[53.45312806874302, 56.20044905385454, 14.728780894243082, 18.11947294132998, 22.047212686755522]\n",
      "mean mse: 32.90980872898523\n",
      "Train [2017-07-01 2017-08-15] Val [2017-08-16 2017-08-31] Test [2017-09-01 2017-09-16]\n",
      "Window Train/Val/Test shape (966, 29) (336, 29) (336, 29)\n",
      "Window Train/Val/Test shape (966, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-08-30] Val [2017-08-31 2017-09-15] Test [2017-09-16 2017-10-01]\n",
      "Window Train/Val/Test shape (1281, 29) (336, 29) (336, 29)\n",
      "Window Train/Val/Test shape (1281, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-09-14] Val [2017-09-15 2017-09-30] Test [2017-10-01 2017-10-16]\n",
      "Window Train/Val/Test shape (1596, 29) (336, 29) (336, 29)\n",
      "Window Train/Val/Test shape (1596, 2) (336, 2) (336, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-09-29] Val [2017-09-30 2017-10-15] Test [2017-10-16 2017-11-10]\n",
      "Window Train/Val/Test shape (1911, 29) (336, 29) (546, 29)\n",
      "Window Train/Val/Test shape (1911, 2) (336, 2) (546, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-10-14] Val [2017-10-15 2017-10-30] Test [2017-10-31 2017-11-10]\n",
      "Window Train/Val/Test shape (2226, 29) (336, 29) (231, 29)\n",
      "Window Train/Val/Test shape (2226, 2) (336, 2) (231, 2)\n",
      "[52.2388701419441, 60.049856285934005, 16.18647157235445, 18.048268905388703, 20.735922466714683]\n",
      "mean mse: 33.451877874467186\n",
      "Train [2017-07-01 2017-08-15] Val [2017-08-16 2017-08-31] Test [2017-09-01 2017-09-16]\n",
      "Window Train/Val/Test shape (966, 29) (336, 29) (336, 29)\n",
      "Window Train/Val/Test shape (966, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-08-30] Val [2017-08-31 2017-09-15] Test [2017-09-16 2017-10-01]\n",
      "Window Train/Val/Test shape (1281, 29) (336, 29) (336, 29)\n",
      "Window Train/Val/Test shape (1281, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-09-14] Val [2017-09-15 2017-09-30] Test [2017-10-01 2017-10-16]\n",
      "Window Train/Val/Test shape (1596, 29) (336, 29) (336, 29)\n",
      "Window Train/Val/Test shape (1596, 2) (336, 2) (336, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-09-29] Val [2017-09-30 2017-10-15] Test [2017-10-16 2017-11-10]\n",
      "Window Train/Val/Test shape (1911, 29) (336, 29) (546, 29)\n",
      "Window Train/Val/Test shape (1911, 2) (336, 2) (546, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-10-14] Val [2017-10-15 2017-10-30] Test [2017-10-31 2017-11-10]\n",
      "Window Train/Val/Test shape (2226, 29) (336, 29) (231, 29)\n",
      "Window Train/Val/Test shape (2226, 2) (336, 2) (231, 2)\n",
      "[50.68301479140356, 56.98667990255626, 29.175457282720714, 19.31980085815283, 19.328832872453365]\n",
      "mean mse: 35.09875714145735\n",
      "Train [2017-07-01 2017-08-15] Val [2017-08-16 2017-08-31] Test [2017-09-01 2017-09-16]\n",
      "Window Train/Val/Test shape (966, 29) (336, 29) (336, 29)\n",
      "Window Train/Val/Test shape (966, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-08-30] Val [2017-08-31 2017-09-15] Test [2017-09-16 2017-10-01]\n",
      "Window Train/Val/Test shape (1281, 29) (336, 29) (336, 29)\n",
      "Window Train/Val/Test shape (1281, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-09-14] Val [2017-09-15 2017-09-30] Test [2017-10-01 2017-10-16]\n",
      "Window Train/Val/Test shape (1596, 29) (336, 29) (336, 29)\n",
      "Window Train/Val/Test shape (1596, 2) (336, 2) (336, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-09-29] Val [2017-09-30 2017-10-15] Test [2017-10-16 2017-11-10]\n",
      "Window Train/Val/Test shape (1911, 29) (336, 29) (546, 29)\n",
      "Window Train/Val/Test shape (1911, 2) (336, 2) (546, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-10-14] Val [2017-10-15 2017-10-30] Test [2017-10-31 2017-11-10]\n",
      "Window Train/Val/Test shape (2226, 29) (336, 29) (231, 29)\n",
      "Window Train/Val/Test shape (2226, 2) (336, 2) (231, 2)\n",
      "[74.33920449464476, 56.12406576164791, 18.583545856531266, 21.77844076126655, 22.158624300992415]\n",
      "mean mse: 38.59677623501658\n",
      "Train [2017-07-01 2017-08-15] Val [2017-08-16 2017-08-31] Test [2017-09-01 2017-09-16]\n",
      "Window Train/Val/Test shape (966, 29) (336, 29) (336, 29)\n",
      "Window Train/Val/Test shape (966, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-08-30] Val [2017-08-31 2017-09-15] Test [2017-09-16 2017-10-01]\n",
      "Window Train/Val/Test shape (1281, 29) (336, 29) (336, 29)\n",
      "Window Train/Val/Test shape (1281, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-09-14] Val [2017-09-15 2017-09-30] Test [2017-10-01 2017-10-16]\n",
      "Window Train/Val/Test shape (1596, 29) (336, 29) (336, 29)\n",
      "Window Train/Val/Test shape (1596, 2) (336, 2) (336, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-09-29] Val [2017-09-30 2017-10-15] Test [2017-10-16 2017-11-10]\n",
      "Window Train/Val/Test shape (1911, 29) (336, 29) (546, 29)\n",
      "Window Train/Val/Test shape (1911, 2) (336, 2) (546, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-10-14] Val [2017-10-15 2017-10-30] Test [2017-10-31 2017-11-10]\n",
      "Window Train/Val/Test shape (2226, 29) (336, 29) (231, 29)\n",
      "Window Train/Val/Test shape (2226, 2) (336, 2) (231, 2)\n",
      "[68.99913534610971, 56.38055688737387, 23.185359882347942, 18.931627499141673, 21.405115926130268]\n",
      "mean mse: 37.78035910822069\n",
      "Train [2017-07-01 2017-08-15] Val [2017-08-16 2017-08-31] Test [2017-09-01 2017-09-16]\n",
      "Window Train/Val/Test shape (966, 29) (336, 29) (336, 29)\n",
      "Window Train/Val/Test shape (966, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-08-30] Val [2017-08-31 2017-09-15] Test [2017-09-16 2017-10-01]\n",
      "Window Train/Val/Test shape (1281, 29) (336, 29) (336, 29)\n",
      "Window Train/Val/Test shape (1281, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-09-14] Val [2017-09-15 2017-09-30] Test [2017-10-01 2017-10-16]\n",
      "Window Train/Val/Test shape (1596, 29) (336, 29) (336, 29)\n",
      "Window Train/Val/Test shape (1596, 2) (336, 2) (336, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-09-29] Val [2017-09-30 2017-10-15] Test [2017-10-16 2017-11-10]\n",
      "Window Train/Val/Test shape (1911, 29) (336, 29) (546, 29)\n",
      "Window Train/Val/Test shape (1911, 2) (336, 2) (546, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-10-14] Val [2017-10-15 2017-10-30] Test [2017-10-31 2017-11-10]\n",
      "Window Train/Val/Test shape (2226, 29) (336, 29) (231, 29)\n",
      "Window Train/Val/Test shape (2226, 2) (336, 2) (231, 2)\n",
      "[59.26558690397047, 61.271558720625144, 18.19570756391178, 18.11752489114745, 18.91123306498491]\n",
      "mean mse: 35.15232222892795\n",
      "Train [2017-07-01 2017-08-15] Val [2017-08-16 2017-08-31] Test [2017-09-01 2017-09-16]\n",
      "Window Train/Val/Test shape (966, 29) (336, 29) (336, 29)\n",
      "Window Train/Val/Test shape (966, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-08-30] Val [2017-08-31 2017-09-15] Test [2017-09-16 2017-10-01]\n",
      "Window Train/Val/Test shape (1281, 29) (336, 29) (336, 29)\n",
      "Window Train/Val/Test shape (1281, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-09-14] Val [2017-09-15 2017-09-30] Test [2017-10-01 2017-10-16]\n",
      "Window Train/Val/Test shape (1596, 29) (336, 29) (336, 29)\n",
      "Window Train/Val/Test shape (1596, 2) (336, 2) (336, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-09-29] Val [2017-09-30 2017-10-15] Test [2017-10-16 2017-11-10]\n",
      "Window Train/Val/Test shape (1911, 29) (336, 29) (546, 29)\n",
      "Window Train/Val/Test shape (1911, 2) (336, 2) (546, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-10-14] Val [2017-10-15 2017-10-30] Test [2017-10-31 2017-11-10]\n",
      "Window Train/Val/Test shape (2226, 29) (336, 29) (231, 29)\n",
      "Window Train/Val/Test shape (2226, 2) (336, 2) (231, 2)\n",
      "[54.55550147555015, 55.658152879845865, 19.329793811616064, 21.071781731089967, 18.48808245734922]\n",
      "mean mse: 33.82066247109025\n",
      "Train [2017-07-01 2017-08-15] Val [2017-08-16 2017-08-31] Test [2017-09-01 2017-09-16]\n",
      "Window Train/Val/Test shape (966, 29) (336, 29) (336, 29)\n",
      "Window Train/Val/Test shape (966, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-08-30] Val [2017-08-31 2017-09-15] Test [2017-09-16 2017-10-01]\n",
      "Window Train/Val/Test shape (1281, 29) (336, 29) (336, 29)\n",
      "Window Train/Val/Test shape (1281, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-09-14] Val [2017-09-15 2017-09-30] Test [2017-10-01 2017-10-16]\n",
      "Window Train/Val/Test shape (1596, 29) (336, 29) (336, 29)\n",
      "Window Train/Val/Test shape (1596, 2) (336, 2) (336, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-09-29] Val [2017-09-30 2017-10-15] Test [2017-10-16 2017-11-10]\n",
      "Window Train/Val/Test shape (1911, 29) (336, 29) (546, 29)\n",
      "Window Train/Val/Test shape (1911, 2) (336, 2) (546, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-10-14] Val [2017-10-15 2017-10-30] Test [2017-10-31 2017-11-10]\n",
      "Window Train/Val/Test shape (2226, 29) (336, 29) (231, 29)\n",
      "Window Train/Val/Test shape (2226, 2) (336, 2) (231, 2)\n",
      "[66.14711478446822, 60.588493264377014, 18.14455703411416, 20.89491906178256, 16.21917118320853]\n",
      "mean mse: 36.3988510655901\n",
      "Train [2017-07-01 2017-08-15] Val [2017-08-16 2017-08-31] Test [2017-09-01 2017-09-16]\n",
      "Window Train/Val/Test shape (966, 29) (336, 29) (336, 29)\n",
      "Window Train/Val/Test shape (966, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-08-30] Val [2017-08-31 2017-09-15] Test [2017-09-16 2017-10-01]\n",
      "Window Train/Val/Test shape (1281, 29) (336, 29) (336, 29)\n",
      "Window Train/Val/Test shape (1281, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-09-14] Val [2017-09-15 2017-09-30] Test [2017-10-01 2017-10-16]\n",
      "Window Train/Val/Test shape (1596, 29) (336, 29) (336, 29)\n",
      "Window Train/Val/Test shape (1596, 2) (336, 2) (336, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-09-29] Val [2017-09-30 2017-10-15] Test [2017-10-16 2017-11-10]\n",
      "Window Train/Val/Test shape (1911, 29) (336, 29) (546, 29)\n",
      "Window Train/Val/Test shape (1911, 2) (336, 2) (546, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-10-14] Val [2017-10-15 2017-10-30] Test [2017-10-31 2017-11-10]\n",
      "Window Train/Val/Test shape (2226, 29) (336, 29) (231, 29)\n",
      "Window Train/Val/Test shape (2226, 2) (336, 2) (231, 2)\n",
      "[53.37050897142446, 60.905765312455095, 24.19545190336101, 22.240140561991826, 21.299333203078234]\n",
      "mean mse: 36.40223999046212\n",
      "Train [2017-07-01 2017-08-15] Val [2017-08-16 2017-08-31] Test [2017-09-01 2017-09-16]\n",
      "Window Train/Val/Test shape (966, 29) (336, 29) (336, 29)\n",
      "Window Train/Val/Test shape (966, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-08-30] Val [2017-08-31 2017-09-15] Test [2017-09-16 2017-10-01]\n",
      "Window Train/Val/Test shape (1281, 29) (336, 29) (336, 29)\n",
      "Window Train/Val/Test shape (1281, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-09-14] Val [2017-09-15 2017-09-30] Test [2017-10-01 2017-10-16]\n",
      "Window Train/Val/Test shape (1596, 29) (336, 29) (336, 29)\n",
      "Window Train/Val/Test shape (1596, 2) (336, 2) (336, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-09-29] Val [2017-09-30 2017-10-15] Test [2017-10-16 2017-11-10]\n",
      "Window Train/Val/Test shape (1911, 29) (336, 29) (546, 29)\n",
      "Window Train/Val/Test shape (1911, 2) (336, 2) (546, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-10-14] Val [2017-10-15 2017-10-30] Test [2017-10-31 2017-11-10]\n",
      "Window Train/Val/Test shape (2226, 29) (336, 29) (231, 29)\n",
      "Window Train/Val/Test shape (2226, 2) (336, 2) (231, 2)\n",
      "[62.37974628078485, 59.214560028356, 18.599518221323358, 17.781257050153375, 25.656056006063547]\n",
      "mean mse: 36.72622751733623\n",
      "Train [2017-07-01 2017-08-15] Val [2017-08-16 2017-08-31] Test [2017-09-01 2017-09-16]\n",
      "Window Train/Val/Test shape (966, 29) (336, 29) (336, 29)\n",
      "Window Train/Val/Test shape (966, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-08-30] Val [2017-08-31 2017-09-15] Test [2017-09-16 2017-10-01]\n",
      "Window Train/Val/Test shape (1281, 29) (336, 29) (336, 29)\n",
      "Window Train/Val/Test shape (1281, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-09-14] Val [2017-09-15 2017-09-30] Test [2017-10-01 2017-10-16]\n",
      "Window Train/Val/Test shape (1596, 29) (336, 29) (336, 29)\n",
      "Window Train/Val/Test shape (1596, 2) (336, 2) (336, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-09-29] Val [2017-09-30 2017-10-15] Test [2017-10-16 2017-11-10]\n",
      "Window Train/Val/Test shape (1911, 29) (336, 29) (546, 29)\n",
      "Window Train/Val/Test shape (1911, 2) (336, 2) (546, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-10-14] Val [2017-10-15 2017-10-30] Test [2017-10-31 2017-11-10]\n",
      "Window Train/Val/Test shape (2226, 29) (336, 29) (231, 29)\n",
      "Window Train/Val/Test shape (2226, 2) (336, 2) (231, 2)\n",
      "[56.02707108765654, 51.028084894549636, 25.947094967347137, 17.637292844776606, 20.36654946765248]\n",
      "mean mse: 34.20121865239648\n",
      "Train [2017-07-01 2017-08-15] Val [2017-08-16 2017-08-31] Test [2017-09-01 2017-09-16]\n",
      "Window Train/Val/Test shape (966, 29) (336, 29) (336, 29)\n",
      "Window Train/Val/Test shape (966, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-08-30] Val [2017-08-31 2017-09-15] Test [2017-09-16 2017-10-01]\n",
      "Window Train/Val/Test shape (1281, 29) (336, 29) (336, 29)\n",
      "Window Train/Val/Test shape (1281, 2) (336, 2) (336, 2)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train [2017-07-01 2017-09-14] Val [2017-09-15 2017-09-30] Test [2017-10-01 2017-10-16]\n",
      "Window Train/Val/Test shape (1596, 29) (336, 29) (336, 29)\n",
      "Window Train/Val/Test shape (1596, 2) (336, 2) (336, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-09-29] Val [2017-09-30 2017-10-15] Test [2017-10-16 2017-11-10]\n",
      "Window Train/Val/Test shape (1911, 29) (336, 29) (546, 29)\n",
      "Window Train/Val/Test shape (1911, 2) (336, 2) (546, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-10-14] Val [2017-10-15 2017-10-30] Test [2017-10-31 2017-11-10]\n",
      "Window Train/Val/Test shape (2226, 29) (336, 29) (231, 29)\n",
      "Window Train/Val/Test shape (2226, 2) (336, 2) (231, 2)\n",
      "[56.886426147483334, 57.41209235253646, 22.707259641597624, 21.512724452610975, 18.45300565633185]\n",
      "mean mse: 35.39430165011204\n",
      "Train [2017-07-01 2017-08-15] Val [2017-08-16 2017-08-31] Test [2017-09-01 2017-09-16]\n",
      "Window Train/Val/Test shape (966, 29) (336, 29) (336, 29)\n",
      "Window Train/Val/Test shape (966, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-08-30] Val [2017-08-31 2017-09-15] Test [2017-09-16 2017-10-01]\n",
      "Window Train/Val/Test shape (1281, 29) (336, 29) (336, 29)\n",
      "Window Train/Val/Test shape (1281, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-09-14] Val [2017-09-15 2017-09-30] Test [2017-10-01 2017-10-16]\n",
      "Window Train/Val/Test shape (1596, 29) (336, 29) (336, 29)\n",
      "Window Train/Val/Test shape (1596, 2) (336, 2) (336, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-09-29] Val [2017-09-30 2017-10-15] Test [2017-10-16 2017-11-10]\n",
      "Window Train/Val/Test shape (1911, 29) (336, 29) (546, 29)\n",
      "Window Train/Val/Test shape (1911, 2) (336, 2) (546, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-10-14] Val [2017-10-15 2017-10-30] Test [2017-10-31 2017-11-10]\n",
      "Window Train/Val/Test shape (2226, 29) (336, 29) (231, 29)\n",
      "Window Train/Val/Test shape (2226, 2) (336, 2) (231, 2)\n",
      "[64.91631460334601, 60.440948512912136, 19.256444785974605, 23.994213951319754, 17.930400179104986]\n",
      "mean mse: 37.307664406531494\n",
      "Train [2017-07-01 2017-08-15] Val [2017-08-16 2017-08-31] Test [2017-09-01 2017-09-16]\n",
      "Window Train/Val/Test shape (966, 29) (336, 29) (336, 29)\n",
      "Window Train/Val/Test shape (966, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-08-30] Val [2017-08-31 2017-09-15] Test [2017-09-16 2017-10-01]\n",
      "Window Train/Val/Test shape (1281, 29) (336, 29) (336, 29)\n",
      "Window Train/Val/Test shape (1281, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-09-14] Val [2017-09-15 2017-09-30] Test [2017-10-01 2017-10-16]\n",
      "Window Train/Val/Test shape (1596, 29) (336, 29) (336, 29)\n",
      "Window Train/Val/Test shape (1596, 2) (336, 2) (336, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-09-29] Val [2017-09-30 2017-10-15] Test [2017-10-16 2017-11-10]\n",
      "Window Train/Val/Test shape (1911, 29) (336, 29) (546, 29)\n",
      "Window Train/Val/Test shape (1911, 2) (336, 2) (546, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-10-14] Val [2017-10-15 2017-10-30] Test [2017-10-31 2017-11-10]\n",
      "Window Train/Val/Test shape (2226, 29) (336, 29) (231, 29)\n",
      "Window Train/Val/Test shape (2226, 2) (336, 2) (231, 2)\n",
      "[69.78520122942598, 64.17408657952777, 18.479227005877494, 20.808937005932204, 19.77168971217251]\n",
      "mean mse: 38.60382830658719\n",
      "Train [2017-07-01 2017-08-15] Val [2017-08-16 2017-08-31] Test [2017-09-01 2017-09-16]\n",
      "Window Train/Val/Test shape (966, 29) (336, 29) (336, 29)\n",
      "Window Train/Val/Test shape (966, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-08-30] Val [2017-08-31 2017-09-15] Test [2017-09-16 2017-10-01]\n",
      "Window Train/Val/Test shape (1281, 29) (336, 29) (336, 29)\n",
      "Window Train/Val/Test shape (1281, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-09-14] Val [2017-09-15 2017-09-30] Test [2017-10-01 2017-10-16]\n",
      "Window Train/Val/Test shape (1596, 29) (336, 29) (336, 29)\n",
      "Window Train/Val/Test shape (1596, 2) (336, 2) (336, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-09-29] Val [2017-09-30 2017-10-15] Test [2017-10-16 2017-11-10]\n",
      "Window Train/Val/Test shape (1911, 29) (336, 29) (546, 29)\n",
      "Window Train/Val/Test shape (1911, 2) (336, 2) (546, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-10-14] Val [2017-10-15 2017-10-30] Test [2017-10-31 2017-11-10]\n",
      "Window Train/Val/Test shape (2226, 29) (336, 29) (231, 29)\n",
      "Window Train/Val/Test shape (2226, 2) (336, 2) (231, 2)\n",
      "[58.315627875422834, 65.03215241565822, 23.60865959151855, 24.313583900227286, 21.240645089369032]\n",
      "mean mse: 38.50213377443918\n",
      "Train [2017-07-01 2017-08-15] Val [2017-08-16 2017-08-31] Test [2017-09-01 2017-09-16]\n",
      "Window Train/Val/Test shape (966, 29) (336, 29) (336, 29)\n",
      "Window Train/Val/Test shape (966, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-08-30] Val [2017-08-31 2017-09-15] Test [2017-09-16 2017-10-01]\n",
      "Window Train/Val/Test shape (1281, 29) (336, 29) (336, 29)\n",
      "Window Train/Val/Test shape (1281, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-09-14] Val [2017-09-15 2017-09-30] Test [2017-10-01 2017-10-16]\n",
      "Window Train/Val/Test shape (1596, 29) (336, 29) (336, 29)\n",
      "Window Train/Val/Test shape (1596, 2) (336, 2) (336, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-09-29] Val [2017-09-30 2017-10-15] Test [2017-10-16 2017-11-10]\n",
      "Window Train/Val/Test shape (1911, 29) (336, 29) (546, 29)\n",
      "Window Train/Val/Test shape (1911, 2) (336, 2) (546, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-10-14] Val [2017-10-15 2017-10-30] Test [2017-10-31 2017-11-10]\n",
      "Window Train/Val/Test shape (2226, 29) (336, 29) (231, 29)\n",
      "Window Train/Val/Test shape (2226, 2) (336, 2) (231, 2)\n",
      "[58.87659673223631, 60.86455380507034, 22.73957701779656, 19.327157631481064, 23.02577734711412]\n",
      "mean mse: 36.96673250673968\n",
      "Train [2017-07-01 2017-08-15] Val [2017-08-16 2017-08-31] Test [2017-09-01 2017-09-16]\n",
      "Window Train/Val/Test shape (966, 29) (336, 29) (336, 29)\n",
      "Window Train/Val/Test shape (966, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-08-30] Val [2017-08-31 2017-09-15] Test [2017-09-16 2017-10-01]\n",
      "Window Train/Val/Test shape (1281, 29) (336, 29) (336, 29)\n",
      "Window Train/Val/Test shape (1281, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-09-14] Val [2017-09-15 2017-09-30] Test [2017-10-01 2017-10-16]\n",
      "Window Train/Val/Test shape (1596, 29) (336, 29) (336, 29)\n",
      "Window Train/Val/Test shape (1596, 2) (336, 2) (336, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-09-29] Val [2017-09-30 2017-10-15] Test [2017-10-16 2017-11-10]\n",
      "Window Train/Val/Test shape (1911, 29) (336, 29) (546, 29)\n",
      "Window Train/Val/Test shape (1911, 2) (336, 2) (546, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-10-14] Val [2017-10-15 2017-10-30] Test [2017-10-31 2017-11-10]\n",
      "Window Train/Val/Test shape (2226, 29) (336, 29) (231, 29)\n",
      "Window Train/Val/Test shape (2226, 2) (336, 2) (231, 2)\n",
      "[63.349282327352114, 63.25749011087288, 19.135732794939173, 15.546818593062081, 22.509625528270238]\n",
      "mean mse: 36.7597898708993\n",
      "Train [2017-07-01 2017-08-15] Val [2017-08-16 2017-08-31] Test [2017-09-01 2017-09-16]\n",
      "Window Train/Val/Test shape (966, 29) (336, 29) (336, 29)\n",
      "Window Train/Val/Test shape (966, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-08-30] Val [2017-08-31 2017-09-15] Test [2017-09-16 2017-10-01]\n",
      "Window Train/Val/Test shape (1281, 29) (336, 29) (336, 29)\n",
      "Window Train/Val/Test shape (1281, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-09-14] Val [2017-09-15 2017-09-30] Test [2017-10-01 2017-10-16]\n",
      "Window Train/Val/Test shape (1596, 29) (336, 29) (336, 29)\n",
      "Window Train/Val/Test shape (1596, 2) (336, 2) (336, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-09-29] Val [2017-09-30 2017-10-15] Test [2017-10-16 2017-11-10]\n",
      "Window Train/Val/Test shape (1911, 29) (336, 29) (546, 29)\n",
      "Window Train/Val/Test shape (1911, 2) (336, 2) (546, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-10-14] Val [2017-10-15 2017-10-30] Test [2017-10-31 2017-11-10]\n",
      "Window Train/Val/Test shape (2226, 29) (336, 29) (231, 29)\n",
      "Window Train/Val/Test shape (2226, 2) (336, 2) (231, 2)\n",
      "[53.74385694233795, 65.22174242664083, 24.729625291605924, 23.6151497504391, 20.8417353011008]\n",
      "mean mse: 37.630421942424924\n",
      "Train [2017-07-01 2017-08-15] Val [2017-08-16 2017-08-31] Test [2017-09-01 2017-09-16]\n",
      "Window Train/Val/Test shape (966, 29) (336, 29) (336, 29)\n",
      "Window Train/Val/Test shape (966, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-08-30] Val [2017-08-31 2017-09-15] Test [2017-09-16 2017-10-01]\n",
      "Window Train/Val/Test shape (1281, 29) (336, 29) (336, 29)\n",
      "Window Train/Val/Test shape (1281, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-09-14] Val [2017-09-15 2017-09-30] Test [2017-10-01 2017-10-16]\n",
      "Window Train/Val/Test shape (1596, 29) (336, 29) (336, 29)\n",
      "Window Train/Val/Test shape (1596, 2) (336, 2) (336, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-09-29] Val [2017-09-30 2017-10-15] Test [2017-10-16 2017-11-10]\n",
      "Window Train/Val/Test shape (1911, 29) (336, 29) (546, 29)\n",
      "Window Train/Val/Test shape (1911, 2) (336, 2) (546, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-10-14] Val [2017-10-15 2017-10-30] Test [2017-10-31 2017-11-10]\n",
      "Window Train/Val/Test shape (2226, 29) (336, 29) (231, 29)\n",
      "Window Train/Val/Test shape (2226, 2) (336, 2) (231, 2)\n",
      "[78.30403589646923, 62.768310073829376, 27.112013169910362, 21.561026166292137, 32.066108128877595]\n",
      "mean mse: 44.36229868707575\n",
      "Train [2017-07-01 2017-08-15] Val [2017-08-16 2017-08-31] Test [2017-09-01 2017-09-16]\n",
      "Window Train/Val/Test shape (966, 29) (336, 29) (336, 29)\n",
      "Window Train/Val/Test shape (966, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-08-30] Val [2017-08-31 2017-09-15] Test [2017-09-16 2017-10-01]\n",
      "Window Train/Val/Test shape (1281, 29) (336, 29) (336, 29)\n",
      "Window Train/Val/Test shape (1281, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-09-14] Val [2017-09-15 2017-09-30] Test [2017-10-01 2017-10-16]\n",
      "Window Train/Val/Test shape (1596, 29) (336, 29) (336, 29)\n",
      "Window Train/Val/Test shape (1596, 2) (336, 2) (336, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-09-29] Val [2017-09-30 2017-10-15] Test [2017-10-16 2017-11-10]\n",
      "Window Train/Val/Test shape (1911, 29) (336, 29) (546, 29)\n",
      "Window Train/Val/Test shape (1911, 2) (336, 2) (546, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-10-14] Val [2017-10-15 2017-10-30] Test [2017-10-31 2017-11-10]\n",
      "Window Train/Val/Test shape (2226, 29) (336, 29) (231, 29)\n",
      "Window Train/Val/Test shape (2226, 2) (336, 2) (231, 2)\n",
      "[74.88977864344287, 50.468040196630305, 21.18602981571129, 21.050344545742508, 19.851251071784016]\n",
      "mean mse: 37.4890888546622\n",
      "Train [2017-07-01 2017-08-15] Val [2017-08-16 2017-08-31] Test [2017-09-01 2017-09-16]\n",
      "Window Train/Val/Test shape (966, 29) (336, 29) (336, 29)\n",
      "Window Train/Val/Test shape (966, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-08-30] Val [2017-08-31 2017-09-15] Test [2017-09-16 2017-10-01]\n",
      "Window Train/Val/Test shape (1281, 29) (336, 29) (336, 29)\n",
      "Window Train/Val/Test shape (1281, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-09-14] Val [2017-09-15 2017-09-30] Test [2017-10-01 2017-10-16]\n",
      "Window Train/Val/Test shape (1596, 29) (336, 29) (336, 29)\n",
      "Window Train/Val/Test shape (1596, 2) (336, 2) (336, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-09-29] Val [2017-09-30 2017-10-15] Test [2017-10-16 2017-11-10]\n",
      "Window Train/Val/Test shape (1911, 29) (336, 29) (546, 29)\n",
      "Window Train/Val/Test shape (1911, 2) (336, 2) (546, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-10-14] Val [2017-10-15 2017-10-30] Test [2017-10-31 2017-11-10]\n",
      "Window Train/Val/Test shape (2226, 29) (336, 29) (231, 29)\n",
      "Window Train/Val/Test shape (2226, 2) (336, 2) (231, 2)\n",
      "[987.931295689933, 521.0934633649847, 350.8162566999191, 222.68899875926647, 143.22159405120408]\n",
      "mean mse: 445.1503217130615\n",
      "Train [2017-07-01 2017-08-15] Val [2017-08-16 2017-08-31] Test [2017-09-01 2017-09-16]\n",
      "Window Train/Val/Test shape (966, 29) (336, 29) (336, 29)\n",
      "Window Train/Val/Test shape (966, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-08-30] Val [2017-08-31 2017-09-15] Test [2017-09-16 2017-10-01]\n",
      "Window Train/Val/Test shape (1281, 29) (336, 29) (336, 29)\n",
      "Window Train/Val/Test shape (1281, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-09-14] Val [2017-09-15 2017-09-30] Test [2017-10-01 2017-10-16]\n",
      "Window Train/Val/Test shape (1596, 29) (336, 29) (336, 29)\n",
      "Window Train/Val/Test shape (1596, 2) (336, 2) (336, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-09-29] Val [2017-09-30 2017-10-15] Test [2017-10-16 2017-11-10]\n",
      "Window Train/Val/Test shape (1911, 29) (336, 29) (546, 29)\n",
      "Window Train/Val/Test shape (1911, 2) (336, 2) (546, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-10-14] Val [2017-10-15 2017-10-30] Test [2017-10-31 2017-11-10]\n",
      "Window Train/Val/Test shape (2226, 29) (336, 29) (231, 29)\n",
      "Window Train/Val/Test shape (2226, 2) (336, 2) (231, 2)\n",
      "[56.507691571691936, 63.24932060159593, 25.91967922672664, 20.071040107086166, 18.31666661716228]\n",
      "mean mse: 36.812879624852584\n",
      "Train [2017-07-01 2017-08-15] Val [2017-08-16 2017-08-31] Test [2017-09-01 2017-09-16]\n",
      "Window Train/Val/Test shape (966, 29) (336, 29) (336, 29)\n",
      "Window Train/Val/Test shape (966, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-08-30] Val [2017-08-31 2017-09-15] Test [2017-09-16 2017-10-01]\n",
      "Window Train/Val/Test shape (1281, 29) (336, 29) (336, 29)\n",
      "Window Train/Val/Test shape (1281, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-09-14] Val [2017-09-15 2017-09-30] Test [2017-10-01 2017-10-16]\n",
      "Window Train/Val/Test shape (1596, 29) (336, 29) (336, 29)\n",
      "Window Train/Val/Test shape (1596, 2) (336, 2) (336, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-09-29] Val [2017-09-30 2017-10-15] Test [2017-10-16 2017-11-10]\n",
      "Window Train/Val/Test shape (1911, 29) (336, 29) (546, 29)\n",
      "Window Train/Val/Test shape (1911, 2) (336, 2) (546, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-10-14] Val [2017-10-15 2017-10-30] Test [2017-10-31 2017-11-10]\n",
      "Window Train/Val/Test shape (2226, 29) (336, 29) (231, 29)\n",
      "Window Train/Val/Test shape (2226, 2) (336, 2) (231, 2)\n",
      "[987.931295689933, 521.0934633649847, 350.8162566999191, 222.68899875926647, 143.22159405120408]\n",
      "mean mse: 445.1503217130615\n",
      "Train [2017-07-01 2017-08-15] Val [2017-08-16 2017-08-31] Test [2017-09-01 2017-09-16]\n",
      "Window Train/Val/Test shape (966, 29) (336, 29) (336, 29)\n",
      "Window Train/Val/Test shape (966, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-08-30] Val [2017-08-31 2017-09-15] Test [2017-09-16 2017-10-01]\n",
      "Window Train/Val/Test shape (1281, 29) (336, 29) (336, 29)\n",
      "Window Train/Val/Test shape (1281, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-09-14] Val [2017-09-15 2017-09-30] Test [2017-10-01 2017-10-16]\n",
      "Window Train/Val/Test shape (1596, 29) (336, 29) (336, 29)\n",
      "Window Train/Val/Test shape (1596, 2) (336, 2) (336, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-09-29] Val [2017-09-30 2017-10-15] Test [2017-10-16 2017-11-10]\n",
      "Window Train/Val/Test shape (1911, 29) (336, 29) (546, 29)\n",
      "Window Train/Val/Test shape (1911, 2) (336, 2) (546, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-10-14] Val [2017-10-15 2017-10-30] Test [2017-10-31 2017-11-10]\n",
      "Window Train/Val/Test shape (2226, 29) (336, 29) (231, 29)\n",
      "Window Train/Val/Test shape (2226, 2) (336, 2) (231, 2)\n",
      "[57.20645346229412, 59.5690833775521, 26.730234986290252, 22.864073468109016, 20.12342865927418]\n",
      "mean mse: 37.29865479070393\n",
      "Train [2017-07-01 2017-08-15] Val [2017-08-16 2017-08-31] Test [2017-09-01 2017-09-16]\n",
      "Window Train/Val/Test shape (966, 29) (336, 29) (336, 29)\n",
      "Window Train/Val/Test shape (966, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-08-30] Val [2017-08-31 2017-09-15] Test [2017-09-16 2017-10-01]\n",
      "Window Train/Val/Test shape (1281, 29) (336, 29) (336, 29)\n",
      "Window Train/Val/Test shape (1281, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-09-14] Val [2017-09-15 2017-09-30] Test [2017-10-01 2017-10-16]\n",
      "Window Train/Val/Test shape (1596, 29) (336, 29) (336, 29)\n",
      "Window Train/Val/Test shape (1596, 2) (336, 2) (336, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-09-29] Val [2017-09-30 2017-10-15] Test [2017-10-16 2017-11-10]\n",
      "Window Train/Val/Test shape (1911, 29) (336, 29) (546, 29)\n",
      "Window Train/Val/Test shape (1911, 2) (336, 2) (546, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-10-14] Val [2017-10-15 2017-10-30] Test [2017-10-31 2017-11-10]\n",
      "Window Train/Val/Test shape (2226, 29) (336, 29) (231, 29)\n",
      "Window Train/Val/Test shape (2226, 2) (336, 2) (231, 2)\n",
      "[66.11112454456818, 55.7218731216245, 17.872841865568322, 20.644532142741838, 22.280764937784294]\n",
      "mean mse: 36.52622732245742\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train [2017-07-01 2017-08-15] Val [2017-08-16 2017-08-31] Test [2017-09-01 2017-09-16]\n",
      "Window Train/Val/Test shape (966, 29) (336, 29) (336, 29)\n",
      "Window Train/Val/Test shape (966, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-08-30] Val [2017-08-31 2017-09-15] Test [2017-09-16 2017-10-01]\n",
      "Window Train/Val/Test shape (1281, 29) (336, 29) (336, 29)\n",
      "Window Train/Val/Test shape (1281, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-09-14] Val [2017-09-15 2017-09-30] Test [2017-10-01 2017-10-16]\n",
      "Window Train/Val/Test shape (1596, 29) (336, 29) (336, 29)\n",
      "Window Train/Val/Test shape (1596, 2) (336, 2) (336, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-09-29] Val [2017-09-30 2017-10-15] Test [2017-10-16 2017-11-10]\n",
      "Window Train/Val/Test shape (1911, 29) (336, 29) (546, 29)\n",
      "Window Train/Val/Test shape (1911, 2) (336, 2) (546, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-10-14] Val [2017-10-15 2017-10-30] Test [2017-10-31 2017-11-10]\n",
      "Window Train/Val/Test shape (2226, 29) (336, 29) (231, 29)\n",
      "Window Train/Val/Test shape (2226, 2) (336, 2) (231, 2)\n",
      "[54.999832859095605, 60.411592036250426, 19.064438081020292, 20.377406217822152, 20.765527896200535]\n",
      "mean mse: 35.123759418077796\n",
      "Train [2017-07-01 2017-08-15] Val [2017-08-16 2017-08-31] Test [2017-09-01 2017-09-16]\n",
      "Window Train/Val/Test shape (966, 29) (336, 29) (336, 29)\n",
      "Window Train/Val/Test shape (966, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-08-30] Val [2017-08-31 2017-09-15] Test [2017-09-16 2017-10-01]\n",
      "Window Train/Val/Test shape (1281, 29) (336, 29) (336, 29)\n",
      "Window Train/Val/Test shape (1281, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-09-14] Val [2017-09-15 2017-09-30] Test [2017-10-01 2017-10-16]\n",
      "Window Train/Val/Test shape (1596, 29) (336, 29) (336, 29)\n",
      "Window Train/Val/Test shape (1596, 2) (336, 2) (336, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-09-29] Val [2017-09-30 2017-10-15] Test [2017-10-16 2017-11-10]\n",
      "Window Train/Val/Test shape (1911, 29) (336, 29) (546, 29)\n",
      "Window Train/Val/Test shape (1911, 2) (336, 2) (546, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-10-14] Val [2017-10-15 2017-10-30] Test [2017-10-31 2017-11-10]\n",
      "Window Train/Val/Test shape (2226, 29) (336, 29) (231, 29)\n",
      "Window Train/Val/Test shape (2226, 2) (336, 2) (231, 2)\n",
      "[57.13737998576337, 55.58599281978921, 24.08799872102872, 18.894410301331614, 22.56511789109197]\n",
      "mean mse: 35.654179943800976\n",
      "Train [2017-07-01 2017-08-15] Val [2017-08-16 2017-08-31] Test [2017-09-01 2017-09-16]\n",
      "Window Train/Val/Test shape (966, 29) (336, 29) (336, 29)\n",
      "Window Train/Val/Test shape (966, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-08-30] Val [2017-08-31 2017-09-15] Test [2017-09-16 2017-10-01]\n",
      "Window Train/Val/Test shape (1281, 29) (336, 29) (336, 29)\n",
      "Window Train/Val/Test shape (1281, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-09-14] Val [2017-09-15 2017-09-30] Test [2017-10-01 2017-10-16]\n",
      "Window Train/Val/Test shape (1596, 29) (336, 29) (336, 29)\n",
      "Window Train/Val/Test shape (1596, 2) (336, 2) (336, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-09-29] Val [2017-09-30 2017-10-15] Test [2017-10-16 2017-11-10]\n",
      "Window Train/Val/Test shape (1911, 29) (336, 29) (546, 29)\n",
      "Window Train/Val/Test shape (1911, 2) (336, 2) (546, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-10-14] Val [2017-10-15 2017-10-30] Test [2017-10-31 2017-11-10]\n",
      "Window Train/Val/Test shape (2226, 29) (336, 29) (231, 29)\n",
      "Window Train/Val/Test shape (2226, 2) (336, 2) (231, 2)\n",
      "[62.71590769480348, 63.188888113628266, 22.885048441352392, 22.838836166670433, 20.87831205258902]\n",
      "mean mse: 38.501398493808715\n",
      "Train [2017-07-01 2017-08-15] Val [2017-08-16 2017-08-31] Test [2017-09-01 2017-09-16]\n",
      "Window Train/Val/Test shape (966, 29) (336, 29) (336, 29)\n",
      "Window Train/Val/Test shape (966, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-08-30] Val [2017-08-31 2017-09-15] Test [2017-09-16 2017-10-01]\n",
      "Window Train/Val/Test shape (1281, 29) (336, 29) (336, 29)\n",
      "Window Train/Val/Test shape (1281, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-09-14] Val [2017-09-15 2017-09-30] Test [2017-10-01 2017-10-16]\n",
      "Window Train/Val/Test shape (1596, 29) (336, 29) (336, 29)\n",
      "Window Train/Val/Test shape (1596, 2) (336, 2) (336, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-09-29] Val [2017-09-30 2017-10-15] Test [2017-10-16 2017-11-10]\n",
      "Window Train/Val/Test shape (1911, 29) (336, 29) (546, 29)\n",
      "Window Train/Val/Test shape (1911, 2) (336, 2) (546, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-10-14] Val [2017-10-15 2017-10-30] Test [2017-10-31 2017-11-10]\n",
      "Window Train/Val/Test shape (2226, 29) (336, 29) (231, 29)\n",
      "Window Train/Val/Test shape (2226, 2) (336, 2) (231, 2)\n",
      "[60.106737935459975, 54.430897324838085, 18.95177094300514, 22.519699208470733, 19.250976632013263]\n",
      "mean mse: 35.05201640875744\n"
     ]
    }
   ],
   "source": [
    "orchestrator24.performHyperparameterOptimization(X_24_preselect[selected_features24], 100)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 138,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'colsample_bytree': 0.7000000000000001,\n",
       " 'gamma': 2.5,\n",
       " 'learning_rate': 0.2,\n",
       " 'max_depth': 7.0,\n",
       " 'min_child_weight': 1.0,\n",
       " 'n_estimators': 550.0,\n",
       " 'subsample': 0.7000000000000001}"
      ]
     },
     "execution_count": 138,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "orchestrator24.bo.best"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 91,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "postselect_params_24 = {'colsample_bytree': 0.70,\n",
    "                         'gamma': 2.5,\n",
    "                         'learning_rate': 0.2,\n",
    "                         'max_depth': 7,\n",
    "                         'min_child_weight': 1.0,\n",
    "                         'n_estimators': 550,\n",
    "                         'subsample': 0.70}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {},
   "outputs": [],
   "source": [
    "# orchestrator24.corr_selected_features_bi_week[selected_features24].to_csv('/Users/Rohil/Documents/iGEM/yemen/intermediary_files/X_2_4_postselect.csv')\n",
    "# orchestrator24.corr_selected_features_bi_week_y.to_csv('/Users/Rohil/Documents/iGEM/yemen/intermediary_files/y_2_4.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 163,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\Rohil\\Anaconda3\\lib\\site-packages\\ipykernel_launcher.py:78: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/indexing.html#indexing-view-versus-copy\n",
      "C:\\Users\\Rohil\\Anaconda3\\lib\\site-packages\\pandas\\core\\frame.py:3694: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame\n",
      "\n",
      "See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/indexing.html#indexing-view-versus-copy\n",
      "  errors=errors)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train [2017-07-01 2017-08-15] Val [2017-08-16 2017-08-31] Test [2017-09-01 2017-09-16]\n",
      "Window Train/Val/Test shape (966, 29) (336, 29) (336, 29)\n",
      "Window Train/Val/Test shape (966, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-08-30] Val [2017-08-31 2017-09-15] Test [2017-09-16 2017-10-01]\n",
      "Window Train/Val/Test shape (1281, 29) (336, 29) (336, 29)\n",
      "Window Train/Val/Test shape (1281, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-09-14] Val [2017-09-15 2017-09-30] Test [2017-10-01 2017-10-16]\n",
      "Window Train/Val/Test shape (1596, 29) (336, 29) (336, 29)\n",
      "Window Train/Val/Test shape (1596, 2) (336, 2) (336, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-09-29] Val [2017-09-30 2017-10-15] Test [2017-10-16 2017-11-10]\n",
      "Window Train/Val/Test shape (1911, 29) (336, 29) (546, 29)\n",
      "Window Train/Val/Test shape (1911, 2) (336, 2) (546, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-10-14] Val [2017-10-15 2017-10-30] Test [2017-10-31 2017-11-10]\n",
      "Window Train/Val/Test shape (2226, 29) (336, 29) (231, 29)\n",
      "Window Train/Val/Test shape (2226, 2) (336, 2) (231, 2)\n",
      "y-test mean 8.732327207993913, y-test std 8.686217979749713\n",
      "cv mse 31.06707844085972, holdout mse19.271375752951002\n"
     ]
    }
   ],
   "source": [
    "orchestrator24.deploy(0.002232, postselect_params_24, None)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [],
   "source": [
    "# running feature selection for 4-6 week model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [],
   "source": [
    "orchestrator46 = Orchestrator(full_features, full_data_4_6, 'week_4_to_6_cases' )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Running TSFresh....\n",
      "Selected features (4011, 15156)\n",
      "Finished running TSFresh....\n"
     ]
    }
   ],
   "source": [
    "orchestrator46.runTsFresh()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Running Feature Selection module ....\n",
      "4773 features with a correlation magnitude greater than 0.97.\n",
      "\n",
      "Original (4011, 15157) and after (4011, 10384)\n",
      "Finished running Feature Selection ....\n"
     ]
    }
   ],
   "source": [
    "orchestrator46.runOptimizeFeatures()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 66,
   "metadata": {},
   "outputs": [],
   "source": [
    "X_46_preselect = orchestrator46.op.corr_selected_features\n",
    "y_46_preselect = orchestrator46.full_data_bi_week[['date', orchestrator46.y_bi_week_label]]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 67,
   "metadata": {},
   "outputs": [],
   "source": [
    "# X_46_preselect.to_csv('/Users/Rohil/Documents/iGEM/yemen/intermediary_files/X_46_preselect.csv')\n",
    "# y_46_preselect.to_csv('/Users/Rohil/Documents/iGEM/yemen/intermediary_files/y_46_preselect.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [],
   "source": [
    "orchestrator46 = Orchestrator(y4_6.dropna().merge(full_features, how = 'left', on = ['gov_iso', 'date']).sort_values('date'), 'week_4_to_6_cases' )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [],
   "source": [
    "X_46_preselect = pd.read_csv('/Users/Rohil/Documents/iGEM/yemen/intermediary_files/X_46_preselect.csv', index_col = 0)\n",
    "X_46_preselect.date = pd.to_datetime(X_46_preselect.date, format = '%Y-%m-%d')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train [2017-07-01 2017-08-15] Val [2017-08-16 2017-08-31] Test [2017-09-01 2017-09-16]\n",
      "Window Train/Val/Test shape (966, 10384) (336, 10384) (336, 10384)\n",
      "Window Train/Val/Test shape (966, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-08-30] Val [2017-08-31 2017-09-15] Test [2017-09-16 2017-10-01]\n",
      "Window Train/Val/Test shape (1281, 10384) (336, 10384) (336, 10384)\n",
      "Window Train/Val/Test shape (1281, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-09-14] Val [2017-09-15 2017-09-30] Test [2017-10-01 2017-10-16]\n",
      "Window Train/Val/Test shape (1596, 10384) (336, 10384) (336, 10384)\n",
      "Window Train/Val/Test shape (1596, 2) (336, 2) (336, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-09-29] Val [2017-09-30 2017-10-15] Test [2017-10-16 2017-11-10]\n",
      "Window Train/Val/Test shape (1911, 10384) (336, 10384) (546, 10384)\n",
      "Window Train/Val/Test shape (1911, 2) (336, 2) (546, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-10-14] Val [2017-10-15 2017-10-30] Test [2017-10-31 2017-11-10]\n",
      "Window Train/Val/Test shape (2226, 10384) (336, 10384) (231, 10384)\n",
      "Window Train/Val/Test shape (2226, 2) (336, 2) (231, 2)\n",
      "[73.84706191063695, 90.79426090164836, 27.09331324957449, 22.238572672968917, 7.273121791528725]\n",
      "mean mse: 44.24926610527149\n",
      "Train [2017-07-01 2017-08-15] Val [2017-08-16 2017-08-31] Test [2017-09-01 2017-09-16]\n",
      "Window Train/Val/Test shape (966, 10384) (336, 10384) (336, 10384)\n",
      "Window Train/Val/Test shape (966, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-08-30] Val [2017-08-31 2017-09-15] Test [2017-09-16 2017-10-01]\n",
      "Window Train/Val/Test shape (1281, 10384) (336, 10384) (336, 10384)\n",
      "Window Train/Val/Test shape (1281, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-09-14] Val [2017-09-15 2017-09-30] Test [2017-10-01 2017-10-16]\n",
      "Window Train/Val/Test shape (1596, 10384) (336, 10384) (336, 10384)\n",
      "Window Train/Val/Test shape (1596, 2) (336, 2) (336, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-09-29] Val [2017-09-30 2017-10-15] Test [2017-10-16 2017-11-10]\n",
      "Window Train/Val/Test shape (1911, 10384) (336, 10384) (546, 10384)\n",
      "Window Train/Val/Test shape (1911, 2) (336, 2) (546, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-10-14] Val [2017-10-15 2017-10-30] Test [2017-10-31 2017-11-10]\n",
      "Window Train/Val/Test shape (2226, 10384) (336, 10384) (231, 10384)\n",
      "Window Train/Val/Test shape (2226, 2) (336, 2) (231, 2)\n",
      "[65.20749348109807, 80.80266326059312, 20.709395170532424, 19.75504314783205, 6.544547120403946]\n",
      "mean mse: 38.603828436091916\n",
      "Train [2017-07-01 2017-08-15] Val [2017-08-16 2017-08-31] Test [2017-09-01 2017-09-16]\n",
      "Window Train/Val/Test shape (966, 10384) (336, 10384) (336, 10384)\n",
      "Window Train/Val/Test shape (966, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-08-30] Val [2017-08-31 2017-09-15] Test [2017-09-16 2017-10-01]\n",
      "Window Train/Val/Test shape (1281, 10384) (336, 10384) (336, 10384)\n",
      "Window Train/Val/Test shape (1281, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-09-14] Val [2017-09-15 2017-09-30] Test [2017-10-01 2017-10-16]\n",
      "Window Train/Val/Test shape (1596, 10384) (336, 10384) (336, 10384)\n",
      "Window Train/Val/Test shape (1596, 2) (336, 2) (336, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-09-29] Val [2017-09-30 2017-10-15] Test [2017-10-16 2017-11-10]\n",
      "Window Train/Val/Test shape (1911, 10384) (336, 10384) (546, 10384)\n",
      "Window Train/Val/Test shape (1911, 2) (336, 2) (546, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-10-14] Val [2017-10-15 2017-10-30] Test [2017-10-31 2017-11-10]\n",
      "Window Train/Val/Test shape (2226, 10384) (336, 10384) (231, 10384)\n",
      "Window Train/Val/Test shape (2226, 2) (336, 2) (231, 2)\n",
      "[83.6796193591947, 53.69223685780681, 27.707631425991902, 18.22510496071776, 4.683239215010511]\n",
      "mean mse: 37.59756636374434\n",
      "Train [2017-07-01 2017-08-15] Val [2017-08-16 2017-08-31] Test [2017-09-01 2017-09-16]\n",
      "Window Train/Val/Test shape (966, 10384) (336, 10384) (336, 10384)\n",
      "Window Train/Val/Test shape (966, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-08-30] Val [2017-08-31 2017-09-15] Test [2017-09-16 2017-10-01]\n",
      "Window Train/Val/Test shape (1281, 10384) (336, 10384) (336, 10384)\n",
      "Window Train/Val/Test shape (1281, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-09-14] Val [2017-09-15 2017-09-30] Test [2017-10-01 2017-10-16]\n",
      "Window Train/Val/Test shape (1596, 10384) (336, 10384) (336, 10384)\n",
      "Window Train/Val/Test shape (1596, 2) (336, 2) (336, 2)\n"
     ]
    }
   ],
   "source": [
    "orchestrator46.performHyperparameterOptimization(X_46_preselect, 25)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "preselect_params_46 = {'colsample_bytree': 0.35,\n",
    "                        'gamma': 4.0,\n",
    "                        'learning_rate': 0.125,\n",
    "                        'max_depth': 11,\n",
    "                        'min_child_weight': 2.0,\n",
    "                        'n_estimators': 350,\n",
    "                        'subsample': 0.95}\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Running custom feature selection module ....\n",
      "Creating baseline model to extract features\n",
      "all features (2688, 10383)\n",
      "13\n",
      "# features 10383 # thresholds 13; thresholds:[0.00783771 0.00507146 0.00461042 0.00414938 0.00368834 0.00322729\n",
      " 0.00276625 0.00230521 0.00184417 0.00138313 0.00092208 0.00046104\n",
      " 0.        ]\n",
      "CV - Window 3\n",
      "Train [2017-07-01 2017-08-15] Val [2017-08-16 2017-08-31] Test [2017-09-01 2017-09-16]\n",
      "Window Train/Val/Test shape (966, 10384) (336, 10384) (336, 10384)\n",
      "Window Train/Val/Test shape (966, 2) (336, 2) (336, 2)\n",
      "X_train_cv  (1302, 10383)\n",
      "3 Thresh=0.00784, n=1, mse: 96.011\n",
      "CV - Window 4\n",
      "Train [2017-07-01 2017-08-30] Val [2017-08-31 2017-09-15] Test [2017-09-16 2017-10-01]\n",
      "Window Train/Val/Test shape (1281, 10384) (336, 10384) (336, 10384)\n",
      "Window Train/Val/Test shape (1281, 2) (336, 2) (336, 2)\n",
      "X_train_cv  (1617, 10383)\n",
      "4 Thresh=0.00784, n=1, mse: 84.528\n",
      "CV - Window 5\n",
      "Train [2017-07-01 2017-09-14] Val [2017-09-15 2017-09-30] Test [2017-10-01 2017-10-16]\n",
      "Window Train/Val/Test shape (1596, 10384) (336, 10384) (336, 10384)\n",
      "Window Train/Val/Test shape (1596, 2) (336, 2) (336, 2)\n",
      "X_train_cv  (1932, 10383)\n",
      "5 Thresh=0.00784, n=1, mse: 87.704\n",
      "CV - Window 6\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-09-29] Val [2017-09-30 2017-10-15] Test [2017-10-16 2017-11-10]\n",
      "Window Train/Val/Test shape (1911, 10384) (336, 10384) (546, 10384)\n",
      "Window Train/Val/Test shape (1911, 2) (336, 2) (546, 2)\n",
      "X_train_cv  (2247, 10383)\n",
      "6 Thresh=0.00784, n=1, mse: 27.307\n",
      "CV - Window 7\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-10-14] Val [2017-10-15 2017-10-30] Test [2017-10-31 2017-11-10]\n",
      "Window Train/Val/Test shape (2226, 10384) (336, 10384) (231, 10384)\n",
      "Window Train/Val/Test shape (2226, 2) (336, 2) (231, 2)\n",
      "X_train_cv  (2562, 10383)\n",
      "7 Thresh=0.00784, n=1, mse: 16.785\n",
      "{'threshold': 0.007837714, 'num_features': 1, 'mse_list': [96.01124097957597, 84.52831216896432, 87.70367842907724, 27.307449291254724, 16.78532946852459], 'mean_mse': 62.46720206747936}\n",
      "\n",
      "\n",
      "CV - Window 3\n",
      "Train [2017-07-01 2017-08-15] Val [2017-08-16 2017-08-31] Test [2017-09-01 2017-09-16]\n",
      "Window Train/Val/Test shape (966, 10384) (336, 10384) (336, 10384)\n",
      "Window Train/Val/Test shape (966, 2) (336, 2) (336, 2)\n",
      "X_train_cv  (1302, 10383)\n",
      "3 Thresh=0.00507, n=2, mse: 72.594\n",
      "CV - Window 4\n",
      "Train [2017-07-01 2017-08-30] Val [2017-08-31 2017-09-15] Test [2017-09-16 2017-10-01]\n",
      "Window Train/Val/Test shape (1281, 10384) (336, 10384) (336, 10384)\n",
      "Window Train/Val/Test shape (1281, 2) (336, 2) (336, 2)\n",
      "X_train_cv  (1617, 10383)\n",
      "4 Thresh=0.00507, n=2, mse: 77.066\n",
      "CV - Window 5\n",
      "Train [2017-07-01 2017-09-14] Val [2017-09-15 2017-09-30] Test [2017-10-01 2017-10-16]\n",
      "Window Train/Val/Test shape (1596, 10384) (336, 10384) (336, 10384)\n",
      "Window Train/Val/Test shape (1596, 2) (336, 2) (336, 2)\n",
      "X_train_cv  (1932, 10383)\n",
      "5 Thresh=0.00507, n=2, mse: 61.034\n",
      "CV - Window 6\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-09-29] Val [2017-09-30 2017-10-15] Test [2017-10-16 2017-11-10]\n",
      "Window Train/Val/Test shape (1911, 10384) (336, 10384) (546, 10384)\n",
      "Window Train/Val/Test shape (1911, 2) (336, 2) (546, 2)\n",
      "X_train_cv  (2247, 10383)\n",
      "6 Thresh=0.00507, n=2, mse: 29.260\n",
      "CV - Window 7\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-10-14] Val [2017-10-15 2017-10-30] Test [2017-10-31 2017-11-10]\n",
      "Window Train/Val/Test shape (2226, 10384) (336, 10384) (231, 10384)\n",
      "Window Train/Val/Test shape (2226, 2) (336, 2) (231, 2)\n",
      "X_train_cv  (2562, 10383)\n",
      "7 Thresh=0.00507, n=2, mse: 37.405\n",
      "{'threshold': 0.0050714617, 'num_features': 2, 'mse_list': [72.5939388033789, 77.06552081754683, 61.03384013681414, 29.259555423985006, 37.404711485696055], 'mean_mse': 55.47151333348419}\n",
      "\n",
      "\n",
      "CV - Window 3\n",
      "Train [2017-07-01 2017-08-15] Val [2017-08-16 2017-08-31] Test [2017-09-01 2017-09-16]\n",
      "Window Train/Val/Test shape (966, 10384) (336, 10384) (336, 10384)\n",
      "Window Train/Val/Test shape (966, 2) (336, 2) (336, 2)\n",
      "X_train_cv  (1302, 10383)\n",
      "3 Thresh=0.00461, n=5, mse: 71.311\n",
      "CV - Window 4\n",
      "Train [2017-07-01 2017-08-30] Val [2017-08-31 2017-09-15] Test [2017-09-16 2017-10-01]\n",
      "Window Train/Val/Test shape (1281, 10384) (336, 10384) (336, 10384)\n",
      "Window Train/Val/Test shape (1281, 2) (336, 2) (336, 2)\n",
      "X_train_cv  (1617, 10383)\n",
      "4 Thresh=0.00461, n=5, mse: 60.223\n",
      "CV - Window 5\n",
      "Train [2017-07-01 2017-09-14] Val [2017-09-15 2017-09-30] Test [2017-10-01 2017-10-16]\n",
      "Window Train/Val/Test shape (1596, 10384) (336, 10384) (336, 10384)\n",
      "Window Train/Val/Test shape (1596, 2) (336, 2) (336, 2)\n",
      "X_train_cv  (1932, 10383)\n",
      "5 Thresh=0.00461, n=5, mse: 72.077\n",
      "CV - Window 6\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-09-29] Val [2017-09-30 2017-10-15] Test [2017-10-16 2017-11-10]\n",
      "Window Train/Val/Test shape (1911, 10384) (336, 10384) (546, 10384)\n",
      "Window Train/Val/Test shape (1911, 2) (336, 2) (546, 2)\n",
      "X_train_cv  (2247, 10383)\n",
      "6 Thresh=0.00461, n=5, mse: 37.085\n",
      "CV - Window 7\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-10-14] Val [2017-10-15 2017-10-30] Test [2017-10-31 2017-11-10]\n",
      "Window Train/Val/Test shape (2226, 10384) (336, 10384) (231, 10384)\n",
      "Window Train/Val/Test shape (2226, 2) (336, 2) (231, 2)\n",
      "X_train_cv  (2562, 10383)\n",
      "7 Thresh=0.00461, n=5, mse: 17.274\n",
      "{'threshold': 0.0046104197, 'num_features': 5, 'mse_list': [71.3112827201162, 60.22254825770118, 72.0766036385655, 37.08527747710178, 17.273832416994942], 'mean_mse': 51.593908902095926}\n",
      "\n",
      "\n",
      "CV - Window 3\n",
      "Train [2017-07-01 2017-08-15] Val [2017-08-16 2017-08-31] Test [2017-09-01 2017-09-16]\n",
      "Window Train/Val/Test shape (966, 10384) (336, 10384) (336, 10384)\n",
      "Window Train/Val/Test shape (966, 2) (336, 2) (336, 2)\n",
      "X_train_cv  (1302, 10383)\n",
      "3 Thresh=0.00415, n=8, mse: 49.503\n",
      "CV - Window 4\n",
      "Train [2017-07-01 2017-08-30] Val [2017-08-31 2017-09-15] Test [2017-09-16 2017-10-01]\n",
      "Window Train/Val/Test shape (1281, 10384) (336, 10384) (336, 10384)\n",
      "Window Train/Val/Test shape (1281, 2) (336, 2) (336, 2)\n",
      "X_train_cv  (1617, 10383)\n",
      "4 Thresh=0.00415, n=8, mse: 45.713\n",
      "CV - Window 5\n",
      "Train [2017-07-01 2017-09-14] Val [2017-09-15 2017-09-30] Test [2017-10-01 2017-10-16]\n",
      "Window Train/Val/Test shape (1596, 10384) (336, 10384) (336, 10384)\n",
      "Window Train/Val/Test shape (1596, 2) (336, 2) (336, 2)\n",
      "X_train_cv  (1932, 10383)\n",
      "5 Thresh=0.00415, n=8, mse: 50.591\n",
      "CV - Window 6\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-09-29] Val [2017-09-30 2017-10-15] Test [2017-10-16 2017-11-10]\n",
      "Window Train/Val/Test shape (1911, 10384) (336, 10384) (546, 10384)\n",
      "Window Train/Val/Test shape (1911, 2) (336, 2) (546, 2)\n",
      "X_train_cv  (2247, 10383)\n",
      "6 Thresh=0.00415, n=8, mse: 18.817\n",
      "CV - Window 7\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-10-14] Val [2017-10-15 2017-10-30] Test [2017-10-31 2017-11-10]\n",
      "Window Train/Val/Test shape (2226, 10384) (336, 10384) (231, 10384)\n",
      "Window Train/Val/Test shape (2226, 2) (336, 2) (231, 2)\n",
      "X_train_cv  (2562, 10383)\n",
      "7 Thresh=0.00415, n=8, mse: 7.821\n",
      "{'threshold': 0.004149378, 'num_features': 8, 'mse_list': [49.50330081621364, 45.71297779305811, 50.59059682087716, 18.81680190436326, 7.821043586043055], 'mean_mse': 34.48894418411104}\n",
      "\n",
      "\n",
      "CV - Window 3\n",
      "Train [2017-07-01 2017-08-15] Val [2017-08-16 2017-08-31] Test [2017-09-01 2017-09-16]\n",
      "Window Train/Val/Test shape (966, 10384) (336, 10384) (336, 10384)\n",
      "Window Train/Val/Test shape (966, 2) (336, 2) (336, 2)\n",
      "X_train_cv  (1302, 10383)\n",
      "3 Thresh=0.00369, n=10, mse: 50.333\n",
      "CV - Window 4\n",
      "Train [2017-07-01 2017-08-30] Val [2017-08-31 2017-09-15] Test [2017-09-16 2017-10-01]\n",
      "Window Train/Val/Test shape (1281, 10384) (336, 10384) (336, 10384)\n",
      "Window Train/Val/Test shape (1281, 2) (336, 2) (336, 2)\n",
      "X_train_cv  (1617, 10383)\n",
      "4 Thresh=0.00369, n=10, mse: 36.486\n",
      "CV - Window 5\n",
      "Train [2017-07-01 2017-09-14] Val [2017-09-15 2017-09-30] Test [2017-10-01 2017-10-16]\n",
      "Window Train/Val/Test shape (1596, 10384) (336, 10384) (336, 10384)\n",
      "Window Train/Val/Test shape (1596, 2) (336, 2) (336, 2)\n",
      "X_train_cv  (1932, 10383)\n",
      "5 Thresh=0.00369, n=10, mse: 37.382\n",
      "CV - Window 6\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-09-29] Val [2017-09-30 2017-10-15] Test [2017-10-16 2017-11-10]\n",
      "Window Train/Val/Test shape (1911, 10384) (336, 10384) (546, 10384)\n",
      "Window Train/Val/Test shape (1911, 2) (336, 2) (546, 2)\n",
      "X_train_cv  (2247, 10383)\n",
      "6 Thresh=0.00369, n=10, mse: 9.648\n",
      "CV - Window 7\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-10-14] Val [2017-10-15 2017-10-30] Test [2017-10-31 2017-11-10]\n",
      "Window Train/Val/Test shape (2226, 10384) (336, 10384) (231, 10384)\n",
      "Window Train/Val/Test shape (2226, 2) (336, 2) (231, 2)\n",
      "X_train_cv  (2562, 10383)\n",
      "7 Thresh=0.00369, n=10, mse: 9.527\n",
      "{'threshold': 0.0036883357, 'num_features': 10, 'mse_list': [50.332832509640696, 36.48606757250619, 37.381766917227935, 9.648404722009701, 9.527214850366652], 'mean_mse': 28.675257314350233}\n",
      "\n",
      "\n",
      "CV - Window 3\n",
      "Train [2017-07-01 2017-08-15] Val [2017-08-16 2017-08-31] Test [2017-09-01 2017-09-16]\n",
      "Window Train/Val/Test shape (966, 10384) (336, 10384) (336, 10384)\n",
      "Window Train/Val/Test shape (966, 2) (336, 2) (336, 2)\n",
      "X_train_cv  (1302, 10383)\n",
      "3 Thresh=0.00323, n=13, mse: 48.110\n",
      "CV - Window 4\n",
      "Train [2017-07-01 2017-08-30] Val [2017-08-31 2017-09-15] Test [2017-09-16 2017-10-01]\n",
      "Window Train/Val/Test shape (1281, 10384) (336, 10384) (336, 10384)\n",
      "Window Train/Val/Test shape (1281, 2) (336, 2) (336, 2)\n",
      "X_train_cv  (1617, 10383)\n",
      "4 Thresh=0.00323, n=13, mse: 33.213\n",
      "CV - Window 5\n",
      "Train [2017-07-01 2017-09-14] Val [2017-09-15 2017-09-30] Test [2017-10-01 2017-10-16]\n",
      "Window Train/Val/Test shape (1596, 10384) (336, 10384) (336, 10384)\n",
      "Window Train/Val/Test shape (1596, 2) (336, 2) (336, 2)\n",
      "X_train_cv  (1932, 10383)\n",
      "5 Thresh=0.00323, n=13, mse: 28.678\n",
      "CV - Window 6\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-09-29] Val [2017-09-30 2017-10-15] Test [2017-10-16 2017-11-10]\n",
      "Window Train/Val/Test shape (1911, 10384) (336, 10384) (546, 10384)\n",
      "Window Train/Val/Test shape (1911, 2) (336, 2) (546, 2)\n",
      "X_train_cv  (2247, 10383)\n",
      "6 Thresh=0.00323, n=13, mse: 8.546\n",
      "CV - Window 7\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-10-14] Val [2017-10-15 2017-10-30] Test [2017-10-31 2017-11-10]\n",
      "Window Train/Val/Test shape (2226, 10384) (336, 10384) (231, 10384)\n",
      "Window Train/Val/Test shape (2226, 2) (336, 2) (231, 2)\n",
      "X_train_cv  (2562, 10383)\n",
      "7 Thresh=0.00323, n=13, mse: 14.759\n",
      "{'threshold': 0.0032272937, 'num_features': 13, 'mse_list': [48.10976106493251, 33.212752818892035, 28.678130564726626, 8.546244689402002, 14.759065688165263], 'mean_mse': 26.66119096522369}\n",
      "\n",
      "\n",
      "CV - Window 3\n",
      "Train [2017-07-01 2017-08-15] Val [2017-08-16 2017-08-31] Test [2017-09-01 2017-09-16]\n",
      "Window Train/Val/Test shape (966, 10384) (336, 10384) (336, 10384)\n",
      "Window Train/Val/Test shape (966, 2) (336, 2) (336, 2)\n",
      "X_train_cv  (1302, 10383)\n",
      "3 Thresh=0.00277, n=19, mse: 32.438\n",
      "CV - Window 4\n",
      "Train [2017-07-01 2017-08-30] Val [2017-08-31 2017-09-15] Test [2017-09-16 2017-10-01]\n",
      "Window Train/Val/Test shape (1281, 10384) (336, 10384) (336, 10384)\n",
      "Window Train/Val/Test shape (1281, 2) (336, 2) (336, 2)\n",
      "X_train_cv  (1617, 10383)\n",
      "4 Thresh=0.00277, n=19, mse: 34.926\n",
      "CV - Window 5\n",
      "Train [2017-07-01 2017-09-14] Val [2017-09-15 2017-09-30] Test [2017-10-01 2017-10-16]\n",
      "Window Train/Val/Test shape (1596, 10384) (336, 10384) (336, 10384)\n",
      "Window Train/Val/Test shape (1596, 2) (336, 2) (336, 2)\n",
      "X_train_cv  (1932, 10383)\n",
      "5 Thresh=0.00277, n=19, mse: 24.065\n",
      "CV - Window 6\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-09-29] Val [2017-09-30 2017-10-15] Test [2017-10-16 2017-11-10]\n",
      "Window Train/Val/Test shape (1911, 10384) (336, 10384) (546, 10384)\n",
      "Window Train/Val/Test shape (1911, 2) (336, 2) (546, 2)\n",
      "X_train_cv  (2247, 10383)\n",
      "6 Thresh=0.00277, n=19, mse: 16.657\n",
      "CV - Window 7\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-10-14] Val [2017-10-15 2017-10-30] Test [2017-10-31 2017-11-10]\n",
      "Window Train/Val/Test shape (2226, 10384) (336, 10384) (231, 10384)\n",
      "Window Train/Val/Test shape (2226, 2) (336, 2) (231, 2)\n",
      "X_train_cv  (2562, 10383)\n",
      "7 Thresh=0.00277, n=19, mse: 5.606\n",
      "{'threshold': 0.0027662518, 'num_features': 19, 'mse_list': [32.438236808872034, 34.92581332507123, 24.064665135517856, 16.65693992275521, 5.6058193726751195], 'mean_mse': 22.73829491297829}\n",
      "\n",
      "\n",
      "CV - Window 3\n",
      "Train [2017-07-01 2017-08-15] Val [2017-08-16 2017-08-31] Test [2017-09-01 2017-09-16]\n",
      "Window Train/Val/Test shape (966, 10384) (336, 10384) (336, 10384)\n",
      "Window Train/Val/Test shape (966, 2) (336, 2) (336, 2)\n",
      "X_train_cv  (1302, 10383)\n",
      "3 Thresh=0.00231, n=25, mse: 28.155\n",
      "CV - Window 4\n",
      "Train [2017-07-01 2017-08-30] Val [2017-08-31 2017-09-15] Test [2017-09-16 2017-10-01]\n",
      "Window Train/Val/Test shape (1281, 10384) (336, 10384) (336, 10384)\n",
      "Window Train/Val/Test shape (1281, 2) (336, 2) (336, 2)\n",
      "X_train_cv  (1617, 10383)\n",
      "4 Thresh=0.00231, n=25, mse: 35.180\n",
      "CV - Window 5\n",
      "Train [2017-07-01 2017-09-14] Val [2017-09-15 2017-09-30] Test [2017-10-01 2017-10-16]\n",
      "Window Train/Val/Test shape (1596, 10384) (336, 10384) (336, 10384)\n",
      "Window Train/Val/Test shape (1596, 2) (336, 2) (336, 2)\n",
      "X_train_cv  (1932, 10383)\n",
      "5 Thresh=0.00231, n=25, mse: 20.702\n",
      "CV - Window 6\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-09-29] Val [2017-09-30 2017-10-15] Test [2017-10-16 2017-11-10]\n",
      "Window Train/Val/Test shape (1911, 10384) (336, 10384) (546, 10384)\n",
      "Window Train/Val/Test shape (1911, 2) (336, 2) (546, 2)\n",
      "X_train_cv  (2247, 10383)\n",
      "6 Thresh=0.00231, n=25, mse: 8.827\n",
      "CV - Window 7\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-10-14] Val [2017-10-15 2017-10-30] Test [2017-10-31 2017-11-10]\n",
      "Window Train/Val/Test shape (2226, 10384) (336, 10384) (231, 10384)\n",
      "Window Train/Val/Test shape (2226, 2) (336, 2) (231, 2)\n",
      "X_train_cv  (2562, 10383)\n",
      "7 Thresh=0.00231, n=25, mse: 7.368\n",
      "{'threshold': 0.0023052099, 'num_features': 25, 'mse_list': [28.15466641078222, 35.17952183346277, 20.702424635368605, 8.827493518843092, 7.367941193836498], 'mean_mse': 20.04640951845864}\n",
      "\n",
      "\n",
      "CV - Window 3\n",
      "Train [2017-07-01 2017-08-15] Val [2017-08-16 2017-08-31] Test [2017-09-01 2017-09-16]\n",
      "Window Train/Val/Test shape (966, 10384) (336, 10384) (336, 10384)\n",
      "Window Train/Val/Test shape (966, 2) (336, 2) (336, 2)\n",
      "X_train_cv  (1302, 10383)\n",
      "3 Thresh=0.00184, n=49, mse: 26.471\n",
      "CV - Window 4\n",
      "Train [2017-07-01 2017-08-30] Val [2017-08-31 2017-09-15] Test [2017-09-16 2017-10-01]\n",
      "Window Train/Val/Test shape (1281, 10384) (336, 10384) (336, 10384)\n",
      "Window Train/Val/Test shape (1281, 2) (336, 2) (336, 2)\n",
      "X_train_cv  (1617, 10383)\n",
      "4 Thresh=0.00184, n=49, mse: 39.943\n",
      "CV - Window 5\n",
      "Train [2017-07-01 2017-09-14] Val [2017-09-15 2017-09-30] Test [2017-10-01 2017-10-16]\n",
      "Window Train/Val/Test shape (1596, 10384) (336, 10384) (336, 10384)\n",
      "Window Train/Val/Test shape (1596, 2) (336, 2) (336, 2)\n",
      "X_train_cv  (1932, 10383)\n",
      "5 Thresh=0.00184, n=49, mse: 15.090\n",
      "CV - Window 6\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-09-29] Val [2017-09-30 2017-10-15] Test [2017-10-16 2017-11-10]\n",
      "Window Train/Val/Test shape (1911, 10384) (336, 10384) (546, 10384)\n",
      "Window Train/Val/Test shape (1911, 2) (336, 2) (546, 2)\n",
      "X_train_cv  (2247, 10383)\n",
      "6 Thresh=0.00184, n=49, mse: 8.448\n",
      "CV - Window 7\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-10-14] Val [2017-10-15 2017-10-30] Test [2017-10-31 2017-11-10]\n",
      "Window Train/Val/Test shape (2226, 10384) (336, 10384) (231, 10384)\n",
      "Window Train/Val/Test shape (2226, 2) (336, 2) (231, 2)\n",
      "X_train_cv  (2562, 10383)\n",
      "7 Thresh=0.00184, n=49, mse: 5.142\n",
      "{'threshold': 0.0018441678, 'num_features': 49, 'mse_list': [26.47072905100062, 39.94320993309955, 15.090127382574819, 8.448456551114802, 5.1421052813103625], 'mean_mse': 19.018925639820033}\n",
      "\n",
      "\n",
      "CV - Window 3\n",
      "Train [2017-07-01 2017-08-15] Val [2017-08-16 2017-08-31] Test [2017-09-01 2017-09-16]\n",
      "Window Train/Val/Test shape (966, 10384) (336, 10384) (336, 10384)\n",
      "Window Train/Val/Test shape (966, 2) (336, 2) (336, 2)\n",
      "X_train_cv  (1302, 10383)\n",
      "3 Thresh=0.00138, n=115, mse: 29.102\n",
      "CV - Window 4\n",
      "Train [2017-07-01 2017-08-30] Val [2017-08-31 2017-09-15] Test [2017-09-16 2017-10-01]\n",
      "Window Train/Val/Test shape (1281, 10384) (336, 10384) (336, 10384)\n",
      "Window Train/Val/Test shape (1281, 2) (336, 2) (336, 2)\n",
      "X_train_cv  (1617, 10383)\n",
      "4 Thresh=0.00138, n=115, mse: 44.975\n",
      "CV - Window 5\n",
      "Train [2017-07-01 2017-09-14] Val [2017-09-15 2017-09-30] Test [2017-10-01 2017-10-16]\n",
      "Window Train/Val/Test shape (1596, 10384) (336, 10384) (336, 10384)\n",
      "Window Train/Val/Test shape (1596, 2) (336, 2) (336, 2)\n",
      "X_train_cv  (1932, 10383)\n",
      "5 Thresh=0.00138, n=115, mse: 18.503\n",
      "CV - Window 6\n",
      "Rolling window to end date\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train [2017-07-01 2017-09-29] Val [2017-09-30 2017-10-15] Test [2017-10-16 2017-11-10]\n",
      "Window Train/Val/Test shape (1911, 10384) (336, 10384) (546, 10384)\n",
      "Window Train/Val/Test shape (1911, 2) (336, 2) (546, 2)\n",
      "X_train_cv  (2247, 10383)\n",
      "6 Thresh=0.00138, n=115, mse: 8.783\n",
      "CV - Window 7\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-10-14] Val [2017-10-15 2017-10-30] Test [2017-10-31 2017-11-10]\n",
      "Window Train/Val/Test shape (2226, 10384) (336, 10384) (231, 10384)\n",
      "Window Train/Val/Test shape (2226, 2) (336, 2) (231, 2)\n",
      "X_train_cv  (2562, 10383)\n",
      "7 Thresh=0.00138, n=115, mse: 5.787\n",
      "{'threshold': 0.0013831259, 'num_features': 115, 'mse_list': [29.10207104571129, 44.97539924161602, 18.50317251635974, 8.783044314656212, 5.787344531037369], 'mean_mse': 21.430206329876125}\n",
      "\n",
      "\n",
      "CV - Window 3\n",
      "Train [2017-07-01 2017-08-15] Val [2017-08-16 2017-08-31] Test [2017-09-01 2017-09-16]\n",
      "Window Train/Val/Test shape (966, 10384) (336, 10384) (336, 10384)\n",
      "Window Train/Val/Test shape (966, 2) (336, 2) (336, 2)\n",
      "X_train_cv  (1302, 10383)\n",
      "3 Thresh=0.00092, n=332, mse: 27.211\n",
      "CV - Window 4\n",
      "Train [2017-07-01 2017-08-30] Val [2017-08-31 2017-09-15] Test [2017-09-16 2017-10-01]\n",
      "Window Train/Val/Test shape (1281, 10384) (336, 10384) (336, 10384)\n",
      "Window Train/Val/Test shape (1281, 2) (336, 2) (336, 2)\n",
      "X_train_cv  (1617, 10383)\n",
      "4 Thresh=0.00092, n=332, mse: 41.168\n",
      "CV - Window 5\n",
      "Train [2017-07-01 2017-09-14] Val [2017-09-15 2017-09-30] Test [2017-10-01 2017-10-16]\n",
      "Window Train/Val/Test shape (1596, 10384) (336, 10384) (336, 10384)\n",
      "Window Train/Val/Test shape (1596, 2) (336, 2) (336, 2)\n",
      "X_train_cv  (1932, 10383)\n",
      "5 Thresh=0.00092, n=332, mse: 17.786\n",
      "CV - Window 6\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-09-29] Val [2017-09-30 2017-10-15] Test [2017-10-16 2017-11-10]\n",
      "Window Train/Val/Test shape (1911, 10384) (336, 10384) (546, 10384)\n",
      "Window Train/Val/Test shape (1911, 2) (336, 2) (546, 2)\n",
      "X_train_cv  (2247, 10383)\n",
      "6 Thresh=0.00092, n=332, mse: 11.324\n",
      "CV - Window 7\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-10-14] Val [2017-10-15 2017-10-30] Test [2017-10-31 2017-11-10]\n",
      "Window Train/Val/Test shape (2226, 10384) (336, 10384) (231, 10384)\n",
      "Window Train/Val/Test shape (2226, 2) (336, 2) (231, 2)\n",
      "X_train_cv  (2562, 10383)\n",
      "7 Thresh=0.00092, n=332, mse: 3.897\n",
      "{'threshold': 0.0009220839, 'num_features': 332, 'mse_list': [27.210938871757673, 41.167749471529206, 17.786065260503218, 11.323789732043483, 3.8967650940723826], 'mean_mse': 20.27706168598119}\n",
      "\n",
      "\n",
      "CV - Window 3\n",
      "Train [2017-07-01 2017-08-15] Val [2017-08-16 2017-08-31] Test [2017-09-01 2017-09-16]\n",
      "Window Train/Val/Test shape (966, 10384) (336, 10384) (336, 10384)\n",
      "Window Train/Val/Test shape (966, 2) (336, 2) (336, 2)\n",
      "X_train_cv  (1302, 10383)\n",
      "3 Thresh=0.00046, n=1585, mse: 26.097\n",
      "CV - Window 4\n",
      "Train [2017-07-01 2017-08-30] Val [2017-08-31 2017-09-15] Test [2017-09-16 2017-10-01]\n",
      "Window Train/Val/Test shape (1281, 10384) (336, 10384) (336, 10384)\n",
      "Window Train/Val/Test shape (1281, 2) (336, 2) (336, 2)\n",
      "X_train_cv  (1617, 10383)\n",
      "4 Thresh=0.00046, n=1585, mse: 28.351\n",
      "CV - Window 5\n",
      "Train [2017-07-01 2017-09-14] Val [2017-09-15 2017-09-30] Test [2017-10-01 2017-10-16]\n",
      "Window Train/Val/Test shape (1596, 10384) (336, 10384) (336, 10384)\n",
      "Window Train/Val/Test shape (1596, 2) (336, 2) (336, 2)\n",
      "X_train_cv  (1932, 10383)\n",
      "5 Thresh=0.00046, n=1585, mse: 16.475\n",
      "CV - Window 6\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-09-29] Val [2017-09-30 2017-10-15] Test [2017-10-16 2017-11-10]\n",
      "Window Train/Val/Test shape (1911, 10384) (336, 10384) (546, 10384)\n",
      "Window Train/Val/Test shape (1911, 2) (336, 2) (546, 2)\n",
      "X_train_cv  (2247, 10383)\n",
      "6 Thresh=0.00046, n=1585, mse: 11.944\n",
      "CV - Window 7\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-10-14] Val [2017-10-15 2017-10-30] Test [2017-10-31 2017-11-10]\n",
      "Window Train/Val/Test shape (2226, 10384) (336, 10384) (231, 10384)\n",
      "Window Train/Val/Test shape (2226, 2) (336, 2) (231, 2)\n",
      "X_train_cv  (2562, 10383)\n",
      "7 Thresh=0.00046, n=1585, mse: 3.559\n",
      "{'threshold': 0.00046104196, 'num_features': 1585, 'mse_list': [26.09729177566052, 28.351267555146727, 16.474959073719884, 11.943921527568078, 3.5585158612346106], 'mean_mse': 17.285191158665963}\n",
      "\n",
      "\n",
      "CV - Window 3\n",
      "Train [2017-07-01 2017-08-15] Val [2017-08-16 2017-08-31] Test [2017-09-01 2017-09-16]\n",
      "Window Train/Val/Test shape (966, 10384) (336, 10384) (336, 10384)\n",
      "Window Train/Val/Test shape (966, 2) (336, 2) (336, 2)\n",
      "X_train_cv  (1302, 10383)\n",
      "3 Thresh=0.00000, n=10383, mse: 40.512\n",
      "CV - Window 4\n",
      "Train [2017-07-01 2017-08-30] Val [2017-08-31 2017-09-15] Test [2017-09-16 2017-10-01]\n",
      "Window Train/Val/Test shape (1281, 10384) (336, 10384) (336, 10384)\n",
      "Window Train/Val/Test shape (1281, 2) (336, 2) (336, 2)\n",
      "X_train_cv  (1617, 10383)\n",
      "4 Thresh=0.00000, n=10383, mse: 33.975\n",
      "CV - Window 5\n",
      "Train [2017-07-01 2017-09-14] Val [2017-09-15 2017-09-30] Test [2017-10-01 2017-10-16]\n",
      "Window Train/Val/Test shape (1596, 10384) (336, 10384) (336, 10384)\n",
      "Window Train/Val/Test shape (1596, 2) (336, 2) (336, 2)\n",
      "X_train_cv  (1932, 10383)\n",
      "5 Thresh=0.00000, n=10383, mse: 24.202\n",
      "CV - Window 6\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-09-29] Val [2017-09-30 2017-10-15] Test [2017-10-16 2017-11-10]\n",
      "Window Train/Val/Test shape (1911, 10384) (336, 10384) (546, 10384)\n",
      "Window Train/Val/Test shape (1911, 2) (336, 2) (546, 2)\n",
      "X_train_cv  (2247, 10383)\n",
      "6 Thresh=0.00000, n=10383, mse: 12.989\n",
      "CV - Window 7\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-10-14] Val [2017-10-15 2017-10-30] Test [2017-10-31 2017-11-10]\n",
      "Window Train/Val/Test shape (2226, 10384) (336, 10384) (231, 10384)\n",
      "Window Train/Val/Test shape (2226, 2) (336, 2) (231, 2)\n",
      "X_train_cv  (2562, 10383)\n",
      "7 Thresh=0.00000, n=10383, mse: 4.994\n",
      "{'threshold': 0.0, 'num_features': 10383, 'mse_list': [40.51220487088585, 33.97469375561881, 24.20229981085328, 12.988900422515346, 4.993733567720106], 'mean_mse': 23.33436648551868}\n",
      "\n",
      "\n",
      "    threshold num_features                                           mse_list  \\\n",
      "0    0.007838            1  [96.01124097957597, 84.52831216896432, 87.7036...   \n",
      "1    0.005071            2  [72.5939388033789, 77.06552081754683, 61.03384...   \n",
      "2    0.004610            5  [71.3112827201162, 60.22254825770118, 72.07660...   \n",
      "3    0.004149            8  [49.50330081621364, 45.71297779305811, 50.5905...   \n",
      "4    0.003688           10  [50.332832509640696, 36.48606757250619, 37.381...   \n",
      "5    0.003227           13  [48.10976106493251, 33.212752818892035, 28.678...   \n",
      "6    0.002766           19  [32.438236808872034, 34.92581332507123, 24.064...   \n",
      "7    0.002305           25  [28.15466641078222, 35.17952183346277, 20.7024...   \n",
      "8    0.001844           49  [26.47072905100062, 39.94320993309955, 15.0901...   \n",
      "9    0.001383          115  [29.10207104571129, 44.97539924161602, 18.5031...   \n",
      "10   0.000922          332  [27.210938871757673, 41.167749471529206, 17.78...   \n",
      "11   0.000461         1585  [26.09729177566052, 28.351267555146727, 16.474...   \n",
      "12   0.000000        10383  [40.51220487088585, 33.97469375561881, 24.2022...   \n",
      "\n",
      "     mean_mse  \n",
      "0   62.467202  \n",
      "1   55.471513  \n",
      "2   51.593909  \n",
      "3   34.488944  \n",
      "4   28.675257  \n",
      "5   26.661191  \n",
      "6   22.738295  \n",
      "7   20.046410  \n",
      "8   19.018926  \n",
      "9   21.430206  \n",
      "10  20.277062  \n",
      "11  17.285191  \n",
      "12  23.334366  \n"
     ]
    }
   ],
   "source": [
    "orchestrator46.runRollingCustomFeatureExtractor(preselect_params_46, X_46_preselect)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {},
   "outputs": [],
   "source": [
    "selected_features46 = ['date'] + list(orchestrator46.rcfe.mcr.feature_importance_df[orchestrator46.rcfe.mcr.feature_importance_df.threshold >= 0.001844].index)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 69,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train [2017-07-01 2017-08-15] Val [2017-08-16 2017-08-31] Test [2017-09-01 2017-09-16]\n",
      "Window Train/Val/Test shape (966, 50) (336, 50) (336, 50)\n",
      "Window Train/Val/Test shape (966, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-08-30] Val [2017-08-31 2017-09-15] Test [2017-09-16 2017-10-01]\n",
      "Window Train/Val/Test shape (1281, 50) (336, 50) (336, 50)\n",
      "Window Train/Val/Test shape (1281, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-09-14] Val [2017-09-15 2017-09-30] Test [2017-10-01 2017-10-16]\n",
      "Window Train/Val/Test shape (1596, 50) (336, 50) (336, 50)\n",
      "Window Train/Val/Test shape (1596, 2) (336, 2) (336, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-09-29] Val [2017-09-30 2017-10-15] Test [2017-10-16 2017-11-10]\n",
      "Window Train/Val/Test shape (1911, 50) (336, 50) (546, 50)\n",
      "Window Train/Val/Test shape (1911, 2) (336, 2) (546, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-10-14] Val [2017-10-15 2017-10-30] Test [2017-10-31 2017-11-10]\n",
      "Window Train/Val/Test shape (2226, 50) (336, 50) (231, 50)\n",
      "Window Train/Val/Test shape (2226, 2) (336, 2) (231, 2)\n",
      "[33.45977689861107, 57.96277904475797, 13.943985173832667, 18.039497586163193, 5.0331133300583355]\n",
      "mean mse: 25.68783040668465\n",
      "Train [2017-07-01 2017-08-15] Val [2017-08-16 2017-08-31] Test [2017-09-01 2017-09-16]\n",
      "Window Train/Val/Test shape (966, 50) (336, 50) (336, 50)\n",
      "Window Train/Val/Test shape (966, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-08-30] Val [2017-08-31 2017-09-15] Test [2017-09-16 2017-10-01]\n",
      "Window Train/Val/Test shape (1281, 50) (336, 50) (336, 50)\n",
      "Window Train/Val/Test shape (1281, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-09-14] Val [2017-09-15 2017-09-30] Test [2017-10-01 2017-10-16]\n",
      "Window Train/Val/Test shape (1596, 50) (336, 50) (336, 50)\n",
      "Window Train/Val/Test shape (1596, 2) (336, 2) (336, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-09-29] Val [2017-09-30 2017-10-15] Test [2017-10-16 2017-11-10]\n",
      "Window Train/Val/Test shape (1911, 50) (336, 50) (546, 50)\n",
      "Window Train/Val/Test shape (1911, 2) (336, 2) (546, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-10-14] Val [2017-10-15 2017-10-30] Test [2017-10-31 2017-11-10]\n",
      "Window Train/Val/Test shape (2226, 50) (336, 50) (231, 50)\n",
      "Window Train/Val/Test shape (2226, 2) (336, 2) (231, 2)\n",
      "[37.87269333750848, 47.286405478348705, 16.353365716146566, 13.14882214634062, 4.063489676964494]\n",
      "mean mse: 23.744955271061777\n",
      "Train [2017-07-01 2017-08-15] Val [2017-08-16 2017-08-31] Test [2017-09-01 2017-09-16]\n",
      "Window Train/Val/Test shape (966, 50) (336, 50) (336, 50)\n",
      "Window Train/Val/Test shape (966, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-08-30] Val [2017-08-31 2017-09-15] Test [2017-09-16 2017-10-01]\n",
      "Window Train/Val/Test shape (1281, 50) (336, 50) (336, 50)\n",
      "Window Train/Val/Test shape (1281, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-09-14] Val [2017-09-15 2017-09-30] Test [2017-10-01 2017-10-16]\n",
      "Window Train/Val/Test shape (1596, 50) (336, 50) (336, 50)\n",
      "Window Train/Val/Test shape (1596, 2) (336, 2) (336, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-09-29] Val [2017-09-30 2017-10-15] Test [2017-10-16 2017-11-10]\n",
      "Window Train/Val/Test shape (1911, 50) (336, 50) (546, 50)\n",
      "Window Train/Val/Test shape (1911, 2) (336, 2) (546, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-10-14] Val [2017-10-15 2017-10-30] Test [2017-10-31 2017-11-10]\n",
      "Window Train/Val/Test shape (2226, 50) (336, 50) (231, 50)\n",
      "Window Train/Val/Test shape (2226, 2) (336, 2) (231, 2)\n",
      "[52.29408852730514, 48.63355531128691, 20.00559513610594, 24.8445007367142, 9.579907926737027]\n",
      "mean mse: 31.071529527629842\n",
      "Train [2017-07-01 2017-08-15] Val [2017-08-16 2017-08-31] Test [2017-09-01 2017-09-16]\n",
      "Window Train/Val/Test shape (966, 50) (336, 50) (336, 50)\n",
      "Window Train/Val/Test shape (966, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-08-30] Val [2017-08-31 2017-09-15] Test [2017-09-16 2017-10-01]\n",
      "Window Train/Val/Test shape (1281, 50) (336, 50) (336, 50)\n",
      "Window Train/Val/Test shape (1281, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-09-14] Val [2017-09-15 2017-09-30] Test [2017-10-01 2017-10-16]\n",
      "Window Train/Val/Test shape (1596, 50) (336, 50) (336, 50)\n",
      "Window Train/Val/Test shape (1596, 2) (336, 2) (336, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-09-29] Val [2017-09-30 2017-10-15] Test [2017-10-16 2017-11-10]\n",
      "Window Train/Val/Test shape (1911, 50) (336, 50) (546, 50)\n",
      "Window Train/Val/Test shape (1911, 2) (336, 2) (546, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-10-14] Val [2017-10-15 2017-10-30] Test [2017-10-31 2017-11-10]\n",
      "Window Train/Val/Test shape (2226, 50) (336, 50) (231, 50)\n",
      "Window Train/Val/Test shape (2226, 2) (336, 2) (231, 2)\n",
      "[40.00451425946685, 45.9110160423758, 13.56994890680667, 15.82841910161796, 4.3559259047660355]\n",
      "mean mse: 23.933964843006667\n",
      "Train [2017-07-01 2017-08-15] Val [2017-08-16 2017-08-31] Test [2017-09-01 2017-09-16]\n",
      "Window Train/Val/Test shape (966, 50) (336, 50) (336, 50)\n",
      "Window Train/Val/Test shape (966, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-08-30] Val [2017-08-31 2017-09-15] Test [2017-09-16 2017-10-01]\n",
      "Window Train/Val/Test shape (1281, 50) (336, 50) (336, 50)\n",
      "Window Train/Val/Test shape (1281, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-09-14] Val [2017-09-15 2017-09-30] Test [2017-10-01 2017-10-16]\n",
      "Window Train/Val/Test shape (1596, 50) (336, 50) (336, 50)\n",
      "Window Train/Val/Test shape (1596, 2) (336, 2) (336, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-09-29] Val [2017-09-30 2017-10-15] Test [2017-10-16 2017-11-10]\n",
      "Window Train/Val/Test shape (1911, 50) (336, 50) (546, 50)\n",
      "Window Train/Val/Test shape (1911, 2) (336, 2) (546, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-10-14] Val [2017-10-15 2017-10-30] Test [2017-10-31 2017-11-10]\n",
      "Window Train/Val/Test shape (2226, 50) (336, 50) (231, 50)\n",
      "Window Train/Val/Test shape (2226, 2) (336, 2) (231, 2)\n",
      "[35.092649687394704, 46.66645703228841, 16.39743873782341, 17.880640158972167, 5.702563073284392]\n",
      "mean mse: 24.347949737952618\n",
      "Train [2017-07-01 2017-08-15] Val [2017-08-16 2017-08-31] Test [2017-09-01 2017-09-16]\n",
      "Window Train/Val/Test shape (966, 50) (336, 50) (336, 50)\n",
      "Window Train/Val/Test shape (966, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-08-30] Val [2017-08-31 2017-09-15] Test [2017-09-16 2017-10-01]\n",
      "Window Train/Val/Test shape (1281, 50) (336, 50) (336, 50)\n",
      "Window Train/Val/Test shape (1281, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-09-14] Val [2017-09-15 2017-09-30] Test [2017-10-01 2017-10-16]\n",
      "Window Train/Val/Test shape (1596, 50) (336, 50) (336, 50)\n",
      "Window Train/Val/Test shape (1596, 2) (336, 2) (336, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-09-29] Val [2017-09-30 2017-10-15] Test [2017-10-16 2017-11-10]\n",
      "Window Train/Val/Test shape (1911, 50) (336, 50) (546, 50)\n",
      "Window Train/Val/Test shape (1911, 2) (336, 2) (546, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-10-14] Val [2017-10-15 2017-10-30] Test [2017-10-31 2017-11-10]\n",
      "Window Train/Val/Test shape (2226, 50) (336, 50) (231, 50)\n",
      "Window Train/Val/Test shape (2226, 2) (336, 2) (231, 2)\n",
      "[35.08579719491909, 38.47742993755911, 10.977850863357084, 15.141513215606949, 4.621486621739962]\n",
      "mean mse: 20.86081556663644\n",
      "Train [2017-07-01 2017-08-15] Val [2017-08-16 2017-08-31] Test [2017-09-01 2017-09-16]\n",
      "Window Train/Val/Test shape (966, 50) (336, 50) (336, 50)\n",
      "Window Train/Val/Test shape (966, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-08-30] Val [2017-08-31 2017-09-15] Test [2017-09-16 2017-10-01]\n",
      "Window Train/Val/Test shape (1281, 50) (336, 50) (336, 50)\n",
      "Window Train/Val/Test shape (1281, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-09-14] Val [2017-09-15 2017-09-30] Test [2017-10-01 2017-10-16]\n",
      "Window Train/Val/Test shape (1596, 50) (336, 50) (336, 50)\n",
      "Window Train/Val/Test shape (1596, 2) (336, 2) (336, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-09-29] Val [2017-09-30 2017-10-15] Test [2017-10-16 2017-11-10]\n",
      "Window Train/Val/Test shape (1911, 50) (336, 50) (546, 50)\n",
      "Window Train/Val/Test shape (1911, 2) (336, 2) (546, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-10-14] Val [2017-10-15 2017-10-30] Test [2017-10-31 2017-11-10]\n",
      "Window Train/Val/Test shape (2226, 50) (336, 50) (231, 50)\n",
      "Window Train/Val/Test shape (2226, 2) (336, 2) (231, 2)\n",
      "[40.947549014148784, 44.149933796481704, 9.30794213104691, 20.87795720721483, 4.773806487027536]\n",
      "mean mse: 24.011437727183953\n",
      "Train [2017-07-01 2017-08-15] Val [2017-08-16 2017-08-31] Test [2017-09-01 2017-09-16]\n",
      "Window Train/Val/Test shape (966, 50) (336, 50) (336, 50)\n",
      "Window Train/Val/Test shape (966, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-08-30] Val [2017-08-31 2017-09-15] Test [2017-09-16 2017-10-01]\n",
      "Window Train/Val/Test shape (1281, 50) (336, 50) (336, 50)\n",
      "Window Train/Val/Test shape (1281, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-09-14] Val [2017-09-15 2017-09-30] Test [2017-10-01 2017-10-16]\n",
      "Window Train/Val/Test shape (1596, 50) (336, 50) (336, 50)\n",
      "Window Train/Val/Test shape (1596, 2) (336, 2) (336, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-09-29] Val [2017-09-30 2017-10-15] Test [2017-10-16 2017-11-10]\n",
      "Window Train/Val/Test shape (1911, 50) (336, 50) (546, 50)\n",
      "Window Train/Val/Test shape (1911, 2) (336, 2) (546, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-10-14] Val [2017-10-15 2017-10-30] Test [2017-10-31 2017-11-10]\n",
      "Window Train/Val/Test shape (2226, 50) (336, 50) (231, 50)\n",
      "Window Train/Val/Test shape (2226, 2) (336, 2) (231, 2)\n",
      "[33.14237388174355, 35.066979809594905, 11.073913023188732, 12.578595801473947, 3.7259362042485833]\n",
      "mean mse: 19.11755974404995\n",
      "Train [2017-07-01 2017-08-15] Val [2017-08-16 2017-08-31] Test [2017-09-01 2017-09-16]\n",
      "Window Train/Val/Test shape (966, 50) (336, 50) (336, 50)\n",
      "Window Train/Val/Test shape (966, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-08-30] Val [2017-08-31 2017-09-15] Test [2017-09-16 2017-10-01]\n",
      "Window Train/Val/Test shape (1281, 50) (336, 50) (336, 50)\n",
      "Window Train/Val/Test shape (1281, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-09-14] Val [2017-09-15 2017-09-30] Test [2017-10-01 2017-10-16]\n",
      "Window Train/Val/Test shape (1596, 50) (336, 50) (336, 50)\n",
      "Window Train/Val/Test shape (1596, 2) (336, 2) (336, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-09-29] Val [2017-09-30 2017-10-15] Test [2017-10-16 2017-11-10]\n",
      "Window Train/Val/Test shape (1911, 50) (336, 50) (546, 50)\n",
      "Window Train/Val/Test shape (1911, 2) (336, 2) (546, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-10-14] Val [2017-10-15 2017-10-30] Test [2017-10-31 2017-11-10]\n",
      "Window Train/Val/Test shape (2226, 50) (336, 50) (231, 50)\n",
      "Window Train/Val/Test shape (2226, 2) (336, 2) (231, 2)\n",
      "[32.626973893446646, 52.3471072898371, 19.761195135197436, 18.00148967690114, 7.570340805601511]\n",
      "mean mse: 26.061421360196768\n",
      "Train [2017-07-01 2017-08-15] Val [2017-08-16 2017-08-31] Test [2017-09-01 2017-09-16]\n",
      "Window Train/Val/Test shape (966, 50) (336, 50) (336, 50)\n",
      "Window Train/Val/Test shape (966, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-08-30] Val [2017-08-31 2017-09-15] Test [2017-09-16 2017-10-01]\n",
      "Window Train/Val/Test shape (1281, 50) (336, 50) (336, 50)\n",
      "Window Train/Val/Test shape (1281, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-09-14] Val [2017-09-15 2017-09-30] Test [2017-10-01 2017-10-16]\n",
      "Window Train/Val/Test shape (1596, 50) (336, 50) (336, 50)\n",
      "Window Train/Val/Test shape (1596, 2) (336, 2) (336, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-09-29] Val [2017-09-30 2017-10-15] Test [2017-10-16 2017-11-10]\n",
      "Window Train/Val/Test shape (1911, 50) (336, 50) (546, 50)\n",
      "Window Train/Val/Test shape (1911, 2) (336, 2) (546, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-10-14] Val [2017-10-15 2017-10-30] Test [2017-10-31 2017-11-10]\n",
      "Window Train/Val/Test shape (2226, 50) (336, 50) (231, 50)\n",
      "Window Train/Val/Test shape (2226, 2) (336, 2) (231, 2)\n",
      "[32.693222927656265, 42.34199297582205, 13.058451238911536, 15.20688942678545, 4.275440857956529]\n",
      "mean mse: 21.515199485426365\n",
      "Train [2017-07-01 2017-08-15] Val [2017-08-16 2017-08-31] Test [2017-09-01 2017-09-16]\n",
      "Window Train/Val/Test shape (966, 50) (336, 50) (336, 50)\n",
      "Window Train/Val/Test shape (966, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-08-30] Val [2017-08-31 2017-09-15] Test [2017-09-16 2017-10-01]\n",
      "Window Train/Val/Test shape (1281, 50) (336, 50) (336, 50)\n",
      "Window Train/Val/Test shape (1281, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-09-14] Val [2017-09-15 2017-09-30] Test [2017-10-01 2017-10-16]\n",
      "Window Train/Val/Test shape (1596, 50) (336, 50) (336, 50)\n",
      "Window Train/Val/Test shape (1596, 2) (336, 2) (336, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-09-29] Val [2017-09-30 2017-10-15] Test [2017-10-16 2017-11-10]\n",
      "Window Train/Val/Test shape (1911, 50) (336, 50) (546, 50)\n",
      "Window Train/Val/Test shape (1911, 2) (336, 2) (546, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-10-14] Val [2017-10-15 2017-10-30] Test [2017-10-31 2017-11-10]\n",
      "Window Train/Val/Test shape (2226, 50) (336, 50) (231, 50)\n",
      "Window Train/Val/Test shape (2226, 2) (336, 2) (231, 2)\n",
      "[44.591508791068115, 31.64451785345774, 11.211149633455694, 22.930537918832748, 5.1578401573600345]\n",
      "mean mse: 23.107110870834862\n",
      "Train [2017-07-01 2017-08-15] Val [2017-08-16 2017-08-31] Test [2017-09-01 2017-09-16]\n",
      "Window Train/Val/Test shape (966, 50) (336, 50) (336, 50)\n",
      "Window Train/Val/Test shape (966, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-08-30] Val [2017-08-31 2017-09-15] Test [2017-09-16 2017-10-01]\n",
      "Window Train/Val/Test shape (1281, 50) (336, 50) (336, 50)\n",
      "Window Train/Val/Test shape (1281, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-09-14] Val [2017-09-15 2017-09-30] Test [2017-10-01 2017-10-16]\n",
      "Window Train/Val/Test shape (1596, 50) (336, 50) (336, 50)\n",
      "Window Train/Val/Test shape (1596, 2) (336, 2) (336, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-09-29] Val [2017-09-30 2017-10-15] Test [2017-10-16 2017-11-10]\n",
      "Window Train/Val/Test shape (1911, 50) (336, 50) (546, 50)\n",
      "Window Train/Val/Test shape (1911, 2) (336, 2) (546, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-10-14] Val [2017-10-15 2017-10-30] Test [2017-10-31 2017-11-10]\n",
      "Window Train/Val/Test shape (2226, 50) (336, 50) (231, 50)\n",
      "Window Train/Val/Test shape (2226, 2) (336, 2) (231, 2)\n",
      "[30.385392078625944, 71.91575604364373, 11.458790541389723, 13.914572862833113, 5.243471070755965]\n",
      "mean mse: 26.583596519449692\n",
      "Train [2017-07-01 2017-08-15] Val [2017-08-16 2017-08-31] Test [2017-09-01 2017-09-16]\n",
      "Window Train/Val/Test shape (966, 50) (336, 50) (336, 50)\n",
      "Window Train/Val/Test shape (966, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-08-30] Val [2017-08-31 2017-09-15] Test [2017-09-16 2017-10-01]\n",
      "Window Train/Val/Test shape (1281, 50) (336, 50) (336, 50)\n",
      "Window Train/Val/Test shape (1281, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-09-14] Val [2017-09-15 2017-09-30] Test [2017-10-01 2017-10-16]\n",
      "Window Train/Val/Test shape (1596, 50) (336, 50) (336, 50)\n",
      "Window Train/Val/Test shape (1596, 2) (336, 2) (336, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-09-29] Val [2017-09-30 2017-10-15] Test [2017-10-16 2017-11-10]\n",
      "Window Train/Val/Test shape (1911, 50) (336, 50) (546, 50)\n",
      "Window Train/Val/Test shape (1911, 2) (336, 2) (546, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-10-14] Val [2017-10-15 2017-10-30] Test [2017-10-31 2017-11-10]\n",
      "Window Train/Val/Test shape (2226, 50) (336, 50) (231, 50)\n",
      "Window Train/Val/Test shape (2226, 2) (336, 2) (231, 2)\n",
      "[37.20272275411458, 38.287152767363445, 10.436882015870301, 14.267825186694061, 5.208501489176808]\n",
      "mean mse: 21.080616842643842\n",
      "Train [2017-07-01 2017-08-15] Val [2017-08-16 2017-08-31] Test [2017-09-01 2017-09-16]\n",
      "Window Train/Val/Test shape (966, 50) (336, 50) (336, 50)\n",
      "Window Train/Val/Test shape (966, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-08-30] Val [2017-08-31 2017-09-15] Test [2017-09-16 2017-10-01]\n",
      "Window Train/Val/Test shape (1281, 50) (336, 50) (336, 50)\n",
      "Window Train/Val/Test shape (1281, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-09-14] Val [2017-09-15 2017-09-30] Test [2017-10-01 2017-10-16]\n",
      "Window Train/Val/Test shape (1596, 50) (336, 50) (336, 50)\n",
      "Window Train/Val/Test shape (1596, 2) (336, 2) (336, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-09-29] Val [2017-09-30 2017-10-15] Test [2017-10-16 2017-11-10]\n",
      "Window Train/Val/Test shape (1911, 50) (336, 50) (546, 50)\n",
      "Window Train/Val/Test shape (1911, 2) (336, 2) (546, 2)\n",
      "Rolling window to end date\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train [2017-07-01 2017-10-14] Val [2017-10-15 2017-10-30] Test [2017-10-31 2017-11-10]\n",
      "Window Train/Val/Test shape (2226, 50) (336, 50) (231, 50)\n",
      "Window Train/Val/Test shape (2226, 2) (336, 2) (231, 2)\n",
      "[36.15453052817052, 50.58751176287239, 14.036072756515258, 15.677892501840095, 5.528601460526408]\n",
      "mean mse: 24.39692180198493\n",
      "Train [2017-07-01 2017-08-15] Val [2017-08-16 2017-08-31] Test [2017-09-01 2017-09-16]\n",
      "Window Train/Val/Test shape (966, 50) (336, 50) (336, 50)\n",
      "Window Train/Val/Test shape (966, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-08-30] Val [2017-08-31 2017-09-15] Test [2017-09-16 2017-10-01]\n",
      "Window Train/Val/Test shape (1281, 50) (336, 50) (336, 50)\n",
      "Window Train/Val/Test shape (1281, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-09-14] Val [2017-09-15 2017-09-30] Test [2017-10-01 2017-10-16]\n",
      "Window Train/Val/Test shape (1596, 50) (336, 50) (336, 50)\n",
      "Window Train/Val/Test shape (1596, 2) (336, 2) (336, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-09-29] Val [2017-09-30 2017-10-15] Test [2017-10-16 2017-11-10]\n",
      "Window Train/Val/Test shape (1911, 50) (336, 50) (546, 50)\n",
      "Window Train/Val/Test shape (1911, 2) (336, 2) (546, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-10-14] Val [2017-10-15 2017-10-30] Test [2017-10-31 2017-11-10]\n",
      "Window Train/Val/Test shape (2226, 50) (336, 50) (231, 50)\n",
      "Window Train/Val/Test shape (2226, 2) (336, 2) (231, 2)\n",
      "[39.93651794577319, 46.71775942856185, 12.106719034405865, 16.243468173306987, 6.07221505146131]\n",
      "mean mse: 24.21533592670184\n",
      "Train [2017-07-01 2017-08-15] Val [2017-08-16 2017-08-31] Test [2017-09-01 2017-09-16]\n",
      "Window Train/Val/Test shape (966, 50) (336, 50) (336, 50)\n",
      "Window Train/Val/Test shape (966, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-08-30] Val [2017-08-31 2017-09-15] Test [2017-09-16 2017-10-01]\n",
      "Window Train/Val/Test shape (1281, 50) (336, 50) (336, 50)\n",
      "Window Train/Val/Test shape (1281, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-09-14] Val [2017-09-15 2017-09-30] Test [2017-10-01 2017-10-16]\n",
      "Window Train/Val/Test shape (1596, 50) (336, 50) (336, 50)\n",
      "Window Train/Val/Test shape (1596, 2) (336, 2) (336, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-09-29] Val [2017-09-30 2017-10-15] Test [2017-10-16 2017-11-10]\n",
      "Window Train/Val/Test shape (1911, 50) (336, 50) (546, 50)\n",
      "Window Train/Val/Test shape (1911, 2) (336, 2) (546, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-10-14] Val [2017-10-15 2017-10-30] Test [2017-10-31 2017-11-10]\n",
      "Window Train/Val/Test shape (2226, 50) (336, 50) (231, 50)\n",
      "Window Train/Val/Test shape (2226, 2) (336, 2) (231, 2)\n",
      "[31.561346898524075, 40.08856085139161, 10.003023721656765, 14.287139170166748, 5.345720334845572]\n",
      "mean mse: 20.257158195316954\n",
      "Train [2017-07-01 2017-08-15] Val [2017-08-16 2017-08-31] Test [2017-09-01 2017-09-16]\n",
      "Window Train/Val/Test shape (966, 50) (336, 50) (336, 50)\n",
      "Window Train/Val/Test shape (966, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-08-30] Val [2017-08-31 2017-09-15] Test [2017-09-16 2017-10-01]\n",
      "Window Train/Val/Test shape (1281, 50) (336, 50) (336, 50)\n",
      "Window Train/Val/Test shape (1281, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-09-14] Val [2017-09-15 2017-09-30] Test [2017-10-01 2017-10-16]\n",
      "Window Train/Val/Test shape (1596, 50) (336, 50) (336, 50)\n",
      "Window Train/Val/Test shape (1596, 2) (336, 2) (336, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-09-29] Val [2017-09-30 2017-10-15] Test [2017-10-16 2017-11-10]\n",
      "Window Train/Val/Test shape (1911, 50) (336, 50) (546, 50)\n",
      "Window Train/Val/Test shape (1911, 2) (336, 2) (546, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-10-14] Val [2017-10-15 2017-10-30] Test [2017-10-31 2017-11-10]\n",
      "Window Train/Val/Test shape (2226, 50) (336, 50) (231, 50)\n",
      "Window Train/Val/Test shape (2226, 2) (336, 2) (231, 2)\n",
      "[36.70969453064271, 42.79040525416265, 13.700378567862675, 13.748545627395947, 4.06749769437633]\n",
      "mean mse: 22.20330433488806\n",
      "Train [2017-07-01 2017-08-15] Val [2017-08-16 2017-08-31] Test [2017-09-01 2017-09-16]\n",
      "Window Train/Val/Test shape (966, 50) (336, 50) (336, 50)\n",
      "Window Train/Val/Test shape (966, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-08-30] Val [2017-08-31 2017-09-15] Test [2017-09-16 2017-10-01]\n",
      "Window Train/Val/Test shape (1281, 50) (336, 50) (336, 50)\n",
      "Window Train/Val/Test shape (1281, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-09-14] Val [2017-09-15 2017-09-30] Test [2017-10-01 2017-10-16]\n",
      "Window Train/Val/Test shape (1596, 50) (336, 50) (336, 50)\n",
      "Window Train/Val/Test shape (1596, 2) (336, 2) (336, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-09-29] Val [2017-09-30 2017-10-15] Test [2017-10-16 2017-11-10]\n",
      "Window Train/Val/Test shape (1911, 50) (336, 50) (546, 50)\n",
      "Window Train/Val/Test shape (1911, 2) (336, 2) (546, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-10-14] Val [2017-10-15 2017-10-30] Test [2017-10-31 2017-11-10]\n",
      "Window Train/Val/Test shape (2226, 50) (336, 50) (231, 50)\n",
      "Window Train/Val/Test shape (2226, 2) (336, 2) (231, 2)\n",
      "[37.43498339993045, 32.40309523876295, 10.956309655517021, 15.35500031955544, 4.505855451680857]\n",
      "mean mse: 20.131048813089343\n",
      "Train [2017-07-01 2017-08-15] Val [2017-08-16 2017-08-31] Test [2017-09-01 2017-09-16]\n",
      "Window Train/Val/Test shape (966, 50) (336, 50) (336, 50)\n",
      "Window Train/Val/Test shape (966, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-08-30] Val [2017-08-31 2017-09-15] Test [2017-09-16 2017-10-01]\n",
      "Window Train/Val/Test shape (1281, 50) (336, 50) (336, 50)\n",
      "Window Train/Val/Test shape (1281, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-09-14] Val [2017-09-15 2017-09-30] Test [2017-10-01 2017-10-16]\n",
      "Window Train/Val/Test shape (1596, 50) (336, 50) (336, 50)\n",
      "Window Train/Val/Test shape (1596, 2) (336, 2) (336, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-09-29] Val [2017-09-30 2017-10-15] Test [2017-10-16 2017-11-10]\n",
      "Window Train/Val/Test shape (1911, 50) (336, 50) (546, 50)\n",
      "Window Train/Val/Test shape (1911, 2) (336, 2) (546, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-10-14] Val [2017-10-15 2017-10-30] Test [2017-10-31 2017-11-10]\n",
      "Window Train/Val/Test shape (2226, 50) (336, 50) (231, 50)\n",
      "Window Train/Val/Test shape (2226, 2) (336, 2) (231, 2)\n",
      "[34.478790447038605, 43.64475728524768, 14.304112996136539, 15.517441270458628, 4.726035569655014]\n",
      "mean mse: 22.534227513707293\n",
      "Train [2017-07-01 2017-08-15] Val [2017-08-16 2017-08-31] Test [2017-09-01 2017-09-16]\n",
      "Window Train/Val/Test shape (966, 50) (336, 50) (336, 50)\n",
      "Window Train/Val/Test shape (966, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-08-30] Val [2017-08-31 2017-09-15] Test [2017-09-16 2017-10-01]\n",
      "Window Train/Val/Test shape (1281, 50) (336, 50) (336, 50)\n",
      "Window Train/Val/Test shape (1281, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-09-14] Val [2017-09-15 2017-09-30] Test [2017-10-01 2017-10-16]\n",
      "Window Train/Val/Test shape (1596, 50) (336, 50) (336, 50)\n",
      "Window Train/Val/Test shape (1596, 2) (336, 2) (336, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-09-29] Val [2017-09-30 2017-10-15] Test [2017-10-16 2017-11-10]\n",
      "Window Train/Val/Test shape (1911, 50) (336, 50) (546, 50)\n",
      "Window Train/Val/Test shape (1911, 2) (336, 2) (546, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-10-14] Val [2017-10-15 2017-10-30] Test [2017-10-31 2017-11-10]\n",
      "Window Train/Val/Test shape (2226, 50) (336, 50) (231, 50)\n",
      "Window Train/Val/Test shape (2226, 2) (336, 2) (231, 2)\n",
      "[33.663368824556194, 43.43859565452402, 11.531661567421242, 16.420496040711285, 4.8981066652512375]\n",
      "mean mse: 21.990445750492793\n",
      "Train [2017-07-01 2017-08-15] Val [2017-08-16 2017-08-31] Test [2017-09-01 2017-09-16]\n",
      "Window Train/Val/Test shape (966, 50) (336, 50) (336, 50)\n",
      "Window Train/Val/Test shape (966, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-08-30] Val [2017-08-31 2017-09-15] Test [2017-09-16 2017-10-01]\n",
      "Window Train/Val/Test shape (1281, 50) (336, 50) (336, 50)\n",
      "Window Train/Val/Test shape (1281, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-09-14] Val [2017-09-15 2017-09-30] Test [2017-10-01 2017-10-16]\n",
      "Window Train/Val/Test shape (1596, 50) (336, 50) (336, 50)\n",
      "Window Train/Val/Test shape (1596, 2) (336, 2) (336, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-09-29] Val [2017-09-30 2017-10-15] Test [2017-10-16 2017-11-10]\n",
      "Window Train/Val/Test shape (1911, 50) (336, 50) (546, 50)\n",
      "Window Train/Val/Test shape (1911, 2) (336, 2) (546, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-10-14] Val [2017-10-15 2017-10-30] Test [2017-10-31 2017-11-10]\n",
      "Window Train/Val/Test shape (2226, 50) (336, 50) (231, 50)\n",
      "Window Train/Val/Test shape (2226, 2) (336, 2) (231, 2)\n",
      "[37.15578523961182, 35.26502670495269, 13.55356826750791, 13.161129920667378, 4.450677205866637]\n",
      "mean mse: 20.717237467721286\n",
      "Train [2017-07-01 2017-08-15] Val [2017-08-16 2017-08-31] Test [2017-09-01 2017-09-16]\n",
      "Window Train/Val/Test shape (966, 50) (336, 50) (336, 50)\n",
      "Window Train/Val/Test shape (966, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-08-30] Val [2017-08-31 2017-09-15] Test [2017-09-16 2017-10-01]\n",
      "Window Train/Val/Test shape (1281, 50) (336, 50) (336, 50)\n",
      "Window Train/Val/Test shape (1281, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-09-14] Val [2017-09-15 2017-09-30] Test [2017-10-01 2017-10-16]\n",
      "Window Train/Val/Test shape (1596, 50) (336, 50) (336, 50)\n",
      "Window Train/Val/Test shape (1596, 2) (336, 2) (336, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-09-29] Val [2017-09-30 2017-10-15] Test [2017-10-16 2017-11-10]\n",
      "Window Train/Val/Test shape (1911, 50) (336, 50) (546, 50)\n",
      "Window Train/Val/Test shape (1911, 2) (336, 2) (546, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-10-14] Val [2017-10-15 2017-10-30] Test [2017-10-31 2017-11-10]\n",
      "Window Train/Val/Test shape (2226, 50) (336, 50) (231, 50)\n",
      "Window Train/Val/Test shape (2226, 2) (336, 2) (231, 2)\n",
      "[26.878920950629453, 40.508507085755845, 17.72987176927888, 12.98247006608768, 3.9675978817802493]\n",
      "mean mse: 20.41347355070642\n",
      "Train [2017-07-01 2017-08-15] Val [2017-08-16 2017-08-31] Test [2017-09-01 2017-09-16]\n",
      "Window Train/Val/Test shape (966, 50) (336, 50) (336, 50)\n",
      "Window Train/Val/Test shape (966, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-08-30] Val [2017-08-31 2017-09-15] Test [2017-09-16 2017-10-01]\n",
      "Window Train/Val/Test shape (1281, 50) (336, 50) (336, 50)\n",
      "Window Train/Val/Test shape (1281, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-09-14] Val [2017-09-15 2017-09-30] Test [2017-10-01 2017-10-16]\n",
      "Window Train/Val/Test shape (1596, 50) (336, 50) (336, 50)\n",
      "Window Train/Val/Test shape (1596, 2) (336, 2) (336, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-09-29] Val [2017-09-30 2017-10-15] Test [2017-10-16 2017-11-10]\n",
      "Window Train/Val/Test shape (1911, 50) (336, 50) (546, 50)\n",
      "Window Train/Val/Test shape (1911, 2) (336, 2) (546, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-10-14] Val [2017-10-15 2017-10-30] Test [2017-10-31 2017-11-10]\n",
      "Window Train/Val/Test shape (2226, 50) (336, 50) (231, 50)\n",
      "Window Train/Val/Test shape (2226, 2) (336, 2) (231, 2)\n",
      "[36.46624025533769, 44.58593736647494, 10.13780150992873, 11.857685752163627, 4.547755819144883]\n",
      "mean mse: 21.519084140609976\n",
      "Train [2017-07-01 2017-08-15] Val [2017-08-16 2017-08-31] Test [2017-09-01 2017-09-16]\n",
      "Window Train/Val/Test shape (966, 50) (336, 50) (336, 50)\n",
      "Window Train/Val/Test shape (966, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-08-30] Val [2017-08-31 2017-09-15] Test [2017-09-16 2017-10-01]\n",
      "Window Train/Val/Test shape (1281, 50) (336, 50) (336, 50)\n",
      "Window Train/Val/Test shape (1281, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-09-14] Val [2017-09-15 2017-09-30] Test [2017-10-01 2017-10-16]\n",
      "Window Train/Val/Test shape (1596, 50) (336, 50) (336, 50)\n",
      "Window Train/Val/Test shape (1596, 2) (336, 2) (336, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-09-29] Val [2017-09-30 2017-10-15] Test [2017-10-16 2017-11-10]\n",
      "Window Train/Val/Test shape (1911, 50) (336, 50) (546, 50)\n",
      "Window Train/Val/Test shape (1911, 2) (336, 2) (546, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-10-14] Val [2017-10-15 2017-10-30] Test [2017-10-31 2017-11-10]\n",
      "Window Train/Val/Test shape (2226, 50) (336, 50) (231, 50)\n",
      "Window Train/Val/Test shape (2226, 2) (336, 2) (231, 2)\n",
      "[35.44090592718096, 35.21406693764105, 12.775683364089703, 17.27447886291244, 4.722117202713148]\n",
      "mean mse: 21.085450458907463\n",
      "Train [2017-07-01 2017-08-15] Val [2017-08-16 2017-08-31] Test [2017-09-01 2017-09-16]\n",
      "Window Train/Val/Test shape (966, 50) (336, 50) (336, 50)\n",
      "Window Train/Val/Test shape (966, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-08-30] Val [2017-08-31 2017-09-15] Test [2017-09-16 2017-10-01]\n",
      "Window Train/Val/Test shape (1281, 50) (336, 50) (336, 50)\n",
      "Window Train/Val/Test shape (1281, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-09-14] Val [2017-09-15 2017-09-30] Test [2017-10-01 2017-10-16]\n",
      "Window Train/Val/Test shape (1596, 50) (336, 50) (336, 50)\n",
      "Window Train/Val/Test shape (1596, 2) (336, 2) (336, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-09-29] Val [2017-09-30 2017-10-15] Test [2017-10-16 2017-11-10]\n",
      "Window Train/Val/Test shape (1911, 50) (336, 50) (546, 50)\n",
      "Window Train/Val/Test shape (1911, 2) (336, 2) (546, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-10-14] Val [2017-10-15 2017-10-30] Test [2017-10-31 2017-11-10]\n",
      "Window Train/Val/Test shape (2226, 50) (336, 50) (231, 50)\n",
      "Window Train/Val/Test shape (2226, 2) (336, 2) (231, 2)\n",
      "[35.86605909484313, 38.25225830099893, 15.256877857872919, 12.111530685841473, 4.098959542185387]\n",
      "mean mse: 21.11713709634837\n",
      "Train [2017-07-01 2017-08-15] Val [2017-08-16 2017-08-31] Test [2017-09-01 2017-09-16]\n",
      "Window Train/Val/Test shape (966, 50) (336, 50) (336, 50)\n",
      "Window Train/Val/Test shape (966, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-08-30] Val [2017-08-31 2017-09-15] Test [2017-09-16 2017-10-01]\n",
      "Window Train/Val/Test shape (1281, 50) (336, 50) (336, 50)\n",
      "Window Train/Val/Test shape (1281, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-09-14] Val [2017-09-15 2017-09-30] Test [2017-10-01 2017-10-16]\n",
      "Window Train/Val/Test shape (1596, 50) (336, 50) (336, 50)\n",
      "Window Train/Val/Test shape (1596, 2) (336, 2) (336, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-09-29] Val [2017-09-30 2017-10-15] Test [2017-10-16 2017-11-10]\n",
      "Window Train/Val/Test shape (1911, 50) (336, 50) (546, 50)\n",
      "Window Train/Val/Test shape (1911, 2) (336, 2) (546, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-10-14] Val [2017-10-15 2017-10-30] Test [2017-10-31 2017-11-10]\n",
      "Window Train/Val/Test shape (2226, 50) (336, 50) (231, 50)\n",
      "Window Train/Val/Test shape (2226, 2) (336, 2) (231, 2)\n",
      "[33.252808889604786, 40.27937291261827, 14.832418236456245, 17.296824000735732, 6.242401733219238]\n",
      "mean mse: 22.380765154526856\n",
      "Train [2017-07-01 2017-08-15] Val [2017-08-16 2017-08-31] Test [2017-09-01 2017-09-16]\n",
      "Window Train/Val/Test shape (966, 50) (336, 50) (336, 50)\n",
      "Window Train/Val/Test shape (966, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-08-30] Val [2017-08-31 2017-09-15] Test [2017-09-16 2017-10-01]\n",
      "Window Train/Val/Test shape (1281, 50) (336, 50) (336, 50)\n",
      "Window Train/Val/Test shape (1281, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-09-14] Val [2017-09-15 2017-09-30] Test [2017-10-01 2017-10-16]\n",
      "Window Train/Val/Test shape (1596, 50) (336, 50) (336, 50)\n",
      "Window Train/Val/Test shape (1596, 2) (336, 2) (336, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-09-29] Val [2017-09-30 2017-10-15] Test [2017-10-16 2017-11-10]\n",
      "Window Train/Val/Test shape (1911, 50) (336, 50) (546, 50)\n",
      "Window Train/Val/Test shape (1911, 2) (336, 2) (546, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-10-14] Val [2017-10-15 2017-10-30] Test [2017-10-31 2017-11-10]\n",
      "Window Train/Val/Test shape (2226, 50) (336, 50) (231, 50)\n",
      "Window Train/Val/Test shape (2226, 2) (336, 2) (231, 2)\n",
      "[37.872632213019884, 43.30740284874838, 11.503189940607145, 11.908449345503545, 3.420485119486718]\n",
      "mean mse: 21.60243189347313\n",
      "Train [2017-07-01 2017-08-15] Val [2017-08-16 2017-08-31] Test [2017-09-01 2017-09-16]\n",
      "Window Train/Val/Test shape (966, 50) (336, 50) (336, 50)\n",
      "Window Train/Val/Test shape (966, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-08-30] Val [2017-08-31 2017-09-15] Test [2017-09-16 2017-10-01]\n",
      "Window Train/Val/Test shape (1281, 50) (336, 50) (336, 50)\n",
      "Window Train/Val/Test shape (1281, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-09-14] Val [2017-09-15 2017-09-30] Test [2017-10-01 2017-10-16]\n",
      "Window Train/Val/Test shape (1596, 50) (336, 50) (336, 50)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Window Train/Val/Test shape (1596, 2) (336, 2) (336, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-09-29] Val [2017-09-30 2017-10-15] Test [2017-10-16 2017-11-10]\n",
      "Window Train/Val/Test shape (1911, 50) (336, 50) (546, 50)\n",
      "Window Train/Val/Test shape (1911, 2) (336, 2) (546, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-10-14] Val [2017-10-15 2017-10-30] Test [2017-10-31 2017-11-10]\n",
      "Window Train/Val/Test shape (2226, 50) (336, 50) (231, 50)\n",
      "Window Train/Val/Test shape (2226, 2) (336, 2) (231, 2)\n",
      "[28.715759025941832, 37.933975704625674, 10.506206857269294, 14.615122373424041, 3.949711031637072]\n",
      "mean mse: 19.14415499857958\n",
      "Train [2017-07-01 2017-08-15] Val [2017-08-16 2017-08-31] Test [2017-09-01 2017-09-16]\n",
      "Window Train/Val/Test shape (966, 50) (336, 50) (336, 50)\n",
      "Window Train/Val/Test shape (966, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-08-30] Val [2017-08-31 2017-09-15] Test [2017-09-16 2017-10-01]\n",
      "Window Train/Val/Test shape (1281, 50) (336, 50) (336, 50)\n",
      "Window Train/Val/Test shape (1281, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-09-14] Val [2017-09-15 2017-09-30] Test [2017-10-01 2017-10-16]\n",
      "Window Train/Val/Test shape (1596, 50) (336, 50) (336, 50)\n",
      "Window Train/Val/Test shape (1596, 2) (336, 2) (336, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-09-29] Val [2017-09-30 2017-10-15] Test [2017-10-16 2017-11-10]\n",
      "Window Train/Val/Test shape (1911, 50) (336, 50) (546, 50)\n",
      "Window Train/Val/Test shape (1911, 2) (336, 2) (546, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-10-14] Val [2017-10-15 2017-10-30] Test [2017-10-31 2017-11-10]\n",
      "Window Train/Val/Test shape (2226, 50) (336, 50) (231, 50)\n",
      "Window Train/Val/Test shape (2226, 2) (336, 2) (231, 2)\n",
      "[43.32567666770227, 38.879131159666045, 16.518132933635844, 17.692705559225743, 5.038744319420454]\n",
      "mean mse: 24.29087812793007\n",
      "Train [2017-07-01 2017-08-15] Val [2017-08-16 2017-08-31] Test [2017-09-01 2017-09-16]\n",
      "Window Train/Val/Test shape (966, 50) (336, 50) (336, 50)\n",
      "Window Train/Val/Test shape (966, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-08-30] Val [2017-08-31 2017-09-15] Test [2017-09-16 2017-10-01]\n",
      "Window Train/Val/Test shape (1281, 50) (336, 50) (336, 50)\n",
      "Window Train/Val/Test shape (1281, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-09-14] Val [2017-09-15 2017-09-30] Test [2017-10-01 2017-10-16]\n",
      "Window Train/Val/Test shape (1596, 50) (336, 50) (336, 50)\n",
      "Window Train/Val/Test shape (1596, 2) (336, 2) (336, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-09-29] Val [2017-09-30 2017-10-15] Test [2017-10-16 2017-11-10]\n",
      "Window Train/Val/Test shape (1911, 50) (336, 50) (546, 50)\n",
      "Window Train/Val/Test shape (1911, 2) (336, 2) (546, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-10-14] Val [2017-10-15 2017-10-30] Test [2017-10-31 2017-11-10]\n",
      "Window Train/Val/Test shape (2226, 50) (336, 50) (231, 50)\n",
      "Window Train/Val/Test shape (2226, 2) (336, 2) (231, 2)\n",
      "[539.6409815294328, 355.77831089297484, 283.344378522231, 125.59562762197143, 105.15474783392483]\n",
      "mean mse: 281.90280928010696\n",
      "Train [2017-07-01 2017-08-15] Val [2017-08-16 2017-08-31] Test [2017-09-01 2017-09-16]\n",
      "Window Train/Val/Test shape (966, 50) (336, 50) (336, 50)\n",
      "Window Train/Val/Test shape (966, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-08-30] Val [2017-08-31 2017-09-15] Test [2017-09-16 2017-10-01]\n",
      "Window Train/Val/Test shape (1281, 50) (336, 50) (336, 50)\n",
      "Window Train/Val/Test shape (1281, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-09-14] Val [2017-09-15 2017-09-30] Test [2017-10-01 2017-10-16]\n",
      "Window Train/Val/Test shape (1596, 50) (336, 50) (336, 50)\n",
      "Window Train/Val/Test shape (1596, 2) (336, 2) (336, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-09-29] Val [2017-09-30 2017-10-15] Test [2017-10-16 2017-11-10]\n",
      "Window Train/Val/Test shape (1911, 50) (336, 50) (546, 50)\n",
      "Window Train/Val/Test shape (1911, 2) (336, 2) (546, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-10-14] Val [2017-10-15 2017-10-30] Test [2017-10-31 2017-11-10]\n",
      "Window Train/Val/Test shape (2226, 50) (336, 50) (231, 50)\n",
      "Window Train/Val/Test shape (2226, 2) (336, 2) (231, 2)\n",
      "[36.12805091201112, 48.297332612324865, 13.214526139025164, 15.040647470364629, 4.679898799892561]\n",
      "mean mse: 23.472091186723667\n",
      "Train [2017-07-01 2017-08-15] Val [2017-08-16 2017-08-31] Test [2017-09-01 2017-09-16]\n",
      "Window Train/Val/Test shape (966, 50) (336, 50) (336, 50)\n",
      "Window Train/Val/Test shape (966, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-08-30] Val [2017-08-31 2017-09-15] Test [2017-09-16 2017-10-01]\n",
      "Window Train/Val/Test shape (1281, 50) (336, 50) (336, 50)\n",
      "Window Train/Val/Test shape (1281, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-09-14] Val [2017-09-15 2017-09-30] Test [2017-10-01 2017-10-16]\n",
      "Window Train/Val/Test shape (1596, 50) (336, 50) (336, 50)\n",
      "Window Train/Val/Test shape (1596, 2) (336, 2) (336, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-09-29] Val [2017-09-30 2017-10-15] Test [2017-10-16 2017-11-10]\n",
      "Window Train/Val/Test shape (1911, 50) (336, 50) (546, 50)\n",
      "Window Train/Val/Test shape (1911, 2) (336, 2) (546, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-10-14] Val [2017-10-15 2017-10-30] Test [2017-10-31 2017-11-10]\n",
      "Window Train/Val/Test shape (2226, 50) (336, 50) (231, 50)\n",
      "Window Train/Val/Test shape (2226, 2) (336, 2) (231, 2)\n",
      "[36.08094541010651, 40.20581631042398, 21.898499257789723, 23.0700169228893, 8.833500039186715]\n",
      "mean mse: 26.01775558807924\n",
      "Train [2017-07-01 2017-08-15] Val [2017-08-16 2017-08-31] Test [2017-09-01 2017-09-16]\n",
      "Window Train/Val/Test shape (966, 50) (336, 50) (336, 50)\n",
      "Window Train/Val/Test shape (966, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-08-30] Val [2017-08-31 2017-09-15] Test [2017-09-16 2017-10-01]\n",
      "Window Train/Val/Test shape (1281, 50) (336, 50) (336, 50)\n",
      "Window Train/Val/Test shape (1281, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-09-14] Val [2017-09-15 2017-09-30] Test [2017-10-01 2017-10-16]\n",
      "Window Train/Val/Test shape (1596, 50) (336, 50) (336, 50)\n",
      "Window Train/Val/Test shape (1596, 2) (336, 2) (336, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-09-29] Val [2017-09-30 2017-10-15] Test [2017-10-16 2017-11-10]\n",
      "Window Train/Val/Test shape (1911, 50) (336, 50) (546, 50)\n",
      "Window Train/Val/Test shape (1911, 2) (336, 2) (546, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-10-14] Val [2017-10-15 2017-10-30] Test [2017-10-31 2017-11-10]\n",
      "Window Train/Val/Test shape (2226, 50) (336, 50) (231, 50)\n",
      "Window Train/Val/Test shape (2226, 2) (336, 2) (231, 2)\n",
      "[32.72379777096954, 38.248662414298366, 13.627346152094375, 13.429680865163224, 6.68564960731317]\n",
      "mean mse: 20.943027361967733\n",
      "Train [2017-07-01 2017-08-15] Val [2017-08-16 2017-08-31] Test [2017-09-01 2017-09-16]\n",
      "Window Train/Val/Test shape (966, 50) (336, 50) (336, 50)\n",
      "Window Train/Val/Test shape (966, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-08-30] Val [2017-08-31 2017-09-15] Test [2017-09-16 2017-10-01]\n",
      "Window Train/Val/Test shape (1281, 50) (336, 50) (336, 50)\n",
      "Window Train/Val/Test shape (1281, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-09-14] Val [2017-09-15 2017-09-30] Test [2017-10-01 2017-10-16]\n",
      "Window Train/Val/Test shape (1596, 50) (336, 50) (336, 50)\n",
      "Window Train/Val/Test shape (1596, 2) (336, 2) (336, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-09-29] Val [2017-09-30 2017-10-15] Test [2017-10-16 2017-11-10]\n",
      "Window Train/Val/Test shape (1911, 50) (336, 50) (546, 50)\n",
      "Window Train/Val/Test shape (1911, 2) (336, 2) (546, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-10-14] Val [2017-10-15 2017-10-30] Test [2017-10-31 2017-11-10]\n",
      "Window Train/Val/Test shape (2226, 50) (336, 50) (231, 50)\n",
      "Window Train/Val/Test shape (2226, 2) (336, 2) (231, 2)\n",
      "[36.572470611266304, 37.99017742751928, 12.104406153051652, 14.164448184238623, 6.411101586189323]\n",
      "mean mse: 21.448520792453035\n",
      "Train [2017-07-01 2017-08-15] Val [2017-08-16 2017-08-31] Test [2017-09-01 2017-09-16]\n",
      "Window Train/Val/Test shape (966, 50) (336, 50) (336, 50)\n",
      "Window Train/Val/Test shape (966, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-08-30] Val [2017-08-31 2017-09-15] Test [2017-09-16 2017-10-01]\n",
      "Window Train/Val/Test shape (1281, 50) (336, 50) (336, 50)\n",
      "Window Train/Val/Test shape (1281, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-09-14] Val [2017-09-15 2017-09-30] Test [2017-10-01 2017-10-16]\n",
      "Window Train/Val/Test shape (1596, 50) (336, 50) (336, 50)\n",
      "Window Train/Val/Test shape (1596, 2) (336, 2) (336, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-09-29] Val [2017-09-30 2017-10-15] Test [2017-10-16 2017-11-10]\n",
      "Window Train/Val/Test shape (1911, 50) (336, 50) (546, 50)\n",
      "Window Train/Val/Test shape (1911, 2) (336, 2) (546, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-10-14] Val [2017-10-15 2017-10-30] Test [2017-10-31 2017-11-10]\n",
      "Window Train/Val/Test shape (2226, 50) (336, 50) (231, 50)\n",
      "Window Train/Val/Test shape (2226, 2) (336, 2) (231, 2)\n",
      "[36.54348281496981, 44.76538289954845, 11.538034828599491, 14.028571907391028, 3.3802531302230134]\n",
      "mean mse: 22.05114511614636\n",
      "Train [2017-07-01 2017-08-15] Val [2017-08-16 2017-08-31] Test [2017-09-01 2017-09-16]\n",
      "Window Train/Val/Test shape (966, 50) (336, 50) (336, 50)\n",
      "Window Train/Val/Test shape (966, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-08-30] Val [2017-08-31 2017-09-15] Test [2017-09-16 2017-10-01]\n",
      "Window Train/Val/Test shape (1281, 50) (336, 50) (336, 50)\n",
      "Window Train/Val/Test shape (1281, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-09-14] Val [2017-09-15 2017-09-30] Test [2017-10-01 2017-10-16]\n",
      "Window Train/Val/Test shape (1596, 50) (336, 50) (336, 50)\n",
      "Window Train/Val/Test shape (1596, 2) (336, 2) (336, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-09-29] Val [2017-09-30 2017-10-15] Test [2017-10-16 2017-11-10]\n",
      "Window Train/Val/Test shape (1911, 50) (336, 50) (546, 50)\n",
      "Window Train/Val/Test shape (1911, 2) (336, 2) (546, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-10-14] Val [2017-10-15 2017-10-30] Test [2017-10-31 2017-11-10]\n",
      "Window Train/Val/Test shape (2226, 50) (336, 50) (231, 50)\n",
      "Window Train/Val/Test shape (2226, 2) (336, 2) (231, 2)\n",
      "[33.34379879143415, 53.32936930892893, 13.188723676156897, 16.948504295492405, 7.609764374572634]\n",
      "mean mse: 24.884032089317007\n",
      "Train [2017-07-01 2017-08-15] Val [2017-08-16 2017-08-31] Test [2017-09-01 2017-09-16]\n",
      "Window Train/Val/Test shape (966, 50) (336, 50) (336, 50)\n",
      "Window Train/Val/Test shape (966, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-08-30] Val [2017-08-31 2017-09-15] Test [2017-09-16 2017-10-01]\n",
      "Window Train/Val/Test shape (1281, 50) (336, 50) (336, 50)\n",
      "Window Train/Val/Test shape (1281, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-09-14] Val [2017-09-15 2017-09-30] Test [2017-10-01 2017-10-16]\n",
      "Window Train/Val/Test shape (1596, 50) (336, 50) (336, 50)\n",
      "Window Train/Val/Test shape (1596, 2) (336, 2) (336, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-09-29] Val [2017-09-30 2017-10-15] Test [2017-10-16 2017-11-10]\n",
      "Window Train/Val/Test shape (1911, 50) (336, 50) (546, 50)\n",
      "Window Train/Val/Test shape (1911, 2) (336, 2) (546, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-10-14] Val [2017-10-15 2017-10-30] Test [2017-10-31 2017-11-10]\n",
      "Window Train/Val/Test shape (2226, 50) (336, 50) (231, 50)\n",
      "Window Train/Val/Test shape (2226, 2) (336, 2) (231, 2)\n",
      "[32.07923690841034, 66.76629845023982, 15.793279013261778, 23.535433405878162, 9.137940710355847]\n",
      "mean mse: 29.46243769762919\n",
      "Train [2017-07-01 2017-08-15] Val [2017-08-16 2017-08-31] Test [2017-09-01 2017-09-16]\n",
      "Window Train/Val/Test shape (966, 50) (336, 50) (336, 50)\n",
      "Window Train/Val/Test shape (966, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-08-30] Val [2017-08-31 2017-09-15] Test [2017-09-16 2017-10-01]\n",
      "Window Train/Val/Test shape (1281, 50) (336, 50) (336, 50)\n",
      "Window Train/Val/Test shape (1281, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-09-14] Val [2017-09-15 2017-09-30] Test [2017-10-01 2017-10-16]\n",
      "Window Train/Val/Test shape (1596, 50) (336, 50) (336, 50)\n",
      "Window Train/Val/Test shape (1596, 2) (336, 2) (336, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-09-29] Val [2017-09-30 2017-10-15] Test [2017-10-16 2017-11-10]\n",
      "Window Train/Val/Test shape (1911, 50) (336, 50) (546, 50)\n",
      "Window Train/Val/Test shape (1911, 2) (336, 2) (546, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-10-14] Val [2017-10-15 2017-10-30] Test [2017-10-31 2017-11-10]\n",
      "Window Train/Val/Test shape (2226, 50) (336, 50) (231, 50)\n",
      "Window Train/Val/Test shape (2226, 2) (336, 2) (231, 2)\n",
      "[35.301815943821175, 33.10849189947271, 13.3641548669471, 19.23082923261061, 4.1640688307032505]\n",
      "mean mse: 21.03387215471097\n",
      "Train [2017-07-01 2017-08-15] Val [2017-08-16 2017-08-31] Test [2017-09-01 2017-09-16]\n",
      "Window Train/Val/Test shape (966, 50) (336, 50) (336, 50)\n",
      "Window Train/Val/Test shape (966, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-08-30] Val [2017-08-31 2017-09-15] Test [2017-09-16 2017-10-01]\n",
      "Window Train/Val/Test shape (1281, 50) (336, 50) (336, 50)\n",
      "Window Train/Val/Test shape (1281, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-09-14] Val [2017-09-15 2017-09-30] Test [2017-10-01 2017-10-16]\n",
      "Window Train/Val/Test shape (1596, 50) (336, 50) (336, 50)\n",
      "Window Train/Val/Test shape (1596, 2) (336, 2) (336, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-09-29] Val [2017-09-30 2017-10-15] Test [2017-10-16 2017-11-10]\n",
      "Window Train/Val/Test shape (1911, 50) (336, 50) (546, 50)\n",
      "Window Train/Val/Test shape (1911, 2) (336, 2) (546, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-10-14] Val [2017-10-15 2017-10-30] Test [2017-10-31 2017-11-10]\n",
      "Window Train/Val/Test shape (2226, 50) (336, 50) (231, 50)\n",
      "Window Train/Val/Test shape (2226, 2) (336, 2) (231, 2)\n",
      "[41.323214846898225, 39.870436289750124, 12.348703089107445, 17.077983267384457, 6.414968161311438]\n",
      "mean mse: 23.407061130890337\n",
      "Train [2017-07-01 2017-08-15] Val [2017-08-16 2017-08-31] Test [2017-09-01 2017-09-16]\n",
      "Window Train/Val/Test shape (966, 50) (336, 50) (336, 50)\n",
      "Window Train/Val/Test shape (966, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-08-30] Val [2017-08-31 2017-09-15] Test [2017-09-16 2017-10-01]\n",
      "Window Train/Val/Test shape (1281, 50) (336, 50) (336, 50)\n",
      "Window Train/Val/Test shape (1281, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-09-14] Val [2017-09-15 2017-09-30] Test [2017-10-01 2017-10-16]\n",
      "Window Train/Val/Test shape (1596, 50) (336, 50) (336, 50)\n",
      "Window Train/Val/Test shape (1596, 2) (336, 2) (336, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-09-29] Val [2017-09-30 2017-10-15] Test [2017-10-16 2017-11-10]\n",
      "Window Train/Val/Test shape (1911, 50) (336, 50) (546, 50)\n",
      "Window Train/Val/Test shape (1911, 2) (336, 2) (546, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-10-14] Val [2017-10-15 2017-10-30] Test [2017-10-31 2017-11-10]\n",
      "Window Train/Val/Test shape (2226, 50) (336, 50) (231, 50)\n",
      "Window Train/Val/Test shape (2226, 2) (336, 2) (231, 2)\n",
      "[30.256666835766737, 49.776242855378314, 16.19019298815303, 13.667965680784787, 8.048097238944532]\n",
      "mean mse: 23.58783311980548\n",
      "Train [2017-07-01 2017-08-15] Val [2017-08-16 2017-08-31] Test [2017-09-01 2017-09-16]\n",
      "Window Train/Val/Test shape (966, 50) (336, 50) (336, 50)\n",
      "Window Train/Val/Test shape (966, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-08-30] Val [2017-08-31 2017-09-15] Test [2017-09-16 2017-10-01]\n",
      "Window Train/Val/Test shape (1281, 50) (336, 50) (336, 50)\n",
      "Window Train/Val/Test shape (1281, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-09-14] Val [2017-09-15 2017-09-30] Test [2017-10-01 2017-10-16]\n",
      "Window Train/Val/Test shape (1596, 50) (336, 50) (336, 50)\n",
      "Window Train/Val/Test shape (1596, 2) (336, 2) (336, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-09-29] Val [2017-09-30 2017-10-15] Test [2017-10-16 2017-11-10]\n",
      "Window Train/Val/Test shape (1911, 50) (336, 50) (546, 50)\n",
      "Window Train/Val/Test shape (1911, 2) (336, 2) (546, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-10-14] Val [2017-10-15 2017-10-30] Test [2017-10-31 2017-11-10]\n",
      "Window Train/Val/Test shape (2226, 50) (336, 50) (231, 50)\n",
      "Window Train/Val/Test shape (2226, 2) (336, 2) (231, 2)\n",
      "[34.27737456022689, 48.694386894791826, 10.944144201841736, 12.124820956527344, 4.205961384759173]\n",
      "mean mse: 22.049337599629393\n",
      "Train [2017-07-01 2017-08-15] Val [2017-08-16 2017-08-31] Test [2017-09-01 2017-09-16]\n",
      "Window Train/Val/Test shape (966, 50) (336, 50) (336, 50)\n",
      "Window Train/Val/Test shape (966, 2) (336, 2) (336, 2)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train [2017-07-01 2017-08-30] Val [2017-08-31 2017-09-15] Test [2017-09-16 2017-10-01]\n",
      "Window Train/Val/Test shape (1281, 50) (336, 50) (336, 50)\n",
      "Window Train/Val/Test shape (1281, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-09-14] Val [2017-09-15 2017-09-30] Test [2017-10-01 2017-10-16]\n",
      "Window Train/Val/Test shape (1596, 50) (336, 50) (336, 50)\n",
      "Window Train/Val/Test shape (1596, 2) (336, 2) (336, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-09-29] Val [2017-09-30 2017-10-15] Test [2017-10-16 2017-11-10]\n",
      "Window Train/Val/Test shape (1911, 50) (336, 50) (546, 50)\n",
      "Window Train/Val/Test shape (1911, 2) (336, 2) (546, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-10-14] Val [2017-10-15 2017-10-30] Test [2017-10-31 2017-11-10]\n",
      "Window Train/Val/Test shape (2226, 50) (336, 50) (231, 50)\n",
      "Window Train/Val/Test shape (2226, 2) (336, 2) (231, 2)\n",
      "[35.26516322226437, 39.218413690133595, 13.38948586832398, 17.244275002214696, 3.597068026065459]\n",
      "mean mse: 21.74288116180042\n",
      "Train [2017-07-01 2017-08-15] Val [2017-08-16 2017-08-31] Test [2017-09-01 2017-09-16]\n",
      "Window Train/Val/Test shape (966, 50) (336, 50) (336, 50)\n",
      "Window Train/Val/Test shape (966, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-08-30] Val [2017-08-31 2017-09-15] Test [2017-09-16 2017-10-01]\n",
      "Window Train/Val/Test shape (1281, 50) (336, 50) (336, 50)\n",
      "Window Train/Val/Test shape (1281, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-09-14] Val [2017-09-15 2017-09-30] Test [2017-10-01 2017-10-16]\n",
      "Window Train/Val/Test shape (1596, 50) (336, 50) (336, 50)\n",
      "Window Train/Val/Test shape (1596, 2) (336, 2) (336, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-09-29] Val [2017-09-30 2017-10-15] Test [2017-10-16 2017-11-10]\n",
      "Window Train/Val/Test shape (1911, 50) (336, 50) (546, 50)\n",
      "Window Train/Val/Test shape (1911, 2) (336, 2) (546, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-10-14] Val [2017-10-15 2017-10-30] Test [2017-10-31 2017-11-10]\n",
      "Window Train/Val/Test shape (2226, 50) (336, 50) (231, 50)\n",
      "Window Train/Val/Test shape (2226, 2) (336, 2) (231, 2)\n",
      "[32.45398277087717, 43.424164549139896, 12.364646525714537, 17.840833485700365, 5.405994707841573]\n",
      "mean mse: 22.297924407854705\n",
      "Train [2017-07-01 2017-08-15] Val [2017-08-16 2017-08-31] Test [2017-09-01 2017-09-16]\n",
      "Window Train/Val/Test shape (966, 50) (336, 50) (336, 50)\n",
      "Window Train/Val/Test shape (966, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-08-30] Val [2017-08-31 2017-09-15] Test [2017-09-16 2017-10-01]\n",
      "Window Train/Val/Test shape (1281, 50) (336, 50) (336, 50)\n",
      "Window Train/Val/Test shape (1281, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-09-14] Val [2017-09-15 2017-09-30] Test [2017-10-01 2017-10-16]\n",
      "Window Train/Val/Test shape (1596, 50) (336, 50) (336, 50)\n",
      "Window Train/Val/Test shape (1596, 2) (336, 2) (336, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-09-29] Val [2017-09-30 2017-10-15] Test [2017-10-16 2017-11-10]\n",
      "Window Train/Val/Test shape (1911, 50) (336, 50) (546, 50)\n",
      "Window Train/Val/Test shape (1911, 2) (336, 2) (546, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-10-14] Val [2017-10-15 2017-10-30] Test [2017-10-31 2017-11-10]\n",
      "Window Train/Val/Test shape (2226, 50) (336, 50) (231, 50)\n",
      "Window Train/Val/Test shape (2226, 2) (336, 2) (231, 2)\n",
      "[44.344148452509785, 44.943988973241005, 15.195376926178298, 18.050295124917213, 7.316252800291015]\n",
      "mean mse: 25.97001245542746\n",
      "Train [2017-07-01 2017-08-15] Val [2017-08-16 2017-08-31] Test [2017-09-01 2017-09-16]\n",
      "Window Train/Val/Test shape (966, 50) (336, 50) (336, 50)\n",
      "Window Train/Val/Test shape (966, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-08-30] Val [2017-08-31 2017-09-15] Test [2017-09-16 2017-10-01]\n",
      "Window Train/Val/Test shape (1281, 50) (336, 50) (336, 50)\n",
      "Window Train/Val/Test shape (1281, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-09-14] Val [2017-09-15 2017-09-30] Test [2017-10-01 2017-10-16]\n",
      "Window Train/Val/Test shape (1596, 50) (336, 50) (336, 50)\n",
      "Window Train/Val/Test shape (1596, 2) (336, 2) (336, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-09-29] Val [2017-09-30 2017-10-15] Test [2017-10-16 2017-11-10]\n",
      "Window Train/Val/Test shape (1911, 50) (336, 50) (546, 50)\n",
      "Window Train/Val/Test shape (1911, 2) (336, 2) (546, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-10-14] Val [2017-10-15 2017-10-30] Test [2017-10-31 2017-11-10]\n",
      "Window Train/Val/Test shape (2226, 50) (336, 50) (231, 50)\n",
      "Window Train/Val/Test shape (2226, 2) (336, 2) (231, 2)\n",
      "[44.871808783254565, 50.126084586708444, 14.208764639337128, 23.654787115807935, 3.9270567332005886]\n",
      "mean mse: 27.357700371661736\n",
      "Train [2017-07-01 2017-08-15] Val [2017-08-16 2017-08-31] Test [2017-09-01 2017-09-16]\n",
      "Window Train/Val/Test shape (966, 50) (336, 50) (336, 50)\n",
      "Window Train/Val/Test shape (966, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-08-30] Val [2017-08-31 2017-09-15] Test [2017-09-16 2017-10-01]\n",
      "Window Train/Val/Test shape (1281, 50) (336, 50) (336, 50)\n",
      "Window Train/Val/Test shape (1281, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-09-14] Val [2017-09-15 2017-09-30] Test [2017-10-01 2017-10-16]\n",
      "Window Train/Val/Test shape (1596, 50) (336, 50) (336, 50)\n",
      "Window Train/Val/Test shape (1596, 2) (336, 2) (336, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-09-29] Val [2017-09-30 2017-10-15] Test [2017-10-16 2017-11-10]\n",
      "Window Train/Val/Test shape (1911, 50) (336, 50) (546, 50)\n",
      "Window Train/Val/Test shape (1911, 2) (336, 2) (546, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-10-14] Val [2017-10-15 2017-10-30] Test [2017-10-31 2017-11-10]\n",
      "Window Train/Val/Test shape (2226, 50) (336, 50) (231, 50)\n",
      "Window Train/Val/Test shape (2226, 2) (336, 2) (231, 2)\n",
      "[35.812296766302104, 44.44316674720302, 16.536775143513285, 13.600325105307109, 4.427811324488328]\n",
      "mean mse: 22.96407501736277\n",
      "Train [2017-07-01 2017-08-15] Val [2017-08-16 2017-08-31] Test [2017-09-01 2017-09-16]\n",
      "Window Train/Val/Test shape (966, 50) (336, 50) (336, 50)\n",
      "Window Train/Val/Test shape (966, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-08-30] Val [2017-08-31 2017-09-15] Test [2017-09-16 2017-10-01]\n",
      "Window Train/Val/Test shape (1281, 50) (336, 50) (336, 50)\n",
      "Window Train/Val/Test shape (1281, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-09-14] Val [2017-09-15 2017-09-30] Test [2017-10-01 2017-10-16]\n",
      "Window Train/Val/Test shape (1596, 50) (336, 50) (336, 50)\n",
      "Window Train/Val/Test shape (1596, 2) (336, 2) (336, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-09-29] Val [2017-09-30 2017-10-15] Test [2017-10-16 2017-11-10]\n",
      "Window Train/Val/Test shape (1911, 50) (336, 50) (546, 50)\n",
      "Window Train/Val/Test shape (1911, 2) (336, 2) (546, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-10-14] Val [2017-10-15 2017-10-30] Test [2017-10-31 2017-11-10]\n",
      "Window Train/Val/Test shape (2226, 50) (336, 50) (231, 50)\n",
      "Window Train/Val/Test shape (2226, 2) (336, 2) (231, 2)\n",
      "[34.49600064322804, 61.386588275371174, 16.353958421103936, 14.28737568838942, 7.134136119933202]\n",
      "mean mse: 26.73161182960515\n",
      "Train [2017-07-01 2017-08-15] Val [2017-08-16 2017-08-31] Test [2017-09-01 2017-09-16]\n",
      "Window Train/Val/Test shape (966, 50) (336, 50) (336, 50)\n",
      "Window Train/Val/Test shape (966, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-08-30] Val [2017-08-31 2017-09-15] Test [2017-09-16 2017-10-01]\n",
      "Window Train/Val/Test shape (1281, 50) (336, 50) (336, 50)\n",
      "Window Train/Val/Test shape (1281, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-09-14] Val [2017-09-15 2017-09-30] Test [2017-10-01 2017-10-16]\n",
      "Window Train/Val/Test shape (1596, 50) (336, 50) (336, 50)\n",
      "Window Train/Val/Test shape (1596, 2) (336, 2) (336, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-09-29] Val [2017-09-30 2017-10-15] Test [2017-10-16 2017-11-10]\n",
      "Window Train/Val/Test shape (1911, 50) (336, 50) (546, 50)\n",
      "Window Train/Val/Test shape (1911, 2) (336, 2) (546, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-10-14] Val [2017-10-15 2017-10-30] Test [2017-10-31 2017-11-10]\n",
      "Window Train/Val/Test shape (2226, 50) (336, 50) (231, 50)\n",
      "Window Train/Val/Test shape (2226, 2) (336, 2) (231, 2)\n",
      "[30.914628353403714, 37.95365348527599, 13.360874743022892, 16.65314159819392, 6.118280916031473]\n",
      "mean mse: 21.000115819185595\n",
      "Train [2017-07-01 2017-08-15] Val [2017-08-16 2017-08-31] Test [2017-09-01 2017-09-16]\n",
      "Window Train/Val/Test shape (966, 50) (336, 50) (336, 50)\n",
      "Window Train/Val/Test shape (966, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-08-30] Val [2017-08-31 2017-09-15] Test [2017-09-16 2017-10-01]\n",
      "Window Train/Val/Test shape (1281, 50) (336, 50) (336, 50)\n",
      "Window Train/Val/Test shape (1281, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-09-14] Val [2017-09-15 2017-09-30] Test [2017-10-01 2017-10-16]\n",
      "Window Train/Val/Test shape (1596, 50) (336, 50) (336, 50)\n",
      "Window Train/Val/Test shape (1596, 2) (336, 2) (336, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-09-29] Val [2017-09-30 2017-10-15] Test [2017-10-16 2017-11-10]\n",
      "Window Train/Val/Test shape (1911, 50) (336, 50) (546, 50)\n",
      "Window Train/Val/Test shape (1911, 2) (336, 2) (546, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-10-14] Val [2017-10-15 2017-10-30] Test [2017-10-31 2017-11-10]\n",
      "Window Train/Val/Test shape (2226, 50) (336, 50) (231, 50)\n",
      "Window Train/Val/Test shape (2226, 2) (336, 2) (231, 2)\n",
      "[539.6409815294328, 355.77831089297484, 283.344378522231, 125.59562762197143, 105.15474783392483]\n",
      "mean mse: 281.90280928010696\n",
      "Train [2017-07-01 2017-08-15] Val [2017-08-16 2017-08-31] Test [2017-09-01 2017-09-16]\n",
      "Window Train/Val/Test shape (966, 50) (336, 50) (336, 50)\n",
      "Window Train/Val/Test shape (966, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-08-30] Val [2017-08-31 2017-09-15] Test [2017-09-16 2017-10-01]\n",
      "Window Train/Val/Test shape (1281, 50) (336, 50) (336, 50)\n",
      "Window Train/Val/Test shape (1281, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-09-14] Val [2017-09-15 2017-09-30] Test [2017-10-01 2017-10-16]\n",
      "Window Train/Val/Test shape (1596, 50) (336, 50) (336, 50)\n",
      "Window Train/Val/Test shape (1596, 2) (336, 2) (336, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-09-29] Val [2017-09-30 2017-10-15] Test [2017-10-16 2017-11-10]\n",
      "Window Train/Val/Test shape (1911, 50) (336, 50) (546, 50)\n",
      "Window Train/Val/Test shape (1911, 2) (336, 2) (546, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-10-14] Val [2017-10-15 2017-10-30] Test [2017-10-31 2017-11-10]\n",
      "Window Train/Val/Test shape (2226, 50) (336, 50) (231, 50)\n",
      "Window Train/Val/Test shape (2226, 2) (336, 2) (231, 2)\n",
      "[27.5731625675555, 52.57247182241004, 13.06965102883044, 14.258053160259296, 3.986770601653666]\n",
      "mean mse: 22.292021836141785\n",
      "Train [2017-07-01 2017-08-15] Val [2017-08-16 2017-08-31] Test [2017-09-01 2017-09-16]\n",
      "Window Train/Val/Test shape (966, 50) (336, 50) (336, 50)\n",
      "Window Train/Val/Test shape (966, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-08-30] Val [2017-08-31 2017-09-15] Test [2017-09-16 2017-10-01]\n",
      "Window Train/Val/Test shape (1281, 50) (336, 50) (336, 50)\n",
      "Window Train/Val/Test shape (1281, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-09-14] Val [2017-09-15 2017-09-30] Test [2017-10-01 2017-10-16]\n",
      "Window Train/Val/Test shape (1596, 50) (336, 50) (336, 50)\n",
      "Window Train/Val/Test shape (1596, 2) (336, 2) (336, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-09-29] Val [2017-09-30 2017-10-15] Test [2017-10-16 2017-11-10]\n",
      "Window Train/Val/Test shape (1911, 50) (336, 50) (546, 50)\n",
      "Window Train/Val/Test shape (1911, 2) (336, 2) (546, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-10-14] Val [2017-10-15 2017-10-30] Test [2017-10-31 2017-11-10]\n",
      "Window Train/Val/Test shape (2226, 50) (336, 50) (231, 50)\n",
      "Window Train/Val/Test shape (2226, 2) (336, 2) (231, 2)\n",
      "[38.35464394144607, 47.143360047260586, 14.01281317363925, 14.11212051879819, 5.1977984279390315]\n",
      "mean mse: 23.764147221816625\n",
      "Train [2017-07-01 2017-08-15] Val [2017-08-16 2017-08-31] Test [2017-09-01 2017-09-16]\n",
      "Window Train/Val/Test shape (966, 50) (336, 50) (336, 50)\n",
      "Window Train/Val/Test shape (966, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-08-30] Val [2017-08-31 2017-09-15] Test [2017-09-16 2017-10-01]\n",
      "Window Train/Val/Test shape (1281, 50) (336, 50) (336, 50)\n",
      "Window Train/Val/Test shape (1281, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-09-14] Val [2017-09-15 2017-09-30] Test [2017-10-01 2017-10-16]\n",
      "Window Train/Val/Test shape (1596, 50) (336, 50) (336, 50)\n",
      "Window Train/Val/Test shape (1596, 2) (336, 2) (336, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-09-29] Val [2017-09-30 2017-10-15] Test [2017-10-16 2017-11-10]\n",
      "Window Train/Val/Test shape (1911, 50) (336, 50) (546, 50)\n",
      "Window Train/Val/Test shape (1911, 2) (336, 2) (546, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-10-14] Val [2017-10-15 2017-10-30] Test [2017-10-31 2017-11-10]\n",
      "Window Train/Val/Test shape (2226, 50) (336, 50) (231, 50)\n",
      "Window Train/Val/Test shape (2226, 2) (336, 2) (231, 2)\n",
      "[37.93224081488833, 44.13544064148056, 15.25272497742237, 14.116096739202233, 4.212358348350596]\n",
      "mean mse: 23.12977230426882\n",
      "Train [2017-07-01 2017-08-15] Val [2017-08-16 2017-08-31] Test [2017-09-01 2017-09-16]\n",
      "Window Train/Val/Test shape (966, 50) (336, 50) (336, 50)\n",
      "Window Train/Val/Test shape (966, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-08-30] Val [2017-08-31 2017-09-15] Test [2017-09-16 2017-10-01]\n",
      "Window Train/Val/Test shape (1281, 50) (336, 50) (336, 50)\n",
      "Window Train/Val/Test shape (1281, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-09-14] Val [2017-09-15 2017-09-30] Test [2017-10-01 2017-10-16]\n",
      "Window Train/Val/Test shape (1596, 50) (336, 50) (336, 50)\n",
      "Window Train/Val/Test shape (1596, 2) (336, 2) (336, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-09-29] Val [2017-09-30 2017-10-15] Test [2017-10-16 2017-11-10]\n",
      "Window Train/Val/Test shape (1911, 50) (336, 50) (546, 50)\n",
      "Window Train/Val/Test shape (1911, 2) (336, 2) (546, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-10-14] Val [2017-10-15 2017-10-30] Test [2017-10-31 2017-11-10]\n",
      "Window Train/Val/Test shape (2226, 50) (336, 50) (231, 50)\n",
      "Window Train/Val/Test shape (2226, 2) (336, 2) (231, 2)\n",
      "[37.88467125159327, 36.25020870911399, 15.562221844642435, 14.608121023374968, 4.502296357962806]\n",
      "mean mse: 21.761503837337493\n",
      "Train [2017-07-01 2017-08-15] Val [2017-08-16 2017-08-31] Test [2017-09-01 2017-09-16]\n",
      "Window Train/Val/Test shape (966, 50) (336, 50) (336, 50)\n",
      "Window Train/Val/Test shape (966, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-08-30] Val [2017-08-31 2017-09-15] Test [2017-09-16 2017-10-01]\n",
      "Window Train/Val/Test shape (1281, 50) (336, 50) (336, 50)\n",
      "Window Train/Val/Test shape (1281, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-09-14] Val [2017-09-15 2017-09-30] Test [2017-10-01 2017-10-16]\n",
      "Window Train/Val/Test shape (1596, 50) (336, 50) (336, 50)\n",
      "Window Train/Val/Test shape (1596, 2) (336, 2) (336, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-09-29] Val [2017-09-30 2017-10-15] Test [2017-10-16 2017-11-10]\n",
      "Window Train/Val/Test shape (1911, 50) (336, 50) (546, 50)\n",
      "Window Train/Val/Test shape (1911, 2) (336, 2) (546, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-10-14] Val [2017-10-15 2017-10-30] Test [2017-10-31 2017-11-10]\n",
      "Window Train/Val/Test shape (2226, 50) (336, 50) (231, 50)\n",
      "Window Train/Val/Test shape (2226, 2) (336, 2) (231, 2)\n",
      "[34.332971601684264, 51.016853337083745, 13.178535791223862, 18.733939113114623, 5.6785952535570745]\n",
      "mean mse: 24.588179019332713\n",
      "Train [2017-07-01 2017-08-15] Val [2017-08-16 2017-08-31] Test [2017-09-01 2017-09-16]\n",
      "Window Train/Val/Test shape (966, 50) (336, 50) (336, 50)\n",
      "Window Train/Val/Test shape (966, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-08-30] Val [2017-08-31 2017-09-15] Test [2017-09-16 2017-10-01]\n",
      "Window Train/Val/Test shape (1281, 50) (336, 50) (336, 50)\n",
      "Window Train/Val/Test shape (1281, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-09-14] Val [2017-09-15 2017-09-30] Test [2017-10-01 2017-10-16]\n",
      "Window Train/Val/Test shape (1596, 50) (336, 50) (336, 50)\n",
      "Window Train/Val/Test shape (1596, 2) (336, 2) (336, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-09-29] Val [2017-09-30 2017-10-15] Test [2017-10-16 2017-11-10]\n",
      "Window Train/Val/Test shape (1911, 50) (336, 50) (546, 50)\n",
      "Window Train/Val/Test shape (1911, 2) (336, 2) (546, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-10-14] Val [2017-10-15 2017-10-30] Test [2017-10-31 2017-11-10]\n",
      "Window Train/Val/Test shape (2226, 50) (336, 50) (231, 50)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Window Train/Val/Test shape (2226, 2) (336, 2) (231, 2)\n",
      "[44.253360810395975, 30.93419705814174, 14.081079314819217, 13.663457691079651, 4.411130792130755]\n",
      "mean mse: 21.468645133313466\n",
      "Train [2017-07-01 2017-08-15] Val [2017-08-16 2017-08-31] Test [2017-09-01 2017-09-16]\n",
      "Window Train/Val/Test shape (966, 50) (336, 50) (336, 50)\n",
      "Window Train/Val/Test shape (966, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-08-30] Val [2017-08-31 2017-09-15] Test [2017-09-16 2017-10-01]\n",
      "Window Train/Val/Test shape (1281, 50) (336, 50) (336, 50)\n",
      "Window Train/Val/Test shape (1281, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-09-14] Val [2017-09-15 2017-09-30] Test [2017-10-01 2017-10-16]\n",
      "Window Train/Val/Test shape (1596, 50) (336, 50) (336, 50)\n",
      "Window Train/Val/Test shape (1596, 2) (336, 2) (336, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-09-29] Val [2017-09-30 2017-10-15] Test [2017-10-16 2017-11-10]\n",
      "Window Train/Val/Test shape (1911, 50) (336, 50) (546, 50)\n",
      "Window Train/Val/Test shape (1911, 2) (336, 2) (546, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-10-14] Val [2017-10-15 2017-10-30] Test [2017-10-31 2017-11-10]\n",
      "Window Train/Val/Test shape (2226, 50) (336, 50) (231, 50)\n",
      "Window Train/Val/Test shape (2226, 2) (336, 2) (231, 2)\n",
      "[35.95136450806362, 38.61787276029984, 11.972068466884433, 20.187975468302074, 5.669635992573879]\n",
      "mean mse: 22.479783439224768\n",
      "Train [2017-07-01 2017-08-15] Val [2017-08-16 2017-08-31] Test [2017-09-01 2017-09-16]\n",
      "Window Train/Val/Test shape (966, 50) (336, 50) (336, 50)\n",
      "Window Train/Val/Test shape (966, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-08-30] Val [2017-08-31 2017-09-15] Test [2017-09-16 2017-10-01]\n",
      "Window Train/Val/Test shape (1281, 50) (336, 50) (336, 50)\n",
      "Window Train/Val/Test shape (1281, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-09-14] Val [2017-09-15 2017-09-30] Test [2017-10-01 2017-10-16]\n",
      "Window Train/Val/Test shape (1596, 50) (336, 50) (336, 50)\n",
      "Window Train/Val/Test shape (1596, 2) (336, 2) (336, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-09-29] Val [2017-09-30 2017-10-15] Test [2017-10-16 2017-11-10]\n",
      "Window Train/Val/Test shape (1911, 50) (336, 50) (546, 50)\n",
      "Window Train/Val/Test shape (1911, 2) (336, 2) (546, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-10-14] Val [2017-10-15 2017-10-30] Test [2017-10-31 2017-11-10]\n",
      "Window Train/Val/Test shape (2226, 50) (336, 50) (231, 50)\n",
      "Window Train/Val/Test shape (2226, 2) (336, 2) (231, 2)\n",
      "[31.557101405673713, 33.50314786649417, 10.810826316645949, 13.539884409557919, 4.6783102259400975]\n",
      "mean mse: 18.817854044862365\n",
      "Train [2017-07-01 2017-08-15] Val [2017-08-16 2017-08-31] Test [2017-09-01 2017-09-16]\n",
      "Window Train/Val/Test shape (966, 50) (336, 50) (336, 50)\n",
      "Window Train/Val/Test shape (966, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-08-30] Val [2017-08-31 2017-09-15] Test [2017-09-16 2017-10-01]\n",
      "Window Train/Val/Test shape (1281, 50) (336, 50) (336, 50)\n",
      "Window Train/Val/Test shape (1281, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-09-14] Val [2017-09-15 2017-09-30] Test [2017-10-01 2017-10-16]\n",
      "Window Train/Val/Test shape (1596, 50) (336, 50) (336, 50)\n",
      "Window Train/Val/Test shape (1596, 2) (336, 2) (336, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-09-29] Val [2017-09-30 2017-10-15] Test [2017-10-16 2017-11-10]\n",
      "Window Train/Val/Test shape (1911, 50) (336, 50) (546, 50)\n",
      "Window Train/Val/Test shape (1911, 2) (336, 2) (546, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-10-14] Val [2017-10-15 2017-10-30] Test [2017-10-31 2017-11-10]\n",
      "Window Train/Val/Test shape (2226, 50) (336, 50) (231, 50)\n",
      "Window Train/Val/Test shape (2226, 2) (336, 2) (231, 2)\n",
      "[31.56950543768508, 45.755155241372144, 13.968374179254727, 11.54152957196765, 4.569635334945928]\n",
      "mean mse: 21.480839953045102\n",
      "Train [2017-07-01 2017-08-15] Val [2017-08-16 2017-08-31] Test [2017-09-01 2017-09-16]\n",
      "Window Train/Val/Test shape (966, 50) (336, 50) (336, 50)\n",
      "Window Train/Val/Test shape (966, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-08-30] Val [2017-08-31 2017-09-15] Test [2017-09-16 2017-10-01]\n",
      "Window Train/Val/Test shape (1281, 50) (336, 50) (336, 50)\n",
      "Window Train/Val/Test shape (1281, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-09-14] Val [2017-09-15 2017-09-30] Test [2017-10-01 2017-10-16]\n",
      "Window Train/Val/Test shape (1596, 50) (336, 50) (336, 50)\n",
      "Window Train/Val/Test shape (1596, 2) (336, 2) (336, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-09-29] Val [2017-09-30 2017-10-15] Test [2017-10-16 2017-11-10]\n",
      "Window Train/Val/Test shape (1911, 50) (336, 50) (546, 50)\n",
      "Window Train/Val/Test shape (1911, 2) (336, 2) (546, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-10-14] Val [2017-10-15 2017-10-30] Test [2017-10-31 2017-11-10]\n",
      "Window Train/Val/Test shape (2226, 50) (336, 50) (231, 50)\n",
      "Window Train/Val/Test shape (2226, 2) (336, 2) (231, 2)\n",
      "[28.441060972499788, 35.89522353470015, 12.937067610398568, 9.374433187029464, 3.7748866269803685]\n",
      "mean mse: 18.084534386321668\n",
      "Train [2017-07-01 2017-08-15] Val [2017-08-16 2017-08-31] Test [2017-09-01 2017-09-16]\n",
      "Window Train/Val/Test shape (966, 50) (336, 50) (336, 50)\n",
      "Window Train/Val/Test shape (966, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-08-30] Val [2017-08-31 2017-09-15] Test [2017-09-16 2017-10-01]\n",
      "Window Train/Val/Test shape (1281, 50) (336, 50) (336, 50)\n",
      "Window Train/Val/Test shape (1281, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-09-14] Val [2017-09-15 2017-09-30] Test [2017-10-01 2017-10-16]\n",
      "Window Train/Val/Test shape (1596, 50) (336, 50) (336, 50)\n",
      "Window Train/Val/Test shape (1596, 2) (336, 2) (336, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-09-29] Val [2017-09-30 2017-10-15] Test [2017-10-16 2017-11-10]\n",
      "Window Train/Val/Test shape (1911, 50) (336, 50) (546, 50)\n",
      "Window Train/Val/Test shape (1911, 2) (336, 2) (546, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-10-14] Val [2017-10-15 2017-10-30] Test [2017-10-31 2017-11-10]\n",
      "Window Train/Val/Test shape (2226, 50) (336, 50) (231, 50)\n",
      "Window Train/Val/Test shape (2226, 2) (336, 2) (231, 2)\n",
      "[33.63202544194414, 38.37443197264514, 11.551318611860289, 15.100614276780464, 4.007873009550611]\n",
      "mean mse: 20.53325266255613\n",
      "Train [2017-07-01 2017-08-15] Val [2017-08-16 2017-08-31] Test [2017-09-01 2017-09-16]\n",
      "Window Train/Val/Test shape (966, 50) (336, 50) (336, 50)\n",
      "Window Train/Val/Test shape (966, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-08-30] Val [2017-08-31 2017-09-15] Test [2017-09-16 2017-10-01]\n",
      "Window Train/Val/Test shape (1281, 50) (336, 50) (336, 50)\n",
      "Window Train/Val/Test shape (1281, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-09-14] Val [2017-09-15 2017-09-30] Test [2017-10-01 2017-10-16]\n",
      "Window Train/Val/Test shape (1596, 50) (336, 50) (336, 50)\n",
      "Window Train/Val/Test shape (1596, 2) (336, 2) (336, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-09-29] Val [2017-09-30 2017-10-15] Test [2017-10-16 2017-11-10]\n",
      "Window Train/Val/Test shape (1911, 50) (336, 50) (546, 50)\n",
      "Window Train/Val/Test shape (1911, 2) (336, 2) (546, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-10-14] Val [2017-10-15 2017-10-30] Test [2017-10-31 2017-11-10]\n",
      "Window Train/Val/Test shape (2226, 50) (336, 50) (231, 50)\n",
      "Window Train/Val/Test shape (2226, 2) (336, 2) (231, 2)\n",
      "[47.10679548921696, 27.76416324730646, 15.023011102713227, 26.035081564900203, 4.582017405446946]\n",
      "mean mse: 24.10221376191676\n",
      "Train [2017-07-01 2017-08-15] Val [2017-08-16 2017-08-31] Test [2017-09-01 2017-09-16]\n",
      "Window Train/Val/Test shape (966, 50) (336, 50) (336, 50)\n",
      "Window Train/Val/Test shape (966, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-08-30] Val [2017-08-31 2017-09-15] Test [2017-09-16 2017-10-01]\n",
      "Window Train/Val/Test shape (1281, 50) (336, 50) (336, 50)\n",
      "Window Train/Val/Test shape (1281, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-09-14] Val [2017-09-15 2017-09-30] Test [2017-10-01 2017-10-16]\n",
      "Window Train/Val/Test shape (1596, 50) (336, 50) (336, 50)\n",
      "Window Train/Val/Test shape (1596, 2) (336, 2) (336, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-09-29] Val [2017-09-30 2017-10-15] Test [2017-10-16 2017-11-10]\n",
      "Window Train/Val/Test shape (1911, 50) (336, 50) (546, 50)\n",
      "Window Train/Val/Test shape (1911, 2) (336, 2) (546, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-10-14] Val [2017-10-15 2017-10-30] Test [2017-10-31 2017-11-10]\n",
      "Window Train/Val/Test shape (2226, 50) (336, 50) (231, 50)\n",
      "Window Train/Val/Test shape (2226, 2) (336, 2) (231, 2)\n",
      "[31.475143665087266, 42.328136897678796, 14.649853368938869, 13.736485886015824, 3.311673990977205]\n",
      "mean mse: 21.100258761739592\n",
      "Train [2017-07-01 2017-08-15] Val [2017-08-16 2017-08-31] Test [2017-09-01 2017-09-16]\n",
      "Window Train/Val/Test shape (966, 50) (336, 50) (336, 50)\n",
      "Window Train/Val/Test shape (966, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-08-30] Val [2017-08-31 2017-09-15] Test [2017-09-16 2017-10-01]\n",
      "Window Train/Val/Test shape (1281, 50) (336, 50) (336, 50)\n",
      "Window Train/Val/Test shape (1281, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-09-14] Val [2017-09-15 2017-09-30] Test [2017-10-01 2017-10-16]\n",
      "Window Train/Val/Test shape (1596, 50) (336, 50) (336, 50)\n",
      "Window Train/Val/Test shape (1596, 2) (336, 2) (336, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-09-29] Val [2017-09-30 2017-10-15] Test [2017-10-16 2017-11-10]\n",
      "Window Train/Val/Test shape (1911, 50) (336, 50) (546, 50)\n",
      "Window Train/Val/Test shape (1911, 2) (336, 2) (546, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-10-14] Val [2017-10-15 2017-10-30] Test [2017-10-31 2017-11-10]\n",
      "Window Train/Val/Test shape (2226, 50) (336, 50) (231, 50)\n",
      "Window Train/Val/Test shape (2226, 2) (336, 2) (231, 2)\n",
      "[43.01401206485955, 41.68978716366461, 14.092667467727646, 20.305542203678986, 8.588578729352568]\n",
      "mean mse: 25.53811752585667\n",
      "Train [2017-07-01 2017-08-15] Val [2017-08-16 2017-08-31] Test [2017-09-01 2017-09-16]\n",
      "Window Train/Val/Test shape (966, 50) (336, 50) (336, 50)\n",
      "Window Train/Val/Test shape (966, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-08-30] Val [2017-08-31 2017-09-15] Test [2017-09-16 2017-10-01]\n",
      "Window Train/Val/Test shape (1281, 50) (336, 50) (336, 50)\n",
      "Window Train/Val/Test shape (1281, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-09-14] Val [2017-09-15 2017-09-30] Test [2017-10-01 2017-10-16]\n",
      "Window Train/Val/Test shape (1596, 50) (336, 50) (336, 50)\n",
      "Window Train/Val/Test shape (1596, 2) (336, 2) (336, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-09-29] Val [2017-09-30 2017-10-15] Test [2017-10-16 2017-11-10]\n",
      "Window Train/Val/Test shape (1911, 50) (336, 50) (546, 50)\n",
      "Window Train/Val/Test shape (1911, 2) (336, 2) (546, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-10-14] Val [2017-10-15 2017-10-30] Test [2017-10-31 2017-11-10]\n",
      "Window Train/Val/Test shape (2226, 50) (336, 50) (231, 50)\n",
      "Window Train/Val/Test shape (2226, 2) (336, 2) (231, 2)\n",
      "[539.6409815294328, 355.77831089297484, 283.344378522231, 125.59562762197143, 105.15474783392483]\n",
      "mean mse: 281.90280928010696\n",
      "Train [2017-07-01 2017-08-15] Val [2017-08-16 2017-08-31] Test [2017-09-01 2017-09-16]\n",
      "Window Train/Val/Test shape (966, 50) (336, 50) (336, 50)\n",
      "Window Train/Val/Test shape (966, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-08-30] Val [2017-08-31 2017-09-15] Test [2017-09-16 2017-10-01]\n",
      "Window Train/Val/Test shape (1281, 50) (336, 50) (336, 50)\n",
      "Window Train/Val/Test shape (1281, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-09-14] Val [2017-09-15 2017-09-30] Test [2017-10-01 2017-10-16]\n",
      "Window Train/Val/Test shape (1596, 50) (336, 50) (336, 50)\n",
      "Window Train/Val/Test shape (1596, 2) (336, 2) (336, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-09-29] Val [2017-09-30 2017-10-15] Test [2017-10-16 2017-11-10]\n",
      "Window Train/Val/Test shape (1911, 50) (336, 50) (546, 50)\n",
      "Window Train/Val/Test shape (1911, 2) (336, 2) (546, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-10-14] Val [2017-10-15 2017-10-30] Test [2017-10-31 2017-11-10]\n",
      "Window Train/Val/Test shape (2226, 50) (336, 50) (231, 50)\n",
      "Window Train/Val/Test shape (2226, 2) (336, 2) (231, 2)\n",
      "[539.6409815294328, 355.77831089297484, 283.344378522231, 125.59562762197143, 105.15474783392483]\n",
      "mean mse: 281.90280928010696\n",
      "Train [2017-07-01 2017-08-15] Val [2017-08-16 2017-08-31] Test [2017-09-01 2017-09-16]\n",
      "Window Train/Val/Test shape (966, 50) (336, 50) (336, 50)\n",
      "Window Train/Val/Test shape (966, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-08-30] Val [2017-08-31 2017-09-15] Test [2017-09-16 2017-10-01]\n",
      "Window Train/Val/Test shape (1281, 50) (336, 50) (336, 50)\n",
      "Window Train/Val/Test shape (1281, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-09-14] Val [2017-09-15 2017-09-30] Test [2017-10-01 2017-10-16]\n",
      "Window Train/Val/Test shape (1596, 50) (336, 50) (336, 50)\n",
      "Window Train/Val/Test shape (1596, 2) (336, 2) (336, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-09-29] Val [2017-09-30 2017-10-15] Test [2017-10-16 2017-11-10]\n",
      "Window Train/Val/Test shape (1911, 50) (336, 50) (546, 50)\n",
      "Window Train/Val/Test shape (1911, 2) (336, 2) (546, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-10-14] Val [2017-10-15 2017-10-30] Test [2017-10-31 2017-11-10]\n",
      "Window Train/Val/Test shape (2226, 50) (336, 50) (231, 50)\n",
      "Window Train/Val/Test shape (2226, 2) (336, 2) (231, 2)\n",
      "[539.6409815294328, 355.77831089297484, 283.344378522231, 125.59562762197143, 105.15474783392483]\n",
      "mean mse: 281.90280928010696\n",
      "Train [2017-07-01 2017-08-15] Val [2017-08-16 2017-08-31] Test [2017-09-01 2017-09-16]\n",
      "Window Train/Val/Test shape (966, 50) (336, 50) (336, 50)\n",
      "Window Train/Val/Test shape (966, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-08-30] Val [2017-08-31 2017-09-15] Test [2017-09-16 2017-10-01]\n",
      "Window Train/Val/Test shape (1281, 50) (336, 50) (336, 50)\n",
      "Window Train/Val/Test shape (1281, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-09-14] Val [2017-09-15 2017-09-30] Test [2017-10-01 2017-10-16]\n",
      "Window Train/Val/Test shape (1596, 50) (336, 50) (336, 50)\n",
      "Window Train/Val/Test shape (1596, 2) (336, 2) (336, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-09-29] Val [2017-09-30 2017-10-15] Test [2017-10-16 2017-11-10]\n",
      "Window Train/Val/Test shape (1911, 50) (336, 50) (546, 50)\n",
      "Window Train/Val/Test shape (1911, 2) (336, 2) (546, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-10-14] Val [2017-10-15 2017-10-30] Test [2017-10-31 2017-11-10]\n",
      "Window Train/Val/Test shape (2226, 50) (336, 50) (231, 50)\n",
      "Window Train/Val/Test shape (2226, 2) (336, 2) (231, 2)\n",
      "[37.048418808249885, 44.4917482356115, 14.401700045622736, 14.912462505604235, 4.620421423278466]\n",
      "mean mse: 23.094950203673367\n",
      "Train [2017-07-01 2017-08-15] Val [2017-08-16 2017-08-31] Test [2017-09-01 2017-09-16]\n",
      "Window Train/Val/Test shape (966, 50) (336, 50) (336, 50)\n",
      "Window Train/Val/Test shape (966, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-08-30] Val [2017-08-31 2017-09-15] Test [2017-09-16 2017-10-01]\n",
      "Window Train/Val/Test shape (1281, 50) (336, 50) (336, 50)\n",
      "Window Train/Val/Test shape (1281, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-09-14] Val [2017-09-15 2017-09-30] Test [2017-10-01 2017-10-16]\n",
      "Window Train/Val/Test shape (1596, 50) (336, 50) (336, 50)\n",
      "Window Train/Val/Test shape (1596, 2) (336, 2) (336, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-09-29] Val [2017-09-30 2017-10-15] Test [2017-10-16 2017-11-10]\n",
      "Window Train/Val/Test shape (1911, 50) (336, 50) (546, 50)\n",
      "Window Train/Val/Test shape (1911, 2) (336, 2) (546, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-10-14] Val [2017-10-15 2017-10-30] Test [2017-10-31 2017-11-10]\n",
      "Window Train/Val/Test shape (2226, 50) (336, 50) (231, 50)\n",
      "Window Train/Val/Test shape (2226, 2) (336, 2) (231, 2)\n",
      "[37.27432330032535, 49.531927706640715, 12.453043598196132, 14.190601833885603, 3.908238792684193]\n",
      "mean mse: 23.4716270463464\n",
      "Train [2017-07-01 2017-08-15] Val [2017-08-16 2017-08-31] Test [2017-09-01 2017-09-16]\n",
      "Window Train/Val/Test shape (966, 50) (336, 50) (336, 50)\n",
      "Window Train/Val/Test shape (966, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-08-30] Val [2017-08-31 2017-09-15] Test [2017-09-16 2017-10-01]\n",
      "Window Train/Val/Test shape (1281, 50) (336, 50) (336, 50)\n",
      "Window Train/Val/Test shape (1281, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-09-14] Val [2017-09-15 2017-09-30] Test [2017-10-01 2017-10-16]\n",
      "Window Train/Val/Test shape (1596, 50) (336, 50) (336, 50)\n",
      "Window Train/Val/Test shape (1596, 2) (336, 2) (336, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-09-29] Val [2017-09-30 2017-10-15] Test [2017-10-16 2017-11-10]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Window Train/Val/Test shape (1911, 50) (336, 50) (546, 50)\n",
      "Window Train/Val/Test shape (1911, 2) (336, 2) (546, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-10-14] Val [2017-10-15 2017-10-30] Test [2017-10-31 2017-11-10]\n",
      "Window Train/Val/Test shape (2226, 50) (336, 50) (231, 50)\n",
      "Window Train/Val/Test shape (2226, 2) (336, 2) (231, 2)\n",
      "[27.853518092763743, 32.56248486031864, 14.502137850007026, 10.185312187699644, 4.226822152454095]\n",
      "mean mse: 17.86605502864863\n",
      "Train [2017-07-01 2017-08-15] Val [2017-08-16 2017-08-31] Test [2017-09-01 2017-09-16]\n",
      "Window Train/Val/Test shape (966, 50) (336, 50) (336, 50)\n",
      "Window Train/Val/Test shape (966, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-08-30] Val [2017-08-31 2017-09-15] Test [2017-09-16 2017-10-01]\n",
      "Window Train/Val/Test shape (1281, 50) (336, 50) (336, 50)\n",
      "Window Train/Val/Test shape (1281, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-09-14] Val [2017-09-15 2017-09-30] Test [2017-10-01 2017-10-16]\n",
      "Window Train/Val/Test shape (1596, 50) (336, 50) (336, 50)\n",
      "Window Train/Val/Test shape (1596, 2) (336, 2) (336, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-09-29] Val [2017-09-30 2017-10-15] Test [2017-10-16 2017-11-10]\n",
      "Window Train/Val/Test shape (1911, 50) (336, 50) (546, 50)\n",
      "Window Train/Val/Test shape (1911, 2) (336, 2) (546, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-10-14] Val [2017-10-15 2017-10-30] Test [2017-10-31 2017-11-10]\n",
      "Window Train/Val/Test shape (2226, 50) (336, 50) (231, 50)\n",
      "Window Train/Val/Test shape (2226, 2) (336, 2) (231, 2)\n",
      "[28.873716793784855, 39.137511877125526, 12.84457948173919, 8.039566901789247, 5.5883241908591925]\n",
      "mean mse: 18.896739849059603\n",
      "Train [2017-07-01 2017-08-15] Val [2017-08-16 2017-08-31] Test [2017-09-01 2017-09-16]\n",
      "Window Train/Val/Test shape (966, 50) (336, 50) (336, 50)\n",
      "Window Train/Val/Test shape (966, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-08-30] Val [2017-08-31 2017-09-15] Test [2017-09-16 2017-10-01]\n",
      "Window Train/Val/Test shape (1281, 50) (336, 50) (336, 50)\n",
      "Window Train/Val/Test shape (1281, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-09-14] Val [2017-09-15 2017-09-30] Test [2017-10-01 2017-10-16]\n",
      "Window Train/Val/Test shape (1596, 50) (336, 50) (336, 50)\n",
      "Window Train/Val/Test shape (1596, 2) (336, 2) (336, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-09-29] Val [2017-09-30 2017-10-15] Test [2017-10-16 2017-11-10]\n",
      "Window Train/Val/Test shape (1911, 50) (336, 50) (546, 50)\n",
      "Window Train/Val/Test shape (1911, 2) (336, 2) (546, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-10-14] Val [2017-10-15 2017-10-30] Test [2017-10-31 2017-11-10]\n",
      "Window Train/Val/Test shape (2226, 50) (336, 50) (231, 50)\n",
      "Window Train/Val/Test shape (2226, 2) (336, 2) (231, 2)\n",
      "[539.6409815294328, 355.77831089297484, 283.344378522231, 125.59562762197143, 105.15474783392483]\n",
      "mean mse: 281.90280928010696\n",
      "Train [2017-07-01 2017-08-15] Val [2017-08-16 2017-08-31] Test [2017-09-01 2017-09-16]\n",
      "Window Train/Val/Test shape (966, 50) (336, 50) (336, 50)\n",
      "Window Train/Val/Test shape (966, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-08-30] Val [2017-08-31 2017-09-15] Test [2017-09-16 2017-10-01]\n",
      "Window Train/Val/Test shape (1281, 50) (336, 50) (336, 50)\n",
      "Window Train/Val/Test shape (1281, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-09-14] Val [2017-09-15 2017-09-30] Test [2017-10-01 2017-10-16]\n",
      "Window Train/Val/Test shape (1596, 50) (336, 50) (336, 50)\n",
      "Window Train/Val/Test shape (1596, 2) (336, 2) (336, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-09-29] Val [2017-09-30 2017-10-15] Test [2017-10-16 2017-11-10]\n",
      "Window Train/Val/Test shape (1911, 50) (336, 50) (546, 50)\n",
      "Window Train/Val/Test shape (1911, 2) (336, 2) (546, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-10-14] Val [2017-10-15 2017-10-30] Test [2017-10-31 2017-11-10]\n",
      "Window Train/Val/Test shape (2226, 50) (336, 50) (231, 50)\n",
      "Window Train/Val/Test shape (2226, 2) (336, 2) (231, 2)\n",
      "[34.05425362549611, 44.51336170828004, 15.899124020521155, 15.036983903287078, 3.285720982314747]\n",
      "mean mse: 22.557888847979825\n",
      "Train [2017-07-01 2017-08-15] Val [2017-08-16 2017-08-31] Test [2017-09-01 2017-09-16]\n",
      "Window Train/Val/Test shape (966, 50) (336, 50) (336, 50)\n",
      "Window Train/Val/Test shape (966, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-08-30] Val [2017-08-31 2017-09-15] Test [2017-09-16 2017-10-01]\n",
      "Window Train/Val/Test shape (1281, 50) (336, 50) (336, 50)\n",
      "Window Train/Val/Test shape (1281, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-09-14] Val [2017-09-15 2017-09-30] Test [2017-10-01 2017-10-16]\n",
      "Window Train/Val/Test shape (1596, 50) (336, 50) (336, 50)\n",
      "Window Train/Val/Test shape (1596, 2) (336, 2) (336, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-09-29] Val [2017-09-30 2017-10-15] Test [2017-10-16 2017-11-10]\n",
      "Window Train/Val/Test shape (1911, 50) (336, 50) (546, 50)\n",
      "Window Train/Val/Test shape (1911, 2) (336, 2) (546, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-10-14] Val [2017-10-15 2017-10-30] Test [2017-10-31 2017-11-10]\n",
      "Window Train/Val/Test shape (2226, 50) (336, 50) (231, 50)\n",
      "Window Train/Val/Test shape (2226, 2) (336, 2) (231, 2)\n",
      "[539.6409815294328, 355.77831089297484, 283.344378522231, 125.59562762197143, 105.15474783392483]\n",
      "mean mse: 281.90280928010696\n",
      "Train [2017-07-01 2017-08-15] Val [2017-08-16 2017-08-31] Test [2017-09-01 2017-09-16]\n",
      "Window Train/Val/Test shape (966, 50) (336, 50) (336, 50)\n",
      "Window Train/Val/Test shape (966, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-08-30] Val [2017-08-31 2017-09-15] Test [2017-09-16 2017-10-01]\n",
      "Window Train/Val/Test shape (1281, 50) (336, 50) (336, 50)\n",
      "Window Train/Val/Test shape (1281, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-09-14] Val [2017-09-15 2017-09-30] Test [2017-10-01 2017-10-16]\n",
      "Window Train/Val/Test shape (1596, 50) (336, 50) (336, 50)\n",
      "Window Train/Val/Test shape (1596, 2) (336, 2) (336, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-09-29] Val [2017-09-30 2017-10-15] Test [2017-10-16 2017-11-10]\n",
      "Window Train/Val/Test shape (1911, 50) (336, 50) (546, 50)\n",
      "Window Train/Val/Test shape (1911, 2) (336, 2) (546, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-10-14] Val [2017-10-15 2017-10-30] Test [2017-10-31 2017-11-10]\n",
      "Window Train/Val/Test shape (2226, 50) (336, 50) (231, 50)\n",
      "Window Train/Val/Test shape (2226, 2) (336, 2) (231, 2)\n",
      "[27.518681609214532, 31.683182763444588, 14.843707837374618, 8.810589702302789, 3.552489801368092]\n",
      "mean mse: 17.281730342740925\n",
      "Train [2017-07-01 2017-08-15] Val [2017-08-16 2017-08-31] Test [2017-09-01 2017-09-16]\n",
      "Window Train/Val/Test shape (966, 50) (336, 50) (336, 50)\n",
      "Window Train/Val/Test shape (966, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-08-30] Val [2017-08-31 2017-09-15] Test [2017-09-16 2017-10-01]\n",
      "Window Train/Val/Test shape (1281, 50) (336, 50) (336, 50)\n",
      "Window Train/Val/Test shape (1281, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-09-14] Val [2017-09-15 2017-09-30] Test [2017-10-01 2017-10-16]\n",
      "Window Train/Val/Test shape (1596, 50) (336, 50) (336, 50)\n",
      "Window Train/Val/Test shape (1596, 2) (336, 2) (336, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-09-29] Val [2017-09-30 2017-10-15] Test [2017-10-16 2017-11-10]\n",
      "Window Train/Val/Test shape (1911, 50) (336, 50) (546, 50)\n",
      "Window Train/Val/Test shape (1911, 2) (336, 2) (546, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-10-14] Val [2017-10-15 2017-10-30] Test [2017-10-31 2017-11-10]\n",
      "Window Train/Val/Test shape (2226, 50) (336, 50) (231, 50)\n",
      "Window Train/Val/Test shape (2226, 2) (336, 2) (231, 2)\n",
      "[539.6409815294328, 355.77831089297484, 283.344378522231, 125.59562762197143, 105.15474783392483]\n",
      "mean mse: 281.90280928010696\n",
      "Train [2017-07-01 2017-08-15] Val [2017-08-16 2017-08-31] Test [2017-09-01 2017-09-16]\n",
      "Window Train/Val/Test shape (966, 50) (336, 50) (336, 50)\n",
      "Window Train/Val/Test shape (966, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-08-30] Val [2017-08-31 2017-09-15] Test [2017-09-16 2017-10-01]\n",
      "Window Train/Val/Test shape (1281, 50) (336, 50) (336, 50)\n",
      "Window Train/Val/Test shape (1281, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-09-14] Val [2017-09-15 2017-09-30] Test [2017-10-01 2017-10-16]\n",
      "Window Train/Val/Test shape (1596, 50) (336, 50) (336, 50)\n",
      "Window Train/Val/Test shape (1596, 2) (336, 2) (336, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-09-29] Val [2017-09-30 2017-10-15] Test [2017-10-16 2017-11-10]\n",
      "Window Train/Val/Test shape (1911, 50) (336, 50) (546, 50)\n",
      "Window Train/Val/Test shape (1911, 2) (336, 2) (546, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-10-14] Val [2017-10-15 2017-10-30] Test [2017-10-31 2017-11-10]\n",
      "Window Train/Val/Test shape (2226, 50) (336, 50) (231, 50)\n",
      "Window Train/Val/Test shape (2226, 2) (336, 2) (231, 2)\n",
      "[33.563606079047744, 39.765235716675114, 13.797361803164145, 13.816629265060605, 4.4728847245497665]\n",
      "mean mse: 21.083143517699476\n",
      "Train [2017-07-01 2017-08-15] Val [2017-08-16 2017-08-31] Test [2017-09-01 2017-09-16]\n",
      "Window Train/Val/Test shape (966, 50) (336, 50) (336, 50)\n",
      "Window Train/Val/Test shape (966, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-08-30] Val [2017-08-31 2017-09-15] Test [2017-09-16 2017-10-01]\n",
      "Window Train/Val/Test shape (1281, 50) (336, 50) (336, 50)\n",
      "Window Train/Val/Test shape (1281, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-09-14] Val [2017-09-15 2017-09-30] Test [2017-10-01 2017-10-16]\n",
      "Window Train/Val/Test shape (1596, 50) (336, 50) (336, 50)\n",
      "Window Train/Val/Test shape (1596, 2) (336, 2) (336, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-09-29] Val [2017-09-30 2017-10-15] Test [2017-10-16 2017-11-10]\n",
      "Window Train/Val/Test shape (1911, 50) (336, 50) (546, 50)\n",
      "Window Train/Val/Test shape (1911, 2) (336, 2) (546, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-10-14] Val [2017-10-15 2017-10-30] Test [2017-10-31 2017-11-10]\n",
      "Window Train/Val/Test shape (2226, 50) (336, 50) (231, 50)\n",
      "Window Train/Val/Test shape (2226, 2) (336, 2) (231, 2)\n",
      "[539.6409815294328, 355.77831089297484, 283.344378522231, 125.59562762197143, 105.15474783392483]\n",
      "mean mse: 281.90280928010696\n",
      "Train [2017-07-01 2017-08-15] Val [2017-08-16 2017-08-31] Test [2017-09-01 2017-09-16]\n",
      "Window Train/Val/Test shape (966, 50) (336, 50) (336, 50)\n",
      "Window Train/Val/Test shape (966, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-08-30] Val [2017-08-31 2017-09-15] Test [2017-09-16 2017-10-01]\n",
      "Window Train/Val/Test shape (1281, 50) (336, 50) (336, 50)\n",
      "Window Train/Val/Test shape (1281, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-09-14] Val [2017-09-15 2017-09-30] Test [2017-10-01 2017-10-16]\n",
      "Window Train/Val/Test shape (1596, 50) (336, 50) (336, 50)\n",
      "Window Train/Val/Test shape (1596, 2) (336, 2) (336, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-09-29] Val [2017-09-30 2017-10-15] Test [2017-10-16 2017-11-10]\n",
      "Window Train/Val/Test shape (1911, 50) (336, 50) (546, 50)\n",
      "Window Train/Val/Test shape (1911, 2) (336, 2) (546, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-10-14] Val [2017-10-15 2017-10-30] Test [2017-10-31 2017-11-10]\n",
      "Window Train/Val/Test shape (2226, 50) (336, 50) (231, 50)\n",
      "Window Train/Val/Test shape (2226, 2) (336, 2) (231, 2)\n",
      "[27.371267549892817, 35.136468555609994, 15.614219671457656, 9.956885571619168, 3.7710895584955137]\n",
      "mean mse: 18.369986181415033\n",
      "Train [2017-07-01 2017-08-15] Val [2017-08-16 2017-08-31] Test [2017-09-01 2017-09-16]\n",
      "Window Train/Val/Test shape (966, 50) (336, 50) (336, 50)\n",
      "Window Train/Val/Test shape (966, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-08-30] Val [2017-08-31 2017-09-15] Test [2017-09-16 2017-10-01]\n",
      "Window Train/Val/Test shape (1281, 50) (336, 50) (336, 50)\n",
      "Window Train/Val/Test shape (1281, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-09-14] Val [2017-09-15 2017-09-30] Test [2017-10-01 2017-10-16]\n",
      "Window Train/Val/Test shape (1596, 50) (336, 50) (336, 50)\n",
      "Window Train/Val/Test shape (1596, 2) (336, 2) (336, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-09-29] Val [2017-09-30 2017-10-15] Test [2017-10-16 2017-11-10]\n",
      "Window Train/Val/Test shape (1911, 50) (336, 50) (546, 50)\n",
      "Window Train/Val/Test shape (1911, 2) (336, 2) (546, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-10-14] Val [2017-10-15 2017-10-30] Test [2017-10-31 2017-11-10]\n",
      "Window Train/Val/Test shape (2226, 50) (336, 50) (231, 50)\n",
      "Window Train/Val/Test shape (2226, 2) (336, 2) (231, 2)\n",
      "[539.6409815294328, 355.77831089297484, 283.344378522231, 125.59562762197143, 105.15474783392483]\n",
      "mean mse: 281.90280928010696\n",
      "Train [2017-07-01 2017-08-15] Val [2017-08-16 2017-08-31] Test [2017-09-01 2017-09-16]\n",
      "Window Train/Val/Test shape (966, 50) (336, 50) (336, 50)\n",
      "Window Train/Val/Test shape (966, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-08-30] Val [2017-08-31 2017-09-15] Test [2017-09-16 2017-10-01]\n",
      "Window Train/Val/Test shape (1281, 50) (336, 50) (336, 50)\n",
      "Window Train/Val/Test shape (1281, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-09-14] Val [2017-09-15 2017-09-30] Test [2017-10-01 2017-10-16]\n",
      "Window Train/Val/Test shape (1596, 50) (336, 50) (336, 50)\n",
      "Window Train/Val/Test shape (1596, 2) (336, 2) (336, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-09-29] Val [2017-09-30 2017-10-15] Test [2017-10-16 2017-11-10]\n",
      "Window Train/Val/Test shape (1911, 50) (336, 50) (546, 50)\n",
      "Window Train/Val/Test shape (1911, 2) (336, 2) (546, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-10-14] Val [2017-10-15 2017-10-30] Test [2017-10-31 2017-11-10]\n",
      "Window Train/Val/Test shape (2226, 50) (336, 50) (231, 50)\n",
      "Window Train/Val/Test shape (2226, 2) (336, 2) (231, 2)\n",
      "[539.6409815294328, 355.77831089297484, 283.344378522231, 125.59562762197143, 105.15474783392483]\n",
      "mean mse: 281.90280928010696\n",
      "Train [2017-07-01 2017-08-15] Val [2017-08-16 2017-08-31] Test [2017-09-01 2017-09-16]\n",
      "Window Train/Val/Test shape (966, 50) (336, 50) (336, 50)\n",
      "Window Train/Val/Test shape (966, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-08-30] Val [2017-08-31 2017-09-15] Test [2017-09-16 2017-10-01]\n",
      "Window Train/Val/Test shape (1281, 50) (336, 50) (336, 50)\n",
      "Window Train/Val/Test shape (1281, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-09-14] Val [2017-09-15 2017-09-30] Test [2017-10-01 2017-10-16]\n",
      "Window Train/Val/Test shape (1596, 50) (336, 50) (336, 50)\n",
      "Window Train/Val/Test shape (1596, 2) (336, 2) (336, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-09-29] Val [2017-09-30 2017-10-15] Test [2017-10-16 2017-11-10]\n",
      "Window Train/Val/Test shape (1911, 50) (336, 50) (546, 50)\n",
      "Window Train/Val/Test shape (1911, 2) (336, 2) (546, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-10-14] Val [2017-10-15 2017-10-30] Test [2017-10-31 2017-11-10]\n",
      "Window Train/Val/Test shape (2226, 50) (336, 50) (231, 50)\n",
      "Window Train/Val/Test shape (2226, 2) (336, 2) (231, 2)\n",
      "[35.90869995688839, 51.772162126609544, 11.141879689468723, 14.643638828005418, 4.3903653393435915]\n",
      "mean mse: 23.571349188063135\n",
      "Train [2017-07-01 2017-08-15] Val [2017-08-16 2017-08-31] Test [2017-09-01 2017-09-16]\n",
      "Window Train/Val/Test shape (966, 50) (336, 50) (336, 50)\n",
      "Window Train/Val/Test shape (966, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-08-30] Val [2017-08-31 2017-09-15] Test [2017-09-16 2017-10-01]\n",
      "Window Train/Val/Test shape (1281, 50) (336, 50) (336, 50)\n",
      "Window Train/Val/Test shape (1281, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-09-14] Val [2017-09-15 2017-09-30] Test [2017-10-01 2017-10-16]\n",
      "Window Train/Val/Test shape (1596, 50) (336, 50) (336, 50)\n",
      "Window Train/Val/Test shape (1596, 2) (336, 2) (336, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-09-29] Val [2017-09-30 2017-10-15] Test [2017-10-16 2017-11-10]\n",
      "Window Train/Val/Test shape (1911, 50) (336, 50) (546, 50)\n",
      "Window Train/Val/Test shape (1911, 2) (336, 2) (546, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-10-14] Val [2017-10-15 2017-10-30] Test [2017-10-31 2017-11-10]\n",
      "Window Train/Val/Test shape (2226, 50) (336, 50) (231, 50)\n",
      "Window Train/Val/Test shape (2226, 2) (336, 2) (231, 2)\n",
      "[32.8534646448876, 45.38998982699017, 13.038050681665306, 13.945017642128617, 5.109087310338434]\n",
      "mean mse: 22.067122021202028\n",
      "Train [2017-07-01 2017-08-15] Val [2017-08-16 2017-08-31] Test [2017-09-01 2017-09-16]\n",
      "Window Train/Val/Test shape (966, 50) (336, 50) (336, 50)\n",
      "Window Train/Val/Test shape (966, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-08-30] Val [2017-08-31 2017-09-15] Test [2017-09-16 2017-10-01]\n",
      "Window Train/Val/Test shape (1281, 50) (336, 50) (336, 50)\n",
      "Window Train/Val/Test shape (1281, 2) (336, 2) (336, 2)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train [2017-07-01 2017-09-14] Val [2017-09-15 2017-09-30] Test [2017-10-01 2017-10-16]\n",
      "Window Train/Val/Test shape (1596, 50) (336, 50) (336, 50)\n",
      "Window Train/Val/Test shape (1596, 2) (336, 2) (336, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-09-29] Val [2017-09-30 2017-10-15] Test [2017-10-16 2017-11-10]\n",
      "Window Train/Val/Test shape (1911, 50) (336, 50) (546, 50)\n",
      "Window Train/Val/Test shape (1911, 2) (336, 2) (546, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-10-14] Val [2017-10-15 2017-10-30] Test [2017-10-31 2017-11-10]\n",
      "Window Train/Val/Test shape (2226, 50) (336, 50) (231, 50)\n",
      "Window Train/Val/Test shape (2226, 2) (336, 2) (231, 2)\n",
      "[539.6409815294328, 355.77831089297484, 283.344378522231, 125.59562762197143, 105.15474783392483]\n",
      "mean mse: 281.90280928010696\n",
      "Train [2017-07-01 2017-08-15] Val [2017-08-16 2017-08-31] Test [2017-09-01 2017-09-16]\n",
      "Window Train/Val/Test shape (966, 50) (336, 50) (336, 50)\n",
      "Window Train/Val/Test shape (966, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-08-30] Val [2017-08-31 2017-09-15] Test [2017-09-16 2017-10-01]\n",
      "Window Train/Val/Test shape (1281, 50) (336, 50) (336, 50)\n",
      "Window Train/Val/Test shape (1281, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-09-14] Val [2017-09-15 2017-09-30] Test [2017-10-01 2017-10-16]\n",
      "Window Train/Val/Test shape (1596, 50) (336, 50) (336, 50)\n",
      "Window Train/Val/Test shape (1596, 2) (336, 2) (336, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-09-29] Val [2017-09-30 2017-10-15] Test [2017-10-16 2017-11-10]\n",
      "Window Train/Val/Test shape (1911, 50) (336, 50) (546, 50)\n",
      "Window Train/Val/Test shape (1911, 2) (336, 2) (546, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-10-14] Val [2017-10-15 2017-10-30] Test [2017-10-31 2017-11-10]\n",
      "Window Train/Val/Test shape (2226, 50) (336, 50) (231, 50)\n",
      "Window Train/Val/Test shape (2226, 2) (336, 2) (231, 2)\n",
      "[539.6409815294328, 355.77831089297484, 283.344378522231, 125.59562762197143, 105.15474783392483]\n",
      "mean mse: 281.90280928010696\n",
      "Train [2017-07-01 2017-08-15] Val [2017-08-16 2017-08-31] Test [2017-09-01 2017-09-16]\n",
      "Window Train/Val/Test shape (966, 50) (336, 50) (336, 50)\n",
      "Window Train/Val/Test shape (966, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-08-30] Val [2017-08-31 2017-09-15] Test [2017-09-16 2017-10-01]\n",
      "Window Train/Val/Test shape (1281, 50) (336, 50) (336, 50)\n",
      "Window Train/Val/Test shape (1281, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-09-14] Val [2017-09-15 2017-09-30] Test [2017-10-01 2017-10-16]\n",
      "Window Train/Val/Test shape (1596, 50) (336, 50) (336, 50)\n",
      "Window Train/Val/Test shape (1596, 2) (336, 2) (336, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-09-29] Val [2017-09-30 2017-10-15] Test [2017-10-16 2017-11-10]\n",
      "Window Train/Val/Test shape (1911, 50) (336, 50) (546, 50)\n",
      "Window Train/Val/Test shape (1911, 2) (336, 2) (546, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-10-14] Val [2017-10-15 2017-10-30] Test [2017-10-31 2017-11-10]\n",
      "Window Train/Val/Test shape (2226, 50) (336, 50) (231, 50)\n",
      "Window Train/Val/Test shape (2226, 2) (336, 2) (231, 2)\n",
      "[539.6409815294328, 355.77831089297484, 283.344378522231, 125.59562762197143, 105.15474783392483]\n",
      "mean mse: 281.90280928010696\n",
      "Train [2017-07-01 2017-08-15] Val [2017-08-16 2017-08-31] Test [2017-09-01 2017-09-16]\n",
      "Window Train/Val/Test shape (966, 50) (336, 50) (336, 50)\n",
      "Window Train/Val/Test shape (966, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-08-30] Val [2017-08-31 2017-09-15] Test [2017-09-16 2017-10-01]\n",
      "Window Train/Val/Test shape (1281, 50) (336, 50) (336, 50)\n",
      "Window Train/Val/Test shape (1281, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-09-14] Val [2017-09-15 2017-09-30] Test [2017-10-01 2017-10-16]\n",
      "Window Train/Val/Test shape (1596, 50) (336, 50) (336, 50)\n",
      "Window Train/Val/Test shape (1596, 2) (336, 2) (336, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-09-29] Val [2017-09-30 2017-10-15] Test [2017-10-16 2017-11-10]\n",
      "Window Train/Val/Test shape (1911, 50) (336, 50) (546, 50)\n",
      "Window Train/Val/Test shape (1911, 2) (336, 2) (546, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-10-14] Val [2017-10-15 2017-10-30] Test [2017-10-31 2017-11-10]\n",
      "Window Train/Val/Test shape (2226, 50) (336, 50) (231, 50)\n",
      "Window Train/Val/Test shape (2226, 2) (336, 2) (231, 2)\n",
      "[34.13729875109375, 50.06522983145499, 14.615081894887417, 17.57183577566323, 4.868269701041299]\n",
      "mean mse: 24.25154319082814\n",
      "Train [2017-07-01 2017-08-15] Val [2017-08-16 2017-08-31] Test [2017-09-01 2017-09-16]\n",
      "Window Train/Val/Test shape (966, 50) (336, 50) (336, 50)\n",
      "Window Train/Val/Test shape (966, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-08-30] Val [2017-08-31 2017-09-15] Test [2017-09-16 2017-10-01]\n",
      "Window Train/Val/Test shape (1281, 50) (336, 50) (336, 50)\n",
      "Window Train/Val/Test shape (1281, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-09-14] Val [2017-09-15 2017-09-30] Test [2017-10-01 2017-10-16]\n",
      "Window Train/Val/Test shape (1596, 50) (336, 50) (336, 50)\n",
      "Window Train/Val/Test shape (1596, 2) (336, 2) (336, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-09-29] Val [2017-09-30 2017-10-15] Test [2017-10-16 2017-11-10]\n",
      "Window Train/Val/Test shape (1911, 50) (336, 50) (546, 50)\n",
      "Window Train/Val/Test shape (1911, 2) (336, 2) (546, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-10-14] Val [2017-10-15 2017-10-30] Test [2017-10-31 2017-11-10]\n",
      "Window Train/Val/Test shape (2226, 50) (336, 50) (231, 50)\n",
      "Window Train/Val/Test shape (2226, 2) (336, 2) (231, 2)\n",
      "[539.6409815294328, 355.77831089297484, 283.344378522231, 125.59562762197143, 105.15474783392483]\n",
      "mean mse: 281.90280928010696\n",
      "Train [2017-07-01 2017-08-15] Val [2017-08-16 2017-08-31] Test [2017-09-01 2017-09-16]\n",
      "Window Train/Val/Test shape (966, 50) (336, 50) (336, 50)\n",
      "Window Train/Val/Test shape (966, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-08-30] Val [2017-08-31 2017-09-15] Test [2017-09-16 2017-10-01]\n",
      "Window Train/Val/Test shape (1281, 50) (336, 50) (336, 50)\n",
      "Window Train/Val/Test shape (1281, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-09-14] Val [2017-09-15 2017-09-30] Test [2017-10-01 2017-10-16]\n",
      "Window Train/Val/Test shape (1596, 50) (336, 50) (336, 50)\n",
      "Window Train/Val/Test shape (1596, 2) (336, 2) (336, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-09-29] Val [2017-09-30 2017-10-15] Test [2017-10-16 2017-11-10]\n",
      "Window Train/Val/Test shape (1911, 50) (336, 50) (546, 50)\n",
      "Window Train/Val/Test shape (1911, 2) (336, 2) (546, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-10-14] Val [2017-10-15 2017-10-30] Test [2017-10-31 2017-11-10]\n",
      "Window Train/Val/Test shape (2226, 50) (336, 50) (231, 50)\n",
      "Window Train/Val/Test shape (2226, 2) (336, 2) (231, 2)\n",
      "[539.6409815294328, 355.77831089297484, 283.344378522231, 125.59562762197143, 105.15474783392483]\n",
      "mean mse: 281.90280928010696\n",
      "Train [2017-07-01 2017-08-15] Val [2017-08-16 2017-08-31] Test [2017-09-01 2017-09-16]\n",
      "Window Train/Val/Test shape (966, 50) (336, 50) (336, 50)\n",
      "Window Train/Val/Test shape (966, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-08-30] Val [2017-08-31 2017-09-15] Test [2017-09-16 2017-10-01]\n",
      "Window Train/Val/Test shape (1281, 50) (336, 50) (336, 50)\n",
      "Window Train/Val/Test shape (1281, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-09-14] Val [2017-09-15 2017-09-30] Test [2017-10-01 2017-10-16]\n",
      "Window Train/Val/Test shape (1596, 50) (336, 50) (336, 50)\n",
      "Window Train/Val/Test shape (1596, 2) (336, 2) (336, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-09-29] Val [2017-09-30 2017-10-15] Test [2017-10-16 2017-11-10]\n",
      "Window Train/Val/Test shape (1911, 50) (336, 50) (546, 50)\n",
      "Window Train/Val/Test shape (1911, 2) (336, 2) (546, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-10-14] Val [2017-10-15 2017-10-30] Test [2017-10-31 2017-11-10]\n",
      "Window Train/Val/Test shape (2226, 50) (336, 50) (231, 50)\n",
      "Window Train/Val/Test shape (2226, 2) (336, 2) (231, 2)\n",
      "[39.62797005152026, 37.49853199557993, 18.544935420082407, 12.080188418687651, 6.178552089236254]\n",
      "mean mse: 22.7860355950213\n",
      "Train [2017-07-01 2017-08-15] Val [2017-08-16 2017-08-31] Test [2017-09-01 2017-09-16]\n",
      "Window Train/Val/Test shape (966, 50) (336, 50) (336, 50)\n",
      "Window Train/Val/Test shape (966, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-08-30] Val [2017-08-31 2017-09-15] Test [2017-09-16 2017-10-01]\n",
      "Window Train/Val/Test shape (1281, 50) (336, 50) (336, 50)\n",
      "Window Train/Val/Test shape (1281, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-09-14] Val [2017-09-15 2017-09-30] Test [2017-10-01 2017-10-16]\n",
      "Window Train/Val/Test shape (1596, 50) (336, 50) (336, 50)\n",
      "Window Train/Val/Test shape (1596, 2) (336, 2) (336, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-09-29] Val [2017-09-30 2017-10-15] Test [2017-10-16 2017-11-10]\n",
      "Window Train/Val/Test shape (1911, 50) (336, 50) (546, 50)\n",
      "Window Train/Val/Test shape (1911, 2) (336, 2) (546, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-10-14] Val [2017-10-15 2017-10-30] Test [2017-10-31 2017-11-10]\n",
      "Window Train/Val/Test shape (2226, 50) (336, 50) (231, 50)\n",
      "Window Train/Val/Test shape (2226, 2) (336, 2) (231, 2)\n",
      "[539.6409815294328, 355.77831089297484, 283.344378522231, 125.59562762197143, 105.15474783392483]\n",
      "mean mse: 281.90280928010696\n",
      "Train [2017-07-01 2017-08-15] Val [2017-08-16 2017-08-31] Test [2017-09-01 2017-09-16]\n",
      "Window Train/Val/Test shape (966, 50) (336, 50) (336, 50)\n",
      "Window Train/Val/Test shape (966, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-08-30] Val [2017-08-31 2017-09-15] Test [2017-09-16 2017-10-01]\n",
      "Window Train/Val/Test shape (1281, 50) (336, 50) (336, 50)\n",
      "Window Train/Val/Test shape (1281, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-09-14] Val [2017-09-15 2017-09-30] Test [2017-10-01 2017-10-16]\n",
      "Window Train/Val/Test shape (1596, 50) (336, 50) (336, 50)\n",
      "Window Train/Val/Test shape (1596, 2) (336, 2) (336, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-09-29] Val [2017-09-30 2017-10-15] Test [2017-10-16 2017-11-10]\n",
      "Window Train/Val/Test shape (1911, 50) (336, 50) (546, 50)\n",
      "Window Train/Val/Test shape (1911, 2) (336, 2) (546, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-10-14] Val [2017-10-15 2017-10-30] Test [2017-10-31 2017-11-10]\n",
      "Window Train/Val/Test shape (2226, 50) (336, 50) (231, 50)\n",
      "Window Train/Val/Test shape (2226, 2) (336, 2) (231, 2)\n",
      "[39.02659558736884, 36.02749386934026, 12.14010762276125, 12.982106162823417, 4.322901925776791]\n",
      "mean mse: 20.899841033614113\n",
      "Train [2017-07-01 2017-08-15] Val [2017-08-16 2017-08-31] Test [2017-09-01 2017-09-16]\n",
      "Window Train/Val/Test shape (966, 50) (336, 50) (336, 50)\n",
      "Window Train/Val/Test shape (966, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-08-30] Val [2017-08-31 2017-09-15] Test [2017-09-16 2017-10-01]\n",
      "Window Train/Val/Test shape (1281, 50) (336, 50) (336, 50)\n",
      "Window Train/Val/Test shape (1281, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-09-14] Val [2017-09-15 2017-09-30] Test [2017-10-01 2017-10-16]\n",
      "Window Train/Val/Test shape (1596, 50) (336, 50) (336, 50)\n",
      "Window Train/Val/Test shape (1596, 2) (336, 2) (336, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-09-29] Val [2017-09-30 2017-10-15] Test [2017-10-16 2017-11-10]\n",
      "Window Train/Val/Test shape (1911, 50) (336, 50) (546, 50)\n",
      "Window Train/Val/Test shape (1911, 2) (336, 2) (546, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-10-14] Val [2017-10-15 2017-10-30] Test [2017-10-31 2017-11-10]\n",
      "Window Train/Val/Test shape (2226, 50) (336, 50) (231, 50)\n",
      "Window Train/Val/Test shape (2226, 2) (336, 2) (231, 2)\n",
      "[33.9191094302531, 62.26777876082116, 15.876344939882127, 15.672276573832457, 8.040323317985731]\n",
      "mean mse: 27.155166604554914\n",
      "Train [2017-07-01 2017-08-15] Val [2017-08-16 2017-08-31] Test [2017-09-01 2017-09-16]\n",
      "Window Train/Val/Test shape (966, 50) (336, 50) (336, 50)\n",
      "Window Train/Val/Test shape (966, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-08-30] Val [2017-08-31 2017-09-15] Test [2017-09-16 2017-10-01]\n",
      "Window Train/Val/Test shape (1281, 50) (336, 50) (336, 50)\n",
      "Window Train/Val/Test shape (1281, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-09-14] Val [2017-09-15 2017-09-30] Test [2017-10-01 2017-10-16]\n",
      "Window Train/Val/Test shape (1596, 50) (336, 50) (336, 50)\n",
      "Window Train/Val/Test shape (1596, 2) (336, 2) (336, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-09-29] Val [2017-09-30 2017-10-15] Test [2017-10-16 2017-11-10]\n",
      "Window Train/Val/Test shape (1911, 50) (336, 50) (546, 50)\n",
      "Window Train/Val/Test shape (1911, 2) (336, 2) (546, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-10-14] Val [2017-10-15 2017-10-30] Test [2017-10-31 2017-11-10]\n",
      "Window Train/Val/Test shape (2226, 50) (336, 50) (231, 50)\n",
      "Window Train/Val/Test shape (2226, 2) (336, 2) (231, 2)\n",
      "[35.607895312645375, 40.06475683089107, 13.29815093592575, 15.807500297326472, 5.478347223813189]\n",
      "mean mse: 22.051330120120372\n",
      "Train [2017-07-01 2017-08-15] Val [2017-08-16 2017-08-31] Test [2017-09-01 2017-09-16]\n",
      "Window Train/Val/Test shape (966, 50) (336, 50) (336, 50)\n",
      "Window Train/Val/Test shape (966, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-08-30] Val [2017-08-31 2017-09-15] Test [2017-09-16 2017-10-01]\n",
      "Window Train/Val/Test shape (1281, 50) (336, 50) (336, 50)\n",
      "Window Train/Val/Test shape (1281, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-09-14] Val [2017-09-15 2017-09-30] Test [2017-10-01 2017-10-16]\n",
      "Window Train/Val/Test shape (1596, 50) (336, 50) (336, 50)\n",
      "Window Train/Val/Test shape (1596, 2) (336, 2) (336, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-09-29] Val [2017-09-30 2017-10-15] Test [2017-10-16 2017-11-10]\n",
      "Window Train/Val/Test shape (1911, 50) (336, 50) (546, 50)\n",
      "Window Train/Val/Test shape (1911, 2) (336, 2) (546, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-10-14] Val [2017-10-15 2017-10-30] Test [2017-10-31 2017-11-10]\n",
      "Window Train/Val/Test shape (2226, 50) (336, 50) (231, 50)\n",
      "Window Train/Val/Test shape (2226, 2) (336, 2) (231, 2)\n",
      "[539.6409815294328, 355.77831089297484, 283.344378522231, 125.59562762197143, 105.15474783392483]\n",
      "mean mse: 281.90280928010696\n",
      "Train [2017-07-01 2017-08-15] Val [2017-08-16 2017-08-31] Test [2017-09-01 2017-09-16]\n",
      "Window Train/Val/Test shape (966, 50) (336, 50) (336, 50)\n",
      "Window Train/Val/Test shape (966, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-08-30] Val [2017-08-31 2017-09-15] Test [2017-09-16 2017-10-01]\n",
      "Window Train/Val/Test shape (1281, 50) (336, 50) (336, 50)\n",
      "Window Train/Val/Test shape (1281, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-09-14] Val [2017-09-15 2017-09-30] Test [2017-10-01 2017-10-16]\n",
      "Window Train/Val/Test shape (1596, 50) (336, 50) (336, 50)\n",
      "Window Train/Val/Test shape (1596, 2) (336, 2) (336, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-09-29] Val [2017-09-30 2017-10-15] Test [2017-10-16 2017-11-10]\n",
      "Window Train/Val/Test shape (1911, 50) (336, 50) (546, 50)\n",
      "Window Train/Val/Test shape (1911, 2) (336, 2) (546, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-10-14] Val [2017-10-15 2017-10-30] Test [2017-10-31 2017-11-10]\n",
      "Window Train/Val/Test shape (2226, 50) (336, 50) (231, 50)\n",
      "Window Train/Val/Test shape (2226, 2) (336, 2) (231, 2)\n",
      "[30.52303042816008, 40.62508513954773, 13.180652254994222, 15.732404480509839, 6.746646710654928]\n",
      "mean mse: 21.361563802773357\n",
      "Train [2017-07-01 2017-08-15] Val [2017-08-16 2017-08-31] Test [2017-09-01 2017-09-16]\n",
      "Window Train/Val/Test shape (966, 50) (336, 50) (336, 50)\n",
      "Window Train/Val/Test shape (966, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-08-30] Val [2017-08-31 2017-09-15] Test [2017-09-16 2017-10-01]\n",
      "Window Train/Val/Test shape (1281, 50) (336, 50) (336, 50)\n",
      "Window Train/Val/Test shape (1281, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-09-14] Val [2017-09-15 2017-09-30] Test [2017-10-01 2017-10-16]\n",
      "Window Train/Val/Test shape (1596, 50) (336, 50) (336, 50)\n",
      "Window Train/Val/Test shape (1596, 2) (336, 2) (336, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-09-29] Val [2017-09-30 2017-10-15] Test [2017-10-16 2017-11-10]\n",
      "Window Train/Val/Test shape (1911, 50) (336, 50) (546, 50)\n",
      "Window Train/Val/Test shape (1911, 2) (336, 2) (546, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-10-14] Val [2017-10-15 2017-10-30] Test [2017-10-31 2017-11-10]\n",
      "Window Train/Val/Test shape (2226, 50) (336, 50) (231, 50)\n",
      "Window Train/Val/Test shape (2226, 2) (336, 2) (231, 2)\n",
      "[539.6409815294328, 355.77831089297484, 283.344378522231, 125.59562762197143, 105.15474783392483]\n",
      "mean mse: 281.90280928010696\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train [2017-07-01 2017-08-15] Val [2017-08-16 2017-08-31] Test [2017-09-01 2017-09-16]\n",
      "Window Train/Val/Test shape (966, 50) (336, 50) (336, 50)\n",
      "Window Train/Val/Test shape (966, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-08-30] Val [2017-08-31 2017-09-15] Test [2017-09-16 2017-10-01]\n",
      "Window Train/Val/Test shape (1281, 50) (336, 50) (336, 50)\n",
      "Window Train/Val/Test shape (1281, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-09-14] Val [2017-09-15 2017-09-30] Test [2017-10-01 2017-10-16]\n",
      "Window Train/Val/Test shape (1596, 50) (336, 50) (336, 50)\n",
      "Window Train/Val/Test shape (1596, 2) (336, 2) (336, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-09-29] Val [2017-09-30 2017-10-15] Test [2017-10-16 2017-11-10]\n",
      "Window Train/Val/Test shape (1911, 50) (336, 50) (546, 50)\n",
      "Window Train/Val/Test shape (1911, 2) (336, 2) (546, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-10-14] Val [2017-10-15 2017-10-30] Test [2017-10-31 2017-11-10]\n",
      "Window Train/Val/Test shape (2226, 50) (336, 50) (231, 50)\n",
      "Window Train/Val/Test shape (2226, 2) (336, 2) (231, 2)\n",
      "[539.6409815294328, 355.77831089297484, 283.344378522231, 125.59562762197143, 105.15474783392483]\n",
      "mean mse: 281.90280928010696\n",
      "Train [2017-07-01 2017-08-15] Val [2017-08-16 2017-08-31] Test [2017-09-01 2017-09-16]\n",
      "Window Train/Val/Test shape (966, 50) (336, 50) (336, 50)\n",
      "Window Train/Val/Test shape (966, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-08-30] Val [2017-08-31 2017-09-15] Test [2017-09-16 2017-10-01]\n",
      "Window Train/Val/Test shape (1281, 50) (336, 50) (336, 50)\n",
      "Window Train/Val/Test shape (1281, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-09-14] Val [2017-09-15 2017-09-30] Test [2017-10-01 2017-10-16]\n",
      "Window Train/Val/Test shape (1596, 50) (336, 50) (336, 50)\n",
      "Window Train/Val/Test shape (1596, 2) (336, 2) (336, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-09-29] Val [2017-09-30 2017-10-15] Test [2017-10-16 2017-11-10]\n",
      "Window Train/Val/Test shape (1911, 50) (336, 50) (546, 50)\n",
      "Window Train/Val/Test shape (1911, 2) (336, 2) (546, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-10-14] Val [2017-10-15 2017-10-30] Test [2017-10-31 2017-11-10]\n",
      "Window Train/Val/Test shape (2226, 50) (336, 50) (231, 50)\n",
      "Window Train/Val/Test shape (2226, 2) (336, 2) (231, 2)\n",
      "[539.6409815294328, 355.77831089297484, 283.344378522231, 125.59562762197143, 105.15474783392483]\n",
      "mean mse: 281.90280928010696\n",
      "Train [2017-07-01 2017-08-15] Val [2017-08-16 2017-08-31] Test [2017-09-01 2017-09-16]\n",
      "Window Train/Val/Test shape (966, 50) (336, 50) (336, 50)\n",
      "Window Train/Val/Test shape (966, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-08-30] Val [2017-08-31 2017-09-15] Test [2017-09-16 2017-10-01]\n",
      "Window Train/Val/Test shape (1281, 50) (336, 50) (336, 50)\n",
      "Window Train/Val/Test shape (1281, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-09-14] Val [2017-09-15 2017-09-30] Test [2017-10-01 2017-10-16]\n",
      "Window Train/Val/Test shape (1596, 50) (336, 50) (336, 50)\n",
      "Window Train/Val/Test shape (1596, 2) (336, 2) (336, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-09-29] Val [2017-09-30 2017-10-15] Test [2017-10-16 2017-11-10]\n",
      "Window Train/Val/Test shape (1911, 50) (336, 50) (546, 50)\n",
      "Window Train/Val/Test shape (1911, 2) (336, 2) (546, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-10-14] Val [2017-10-15 2017-10-30] Test [2017-10-31 2017-11-10]\n",
      "Window Train/Val/Test shape (2226, 50) (336, 50) (231, 50)\n",
      "Window Train/Val/Test shape (2226, 2) (336, 2) (231, 2)\n",
      "[539.6409815294328, 355.77831089297484, 283.344378522231, 125.59562762197143, 105.15474783392483]\n",
      "mean mse: 281.90280928010696\n",
      "Train [2017-07-01 2017-08-15] Val [2017-08-16 2017-08-31] Test [2017-09-01 2017-09-16]\n",
      "Window Train/Val/Test shape (966, 50) (336, 50) (336, 50)\n",
      "Window Train/Val/Test shape (966, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-08-30] Val [2017-08-31 2017-09-15] Test [2017-09-16 2017-10-01]\n",
      "Window Train/Val/Test shape (1281, 50) (336, 50) (336, 50)\n",
      "Window Train/Val/Test shape (1281, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-09-14] Val [2017-09-15 2017-09-30] Test [2017-10-01 2017-10-16]\n",
      "Window Train/Val/Test shape (1596, 50) (336, 50) (336, 50)\n",
      "Window Train/Val/Test shape (1596, 2) (336, 2) (336, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-09-29] Val [2017-09-30 2017-10-15] Test [2017-10-16 2017-11-10]\n",
      "Window Train/Val/Test shape (1911, 50) (336, 50) (546, 50)\n",
      "Window Train/Val/Test shape (1911, 2) (336, 2) (546, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-10-14] Val [2017-10-15 2017-10-30] Test [2017-10-31 2017-11-10]\n",
      "Window Train/Val/Test shape (2226, 50) (336, 50) (231, 50)\n",
      "Window Train/Val/Test shape (2226, 2) (336, 2) (231, 2)\n",
      "[539.6409815294328, 355.77831089297484, 283.344378522231, 125.59562762197143, 105.15474783392483]\n",
      "mean mse: 281.90280928010696\n"
     ]
    }
   ],
   "source": [
    "orchestrator46.performHyperparameterOptimization(X_46_preselect[selected_features46], 100)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 70,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'colsample_bytree': 0.6000000000000001,\n",
       " 'gamma': 3.5,\n",
       " 'learning_rate': 0.025,\n",
       " 'max_depth': 9.0,\n",
       " 'min_child_weight': 8.0,\n",
       " 'n_estimators': 100.0,\n",
       " 'subsample': 0.7000000000000001}"
      ]
     },
     "execution_count": 70,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "orchestrator46.bo.best"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {},
   "outputs": [],
   "source": [
    "orchestrator46.corr_selected_features_bi_week[selected_features46].to_csv('/Users/Rohil/Documents/iGEM/yemen/intermediary_files/X_4_6_postselect.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "orchestrator46.corr_selected_features_bi_week_y.to_csv('/Users/Rohil/Documents/iGEM/yemen/intermediary_files/y_4_6.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 93,
   "metadata": {},
   "outputs": [],
   "source": [
    "postselect_params_46 = {'colsample_bytree': 0.60,\n",
    "                         'gamma': 3.5,\n",
    "                         'learning_rate': 0.025,\n",
    "                         'max_depth': 9,\n",
    "                         'min_child_weight': 8.0,\n",
    "                         'n_estimators': 100,\n",
    "                         'subsample': 0.70}\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 164,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\Rohil\\Anaconda3\\lib\\site-packages\\ipykernel_launcher.py:78: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/indexing.html#indexing-view-versus-copy\n",
      "C:\\Users\\Rohil\\Anaconda3\\lib\\site-packages\\pandas\\core\\frame.py:3694: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame\n",
      "\n",
      "See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/indexing.html#indexing-view-versus-copy\n",
      "  errors=errors)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train [2017-07-01 2017-08-15] Val [2017-08-16 2017-08-31] Test [2017-09-01 2017-09-16]\n",
      "Window Train/Val/Test shape (966, 50) (336, 50) (336, 50)\n",
      "Window Train/Val/Test shape (966, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-08-30] Val [2017-08-31 2017-09-15] Test [2017-09-16 2017-10-01]\n",
      "Window Train/Val/Test shape (1281, 50) (336, 50) (336, 50)\n",
      "Window Train/Val/Test shape (1281, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-09-14] Val [2017-09-15 2017-09-30] Test [2017-10-01 2017-10-16]\n",
      "Window Train/Val/Test shape (1596, 50) (336, 50) (336, 50)\n",
      "Window Train/Val/Test shape (1596, 2) (336, 2) (336, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-09-29] Val [2017-09-30 2017-10-15] Test [2017-10-16 2017-11-10]\n",
      "Window Train/Val/Test shape (1911, 50) (336, 50) (546, 50)\n",
      "Window Train/Val/Test shape (1911, 2) (336, 2) (546, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-10-14] Val [2017-10-15 2017-10-30] Test [2017-10-31 2017-11-10]\n",
      "Window Train/Val/Test shape (2226, 50) (336, 50) (231, 50)\n",
      "Window Train/Val/Test shape (2226, 2) (336, 2) (231, 2)\n",
      "y-test mean 7.30661865609446, y-test std 7.669725569042983\n",
      "cv mse 17.092727946299544, holdout mse21.663910137906797\n"
     ]
    }
   ],
   "source": [
    "orchestrator46.deploy(0.001844, postselect_params_46, None)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {},
   "outputs": [],
   "source": [
    "# running feature selection for 6-8 week model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {},
   "outputs": [],
   "source": [
    "orchestrator68 = Orchestrator(full_features, full_data_6_8, 'week_6_to_8_cases' )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Running TSFresh....\n",
      "Selected features (3717, 15087)\n",
      "Finished running TSFresh....\n"
     ]
    }
   ],
   "source": [
    "orchestrator68.runTsFresh()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Running Feature Selection module ....\n",
      "4719 features with a correlation magnitude greater than 0.97.\n",
      "\n",
      "Original (3717, 15088) and after (3717, 10369)\n",
      "Finished running Feature Selection ....\n"
     ]
    }
   ],
   "source": [
    "orchestrator68.runOptimizeFeatures()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 68,
   "metadata": {},
   "outputs": [],
   "source": [
    "X_68_preselect = orchestrator68.op.corr_selected_features\n",
    "y_68_preselect = orchestrator68.full_data_bi_week[['date', orchestrator68.y_bi_week_label]]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 69,
   "metadata": {},
   "outputs": [],
   "source": [
    "# X_68_preselect.to_csv('/Users/Rohil/Documents/iGEM/yemen/intermediary_files/X_68_preselect.csv')\n",
    "# y_68_preselect.to_csv('/Users/Rohil/Documents/iGEM/yemen/intermediary_files/y_68_preselect.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [],
   "source": [
    "orchestrator68 = Orchestrator(y6_8.dropna().merge(full_features, how = 'left', on = ['gov_iso', 'date']).sort_values('date'), 'week_6_to_8_cases' )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [],
   "source": [
    "# X_68_preselect = pd.read_csv('/Users/Rohil/Documents/iGEM/yemen/intermediary_files/X_68_preselect.csv', index_col = 0)\n",
    "# X_68_preselect.date = pd.to_datetime(X_68_preselect.date, format = '%Y-%m-%d')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 108,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train [2017-07-01 2017-08-15] Val [2017-08-16 2017-08-31] Test [2017-09-01 2017-09-16]\n",
      "Window Train/Val/Test shape (966, 10369) (336, 10369) (336, 10369)\n",
      "Window Train/Val/Test shape (966, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-08-30] Val [2017-08-31 2017-09-15] Test [2017-09-16 2017-10-01]\n",
      "Window Train/Val/Test shape (1281, 10369) (336, 10369) (336, 10369)\n",
      "Window Train/Val/Test shape (1281, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-09-14] Val [2017-09-15 2017-09-30] Test [2017-10-01 2017-10-16]\n",
      "Window Train/Val/Test shape (1596, 10369) (336, 10369) (336, 10369)\n",
      "Window Train/Val/Test shape (1596, 2) (336, 2) (336, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-09-29] Val [2017-09-30 2017-10-15] Test [2017-10-16 2017-11-10]\n",
      "Window Train/Val/Test shape (1911, 10369) (336, 10369) (546, 10369)\n",
      "Window Train/Val/Test shape (1911, 2) (336, 2) (546, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-10-14] Val [2017-10-15 2017-10-30] Test [2017-10-31 2017-11-10]\n",
      "Window Train/Val/Test shape (2226, 10369) (336, 10369) (231, 10369)\n",
      "Window Train/Val/Test shape (2226, 2) (336, 2) (231, 2)\n",
      "[33.31603314354542, 47.028121603984424, 21.695481314778256, 14.401751971928052, 11.666859051489427]\n",
      "mean mse: 25.621649417145115\n",
      "Train [2017-07-01 2017-08-15] Val [2017-08-16 2017-08-31] Test [2017-09-01 2017-09-16]\n",
      "Window Train/Val/Test shape (966, 10369) (336, 10369) (336, 10369)\n",
      "Window Train/Val/Test shape (966, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-08-30] Val [2017-08-31 2017-09-15] Test [2017-09-16 2017-10-01]\n",
      "Window Train/Val/Test shape (1281, 10369) (336, 10369) (336, 10369)\n",
      "Window Train/Val/Test shape (1281, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-09-14] Val [2017-09-15 2017-09-30] Test [2017-10-01 2017-10-16]\n",
      "Window Train/Val/Test shape (1596, 10369) (336, 10369) (336, 10369)\n",
      "Window Train/Val/Test shape (1596, 2) (336, 2) (336, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-09-29] Val [2017-09-30 2017-10-15] Test [2017-10-16 2017-11-10]\n",
      "Window Train/Val/Test shape (1911, 10369) (336, 10369) (546, 10369)\n",
      "Window Train/Val/Test shape (1911, 2) (336, 2) (546, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-10-14] Val [2017-10-15 2017-10-30] Test [2017-10-31 2017-11-10]\n",
      "Window Train/Val/Test shape (2226, 10369) (336, 10369) (231, 10369)\n",
      "Window Train/Val/Test shape (2226, 2) (336, 2) (231, 2)\n",
      "[31.043038586267325, 37.10613585179067, 26.259731031452908, 11.538582782009277, 8.212363837088786]\n",
      "mean mse: 22.83197041772179\n",
      "Train [2017-07-01 2017-08-15] Val [2017-08-16 2017-08-31] Test [2017-09-01 2017-09-16]\n",
      "Window Train/Val/Test shape (966, 10369) (336, 10369) (336, 10369)\n",
      "Window Train/Val/Test shape (966, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-08-30] Val [2017-08-31 2017-09-15] Test [2017-09-16 2017-10-01]\n",
      "Window Train/Val/Test shape (1281, 10369) (336, 10369) (336, 10369)\n",
      "Window Train/Val/Test shape (1281, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-09-14] Val [2017-09-15 2017-09-30] Test [2017-10-01 2017-10-16]\n",
      "Window Train/Val/Test shape (1596, 10369) (336, 10369) (336, 10369)\n",
      "Window Train/Val/Test shape (1596, 2) (336, 2) (336, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-09-29] Val [2017-09-30 2017-10-15] Test [2017-10-16 2017-11-10]\n",
      "Window Train/Val/Test shape (1911, 10369) (336, 10369) (546, 10369)\n",
      "Window Train/Val/Test shape (1911, 2) (336, 2) (546, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-10-14] Val [2017-10-15 2017-10-30] Test [2017-10-31 2017-11-10]\n",
      "Window Train/Val/Test shape (2226, 10369) (336, 10369) (231, 10369)\n",
      "Window Train/Val/Test shape (2226, 2) (336, 2) (231, 2)\n",
      "[36.607347168253305, 34.09674549127934, 30.017124528514934, 12.828260924399332, 7.428701886894681]\n",
      "mean mse: 24.19563599986832\n",
      "Train [2017-07-01 2017-08-15] Val [2017-08-16 2017-08-31] Test [2017-09-01 2017-09-16]\n",
      "Window Train/Val/Test shape (966, 10369) (336, 10369) (336, 10369)\n",
      "Window Train/Val/Test shape (966, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-08-30] Val [2017-08-31 2017-09-15] Test [2017-09-16 2017-10-01]\n",
      "Window Train/Val/Test shape (1281, 10369) (336, 10369) (336, 10369)\n",
      "Window Train/Val/Test shape (1281, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-09-14] Val [2017-09-15 2017-09-30] Test [2017-10-01 2017-10-16]\n",
      "Window Train/Val/Test shape (1596, 10369) (336, 10369) (336, 10369)\n",
      "Window Train/Val/Test shape (1596, 2) (336, 2) (336, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-09-29] Val [2017-09-30 2017-10-15] Test [2017-10-16 2017-11-10]\n",
      "Window Train/Val/Test shape (1911, 10369) (336, 10369) (546, 10369)\n",
      "Window Train/Val/Test shape (1911, 2) (336, 2) (546, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-10-14] Val [2017-10-15 2017-10-30] Test [2017-10-31 2017-11-10]\n",
      "Window Train/Val/Test shape (2226, 10369) (336, 10369) (231, 10369)\n",
      "Window Train/Val/Test shape (2226, 2) (336, 2) (231, 2)\n",
      "[39.15408245579618, 39.82718681003128, 23.284190771403022, 11.541042927556187, 10.139358429111448]\n",
      "mean mse: 24.789172278779624\n",
      "Train [2017-07-01 2017-08-15] Val [2017-08-16 2017-08-31] Test [2017-09-01 2017-09-16]\n",
      "Window Train/Val/Test shape (966, 10369) (336, 10369) (336, 10369)\n",
      "Window Train/Val/Test shape (966, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-08-30] Val [2017-08-31 2017-09-15] Test [2017-09-16 2017-10-01]\n",
      "Window Train/Val/Test shape (1281, 10369) (336, 10369) (336, 10369)\n",
      "Window Train/Val/Test shape (1281, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-09-14] Val [2017-09-15 2017-09-30] Test [2017-10-01 2017-10-16]\n",
      "Window Train/Val/Test shape (1596, 10369) (336, 10369) (336, 10369)\n",
      "Window Train/Val/Test shape (1596, 2) (336, 2) (336, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-09-29] Val [2017-09-30 2017-10-15] Test [2017-10-16 2017-11-10]\n",
      "Window Train/Val/Test shape (1911, 10369) (336, 10369) (546, 10369)\n",
      "Window Train/Val/Test shape (1911, 2) (336, 2) (546, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-10-14] Val [2017-10-15 2017-10-30] Test [2017-10-31 2017-11-10]\n",
      "Window Train/Val/Test shape (2226, 10369) (336, 10369) (231, 10369)\n",
      "Window Train/Val/Test shape (2226, 2) (336, 2) (231, 2)\n",
      "[33.61394476370585, 42.59387421994964, 19.940771914900363, 11.181453413339913, 8.890957473790774]\n",
      "mean mse: 23.24420035713731\n",
      "Train [2017-07-01 2017-08-15] Val [2017-08-16 2017-08-31] Test [2017-09-01 2017-09-16]\n",
      "Window Train/Val/Test shape (966, 10369) (336, 10369) (336, 10369)\n",
      "Window Train/Val/Test shape (966, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-08-30] Val [2017-08-31 2017-09-15] Test [2017-09-16 2017-10-01]\n",
      "Window Train/Val/Test shape (1281, 10369) (336, 10369) (336, 10369)\n",
      "Window Train/Val/Test shape (1281, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-09-14] Val [2017-09-15 2017-09-30] Test [2017-10-01 2017-10-16]\n",
      "Window Train/Val/Test shape (1596, 10369) (336, 10369) (336, 10369)\n",
      "Window Train/Val/Test shape (1596, 2) (336, 2) (336, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-09-29] Val [2017-09-30 2017-10-15] Test [2017-10-16 2017-11-10]\n",
      "Window Train/Val/Test shape (1911, 10369) (336, 10369) (546, 10369)\n",
      "Window Train/Val/Test shape (1911, 2) (336, 2) (546, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-10-14] Val [2017-10-15 2017-10-30] Test [2017-10-31 2017-11-10]\n",
      "Window Train/Val/Test shape (2226, 10369) (336, 10369) (231, 10369)\n",
      "Window Train/Val/Test shape (2226, 2) (336, 2) (231, 2)\n",
      "[32.59834082683749, 38.6291880047513, 22.488301748347503, 12.679731724249041, 8.449337053424138]\n",
      "mean mse: 22.968979871521896\n",
      "Train [2017-07-01 2017-08-15] Val [2017-08-16 2017-08-31] Test [2017-09-01 2017-09-16]\n",
      "Window Train/Val/Test shape (966, 10369) (336, 10369) (336, 10369)\n",
      "Window Train/Val/Test shape (966, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-08-30] Val [2017-08-31 2017-09-15] Test [2017-09-16 2017-10-01]\n",
      "Window Train/Val/Test shape (1281, 10369) (336, 10369) (336, 10369)\n",
      "Window Train/Val/Test shape (1281, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-09-14] Val [2017-09-15 2017-09-30] Test [2017-10-01 2017-10-16]\n",
      "Window Train/Val/Test shape (1596, 10369) (336, 10369) (336, 10369)\n",
      "Window Train/Val/Test shape (1596, 2) (336, 2) (336, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-09-29] Val [2017-09-30 2017-10-15] Test [2017-10-16 2017-11-10]\n",
      "Window Train/Val/Test shape (1911, 10369) (336, 10369) (546, 10369)\n",
      "Window Train/Val/Test shape (1911, 2) (336, 2) (546, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-10-14] Val [2017-10-15 2017-10-30] Test [2017-10-31 2017-11-10]\n",
      "Window Train/Val/Test shape (2226, 10369) (336, 10369) (231, 10369)\n",
      "Window Train/Val/Test shape (2226, 2) (336, 2) (231, 2)\n",
      "[44.80117934714682, 55.96514649305077, 25.17762281533056, 12.933618832504806, 11.02711460430556]\n",
      "mean mse: 29.980936418467707\n",
      "Train [2017-07-01 2017-08-15] Val [2017-08-16 2017-08-31] Test [2017-09-01 2017-09-16]\n",
      "Window Train/Val/Test shape (966, 10369) (336, 10369) (336, 10369)\n",
      "Window Train/Val/Test shape (966, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-08-30] Val [2017-08-31 2017-09-15] Test [2017-09-16 2017-10-01]\n",
      "Window Train/Val/Test shape (1281, 10369) (336, 10369) (336, 10369)\n",
      "Window Train/Val/Test shape (1281, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-09-14] Val [2017-09-15 2017-09-30] Test [2017-10-01 2017-10-16]\n",
      "Window Train/Val/Test shape (1596, 10369) (336, 10369) (336, 10369)\n",
      "Window Train/Val/Test shape (1596, 2) (336, 2) (336, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-09-29] Val [2017-09-30 2017-10-15] Test [2017-10-16 2017-11-10]\n",
      "Window Train/Val/Test shape (1911, 10369) (336, 10369) (546, 10369)\n",
      "Window Train/Val/Test shape (1911, 2) (336, 2) (546, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-10-14] Val [2017-10-15 2017-10-30] Test [2017-10-31 2017-11-10]\n",
      "Window Train/Val/Test shape (2226, 10369) (336, 10369) (231, 10369)\n",
      "Window Train/Val/Test shape (2226, 2) (336, 2) (231, 2)\n",
      "[42.913771576518045, 38.846781323148875, 25.881113551861723, 11.53259875040904, 7.6750266470126185]\n",
      "mean mse: 25.36985836979006\n",
      "Train [2017-07-01 2017-08-15] Val [2017-08-16 2017-08-31] Test [2017-09-01 2017-09-16]\n",
      "Window Train/Val/Test shape (966, 10369) (336, 10369) (336, 10369)\n",
      "Window Train/Val/Test shape (966, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-08-30] Val [2017-08-31 2017-09-15] Test [2017-09-16 2017-10-01]\n",
      "Window Train/Val/Test shape (1281, 10369) (336, 10369) (336, 10369)\n",
      "Window Train/Val/Test shape (1281, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-09-14] Val [2017-09-15 2017-09-30] Test [2017-10-01 2017-10-16]\n",
      "Window Train/Val/Test shape (1596, 10369) (336, 10369) (336, 10369)\n",
      "Window Train/Val/Test shape (1596, 2) (336, 2) (336, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-09-29] Val [2017-09-30 2017-10-15] Test [2017-10-16 2017-11-10]\n",
      "Window Train/Val/Test shape (1911, 10369) (336, 10369) (546, 10369)\n",
      "Window Train/Val/Test shape (1911, 2) (336, 2) (546, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-10-14] Val [2017-10-15 2017-10-30] Test [2017-10-31 2017-11-10]\n",
      "Window Train/Val/Test shape (2226, 10369) (336, 10369) (231, 10369)\n",
      "Window Train/Val/Test shape (2226, 2) (336, 2) (231, 2)\n",
      "[44.23693433271224, 29.999132819454292, 22.672368166109898, 11.092278610911647, 9.480408475126398]\n",
      "mean mse: 23.496224480862892\n",
      "Train [2017-07-01 2017-08-15] Val [2017-08-16 2017-08-31] Test [2017-09-01 2017-09-16]\n",
      "Window Train/Val/Test shape (966, 10369) (336, 10369) (336, 10369)\n",
      "Window Train/Val/Test shape (966, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-08-30] Val [2017-08-31 2017-09-15] Test [2017-09-16 2017-10-01]\n",
      "Window Train/Val/Test shape (1281, 10369) (336, 10369) (336, 10369)\n",
      "Window Train/Val/Test shape (1281, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-09-14] Val [2017-09-15 2017-09-30] Test [2017-10-01 2017-10-16]\n",
      "Window Train/Val/Test shape (1596, 10369) (336, 10369) (336, 10369)\n",
      "Window Train/Val/Test shape (1596, 2) (336, 2) (336, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-09-29] Val [2017-09-30 2017-10-15] Test [2017-10-16 2017-11-10]\n",
      "Window Train/Val/Test shape (1911, 10369) (336, 10369) (546, 10369)\n",
      "Window Train/Val/Test shape (1911, 2) (336, 2) (546, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-10-14] Val [2017-10-15 2017-10-30] Test [2017-10-31 2017-11-10]\n",
      "Window Train/Val/Test shape (2226, 10369) (336, 10369) (231, 10369)\n",
      "Window Train/Val/Test shape (2226, 2) (336, 2) (231, 2)\n",
      "[28.40836974802252, 35.8863389467417, 24.460533837013607, 10.573303417064253, 8.42879162025631]\n",
      "mean mse: 21.55146751381968\n",
      "Train [2017-07-01 2017-08-15] Val [2017-08-16 2017-08-31] Test [2017-09-01 2017-09-16]\n",
      "Window Train/Val/Test shape (966, 10369) (336, 10369) (336, 10369)\n",
      "Window Train/Val/Test shape (966, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-08-30] Val [2017-08-31 2017-09-15] Test [2017-09-16 2017-10-01]\n",
      "Window Train/Val/Test shape (1281, 10369) (336, 10369) (336, 10369)\n",
      "Window Train/Val/Test shape (1281, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-09-14] Val [2017-09-15 2017-09-30] Test [2017-10-01 2017-10-16]\n",
      "Window Train/Val/Test shape (1596, 10369) (336, 10369) (336, 10369)\n",
      "Window Train/Val/Test shape (1596, 2) (336, 2) (336, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-09-29] Val [2017-09-30 2017-10-15] Test [2017-10-16 2017-11-10]\n",
      "Window Train/Val/Test shape (1911, 10369) (336, 10369) (546, 10369)\n",
      "Window Train/Val/Test shape (1911, 2) (336, 2) (546, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-10-14] Val [2017-10-15 2017-10-30] Test [2017-10-31 2017-11-10]\n",
      "Window Train/Val/Test shape (2226, 10369) (336, 10369) (231, 10369)\n",
      "Window Train/Val/Test shape (2226, 2) (336, 2) (231, 2)\n",
      "[36.62059969606181, 48.47749870500373, 25.885189390937178, 12.545517666503212, 10.513807052691568]\n",
      "mean mse: 26.808522502239505\n",
      "Train [2017-07-01 2017-08-15] Val [2017-08-16 2017-08-31] Test [2017-09-01 2017-09-16]\n",
      "Window Train/Val/Test shape (966, 10369) (336, 10369) (336, 10369)\n",
      "Window Train/Val/Test shape (966, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-08-30] Val [2017-08-31 2017-09-15] Test [2017-09-16 2017-10-01]\n",
      "Window Train/Val/Test shape (1281, 10369) (336, 10369) (336, 10369)\n",
      "Window Train/Val/Test shape (1281, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-09-14] Val [2017-09-15 2017-09-30] Test [2017-10-01 2017-10-16]\n",
      "Window Train/Val/Test shape (1596, 10369) (336, 10369) (336, 10369)\n",
      "Window Train/Val/Test shape (1596, 2) (336, 2) (336, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-09-29] Val [2017-09-30 2017-10-15] Test [2017-10-16 2017-11-10]\n",
      "Window Train/Val/Test shape (1911, 10369) (336, 10369) (546, 10369)\n",
      "Window Train/Val/Test shape (1911, 2) (336, 2) (546, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-10-14] Val [2017-10-15 2017-10-30] Test [2017-10-31 2017-11-10]\n",
      "Window Train/Val/Test shape (2226, 10369) (336, 10369) (231, 10369)\n",
      "Window Train/Val/Test shape (2226, 2) (336, 2) (231, 2)\n",
      "[33.27685938362271, 40.4055513725899, 21.133539911171958, 13.076231734412675, 5.932412816674341]\n",
      "mean mse: 22.764919043694313\n",
      "Train [2017-07-01 2017-08-15] Val [2017-08-16 2017-08-31] Test [2017-09-01 2017-09-16]\n",
      "Window Train/Val/Test shape (966, 10369) (336, 10369) (336, 10369)\n",
      "Window Train/Val/Test shape (966, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-08-30] Val [2017-08-31 2017-09-15] Test [2017-09-16 2017-10-01]\n",
      "Window Train/Val/Test shape (1281, 10369) (336, 10369) (336, 10369)\n",
      "Window Train/Val/Test shape (1281, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-09-14] Val [2017-09-15 2017-09-30] Test [2017-10-01 2017-10-16]\n",
      "Window Train/Val/Test shape (1596, 10369) (336, 10369) (336, 10369)\n",
      "Window Train/Val/Test shape (1596, 2) (336, 2) (336, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-09-29] Val [2017-09-30 2017-10-15] Test [2017-10-16 2017-11-10]\n",
      "Window Train/Val/Test shape (1911, 10369) (336, 10369) (546, 10369)\n",
      "Window Train/Val/Test shape (1911, 2) (336, 2) (546, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-10-14] Val [2017-10-15 2017-10-30] Test [2017-10-31 2017-11-10]\n",
      "Window Train/Val/Test shape (2226, 10369) (336, 10369) (231, 10369)\n",
      "Window Train/Val/Test shape (2226, 2) (336, 2) (231, 2)\n",
      "[37.64122849266827, 38.86616968111006, 22.258448089569875, 10.786391998212583, 7.470597175059091]\n",
      "mean mse: 23.404567087323976\n",
      "Train [2017-07-01 2017-08-15] Val [2017-08-16 2017-08-31] Test [2017-09-01 2017-09-16]\n",
      "Window Train/Val/Test shape (966, 10369) (336, 10369) (336, 10369)\n",
      "Window Train/Val/Test shape (966, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-08-30] Val [2017-08-31 2017-09-15] Test [2017-09-16 2017-10-01]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Window Train/Val/Test shape (1281, 10369) (336, 10369) (336, 10369)\n",
      "Window Train/Val/Test shape (1281, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-09-14] Val [2017-09-15 2017-09-30] Test [2017-10-01 2017-10-16]\n",
      "Window Train/Val/Test shape (1596, 10369) (336, 10369) (336, 10369)\n",
      "Window Train/Val/Test shape (1596, 2) (336, 2) (336, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-09-29] Val [2017-09-30 2017-10-15] Test [2017-10-16 2017-11-10]\n",
      "Window Train/Val/Test shape (1911, 10369) (336, 10369) (546, 10369)\n",
      "Window Train/Val/Test shape (1911, 2) (336, 2) (546, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-10-14] Val [2017-10-15 2017-10-30] Test [2017-10-31 2017-11-10]\n",
      "Window Train/Val/Test shape (2226, 10369) (336, 10369) (231, 10369)\n",
      "Window Train/Val/Test shape (2226, 2) (336, 2) (231, 2)\n",
      "[43.957924715230675, 31.25903012556706, 22.037100089478308, 11.4338612140947, 11.694577860095723]\n",
      "mean mse: 24.076498800893297\n",
      "Train [2017-07-01 2017-08-15] Val [2017-08-16 2017-08-31] Test [2017-09-01 2017-09-16]\n",
      "Window Train/Val/Test shape (966, 10369) (336, 10369) (336, 10369)\n",
      "Window Train/Val/Test shape (966, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-08-30] Val [2017-08-31 2017-09-15] Test [2017-09-16 2017-10-01]\n",
      "Window Train/Val/Test shape (1281, 10369) (336, 10369) (336, 10369)\n",
      "Window Train/Val/Test shape (1281, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-09-14] Val [2017-09-15 2017-09-30] Test [2017-10-01 2017-10-16]\n",
      "Window Train/Val/Test shape (1596, 10369) (336, 10369) (336, 10369)\n",
      "Window Train/Val/Test shape (1596, 2) (336, 2) (336, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-09-29] Val [2017-09-30 2017-10-15] Test [2017-10-16 2017-11-10]\n",
      "Window Train/Val/Test shape (1911, 10369) (336, 10369) (546, 10369)\n",
      "Window Train/Val/Test shape (1911, 2) (336, 2) (546, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-10-14] Val [2017-10-15 2017-10-30] Test [2017-10-31 2017-11-10]\n",
      "Window Train/Val/Test shape (2226, 10369) (336, 10369) (231, 10369)\n",
      "Window Train/Val/Test shape (2226, 2) (336, 2) (231, 2)\n",
      "[57.42606856380711, 35.59494593474541, 29.165499606646385, 18.888955433194752, 8.324658814511988]\n",
      "mean mse: 29.88002567058113\n",
      "Train [2017-07-01 2017-08-15] Val [2017-08-16 2017-08-31] Test [2017-09-01 2017-09-16]\n",
      "Window Train/Val/Test shape (966, 10369) (336, 10369) (336, 10369)\n",
      "Window Train/Val/Test shape (966, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-08-30] Val [2017-08-31 2017-09-15] Test [2017-09-16 2017-10-01]\n",
      "Window Train/Val/Test shape (1281, 10369) (336, 10369) (336, 10369)\n",
      "Window Train/Val/Test shape (1281, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-09-14] Val [2017-09-15 2017-09-30] Test [2017-10-01 2017-10-16]\n",
      "Window Train/Val/Test shape (1596, 10369) (336, 10369) (336, 10369)\n",
      "Window Train/Val/Test shape (1596, 2) (336, 2) (336, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-09-29] Val [2017-09-30 2017-10-15] Test [2017-10-16 2017-11-10]\n",
      "Window Train/Val/Test shape (1911, 10369) (336, 10369) (546, 10369)\n",
      "Window Train/Val/Test shape (1911, 2) (336, 2) (546, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-10-14] Val [2017-10-15 2017-10-30] Test [2017-10-31 2017-11-10]\n",
      "Window Train/Val/Test shape (2226, 10369) (336, 10369) (231, 10369)\n",
      "Window Train/Val/Test shape (2226, 2) (336, 2) (231, 2)\n",
      "[34.87777359485375, 39.3684953512841, 23.401260101876968, 11.912687029571902, 8.918259916959746]\n",
      "mean mse: 23.695695198909295\n",
      "Train [2017-07-01 2017-08-15] Val [2017-08-16 2017-08-31] Test [2017-09-01 2017-09-16]\n",
      "Window Train/Val/Test shape (966, 10369) (336, 10369) (336, 10369)\n",
      "Window Train/Val/Test shape (966, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-08-30] Val [2017-08-31 2017-09-15] Test [2017-09-16 2017-10-01]\n",
      "Window Train/Val/Test shape (1281, 10369) (336, 10369) (336, 10369)\n",
      "Window Train/Val/Test shape (1281, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-09-14] Val [2017-09-15 2017-09-30] Test [2017-10-01 2017-10-16]\n",
      "Window Train/Val/Test shape (1596, 10369) (336, 10369) (336, 10369)\n",
      "Window Train/Val/Test shape (1596, 2) (336, 2) (336, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-09-29] Val [2017-09-30 2017-10-15] Test [2017-10-16 2017-11-10]\n",
      "Window Train/Val/Test shape (1911, 10369) (336, 10369) (546, 10369)\n",
      "Window Train/Val/Test shape (1911, 2) (336, 2) (546, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-10-14] Val [2017-10-15 2017-10-30] Test [2017-10-31 2017-11-10]\n",
      "Window Train/Val/Test shape (2226, 10369) (336, 10369) (231, 10369)\n",
      "Window Train/Val/Test shape (2226, 2) (336, 2) (231, 2)\n",
      "[43.56880560530159, 33.838359207288576, 34.28252325573978, 14.456325690303789, 11.707218250574062]\n",
      "mean mse: 27.57064640184156\n",
      "Train [2017-07-01 2017-08-15] Val [2017-08-16 2017-08-31] Test [2017-09-01 2017-09-16]\n",
      "Window Train/Val/Test shape (966, 10369) (336, 10369) (336, 10369)\n",
      "Window Train/Val/Test shape (966, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-08-30] Val [2017-08-31 2017-09-15] Test [2017-09-16 2017-10-01]\n",
      "Window Train/Val/Test shape (1281, 10369) (336, 10369) (336, 10369)\n",
      "Window Train/Val/Test shape (1281, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-09-14] Val [2017-09-15 2017-09-30] Test [2017-10-01 2017-10-16]\n",
      "Window Train/Val/Test shape (1596, 10369) (336, 10369) (336, 10369)\n",
      "Window Train/Val/Test shape (1596, 2) (336, 2) (336, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-09-29] Val [2017-09-30 2017-10-15] Test [2017-10-16 2017-11-10]\n",
      "Window Train/Val/Test shape (1911, 10369) (336, 10369) (546, 10369)\n",
      "Window Train/Val/Test shape (1911, 2) (336, 2) (546, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-10-14] Val [2017-10-15 2017-10-30] Test [2017-10-31 2017-11-10]\n",
      "Window Train/Val/Test shape (2226, 10369) (336, 10369) (231, 10369)\n",
      "Window Train/Val/Test shape (2226, 2) (336, 2) (231, 2)\n",
      "[48.94776651937896, 50.73035057079499, 43.0439123235146, 14.613076389171823, 7.705504074427434]\n",
      "mean mse: 33.00812197545756\n",
      "Train [2017-07-01 2017-08-15] Val [2017-08-16 2017-08-31] Test [2017-09-01 2017-09-16]\n",
      "Window Train/Val/Test shape (966, 10369) (336, 10369) (336, 10369)\n",
      "Window Train/Val/Test shape (966, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-08-30] Val [2017-08-31 2017-09-15] Test [2017-09-16 2017-10-01]\n",
      "Window Train/Val/Test shape (1281, 10369) (336, 10369) (336, 10369)\n",
      "Window Train/Val/Test shape (1281, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-09-14] Val [2017-09-15 2017-09-30] Test [2017-10-01 2017-10-16]\n",
      "Window Train/Val/Test shape (1596, 10369) (336, 10369) (336, 10369)\n",
      "Window Train/Val/Test shape (1596, 2) (336, 2) (336, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-09-29] Val [2017-09-30 2017-10-15] Test [2017-10-16 2017-11-10]\n",
      "Window Train/Val/Test shape (1911, 10369) (336, 10369) (546, 10369)\n",
      "Window Train/Val/Test shape (1911, 2) (336, 2) (546, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-10-14] Val [2017-10-15 2017-10-30] Test [2017-10-31 2017-11-10]\n",
      "Window Train/Val/Test shape (2226, 10369) (336, 10369) (231, 10369)\n",
      "Window Train/Val/Test shape (2226, 2) (336, 2) (231, 2)\n",
      "[38.87820266074499, 40.420251737684126, 25.064012425776873, 16.929464256774356, 10.899438912777079]\n",
      "mean mse: 26.438273998751487\n",
      "Train [2017-07-01 2017-08-15] Val [2017-08-16 2017-08-31] Test [2017-09-01 2017-09-16]\n",
      "Window Train/Val/Test shape (966, 10369) (336, 10369) (336, 10369)\n",
      "Window Train/Val/Test shape (966, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-08-30] Val [2017-08-31 2017-09-15] Test [2017-09-16 2017-10-01]\n",
      "Window Train/Val/Test shape (1281, 10369) (336, 10369) (336, 10369)\n",
      "Window Train/Val/Test shape (1281, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-09-14] Val [2017-09-15 2017-09-30] Test [2017-10-01 2017-10-16]\n",
      "Window Train/Val/Test shape (1596, 10369) (336, 10369) (336, 10369)\n",
      "Window Train/Val/Test shape (1596, 2) (336, 2) (336, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-09-29] Val [2017-09-30 2017-10-15] Test [2017-10-16 2017-11-10]\n",
      "Window Train/Val/Test shape (1911, 10369) (336, 10369) (546, 10369)\n",
      "Window Train/Val/Test shape (1911, 2) (336, 2) (546, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-10-14] Val [2017-10-15 2017-10-30] Test [2017-10-31 2017-11-10]\n",
      "Window Train/Val/Test shape (2226, 10369) (336, 10369) (231, 10369)\n",
      "Window Train/Val/Test shape (2226, 2) (336, 2) (231, 2)\n",
      "[28.34138730703982, 39.176634889219464, 21.15239926922582, 11.942416520856973, 9.017020889108595]\n",
      "mean mse: 21.925971775090133\n"
     ]
    }
   ],
   "source": [
    "orchestrator68.performHyperparameterOptimization(X_68_preselect, 20)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 120,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'colsample_bytree': 0.45,\n",
       " 'gamma': 3.0,\n",
       " 'learning_rate': 0.15000000000000002,\n",
       " 'max_depth': 7.0,\n",
       " 'min_child_weight': 10.0,\n",
       " 'n_estimators': 250.0,\n",
       " 'subsample': 0.7000000000000001}"
      ]
     },
     "execution_count": 120,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "orchestrator68.bo.best"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "preselect_params_68= {'colsample_bytree': 0.45,\n",
    "                         'gamma': 3.0,\n",
    "                         'learning_rate': 0.152,\n",
    "                         'max_depth': 7,\n",
    "                         'min_child_weight': 10.0,\n",
    "                         'n_estimators': 250,\n",
    "                         'subsample': 0.70}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Running custom feature selection module ....\n",
      "Creating baseline model to extract features\n",
      "all features (2688, 10368)\n",
      "10\n",
      "# features 10368 # thresholds 10; thresholds:[0.00726392 0.0066586  0.00423729 0.00363196 0.00302663 0.00242131\n",
      " 0.00181598 0.00121065 0.00060533 0.        ]\n",
      "CV - Window 3\n",
      "Train [2017-07-01 2017-08-15] Val [2017-08-16 2017-08-31] Test [2017-09-01 2017-09-16]\n",
      "Window Train/Val/Test shape (966, 10369) (336, 10369) (336, 10369)\n",
      "Window Train/Val/Test shape (966, 2) (336, 2) (336, 2)\n",
      "X_train_cv  (1302, 10368)\n",
      "3 Thresh=0.00726, n=1, mse: 48.590\n",
      "CV - Window 4\n",
      "Train [2017-07-01 2017-08-30] Val [2017-08-31 2017-09-15] Test [2017-09-16 2017-10-01]\n",
      "Window Train/Val/Test shape (1281, 10369) (336, 10369) (336, 10369)\n",
      "Window Train/Val/Test shape (1281, 2) (336, 2) (336, 2)\n",
      "X_train_cv  (1617, 10368)\n",
      "4 Thresh=0.00726, n=1, mse: 19.354\n",
      "CV - Window 5\n",
      "Train [2017-07-01 2017-09-14] Val [2017-09-15 2017-09-30] Test [2017-10-01 2017-10-16]\n",
      "Window Train/Val/Test shape (1596, 10369) (336, 10369) (336, 10369)\n",
      "Window Train/Val/Test shape (1596, 2) (336, 2) (336, 2)\n",
      "X_train_cv  (1932, 10368)\n",
      "5 Thresh=0.00726, n=1, mse: 21.544\n",
      "CV - Window 6\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-09-29] Val [2017-09-30 2017-10-15] Test [2017-10-16 2017-11-10]\n",
      "Window Train/Val/Test shape (1911, 10369) (336, 10369) (546, 10369)\n",
      "Window Train/Val/Test shape (1911, 2) (336, 2) (546, 2)\n",
      "X_train_cv  (2247, 10368)\n",
      "6 Thresh=0.00726, n=1, mse: 27.462\n",
      "CV - Window 7\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-10-14] Val [2017-10-15 2017-10-30] Test [2017-10-31 2017-11-10]\n",
      "Window Train/Val/Test shape (2226, 10369) (336, 10369) (231, 10369)\n",
      "Window Train/Val/Test shape (2226, 2) (336, 2) (231, 2)\n",
      "X_train_cv  (2562, 10368)\n",
      "7 Thresh=0.00726, n=1, mse: 26.990\n",
      "{'threshold': 0.0072639226, 'num_features': 1, 'mse_list': [48.590345694993744, 19.35422173603673, 21.54359139431796, 27.461960700267902, 26.990083029643223], 'mean_mse': 28.78804051105191}\n",
      "\n",
      "\n",
      "CV - Window 3\n",
      "Train [2017-07-01 2017-08-15] Val [2017-08-16 2017-08-31] Test [2017-09-01 2017-09-16]\n",
      "Window Train/Val/Test shape (966, 10369) (336, 10369) (336, 10369)\n",
      "Window Train/Val/Test shape (966, 2) (336, 2) (336, 2)\n",
      "X_train_cv  (1302, 10368)\n",
      "3 Thresh=0.00666, n=2, mse: 53.826\n",
      "CV - Window 4\n",
      "Train [2017-07-01 2017-08-30] Val [2017-08-31 2017-09-15] Test [2017-09-16 2017-10-01]\n",
      "Window Train/Val/Test shape (1281, 10369) (336, 10369) (336, 10369)\n",
      "Window Train/Val/Test shape (1281, 2) (336, 2) (336, 2)\n",
      "X_train_cv  (1617, 10368)\n",
      "4 Thresh=0.00666, n=2, mse: 30.542\n",
      "CV - Window 5\n",
      "Train [2017-07-01 2017-09-14] Val [2017-09-15 2017-09-30] Test [2017-10-01 2017-10-16]\n",
      "Window Train/Val/Test shape (1596, 10369) (336, 10369) (336, 10369)\n",
      "Window Train/Val/Test shape (1596, 2) (336, 2) (336, 2)\n",
      "X_train_cv  (1932, 10368)\n",
      "5 Thresh=0.00666, n=2, mse: 21.020\n",
      "CV - Window 6\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-09-29] Val [2017-09-30 2017-10-15] Test [2017-10-16 2017-11-10]\n",
      "Window Train/Val/Test shape (1911, 10369) (336, 10369) (546, 10369)\n",
      "Window Train/Val/Test shape (1911, 2) (336, 2) (546, 2)\n",
      "X_train_cv  (2247, 10368)\n",
      "6 Thresh=0.00666, n=2, mse: 22.825\n",
      "CV - Window 7\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-10-14] Val [2017-10-15 2017-10-30] Test [2017-10-31 2017-11-10]\n",
      "Window Train/Val/Test shape (2226, 10369) (336, 10369) (231, 10369)\n",
      "Window Train/Val/Test shape (2226, 2) (336, 2) (231, 2)\n",
      "X_train_cv  (2562, 10368)\n",
      "7 Thresh=0.00666, n=2, mse: 15.994\n",
      "{'threshold': 0.0066585955, 'num_features': 2, 'mse_list': [53.82605493473728, 30.542101385357643, 21.020081459327045, 22.825334710159357, 15.993637019768615], 'mean_mse': 28.84144190186999}\n",
      "\n",
      "\n",
      "CV - Window 3\n",
      "Train [2017-07-01 2017-08-15] Val [2017-08-16 2017-08-31] Test [2017-09-01 2017-09-16]\n",
      "Window Train/Val/Test shape (966, 10369) (336, 10369) (336, 10369)\n",
      "Window Train/Val/Test shape (966, 2) (336, 2) (336, 2)\n",
      "X_train_cv  (1302, 10368)\n",
      "3 Thresh=0.00424, n=4, mse: 87.554\n",
      "CV - Window 4\n",
      "Train [2017-07-01 2017-08-30] Val [2017-08-31 2017-09-15] Test [2017-09-16 2017-10-01]\n",
      "Window Train/Val/Test shape (1281, 10369) (336, 10369) (336, 10369)\n",
      "Window Train/Val/Test shape (1281, 2) (336, 2) (336, 2)\n",
      "X_train_cv  (1617, 10368)\n",
      "4 Thresh=0.00424, n=4, mse: 46.822\n",
      "CV - Window 5\n",
      "Train [2017-07-01 2017-09-14] Val [2017-09-15 2017-09-30] Test [2017-10-01 2017-10-16]\n",
      "Window Train/Val/Test shape (1596, 10369) (336, 10369) (336, 10369)\n",
      "Window Train/Val/Test shape (1596, 2) (336, 2) (336, 2)\n",
      "X_train_cv  (1932, 10368)\n",
      "5 Thresh=0.00424, n=4, mse: 39.553\n",
      "CV - Window 6\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-09-29] Val [2017-09-30 2017-10-15] Test [2017-10-16 2017-11-10]\n",
      "Window Train/Val/Test shape (1911, 10369) (336, 10369) (546, 10369)\n",
      "Window Train/Val/Test shape (1911, 2) (336, 2) (546, 2)\n",
      "X_train_cv  (2247, 10368)\n",
      "6 Thresh=0.00424, n=4, mse: 24.293\n",
      "CV - Window 7\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-10-14] Val [2017-10-15 2017-10-30] Test [2017-10-31 2017-11-10]\n",
      "Window Train/Val/Test shape (2226, 10369) (336, 10369) (231, 10369)\n",
      "Window Train/Val/Test shape (2226, 2) (336, 2) (231, 2)\n",
      "X_train_cv  (2562, 10368)\n",
      "7 Thresh=0.00424, n=4, mse: 18.442\n",
      "{'threshold': 0.004237288, 'num_features': 4, 'mse_list': [87.55425827785686, 46.82209172792178, 39.55300560589134, 24.293358291597468, 18.441553624755173], 'mean_mse': 43.33285350560452}\n",
      "\n",
      "\n",
      "CV - Window 3\n",
      "Train [2017-07-01 2017-08-15] Val [2017-08-16 2017-08-31] Test [2017-09-01 2017-09-16]\n",
      "Window Train/Val/Test shape (966, 10369) (336, 10369) (336, 10369)\n",
      "Window Train/Val/Test shape (966, 2) (336, 2) (336, 2)\n",
      "X_train_cv  (1302, 10368)\n",
      "3 Thresh=0.00363, n=10, mse: 28.052\n",
      "CV - Window 4\n",
      "Train [2017-07-01 2017-08-30] Val [2017-08-31 2017-09-15] Test [2017-09-16 2017-10-01]\n",
      "Window Train/Val/Test shape (1281, 10369) (336, 10369) (336, 10369)\n",
      "Window Train/Val/Test shape (1281, 2) (336, 2) (336, 2)\n",
      "X_train_cv  (1617, 10368)\n",
      "4 Thresh=0.00363, n=10, mse: 27.626\n",
      "CV - Window 5\n",
      "Train [2017-07-01 2017-09-14] Val [2017-09-15 2017-09-30] Test [2017-10-01 2017-10-16]\n",
      "Window Train/Val/Test shape (1596, 10369) (336, 10369) (336, 10369)\n",
      "Window Train/Val/Test shape (1596, 2) (336, 2) (336, 2)\n",
      "X_train_cv  (1932, 10368)\n",
      "5 Thresh=0.00363, n=10, mse: 26.708\n",
      "CV - Window 6\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-09-29] Val [2017-09-30 2017-10-15] Test [2017-10-16 2017-11-10]\n",
      "Window Train/Val/Test shape (1911, 10369) (336, 10369) (546, 10369)\n",
      "Window Train/Val/Test shape (1911, 2) (336, 2) (546, 2)\n",
      "X_train_cv  (2247, 10368)\n",
      "6 Thresh=0.00363, n=10, mse: 17.625\n",
      "CV - Window 7\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-10-14] Val [2017-10-15 2017-10-30] Test [2017-10-31 2017-11-10]\n",
      "Window Train/Val/Test shape (2226, 10369) (336, 10369) (231, 10369)\n",
      "Window Train/Val/Test shape (2226, 2) (336, 2) (231, 2)\n",
      "X_train_cv  (2562, 10368)\n",
      "7 Thresh=0.00363, n=10, mse: 7.656\n",
      "{'threshold': 0.0036319613, 'num_features': 10, 'mse_list': [28.051858730132977, 27.625896577647474, 26.707845402813785, 17.624683775442776, 7.656352217177794], 'mean_mse': 21.533327340642963}\n",
      "\n",
      "\n",
      "CV - Window 3\n",
      "Train [2017-07-01 2017-08-15] Val [2017-08-16 2017-08-31] Test [2017-09-01 2017-09-16]\n",
      "Window Train/Val/Test shape (966, 10369) (336, 10369) (336, 10369)\n",
      "Window Train/Val/Test shape (966, 2) (336, 2) (336, 2)\n",
      "X_train_cv  (1302, 10368)\n",
      "3 Thresh=0.00303, n=12, mse: 38.078\n",
      "CV - Window 4\n",
      "Train [2017-07-01 2017-08-30] Val [2017-08-31 2017-09-15] Test [2017-09-16 2017-10-01]\n",
      "Window Train/Val/Test shape (1281, 10369) (336, 10369) (336, 10369)\n",
      "Window Train/Val/Test shape (1281, 2) (336, 2) (336, 2)\n",
      "X_train_cv  (1617, 10368)\n",
      "4 Thresh=0.00303, n=12, mse: 38.322\n",
      "CV - Window 5\n",
      "Train [2017-07-01 2017-09-14] Val [2017-09-15 2017-09-30] Test [2017-10-01 2017-10-16]\n",
      "Window Train/Val/Test shape (1596, 10369) (336, 10369) (336, 10369)\n",
      "Window Train/Val/Test shape (1596, 2) (336, 2) (336, 2)\n",
      "X_train_cv  (1932, 10368)\n",
      "5 Thresh=0.00303, n=12, mse: 18.896\n",
      "CV - Window 6\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-09-29] Val [2017-09-30 2017-10-15] Test [2017-10-16 2017-11-10]\n",
      "Window Train/Val/Test shape (1911, 10369) (336, 10369) (546, 10369)\n",
      "Window Train/Val/Test shape (1911, 2) (336, 2) (546, 2)\n",
      "X_train_cv  (2247, 10368)\n",
      "6 Thresh=0.00303, n=12, mse: 11.934\n",
      "CV - Window 7\n",
      "Rolling window to end date\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train [2017-07-01 2017-10-14] Val [2017-10-15 2017-10-30] Test [2017-10-31 2017-11-10]\n",
      "Window Train/Val/Test shape (2226, 10369) (336, 10369) (231, 10369)\n",
      "Window Train/Val/Test shape (2226, 2) (336, 2) (231, 2)\n",
      "X_train_cv  (2562, 10368)\n",
      "7 Thresh=0.00303, n=12, mse: 8.174\n",
      "{'threshold': 0.0030266345, 'num_features': 12, 'mse_list': [38.078019850613536, 38.322129399977506, 18.896325772319987, 11.933688321862256, 8.173906015441437], 'mean_mse': 23.080813872042942}\n",
      "\n",
      "\n",
      "CV - Window 3\n",
      "Train [2017-07-01 2017-08-15] Val [2017-08-16 2017-08-31] Test [2017-09-01 2017-09-16]\n",
      "Window Train/Val/Test shape (966, 10369) (336, 10369) (336, 10369)\n",
      "Window Train/Val/Test shape (966, 2) (336, 2) (336, 2)\n",
      "X_train_cv  (1302, 10368)\n",
      "3 Thresh=0.00242, n=22, mse: 24.077\n",
      "CV - Window 4\n",
      "Train [2017-07-01 2017-08-30] Val [2017-08-31 2017-09-15] Test [2017-09-16 2017-10-01]\n",
      "Window Train/Val/Test shape (1281, 10369) (336, 10369) (336, 10369)\n",
      "Window Train/Val/Test shape (1281, 2) (336, 2) (336, 2)\n",
      "X_train_cv  (1617, 10368)\n",
      "4 Thresh=0.00242, n=22, mse: 40.704\n",
      "CV - Window 5\n",
      "Train [2017-07-01 2017-09-14] Val [2017-09-15 2017-09-30] Test [2017-10-01 2017-10-16]\n",
      "Window Train/Val/Test shape (1596, 10369) (336, 10369) (336, 10369)\n",
      "Window Train/Val/Test shape (1596, 2) (336, 2) (336, 2)\n",
      "X_train_cv  (1932, 10368)\n",
      "5 Thresh=0.00242, n=22, mse: 14.402\n",
      "CV - Window 6\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-09-29] Val [2017-09-30 2017-10-15] Test [2017-10-16 2017-11-10]\n",
      "Window Train/Val/Test shape (1911, 10369) (336, 10369) (546, 10369)\n",
      "Window Train/Val/Test shape (1911, 2) (336, 2) (546, 2)\n",
      "X_train_cv  (2247, 10368)\n",
      "6 Thresh=0.00242, n=22, mse: 8.679\n",
      "CV - Window 7\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-10-14] Val [2017-10-15 2017-10-30] Test [2017-10-31 2017-11-10]\n",
      "Window Train/Val/Test shape (2226, 10369) (336, 10369) (231, 10369)\n",
      "Window Train/Val/Test shape (2226, 2) (336, 2) (231, 2)\n",
      "X_train_cv  (2562, 10368)\n",
      "7 Thresh=0.00242, n=22, mse: 6.603\n",
      "{'threshold': 0.0024213076, 'num_features': 22, 'mse_list': [24.077313937734214, 40.704113919699125, 14.401663664258816, 8.679257370378842, 6.602716497748316], 'mean_mse': 18.893013077963865}\n",
      "\n",
      "\n",
      "CV - Window 3\n",
      "Train [2017-07-01 2017-08-15] Val [2017-08-16 2017-08-31] Test [2017-09-01 2017-09-16]\n",
      "Window Train/Val/Test shape (966, 10369) (336, 10369) (336, 10369)\n",
      "Window Train/Val/Test shape (966, 2) (336, 2) (336, 2)\n",
      "X_train_cv  (1302, 10368)\n",
      "3 Thresh=0.00182, n=60, mse: 26.927\n",
      "CV - Window 4\n",
      "Train [2017-07-01 2017-08-30] Val [2017-08-31 2017-09-15] Test [2017-09-16 2017-10-01]\n",
      "Window Train/Val/Test shape (1281, 10369) (336, 10369) (336, 10369)\n",
      "Window Train/Val/Test shape (1281, 2) (336, 2) (336, 2)\n",
      "X_train_cv  (1617, 10368)\n",
      "4 Thresh=0.00182, n=60, mse: 30.184\n",
      "CV - Window 5\n",
      "Train [2017-07-01 2017-09-14] Val [2017-09-15 2017-09-30] Test [2017-10-01 2017-10-16]\n",
      "Window Train/Val/Test shape (1596, 10369) (336, 10369) (336, 10369)\n",
      "Window Train/Val/Test shape (1596, 2) (336, 2) (336, 2)\n",
      "X_train_cv  (1932, 10368)\n",
      "5 Thresh=0.00182, n=60, mse: 15.716\n",
      "CV - Window 6\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-09-29] Val [2017-09-30 2017-10-15] Test [2017-10-16 2017-11-10]\n",
      "Window Train/Val/Test shape (1911, 10369) (336, 10369) (546, 10369)\n",
      "Window Train/Val/Test shape (1911, 2) (336, 2) (546, 2)\n",
      "X_train_cv  (2247, 10368)\n",
      "6 Thresh=0.00182, n=60, mse: 7.003\n",
      "CV - Window 7\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-10-14] Val [2017-10-15 2017-10-30] Test [2017-10-31 2017-11-10]\n",
      "Window Train/Val/Test shape (2226, 10369) (336, 10369) (231, 10369)\n",
      "Window Train/Val/Test shape (2226, 2) (336, 2) (231, 2)\n",
      "X_train_cv  (2562, 10368)\n",
      "7 Thresh=0.00182, n=60, mse: 6.920\n",
      "{'threshold': 0.0018159806, 'num_features': 60, 'mse_list': [26.927297564523478, 30.18431732565236, 15.715821938952507, 7.0030048098869795, 6.920015422591187], 'mean_mse': 17.3500914123213}\n",
      "\n",
      "\n",
      "CV - Window 3\n",
      "Train [2017-07-01 2017-08-15] Val [2017-08-16 2017-08-31] Test [2017-09-01 2017-09-16]\n",
      "Window Train/Val/Test shape (966, 10369) (336, 10369) (336, 10369)\n",
      "Window Train/Val/Test shape (966, 2) (336, 2) (336, 2)\n",
      "X_train_cv  (1302, 10368)\n",
      "3 Thresh=0.00121, n=225, mse: 32.756\n",
      "CV - Window 4\n",
      "Train [2017-07-01 2017-08-30] Val [2017-08-31 2017-09-15] Test [2017-09-16 2017-10-01]\n",
      "Window Train/Val/Test shape (1281, 10369) (336, 10369) (336, 10369)\n",
      "Window Train/Val/Test shape (1281, 2) (336, 2) (336, 2)\n",
      "X_train_cv  (1617, 10368)\n",
      "4 Thresh=0.00121, n=225, mse: 28.130\n",
      "CV - Window 5\n",
      "Train [2017-07-01 2017-09-14] Val [2017-09-15 2017-09-30] Test [2017-10-01 2017-10-16]\n",
      "Window Train/Val/Test shape (1596, 10369) (336, 10369) (336, 10369)\n",
      "Window Train/Val/Test shape (1596, 2) (336, 2) (336, 2)\n",
      "X_train_cv  (1932, 10368)\n",
      "5 Thresh=0.00121, n=225, mse: 16.315\n",
      "CV - Window 6\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-09-29] Val [2017-09-30 2017-10-15] Test [2017-10-16 2017-11-10]\n",
      "Window Train/Val/Test shape (1911, 10369) (336, 10369) (546, 10369)\n",
      "Window Train/Val/Test shape (1911, 2) (336, 2) (546, 2)\n",
      "X_train_cv  (2247, 10368)\n",
      "6 Thresh=0.00121, n=225, mse: 11.311\n",
      "CV - Window 7\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-10-14] Val [2017-10-15 2017-10-30] Test [2017-10-31 2017-11-10]\n",
      "Window Train/Val/Test shape (2226, 10369) (336, 10369) (231, 10369)\n",
      "Window Train/Val/Test shape (2226, 2) (336, 2) (231, 2)\n",
      "X_train_cv  (2562, 10368)\n",
      "7 Thresh=0.00121, n=225, mse: 8.032\n",
      "{'threshold': 0.0012106538, 'num_features': 225, 'mse_list': [32.7562837260561, 28.12962199353752, 16.314588814059427, 11.311103940813284, 8.032068005407874], 'mean_mse': 19.30873329597484}\n",
      "\n",
      "\n",
      "CV - Window 3\n",
      "Train [2017-07-01 2017-08-15] Val [2017-08-16 2017-08-31] Test [2017-09-01 2017-09-16]\n",
      "Window Train/Val/Test shape (966, 10369) (336, 10369) (336, 10369)\n",
      "Window Train/Val/Test shape (966, 2) (336, 2) (336, 2)\n",
      "X_train_cv  (1302, 10368)\n",
      "3 Thresh=0.00061, n=1310, mse: 26.056\n",
      "CV - Window 4\n",
      "Train [2017-07-01 2017-08-30] Val [2017-08-31 2017-09-15] Test [2017-09-16 2017-10-01]\n",
      "Window Train/Val/Test shape (1281, 10369) (336, 10369) (336, 10369)\n",
      "Window Train/Val/Test shape (1281, 2) (336, 2) (336, 2)\n",
      "X_train_cv  (1617, 10368)\n",
      "4 Thresh=0.00061, n=1310, mse: 36.072\n",
      "CV - Window 5\n",
      "Train [2017-07-01 2017-09-14] Val [2017-09-15 2017-09-30] Test [2017-10-01 2017-10-16]\n",
      "Window Train/Val/Test shape (1596, 10369) (336, 10369) (336, 10369)\n",
      "Window Train/Val/Test shape (1596, 2) (336, 2) (336, 2)\n",
      "X_train_cv  (1932, 10368)\n",
      "5 Thresh=0.00061, n=1310, mse: 21.780\n",
      "CV - Window 6\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-09-29] Val [2017-09-30 2017-10-15] Test [2017-10-16 2017-11-10]\n",
      "Window Train/Val/Test shape (1911, 10369) (336, 10369) (546, 10369)\n",
      "Window Train/Val/Test shape (1911, 2) (336, 2) (546, 2)\n",
      "X_train_cv  (2247, 10368)\n",
      "6 Thresh=0.00061, n=1310, mse: 9.389\n",
      "CV - Window 7\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-10-14] Val [2017-10-15 2017-10-30] Test [2017-10-31 2017-11-10]\n",
      "Window Train/Val/Test shape (2226, 10369) (336, 10369) (231, 10369)\n",
      "Window Train/Val/Test shape (2226, 2) (336, 2) (231, 2)\n",
      "X_train_cv  (2562, 10368)\n",
      "7 Thresh=0.00061, n=1310, mse: 9.230\n",
      "{'threshold': 0.0006053269, 'num_features': 1310, 'mse_list': [26.0555038377847, 36.0719404055451, 21.780336224265994, 9.388977001753167, 9.229865617749768], 'mean_mse': 20.505324617419742}\n",
      "\n",
      "\n",
      "CV - Window 3\n",
      "Train [2017-07-01 2017-08-15] Val [2017-08-16 2017-08-31] Test [2017-09-01 2017-09-16]\n",
      "Window Train/Val/Test shape (966, 10369) (336, 10369) (336, 10369)\n",
      "Window Train/Val/Test shape (966, 2) (336, 2) (336, 2)\n",
      "X_train_cv  (1302, 10368)\n",
      "3 Thresh=0.00000, n=10368, mse: 21.971\n",
      "CV - Window 4\n",
      "Train [2017-07-01 2017-08-30] Val [2017-08-31 2017-09-15] Test [2017-09-16 2017-10-01]\n",
      "Window Train/Val/Test shape (1281, 10369) (336, 10369) (336, 10369)\n",
      "Window Train/Val/Test shape (1281, 2) (336, 2) (336, 2)\n",
      "X_train_cv  (1617, 10368)\n",
      "4 Thresh=0.00000, n=10368, mse: 31.416\n",
      "CV - Window 5\n",
      "Train [2017-07-01 2017-09-14] Val [2017-09-15 2017-09-30] Test [2017-10-01 2017-10-16]\n",
      "Window Train/Val/Test shape (1596, 10369) (336, 10369) (336, 10369)\n",
      "Window Train/Val/Test shape (1596, 2) (336, 2) (336, 2)\n",
      "X_train_cv  (1932, 10368)\n",
      "5 Thresh=0.00000, n=10368, mse: 18.557\n",
      "CV - Window 6\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-09-29] Val [2017-09-30 2017-10-15] Test [2017-10-16 2017-11-10]\n",
      "Window Train/Val/Test shape (1911, 10369) (336, 10369) (546, 10369)\n",
      "Window Train/Val/Test shape (1911, 2) (336, 2) (546, 2)\n",
      "X_train_cv  (2247, 10368)\n",
      "6 Thresh=0.00000, n=10368, mse: 10.236\n",
      "CV - Window 7\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-10-14] Val [2017-10-15 2017-10-30] Test [2017-10-31 2017-11-10]\n",
      "Window Train/Val/Test shape (2226, 10369) (336, 10369) (231, 10369)\n",
      "Window Train/Val/Test shape (2226, 2) (336, 2) (231, 2)\n",
      "X_train_cv  (2562, 10368)\n",
      "7 Thresh=0.00000, n=10368, mse: 9.567\n",
      "{'threshold': 0.0, 'num_features': 10368, 'mse_list': [21.97052110725286, 31.415998767193038, 18.556845022390483, 10.236157845186723, 9.567022355335618], 'mean_mse': 18.349309019471747}\n",
      "\n",
      "\n",
      "   threshold num_features                                           mse_list  \\\n",
      "0   0.007264            1  [48.590345694993744, 19.35422173603673, 21.543...   \n",
      "1   0.006659            2  [53.82605493473728, 30.542101385357643, 21.020...   \n",
      "2   0.004237            4  [87.55425827785686, 46.82209172792178, 39.5530...   \n",
      "3   0.003632           10  [28.051858730132977, 27.625896577647474, 26.70...   \n",
      "4   0.003027           12  [38.078019850613536, 38.322129399977506, 18.89...   \n",
      "5   0.002421           22  [24.077313937734214, 40.704113919699125, 14.40...   \n",
      "6   0.001816           60  [26.927297564523478, 30.18431732565236, 15.715...   \n",
      "7   0.001211          225  [32.7562837260561, 28.12962199353752, 16.31458...   \n",
      "8   0.000605         1310  [26.0555038377847, 36.0719404055451, 21.780336...   \n",
      "9   0.000000        10368  [21.97052110725286, 31.415998767193038, 18.556...   \n",
      "\n",
      "    mean_mse  \n",
      "0  28.788041  \n",
      "1  28.841442  \n",
      "2  43.332854  \n",
      "3  21.533327  \n",
      "4  23.080814  \n",
      "5  18.893013  \n",
      "6  17.350091  \n",
      "7  19.308733  \n",
      "8  20.505325  \n",
      "9  18.349309  \n"
     ]
    }
   ],
   "source": [
    "orchestrator68.runRollingCustomFeatureExtractor(preselect_params_68, X_68_preselect)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>threshold</th>\n",
       "      <th>num_features</th>\n",
       "      <th>mse_list</th>\n",
       "      <th>mean_mse</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0.007264</td>\n",
       "      <td>1</td>\n",
       "      <td>[48.590345694993744, 19.35422173603673, 21.543...</td>\n",
       "      <td>28.788041</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0.006659</td>\n",
       "      <td>2</td>\n",
       "      <td>[53.82605493473728, 30.542101385357643, 21.020...</td>\n",
       "      <td>28.841442</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0.004237</td>\n",
       "      <td>4</td>\n",
       "      <td>[87.55425827785686, 46.82209172792178, 39.5530...</td>\n",
       "      <td>43.332854</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0.003632</td>\n",
       "      <td>10</td>\n",
       "      <td>[28.051858730132977, 27.625896577647474, 26.70...</td>\n",
       "      <td>21.533327</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0.003027</td>\n",
       "      <td>12</td>\n",
       "      <td>[38.078019850613536, 38.322129399977506, 18.89...</td>\n",
       "      <td>23.080814</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>0.002421</td>\n",
       "      <td>22</td>\n",
       "      <td>[24.077313937734214, 40.704113919699125, 14.40...</td>\n",
       "      <td>18.893013</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>0.001816</td>\n",
       "      <td>60</td>\n",
       "      <td>[26.927297564523478, 30.18431732565236, 15.715...</td>\n",
       "      <td>17.350091</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>0.001211</td>\n",
       "      <td>225</td>\n",
       "      <td>[32.7562837260561, 28.12962199353752, 16.31458...</td>\n",
       "      <td>19.308733</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>0.000605</td>\n",
       "      <td>1310</td>\n",
       "      <td>[26.0555038377847, 36.0719404055451, 21.780336...</td>\n",
       "      <td>20.505325</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>0.000000</td>\n",
       "      <td>10368</td>\n",
       "      <td>[21.97052110725286, 31.415998767193038, 18.556...</td>\n",
       "      <td>18.349309</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   threshold num_features                                           mse_list  \\\n",
       "0   0.007264            1  [48.590345694993744, 19.35422173603673, 21.543...   \n",
       "1   0.006659            2  [53.82605493473728, 30.542101385357643, 21.020...   \n",
       "2   0.004237            4  [87.55425827785686, 46.82209172792178, 39.5530...   \n",
       "3   0.003632           10  [28.051858730132977, 27.625896577647474, 26.70...   \n",
       "4   0.003027           12  [38.078019850613536, 38.322129399977506, 18.89...   \n",
       "5   0.002421           22  [24.077313937734214, 40.704113919699125, 14.40...   \n",
       "6   0.001816           60  [26.927297564523478, 30.18431732565236, 15.715...   \n",
       "7   0.001211          225  [32.7562837260561, 28.12962199353752, 16.31458...   \n",
       "8   0.000605         1310  [26.0555038377847, 36.0719404055451, 21.780336...   \n",
       "9   0.000000        10368  [21.97052110725286, 31.415998767193038, 18.556...   \n",
       "\n",
       "    mean_mse  \n",
       "0  28.788041  \n",
       "1  28.841442  \n",
       "2  43.332854  \n",
       "3  21.533327  \n",
       "4  23.080814  \n",
       "5  18.893013  \n",
       "6  17.350091  \n",
       "7  19.308733  \n",
       "8  20.505325  \n",
       "9  18.349309  "
      ]
     },
     "execution_count": 44,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "orchestrator68.rcfe.summary"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 112,
   "metadata": {},
   "outputs": [],
   "source": [
    "selected_features68 = ['date'] + list(orchestrator68.rcfe.mcr.feature_importance_df[orchestrator68.rcfe.mcr.feature_importance_df.threshold >= 0.001816].index)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 114,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train [2017-07-01 2017-08-15] Val [2017-08-16 2017-08-31] Test [2017-09-01 2017-09-16]\n",
      "Window Train/Val/Test shape (966, 61) (336, 61) (336, 61)\n",
      "Window Train/Val/Test shape (966, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-08-30] Val [2017-08-31 2017-09-15] Test [2017-09-16 2017-10-01]\n",
      "Window Train/Val/Test shape (1281, 61) (336, 61) (336, 61)\n",
      "Window Train/Val/Test shape (1281, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-09-14] Val [2017-09-15 2017-09-30] Test [2017-10-01 2017-10-16]\n",
      "Window Train/Val/Test shape (1596, 61) (336, 61) (336, 61)\n",
      "Window Train/Val/Test shape (1596, 2) (336, 2) (336, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-09-29] Val [2017-09-30 2017-10-15] Test [2017-10-16 2017-11-10]\n",
      "Window Train/Val/Test shape (1911, 61) (336, 61) (546, 61)\n",
      "Window Train/Val/Test shape (1911, 2) (336, 2) (546, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-10-14] Val [2017-10-15 2017-10-30] Test [2017-10-31 2017-11-10]\n",
      "Window Train/Val/Test shape (2226, 61) (336, 61) (231, 61)\n",
      "Window Train/Val/Test shape (2226, 2) (336, 2) (231, 2)\n",
      "[32.75870563631723, 52.29361981070505, 16.045187580326257, 7.7966212830173784, 7.5691172730782705]\n",
      "mean mse: 23.29265031668884\n",
      "Train [2017-07-01 2017-08-15] Val [2017-08-16 2017-08-31] Test [2017-09-01 2017-09-16]\n",
      "Window Train/Val/Test shape (966, 61) (336, 61) (336, 61)\n",
      "Window Train/Val/Test shape (966, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-08-30] Val [2017-08-31 2017-09-15] Test [2017-09-16 2017-10-01]\n",
      "Window Train/Val/Test shape (1281, 61) (336, 61) (336, 61)\n",
      "Window Train/Val/Test shape (1281, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-09-14] Val [2017-09-15 2017-09-30] Test [2017-10-01 2017-10-16]\n",
      "Window Train/Val/Test shape (1596, 61) (336, 61) (336, 61)\n",
      "Window Train/Val/Test shape (1596, 2) (336, 2) (336, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-09-29] Val [2017-09-30 2017-10-15] Test [2017-10-16 2017-11-10]\n",
      "Window Train/Val/Test shape (1911, 61) (336, 61) (546, 61)\n",
      "Window Train/Val/Test shape (1911, 2) (336, 2) (546, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-10-14] Val [2017-10-15 2017-10-30] Test [2017-10-31 2017-11-10]\n",
      "Window Train/Val/Test shape (2226, 61) (336, 61) (231, 61)\n",
      "Window Train/Val/Test shape (2226, 2) (336, 2) (231, 2)\n",
      "[22.388813743819696, 49.36033194706337, 20.53888956898424, 11.446451194776568, 7.1887585815790525]\n",
      "mean mse: 22.184649007244587\n",
      "Train [2017-07-01 2017-08-15] Val [2017-08-16 2017-08-31] Test [2017-09-01 2017-09-16]\n",
      "Window Train/Val/Test shape (966, 61) (336, 61) (336, 61)\n",
      "Window Train/Val/Test shape (966, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-08-30] Val [2017-08-31 2017-09-15] Test [2017-09-16 2017-10-01]\n",
      "Window Train/Val/Test shape (1281, 61) (336, 61) (336, 61)\n",
      "Window Train/Val/Test shape (1281, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-09-14] Val [2017-09-15 2017-09-30] Test [2017-10-01 2017-10-16]\n",
      "Window Train/Val/Test shape (1596, 61) (336, 61) (336, 61)\n",
      "Window Train/Val/Test shape (1596, 2) (336, 2) (336, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-09-29] Val [2017-09-30 2017-10-15] Test [2017-10-16 2017-11-10]\n",
      "Window Train/Val/Test shape (1911, 61) (336, 61) (546, 61)\n",
      "Window Train/Val/Test shape (1911, 2) (336, 2) (546, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-10-14] Val [2017-10-15 2017-10-30] Test [2017-10-31 2017-11-10]\n",
      "Window Train/Val/Test shape (2226, 61) (336, 61) (231, 61)\n",
      "Window Train/Val/Test shape (2226, 2) (336, 2) (231, 2)\n",
      "[24.01100618382598, 34.98461413991447, 18.555522594625987, 6.730157874554207, 7.007137300833739]\n",
      "mean mse: 18.257687618750875\n",
      "Train [2017-07-01 2017-08-15] Val [2017-08-16 2017-08-31] Test [2017-09-01 2017-09-16]\n",
      "Window Train/Val/Test shape (966, 61) (336, 61) (336, 61)\n",
      "Window Train/Val/Test shape (966, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-08-30] Val [2017-08-31 2017-09-15] Test [2017-09-16 2017-10-01]\n",
      "Window Train/Val/Test shape (1281, 61) (336, 61) (336, 61)\n",
      "Window Train/Val/Test shape (1281, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-09-14] Val [2017-09-15 2017-09-30] Test [2017-10-01 2017-10-16]\n",
      "Window Train/Val/Test shape (1596, 61) (336, 61) (336, 61)\n",
      "Window Train/Val/Test shape (1596, 2) (336, 2) (336, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-09-29] Val [2017-09-30 2017-10-15] Test [2017-10-16 2017-11-10]\n",
      "Window Train/Val/Test shape (1911, 61) (336, 61) (546, 61)\n",
      "Window Train/Val/Test shape (1911, 2) (336, 2) (546, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-10-14] Val [2017-10-15 2017-10-30] Test [2017-10-31 2017-11-10]\n",
      "Window Train/Val/Test shape (2226, 61) (336, 61) (231, 61)\n",
      "Window Train/Val/Test shape (2226, 2) (336, 2) (231, 2)\n",
      "[19.183227615972125, 40.26794986332283, 15.548799876677776, 8.689237316250626, 7.551026441542947]\n",
      "mean mse: 18.248048222753262\n",
      "Train [2017-07-01 2017-08-15] Val [2017-08-16 2017-08-31] Test [2017-09-01 2017-09-16]\n",
      "Window Train/Val/Test shape (966, 61) (336, 61) (336, 61)\n",
      "Window Train/Val/Test shape (966, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-08-30] Val [2017-08-31 2017-09-15] Test [2017-09-16 2017-10-01]\n",
      "Window Train/Val/Test shape (1281, 61) (336, 61) (336, 61)\n",
      "Window Train/Val/Test shape (1281, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-09-14] Val [2017-09-15 2017-09-30] Test [2017-10-01 2017-10-16]\n",
      "Window Train/Val/Test shape (1596, 61) (336, 61) (336, 61)\n",
      "Window Train/Val/Test shape (1596, 2) (336, 2) (336, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-09-29] Val [2017-09-30 2017-10-15] Test [2017-10-16 2017-11-10]\n",
      "Window Train/Val/Test shape (1911, 61) (336, 61) (546, 61)\n",
      "Window Train/Val/Test shape (1911, 2) (336, 2) (546, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-10-14] Val [2017-10-15 2017-10-30] Test [2017-10-31 2017-11-10]\n",
      "Window Train/Val/Test shape (2226, 61) (336, 61) (231, 61)\n",
      "Window Train/Val/Test shape (2226, 2) (336, 2) (231, 2)\n",
      "[21.86784472808541, 31.400898415219228, 11.65840152638641, 9.659349813356908, 7.849480693606577]\n",
      "mean mse: 16.487195035330906\n",
      "Train [2017-07-01 2017-08-15] Val [2017-08-16 2017-08-31] Test [2017-09-01 2017-09-16]\n",
      "Window Train/Val/Test shape (966, 61) (336, 61) (336, 61)\n",
      "Window Train/Val/Test shape (966, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-08-30] Val [2017-08-31 2017-09-15] Test [2017-09-16 2017-10-01]\n",
      "Window Train/Val/Test shape (1281, 61) (336, 61) (336, 61)\n",
      "Window Train/Val/Test shape (1281, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-09-14] Val [2017-09-15 2017-09-30] Test [2017-10-01 2017-10-16]\n",
      "Window Train/Val/Test shape (1596, 61) (336, 61) (336, 61)\n",
      "Window Train/Val/Test shape (1596, 2) (336, 2) (336, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-09-29] Val [2017-09-30 2017-10-15] Test [2017-10-16 2017-11-10]\n",
      "Window Train/Val/Test shape (1911, 61) (336, 61) (546, 61)\n",
      "Window Train/Val/Test shape (1911, 2) (336, 2) (546, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-10-14] Val [2017-10-15 2017-10-30] Test [2017-10-31 2017-11-10]\n",
      "Window Train/Val/Test shape (2226, 61) (336, 61) (231, 61)\n",
      "Window Train/Val/Test shape (2226, 2) (336, 2) (231, 2)\n",
      "[39.86191865577181, 47.302408831408925, 20.26027714315483, 9.291976822678647, 10.564596350792817]\n",
      "mean mse: 25.456235560761407\n",
      "Train [2017-07-01 2017-08-15] Val [2017-08-16 2017-08-31] Test [2017-09-01 2017-09-16]\n",
      "Window Train/Val/Test shape (966, 61) (336, 61) (336, 61)\n",
      "Window Train/Val/Test shape (966, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-08-30] Val [2017-08-31 2017-09-15] Test [2017-09-16 2017-10-01]\n",
      "Window Train/Val/Test shape (1281, 61) (336, 61) (336, 61)\n",
      "Window Train/Val/Test shape (1281, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-09-14] Val [2017-09-15 2017-09-30] Test [2017-10-01 2017-10-16]\n",
      "Window Train/Val/Test shape (1596, 61) (336, 61) (336, 61)\n",
      "Window Train/Val/Test shape (1596, 2) (336, 2) (336, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-09-29] Val [2017-09-30 2017-10-15] Test [2017-10-16 2017-11-10]\n",
      "Window Train/Val/Test shape (1911, 61) (336, 61) (546, 61)\n",
      "Window Train/Val/Test shape (1911, 2) (336, 2) (546, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-10-14] Val [2017-10-15 2017-10-30] Test [2017-10-31 2017-11-10]\n",
      "Window Train/Val/Test shape (2226, 61) (336, 61) (231, 61)\n",
      "Window Train/Val/Test shape (2226, 2) (336, 2) (231, 2)\n",
      "[32.22125654310505, 32.0543354468084, 20.217058733783432, 6.28083597025118, 6.668937415114118]\n",
      "mean mse: 19.488484821812435\n",
      "Train [2017-07-01 2017-08-15] Val [2017-08-16 2017-08-31] Test [2017-09-01 2017-09-16]\n",
      "Window Train/Val/Test shape (966, 61) (336, 61) (336, 61)\n",
      "Window Train/Val/Test shape (966, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-08-30] Val [2017-08-31 2017-09-15] Test [2017-09-16 2017-10-01]\n",
      "Window Train/Val/Test shape (1281, 61) (336, 61) (336, 61)\n",
      "Window Train/Val/Test shape (1281, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-09-14] Val [2017-09-15 2017-09-30] Test [2017-10-01 2017-10-16]\n",
      "Window Train/Val/Test shape (1596, 61) (336, 61) (336, 61)\n",
      "Window Train/Val/Test shape (1596, 2) (336, 2) (336, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-09-29] Val [2017-09-30 2017-10-15] Test [2017-10-16 2017-11-10]\n",
      "Window Train/Val/Test shape (1911, 61) (336, 61) (546, 61)\n",
      "Window Train/Val/Test shape (1911, 2) (336, 2) (546, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-10-14] Val [2017-10-15 2017-10-30] Test [2017-10-31 2017-11-10]\n",
      "Window Train/Val/Test shape (2226, 61) (336, 61) (231, 61)\n",
      "Window Train/Val/Test shape (2226, 2) (336, 2) (231, 2)\n",
      "[30.83508079988865, 38.44490088887441, 17.459515368630022, 7.443323383598435, 8.165960330252728]\n",
      "mean mse: 20.46975615424885\n",
      "Train [2017-07-01 2017-08-15] Val [2017-08-16 2017-08-31] Test [2017-09-01 2017-09-16]\n",
      "Window Train/Val/Test shape (966, 61) (336, 61) (336, 61)\n",
      "Window Train/Val/Test shape (966, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-08-30] Val [2017-08-31 2017-09-15] Test [2017-09-16 2017-10-01]\n",
      "Window Train/Val/Test shape (1281, 61) (336, 61) (336, 61)\n",
      "Window Train/Val/Test shape (1281, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-09-14] Val [2017-09-15 2017-09-30] Test [2017-10-01 2017-10-16]\n",
      "Window Train/Val/Test shape (1596, 61) (336, 61) (336, 61)\n",
      "Window Train/Val/Test shape (1596, 2) (336, 2) (336, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-09-29] Val [2017-09-30 2017-10-15] Test [2017-10-16 2017-11-10]\n",
      "Window Train/Val/Test shape (1911, 61) (336, 61) (546, 61)\n",
      "Window Train/Val/Test shape (1911, 2) (336, 2) (546, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-10-14] Val [2017-10-15 2017-10-30] Test [2017-10-31 2017-11-10]\n",
      "Window Train/Val/Test shape (2226, 61) (336, 61) (231, 61)\n",
      "Window Train/Val/Test shape (2226, 2) (336, 2) (231, 2)\n",
      "[32.31291622040297, 45.60982161496191, 18.417954807288464, 11.554946950193568, 8.857271239446334]\n",
      "mean mse: 23.350582166458647\n",
      "Train [2017-07-01 2017-08-15] Val [2017-08-16 2017-08-31] Test [2017-09-01 2017-09-16]\n",
      "Window Train/Val/Test shape (966, 61) (336, 61) (336, 61)\n",
      "Window Train/Val/Test shape (966, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-08-30] Val [2017-08-31 2017-09-15] Test [2017-09-16 2017-10-01]\n",
      "Window Train/Val/Test shape (1281, 61) (336, 61) (336, 61)\n",
      "Window Train/Val/Test shape (1281, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-09-14] Val [2017-09-15 2017-09-30] Test [2017-10-01 2017-10-16]\n",
      "Window Train/Val/Test shape (1596, 61) (336, 61) (336, 61)\n",
      "Window Train/Val/Test shape (1596, 2) (336, 2) (336, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-09-29] Val [2017-09-30 2017-10-15] Test [2017-10-16 2017-11-10]\n",
      "Window Train/Val/Test shape (1911, 61) (336, 61) (546, 61)\n",
      "Window Train/Val/Test shape (1911, 2) (336, 2) (546, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-10-14] Val [2017-10-15 2017-10-30] Test [2017-10-31 2017-11-10]\n",
      "Window Train/Val/Test shape (2226, 61) (336, 61) (231, 61)\n",
      "Window Train/Val/Test shape (2226, 2) (336, 2) (231, 2)\n",
      "[23.432444473346838, 37.45642619056596, 14.939750495307, 8.511873207529655, 9.442541410845537]\n",
      "mean mse: 18.756607155519\n",
      "Train [2017-07-01 2017-08-15] Val [2017-08-16 2017-08-31] Test [2017-09-01 2017-09-16]\n",
      "Window Train/Val/Test shape (966, 61) (336, 61) (336, 61)\n",
      "Window Train/Val/Test shape (966, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-08-30] Val [2017-08-31 2017-09-15] Test [2017-09-16 2017-10-01]\n",
      "Window Train/Val/Test shape (1281, 61) (336, 61) (336, 61)\n",
      "Window Train/Val/Test shape (1281, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-09-14] Val [2017-09-15 2017-09-30] Test [2017-10-01 2017-10-16]\n",
      "Window Train/Val/Test shape (1596, 61) (336, 61) (336, 61)\n",
      "Window Train/Val/Test shape (1596, 2) (336, 2) (336, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-09-29] Val [2017-09-30 2017-10-15] Test [2017-10-16 2017-11-10]\n",
      "Window Train/Val/Test shape (1911, 61) (336, 61) (546, 61)\n",
      "Window Train/Val/Test shape (1911, 2) (336, 2) (546, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-10-14] Val [2017-10-15 2017-10-30] Test [2017-10-31 2017-11-10]\n",
      "Window Train/Val/Test shape (2226, 61) (336, 61) (231, 61)\n",
      "Window Train/Val/Test shape (2226, 2) (336, 2) (231, 2)\n",
      "[32.763697855750294, 39.24335534215045, 19.753811960309285, 13.465752359484968, 8.895074687079456]\n",
      "mean mse: 22.824338440954893\n",
      "Train [2017-07-01 2017-08-15] Val [2017-08-16 2017-08-31] Test [2017-09-01 2017-09-16]\n",
      "Window Train/Val/Test shape (966, 61) (336, 61) (336, 61)\n",
      "Window Train/Val/Test shape (966, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-08-30] Val [2017-08-31 2017-09-15] Test [2017-09-16 2017-10-01]\n",
      "Window Train/Val/Test shape (1281, 61) (336, 61) (336, 61)\n",
      "Window Train/Val/Test shape (1281, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-09-14] Val [2017-09-15 2017-09-30] Test [2017-10-01 2017-10-16]\n",
      "Window Train/Val/Test shape (1596, 61) (336, 61) (336, 61)\n",
      "Window Train/Val/Test shape (1596, 2) (336, 2) (336, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-09-29] Val [2017-09-30 2017-10-15] Test [2017-10-16 2017-11-10]\n",
      "Window Train/Val/Test shape (1911, 61) (336, 61) (546, 61)\n",
      "Window Train/Val/Test shape (1911, 2) (336, 2) (546, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-10-14] Val [2017-10-15 2017-10-30] Test [2017-10-31 2017-11-10]\n",
      "Window Train/Val/Test shape (2226, 61) (336, 61) (231, 61)\n",
      "Window Train/Val/Test shape (2226, 2) (336, 2) (231, 2)\n",
      "[26.47103290291479, 48.04865943174697, 16.92143794205122, 9.158430930076888, 7.426902282260867]\n",
      "mean mse: 21.60529269781015\n",
      "Train [2017-07-01 2017-08-15] Val [2017-08-16 2017-08-31] Test [2017-09-01 2017-09-16]\n",
      "Window Train/Val/Test shape (966, 61) (336, 61) (336, 61)\n",
      "Window Train/Val/Test shape (966, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-08-30] Val [2017-08-31 2017-09-15] Test [2017-09-16 2017-10-01]\n",
      "Window Train/Val/Test shape (1281, 61) (336, 61) (336, 61)\n",
      "Window Train/Val/Test shape (1281, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-09-14] Val [2017-09-15 2017-09-30] Test [2017-10-01 2017-10-16]\n",
      "Window Train/Val/Test shape (1596, 61) (336, 61) (336, 61)\n",
      "Window Train/Val/Test shape (1596, 2) (336, 2) (336, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-09-29] Val [2017-09-30 2017-10-15] Test [2017-10-16 2017-11-10]\n",
      "Window Train/Val/Test shape (1911, 61) (336, 61) (546, 61)\n",
      "Window Train/Val/Test shape (1911, 2) (336, 2) (546, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-10-14] Val [2017-10-15 2017-10-30] Test [2017-10-31 2017-11-10]\n",
      "Window Train/Val/Test shape (2226, 61) (336, 61) (231, 61)\n",
      "Window Train/Val/Test shape (2226, 2) (336, 2) (231, 2)\n",
      "[25.90238441449526, 42.7004163694252, 15.311856435916875, 9.306335864945739, 6.60821990490256]\n",
      "mean mse: 19.965842597937126\n",
      "Train [2017-07-01 2017-08-15] Val [2017-08-16 2017-08-31] Test [2017-09-01 2017-09-16]\n",
      "Window Train/Val/Test shape (966, 61) (336, 61) (336, 61)\n",
      "Window Train/Val/Test shape (966, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-08-30] Val [2017-08-31 2017-09-15] Test [2017-09-16 2017-10-01]\n",
      "Window Train/Val/Test shape (1281, 61) (336, 61) (336, 61)\n",
      "Window Train/Val/Test shape (1281, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-09-14] Val [2017-09-15 2017-09-30] Test [2017-10-01 2017-10-16]\n",
      "Window Train/Val/Test shape (1596, 61) (336, 61) (336, 61)\n",
      "Window Train/Val/Test shape (1596, 2) (336, 2) (336, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-09-29] Val [2017-09-30 2017-10-15] Test [2017-10-16 2017-11-10]\n",
      "Window Train/Val/Test shape (1911, 61) (336, 61) (546, 61)\n",
      "Window Train/Val/Test shape (1911, 2) (336, 2) (546, 2)\n",
      "Rolling window to end date\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train [2017-07-01 2017-10-14] Val [2017-10-15 2017-10-30] Test [2017-10-31 2017-11-10]\n",
      "Window Train/Val/Test shape (2226, 61) (336, 61) (231, 61)\n",
      "Window Train/Val/Test shape (2226, 2) (336, 2) (231, 2)\n",
      "[29.226584113268668, 35.75855248007927, 20.1033694288779, 9.890816458695376, 6.707771890567726]\n",
      "mean mse: 20.33741887429779\n",
      "Train [2017-07-01 2017-08-15] Val [2017-08-16 2017-08-31] Test [2017-09-01 2017-09-16]\n",
      "Window Train/Val/Test shape (966, 61) (336, 61) (336, 61)\n",
      "Window Train/Val/Test shape (966, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-08-30] Val [2017-08-31 2017-09-15] Test [2017-09-16 2017-10-01]\n",
      "Window Train/Val/Test shape (1281, 61) (336, 61) (336, 61)\n",
      "Window Train/Val/Test shape (1281, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-09-14] Val [2017-09-15 2017-09-30] Test [2017-10-01 2017-10-16]\n",
      "Window Train/Val/Test shape (1596, 61) (336, 61) (336, 61)\n",
      "Window Train/Val/Test shape (1596, 2) (336, 2) (336, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-09-29] Val [2017-09-30 2017-10-15] Test [2017-10-16 2017-11-10]\n",
      "Window Train/Val/Test shape (1911, 61) (336, 61) (546, 61)\n",
      "Window Train/Val/Test shape (1911, 2) (336, 2) (546, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-10-14] Val [2017-10-15 2017-10-30] Test [2017-10-31 2017-11-10]\n",
      "Window Train/Val/Test shape (2226, 61) (336, 61) (231, 61)\n",
      "Window Train/Val/Test shape (2226, 2) (336, 2) (231, 2)\n",
      "[33.22571234937316, 41.33032459870521, 18.349884717663347, 9.147958808826594, 6.43563874429792]\n",
      "mean mse: 21.697903843773243\n",
      "Train [2017-07-01 2017-08-15] Val [2017-08-16 2017-08-31] Test [2017-09-01 2017-09-16]\n",
      "Window Train/Val/Test shape (966, 61) (336, 61) (336, 61)\n",
      "Window Train/Val/Test shape (966, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-08-30] Val [2017-08-31 2017-09-15] Test [2017-09-16 2017-10-01]\n",
      "Window Train/Val/Test shape (1281, 61) (336, 61) (336, 61)\n",
      "Window Train/Val/Test shape (1281, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-09-14] Val [2017-09-15 2017-09-30] Test [2017-10-01 2017-10-16]\n",
      "Window Train/Val/Test shape (1596, 61) (336, 61) (336, 61)\n",
      "Window Train/Val/Test shape (1596, 2) (336, 2) (336, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-09-29] Val [2017-09-30 2017-10-15] Test [2017-10-16 2017-11-10]\n",
      "Window Train/Val/Test shape (1911, 61) (336, 61) (546, 61)\n",
      "Window Train/Val/Test shape (1911, 2) (336, 2) (546, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-10-14] Val [2017-10-15 2017-10-30] Test [2017-10-31 2017-11-10]\n",
      "Window Train/Val/Test shape (2226, 61) (336, 61) (231, 61)\n",
      "Window Train/Val/Test shape (2226, 2) (336, 2) (231, 2)\n",
      "[31.294727466339594, 31.570819021566884, 13.794780617355205, 8.599129799417, 5.473307338108678]\n",
      "mean mse: 18.146552848557473\n",
      "Train [2017-07-01 2017-08-15] Val [2017-08-16 2017-08-31] Test [2017-09-01 2017-09-16]\n",
      "Window Train/Val/Test shape (966, 61) (336, 61) (336, 61)\n",
      "Window Train/Val/Test shape (966, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-08-30] Val [2017-08-31 2017-09-15] Test [2017-09-16 2017-10-01]\n",
      "Window Train/Val/Test shape (1281, 61) (336, 61) (336, 61)\n",
      "Window Train/Val/Test shape (1281, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-09-14] Val [2017-09-15 2017-09-30] Test [2017-10-01 2017-10-16]\n",
      "Window Train/Val/Test shape (1596, 61) (336, 61) (336, 61)\n",
      "Window Train/Val/Test shape (1596, 2) (336, 2) (336, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-09-29] Val [2017-09-30 2017-10-15] Test [2017-10-16 2017-11-10]\n",
      "Window Train/Val/Test shape (1911, 61) (336, 61) (546, 61)\n",
      "Window Train/Val/Test shape (1911, 2) (336, 2) (546, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-10-14] Val [2017-10-15 2017-10-30] Test [2017-10-31 2017-11-10]\n",
      "Window Train/Val/Test shape (2226, 61) (336, 61) (231, 61)\n",
      "Window Train/Val/Test shape (2226, 2) (336, 2) (231, 2)\n",
      "[23.623231470559343, 39.79913464926841, 20.246443289171967, 14.909610162453966, 7.817391352559388]\n",
      "mean mse: 21.279162184802612\n",
      "Train [2017-07-01 2017-08-15] Val [2017-08-16 2017-08-31] Test [2017-09-01 2017-09-16]\n",
      "Window Train/Val/Test shape (966, 61) (336, 61) (336, 61)\n",
      "Window Train/Val/Test shape (966, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-08-30] Val [2017-08-31 2017-09-15] Test [2017-09-16 2017-10-01]\n",
      "Window Train/Val/Test shape (1281, 61) (336, 61) (336, 61)\n",
      "Window Train/Val/Test shape (1281, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-09-14] Val [2017-09-15 2017-09-30] Test [2017-10-01 2017-10-16]\n",
      "Window Train/Val/Test shape (1596, 61) (336, 61) (336, 61)\n",
      "Window Train/Val/Test shape (1596, 2) (336, 2) (336, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-09-29] Val [2017-09-30 2017-10-15] Test [2017-10-16 2017-11-10]\n",
      "Window Train/Val/Test shape (1911, 61) (336, 61) (546, 61)\n",
      "Window Train/Val/Test shape (1911, 2) (336, 2) (546, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-10-14] Val [2017-10-15 2017-10-30] Test [2017-10-31 2017-11-10]\n",
      "Window Train/Val/Test shape (2226, 61) (336, 61) (231, 61)\n",
      "Window Train/Val/Test shape (2226, 2) (336, 2) (231, 2)\n",
      "[19.92718297830212, 41.55471399605277, 16.15829625586096, 11.045718014075696, 7.153586792713949]\n",
      "mean mse: 19.167899607401104\n",
      "Train [2017-07-01 2017-08-15] Val [2017-08-16 2017-08-31] Test [2017-09-01 2017-09-16]\n",
      "Window Train/Val/Test shape (966, 61) (336, 61) (336, 61)\n",
      "Window Train/Val/Test shape (966, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-08-30] Val [2017-08-31 2017-09-15] Test [2017-09-16 2017-10-01]\n",
      "Window Train/Val/Test shape (1281, 61) (336, 61) (336, 61)\n",
      "Window Train/Val/Test shape (1281, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-09-14] Val [2017-09-15 2017-09-30] Test [2017-10-01 2017-10-16]\n",
      "Window Train/Val/Test shape (1596, 61) (336, 61) (336, 61)\n",
      "Window Train/Val/Test shape (1596, 2) (336, 2) (336, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-09-29] Val [2017-09-30 2017-10-15] Test [2017-10-16 2017-11-10]\n",
      "Window Train/Val/Test shape (1911, 61) (336, 61) (546, 61)\n",
      "Window Train/Val/Test shape (1911, 2) (336, 2) (546, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-10-14] Val [2017-10-15 2017-10-30] Test [2017-10-31 2017-11-10]\n",
      "Window Train/Val/Test shape (2226, 61) (336, 61) (231, 61)\n",
      "Window Train/Val/Test shape (2226, 2) (336, 2) (231, 2)\n",
      "[29.925670171388315, 44.41923281157026, 18.50434221786282, 11.546472704328199, 7.562749359372023]\n",
      "mean mse: 22.391693452904324\n",
      "Train [2017-07-01 2017-08-15] Val [2017-08-16 2017-08-31] Test [2017-09-01 2017-09-16]\n",
      "Window Train/Val/Test shape (966, 61) (336, 61) (336, 61)\n",
      "Window Train/Val/Test shape (966, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-08-30] Val [2017-08-31 2017-09-15] Test [2017-09-16 2017-10-01]\n",
      "Window Train/Val/Test shape (1281, 61) (336, 61) (336, 61)\n",
      "Window Train/Val/Test shape (1281, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-09-14] Val [2017-09-15 2017-09-30] Test [2017-10-01 2017-10-16]\n",
      "Window Train/Val/Test shape (1596, 61) (336, 61) (336, 61)\n",
      "Window Train/Val/Test shape (1596, 2) (336, 2) (336, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-09-29] Val [2017-09-30 2017-10-15] Test [2017-10-16 2017-11-10]\n",
      "Window Train/Val/Test shape (1911, 61) (336, 61) (546, 61)\n",
      "Window Train/Val/Test shape (1911, 2) (336, 2) (546, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-10-14] Val [2017-10-15 2017-10-30] Test [2017-10-31 2017-11-10]\n",
      "Window Train/Val/Test shape (2226, 61) (336, 61) (231, 61)\n",
      "Window Train/Val/Test shape (2226, 2) (336, 2) (231, 2)\n",
      "[25.76732082659774, 35.396082624976195, 14.607687404851303, 7.132723782102728, 5.97562844797145]\n",
      "mean mse: 17.77588861729988\n",
      "Train [2017-07-01 2017-08-15] Val [2017-08-16 2017-08-31] Test [2017-09-01 2017-09-16]\n",
      "Window Train/Val/Test shape (966, 61) (336, 61) (336, 61)\n",
      "Window Train/Val/Test shape (966, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-08-30] Val [2017-08-31 2017-09-15] Test [2017-09-16 2017-10-01]\n",
      "Window Train/Val/Test shape (1281, 61) (336, 61) (336, 61)\n",
      "Window Train/Val/Test shape (1281, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-09-14] Val [2017-09-15 2017-09-30] Test [2017-10-01 2017-10-16]\n",
      "Window Train/Val/Test shape (1596, 61) (336, 61) (336, 61)\n",
      "Window Train/Val/Test shape (1596, 2) (336, 2) (336, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-09-29] Val [2017-09-30 2017-10-15] Test [2017-10-16 2017-11-10]\n",
      "Window Train/Val/Test shape (1911, 61) (336, 61) (546, 61)\n",
      "Window Train/Val/Test shape (1911, 2) (336, 2) (546, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-10-14] Val [2017-10-15 2017-10-30] Test [2017-10-31 2017-11-10]\n",
      "Window Train/Val/Test shape (2226, 61) (336, 61) (231, 61)\n",
      "Window Train/Val/Test shape (2226, 2) (336, 2) (231, 2)\n",
      "[22.14025764092321, 26.999892158528382, 15.199719984864679, 7.408358637946547, 6.1997630575690055]\n",
      "mean mse: 15.589598295966365\n",
      "Train [2017-07-01 2017-08-15] Val [2017-08-16 2017-08-31] Test [2017-09-01 2017-09-16]\n",
      "Window Train/Val/Test shape (966, 61) (336, 61) (336, 61)\n",
      "Window Train/Val/Test shape (966, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-08-30] Val [2017-08-31 2017-09-15] Test [2017-09-16 2017-10-01]\n",
      "Window Train/Val/Test shape (1281, 61) (336, 61) (336, 61)\n",
      "Window Train/Val/Test shape (1281, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-09-14] Val [2017-09-15 2017-09-30] Test [2017-10-01 2017-10-16]\n",
      "Window Train/Val/Test shape (1596, 61) (336, 61) (336, 61)\n",
      "Window Train/Val/Test shape (1596, 2) (336, 2) (336, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-09-29] Val [2017-09-30 2017-10-15] Test [2017-10-16 2017-11-10]\n",
      "Window Train/Val/Test shape (1911, 61) (336, 61) (546, 61)\n",
      "Window Train/Val/Test shape (1911, 2) (336, 2) (546, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-10-14] Val [2017-10-15 2017-10-30] Test [2017-10-31 2017-11-10]\n",
      "Window Train/Val/Test shape (2226, 61) (336, 61) (231, 61)\n",
      "Window Train/Val/Test shape (2226, 2) (336, 2) (231, 2)\n",
      "[26.095503294086047, 38.473839458712185, 16.29867298651298, 9.812831886955925, 6.5832135098340565]\n",
      "mean mse: 19.45281222722024\n",
      "Train [2017-07-01 2017-08-15] Val [2017-08-16 2017-08-31] Test [2017-09-01 2017-09-16]\n",
      "Window Train/Val/Test shape (966, 61) (336, 61) (336, 61)\n",
      "Window Train/Val/Test shape (966, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-08-30] Val [2017-08-31 2017-09-15] Test [2017-09-16 2017-10-01]\n",
      "Window Train/Val/Test shape (1281, 61) (336, 61) (336, 61)\n",
      "Window Train/Val/Test shape (1281, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-09-14] Val [2017-09-15 2017-09-30] Test [2017-10-01 2017-10-16]\n",
      "Window Train/Val/Test shape (1596, 61) (336, 61) (336, 61)\n",
      "Window Train/Val/Test shape (1596, 2) (336, 2) (336, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-09-29] Val [2017-09-30 2017-10-15] Test [2017-10-16 2017-11-10]\n",
      "Window Train/Val/Test shape (1911, 61) (336, 61) (546, 61)\n",
      "Window Train/Val/Test shape (1911, 2) (336, 2) (546, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-10-14] Val [2017-10-15 2017-10-30] Test [2017-10-31 2017-11-10]\n",
      "Window Train/Val/Test shape (2226, 61) (336, 61) (231, 61)\n",
      "Window Train/Val/Test shape (2226, 2) (336, 2) (231, 2)\n",
      "[27.93275738917137, 39.473413673190336, 13.754918364238563, 9.827213294110754, 6.857930767588344]\n",
      "mean mse: 19.56924669765987\n",
      "Train [2017-07-01 2017-08-15] Val [2017-08-16 2017-08-31] Test [2017-09-01 2017-09-16]\n",
      "Window Train/Val/Test shape (966, 61) (336, 61) (336, 61)\n",
      "Window Train/Val/Test shape (966, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-08-30] Val [2017-08-31 2017-09-15] Test [2017-09-16 2017-10-01]\n",
      "Window Train/Val/Test shape (1281, 61) (336, 61) (336, 61)\n",
      "Window Train/Val/Test shape (1281, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-09-14] Val [2017-09-15 2017-09-30] Test [2017-10-01 2017-10-16]\n",
      "Window Train/Val/Test shape (1596, 61) (336, 61) (336, 61)\n",
      "Window Train/Val/Test shape (1596, 2) (336, 2) (336, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-09-29] Val [2017-09-30 2017-10-15] Test [2017-10-16 2017-11-10]\n",
      "Window Train/Val/Test shape (1911, 61) (336, 61) (546, 61)\n",
      "Window Train/Val/Test shape (1911, 2) (336, 2) (546, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-10-14] Val [2017-10-15 2017-10-30] Test [2017-10-31 2017-11-10]\n",
      "Window Train/Val/Test shape (2226, 61) (336, 61) (231, 61)\n",
      "Window Train/Val/Test shape (2226, 2) (336, 2) (231, 2)\n",
      "[28.20083015875585, 30.200376940477955, 15.871716013612435, 6.734248095393608, 6.80739118724158]\n",
      "mean mse: 17.562912479096283\n",
      "Train [2017-07-01 2017-08-15] Val [2017-08-16 2017-08-31] Test [2017-09-01 2017-09-16]\n",
      "Window Train/Val/Test shape (966, 61) (336, 61) (336, 61)\n",
      "Window Train/Val/Test shape (966, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-08-30] Val [2017-08-31 2017-09-15] Test [2017-09-16 2017-10-01]\n",
      "Window Train/Val/Test shape (1281, 61) (336, 61) (336, 61)\n",
      "Window Train/Val/Test shape (1281, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-09-14] Val [2017-09-15 2017-09-30] Test [2017-10-01 2017-10-16]\n",
      "Window Train/Val/Test shape (1596, 61) (336, 61) (336, 61)\n",
      "Window Train/Val/Test shape (1596, 2) (336, 2) (336, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-09-29] Val [2017-09-30 2017-10-15] Test [2017-10-16 2017-11-10]\n",
      "Window Train/Val/Test shape (1911, 61) (336, 61) (546, 61)\n",
      "Window Train/Val/Test shape (1911, 2) (336, 2) (546, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-10-14] Val [2017-10-15 2017-10-30] Test [2017-10-31 2017-11-10]\n",
      "Window Train/Val/Test shape (2226, 61) (336, 61) (231, 61)\n",
      "Window Train/Val/Test shape (2226, 2) (336, 2) (231, 2)\n",
      "[23.310614753759683, 33.01546089788033, 11.736070359642431, 7.947362854275091, 6.38314132934812]\n",
      "mean mse: 16.478530038981127\n",
      "Train [2017-07-01 2017-08-15] Val [2017-08-16 2017-08-31] Test [2017-09-01 2017-09-16]\n",
      "Window Train/Val/Test shape (966, 61) (336, 61) (336, 61)\n",
      "Window Train/Val/Test shape (966, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-08-30] Val [2017-08-31 2017-09-15] Test [2017-09-16 2017-10-01]\n",
      "Window Train/Val/Test shape (1281, 61) (336, 61) (336, 61)\n",
      "Window Train/Val/Test shape (1281, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-09-14] Val [2017-09-15 2017-09-30] Test [2017-10-01 2017-10-16]\n",
      "Window Train/Val/Test shape (1596, 61) (336, 61) (336, 61)\n",
      "Window Train/Val/Test shape (1596, 2) (336, 2) (336, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-09-29] Val [2017-09-30 2017-10-15] Test [2017-10-16 2017-11-10]\n",
      "Window Train/Val/Test shape (1911, 61) (336, 61) (546, 61)\n",
      "Window Train/Val/Test shape (1911, 2) (336, 2) (546, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-10-14] Val [2017-10-15 2017-10-30] Test [2017-10-31 2017-11-10]\n",
      "Window Train/Val/Test shape (2226, 61) (336, 61) (231, 61)\n",
      "Window Train/Val/Test shape (2226, 2) (336, 2) (231, 2)\n",
      "[21.801843895487256, 40.15853480070475, 13.375019911404488, 8.025063101934165, 6.223725222489093]\n",
      "mean mse: 17.91683738640395\n",
      "Train [2017-07-01 2017-08-15] Val [2017-08-16 2017-08-31] Test [2017-09-01 2017-09-16]\n",
      "Window Train/Val/Test shape (966, 61) (336, 61) (336, 61)\n",
      "Window Train/Val/Test shape (966, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-08-30] Val [2017-08-31 2017-09-15] Test [2017-09-16 2017-10-01]\n",
      "Window Train/Val/Test shape (1281, 61) (336, 61) (336, 61)\n",
      "Window Train/Val/Test shape (1281, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-09-14] Val [2017-09-15 2017-09-30] Test [2017-10-01 2017-10-16]\n",
      "Window Train/Val/Test shape (1596, 61) (336, 61) (336, 61)\n",
      "Window Train/Val/Test shape (1596, 2) (336, 2) (336, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-09-29] Val [2017-09-30 2017-10-15] Test [2017-10-16 2017-11-10]\n",
      "Window Train/Val/Test shape (1911, 61) (336, 61) (546, 61)\n",
      "Window Train/Val/Test shape (1911, 2) (336, 2) (546, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-10-14] Val [2017-10-15 2017-10-30] Test [2017-10-31 2017-11-10]\n",
      "Window Train/Val/Test shape (2226, 61) (336, 61) (231, 61)\n",
      "Window Train/Val/Test shape (2226, 2) (336, 2) (231, 2)\n",
      "[25.258654919619737, 37.02989439397989, 12.39471225811296, 7.942186595951844, 7.531569259345851]\n",
      "mean mse: 18.03140348540206\n",
      "Train [2017-07-01 2017-08-15] Val [2017-08-16 2017-08-31] Test [2017-09-01 2017-09-16]\n",
      "Window Train/Val/Test shape (966, 61) (336, 61) (336, 61)\n",
      "Window Train/Val/Test shape (966, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-08-30] Val [2017-08-31 2017-09-15] Test [2017-09-16 2017-10-01]\n",
      "Window Train/Val/Test shape (1281, 61) (336, 61) (336, 61)\n",
      "Window Train/Val/Test shape (1281, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-09-14] Val [2017-09-15 2017-09-30] Test [2017-10-01 2017-10-16]\n",
      "Window Train/Val/Test shape (1596, 61) (336, 61) (336, 61)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Window Train/Val/Test shape (1596, 2) (336, 2) (336, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-09-29] Val [2017-09-30 2017-10-15] Test [2017-10-16 2017-11-10]\n",
      "Window Train/Val/Test shape (1911, 61) (336, 61) (546, 61)\n",
      "Window Train/Val/Test shape (1911, 2) (336, 2) (546, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-10-14] Val [2017-10-15 2017-10-30] Test [2017-10-31 2017-11-10]\n",
      "Window Train/Val/Test shape (2226, 61) (336, 61) (231, 61)\n",
      "Window Train/Val/Test shape (2226, 2) (336, 2) (231, 2)\n",
      "[23.76567823406257, 33.76718677905179, 11.815849485778788, 8.252549807319696, 7.186965005905659]\n",
      "mean mse: 16.9576458624237\n",
      "Train [2017-07-01 2017-08-15] Val [2017-08-16 2017-08-31] Test [2017-09-01 2017-09-16]\n",
      "Window Train/Val/Test shape (966, 61) (336, 61) (336, 61)\n",
      "Window Train/Val/Test shape (966, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-08-30] Val [2017-08-31 2017-09-15] Test [2017-09-16 2017-10-01]\n",
      "Window Train/Val/Test shape (1281, 61) (336, 61) (336, 61)\n",
      "Window Train/Val/Test shape (1281, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-09-14] Val [2017-09-15 2017-09-30] Test [2017-10-01 2017-10-16]\n",
      "Window Train/Val/Test shape (1596, 61) (336, 61) (336, 61)\n",
      "Window Train/Val/Test shape (1596, 2) (336, 2) (336, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-09-29] Val [2017-09-30 2017-10-15] Test [2017-10-16 2017-11-10]\n",
      "Window Train/Val/Test shape (1911, 61) (336, 61) (546, 61)\n",
      "Window Train/Val/Test shape (1911, 2) (336, 2) (546, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-10-14] Val [2017-10-15 2017-10-30] Test [2017-10-31 2017-11-10]\n",
      "Window Train/Val/Test shape (2226, 61) (336, 61) (231, 61)\n",
      "Window Train/Val/Test shape (2226, 2) (336, 2) (231, 2)\n",
      "[25.68145585025064, 48.985751981863075, 19.061013900868016, 10.341050594514394, 7.65601541068261]\n",
      "mean mse: 22.34505754763575\n",
      "Train [2017-07-01 2017-08-15] Val [2017-08-16 2017-08-31] Test [2017-09-01 2017-09-16]\n",
      "Window Train/Val/Test shape (966, 61) (336, 61) (336, 61)\n",
      "Window Train/Val/Test shape (966, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-08-30] Val [2017-08-31 2017-09-15] Test [2017-09-16 2017-10-01]\n",
      "Window Train/Val/Test shape (1281, 61) (336, 61) (336, 61)\n",
      "Window Train/Val/Test shape (1281, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-09-14] Val [2017-09-15 2017-09-30] Test [2017-10-01 2017-10-16]\n",
      "Window Train/Val/Test shape (1596, 61) (336, 61) (336, 61)\n",
      "Window Train/Val/Test shape (1596, 2) (336, 2) (336, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-09-29] Val [2017-09-30 2017-10-15] Test [2017-10-16 2017-11-10]\n",
      "Window Train/Val/Test shape (1911, 61) (336, 61) (546, 61)\n",
      "Window Train/Val/Test shape (1911, 2) (336, 2) (546, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-10-14] Val [2017-10-15 2017-10-30] Test [2017-10-31 2017-11-10]\n",
      "Window Train/Val/Test shape (2226, 61) (336, 61) (231, 61)\n",
      "Window Train/Val/Test shape (2226, 2) (336, 2) (231, 2)\n",
      "[24.57883004669548, 30.259579551562318, 18.024009855335724, 7.28310195543274, 6.230699720184274]\n",
      "mean mse: 17.27524422584211\n",
      "Train [2017-07-01 2017-08-15] Val [2017-08-16 2017-08-31] Test [2017-09-01 2017-09-16]\n",
      "Window Train/Val/Test shape (966, 61) (336, 61) (336, 61)\n",
      "Window Train/Val/Test shape (966, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-08-30] Val [2017-08-31 2017-09-15] Test [2017-09-16 2017-10-01]\n",
      "Window Train/Val/Test shape (1281, 61) (336, 61) (336, 61)\n",
      "Window Train/Val/Test shape (1281, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-09-14] Val [2017-09-15 2017-09-30] Test [2017-10-01 2017-10-16]\n",
      "Window Train/Val/Test shape (1596, 61) (336, 61) (336, 61)\n",
      "Window Train/Val/Test shape (1596, 2) (336, 2) (336, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-09-29] Val [2017-09-30 2017-10-15] Test [2017-10-16 2017-11-10]\n",
      "Window Train/Val/Test shape (1911, 61) (336, 61) (546, 61)\n",
      "Window Train/Val/Test shape (1911, 2) (336, 2) (546, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-10-14] Val [2017-10-15 2017-10-30] Test [2017-10-31 2017-11-10]\n",
      "Window Train/Val/Test shape (2226, 61) (336, 61) (231, 61)\n",
      "Window Train/Val/Test shape (2226, 2) (336, 2) (231, 2)\n",
      "[23.5619251136067, 33.67376144663912, 14.218054576852875, 6.480661955216756, 6.855519235677008]\n",
      "mean mse: 16.957984465598493\n",
      "Train [2017-07-01 2017-08-15] Val [2017-08-16 2017-08-31] Test [2017-09-01 2017-09-16]\n",
      "Window Train/Val/Test shape (966, 61) (336, 61) (336, 61)\n",
      "Window Train/Val/Test shape (966, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-08-30] Val [2017-08-31 2017-09-15] Test [2017-09-16 2017-10-01]\n",
      "Window Train/Val/Test shape (1281, 61) (336, 61) (336, 61)\n",
      "Window Train/Val/Test shape (1281, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-09-14] Val [2017-09-15 2017-09-30] Test [2017-10-01 2017-10-16]\n",
      "Window Train/Val/Test shape (1596, 61) (336, 61) (336, 61)\n",
      "Window Train/Val/Test shape (1596, 2) (336, 2) (336, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-09-29] Val [2017-09-30 2017-10-15] Test [2017-10-16 2017-11-10]\n",
      "Window Train/Val/Test shape (1911, 61) (336, 61) (546, 61)\n",
      "Window Train/Val/Test shape (1911, 2) (336, 2) (546, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-10-14] Val [2017-10-15 2017-10-30] Test [2017-10-31 2017-11-10]\n",
      "Window Train/Val/Test shape (2226, 61) (336, 61) (231, 61)\n",
      "Window Train/Val/Test shape (2226, 2) (336, 2) (231, 2)\n",
      "[23.627839252528258, 42.822354999281906, 15.30771710557197, 9.165271883551027, 6.884181713101728]\n",
      "mean mse: 19.561472990806976\n",
      "Train [2017-07-01 2017-08-15] Val [2017-08-16 2017-08-31] Test [2017-09-01 2017-09-16]\n",
      "Window Train/Val/Test shape (966, 61) (336, 61) (336, 61)\n",
      "Window Train/Val/Test shape (966, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-08-30] Val [2017-08-31 2017-09-15] Test [2017-09-16 2017-10-01]\n",
      "Window Train/Val/Test shape (1281, 61) (336, 61) (336, 61)\n",
      "Window Train/Val/Test shape (1281, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-09-14] Val [2017-09-15 2017-09-30] Test [2017-10-01 2017-10-16]\n",
      "Window Train/Val/Test shape (1596, 61) (336, 61) (336, 61)\n",
      "Window Train/Val/Test shape (1596, 2) (336, 2) (336, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-09-29] Val [2017-09-30 2017-10-15] Test [2017-10-16 2017-11-10]\n",
      "Window Train/Val/Test shape (1911, 61) (336, 61) (546, 61)\n",
      "Window Train/Val/Test shape (1911, 2) (336, 2) (546, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-10-14] Val [2017-10-15 2017-10-30] Test [2017-10-31 2017-11-10]\n",
      "Window Train/Val/Test shape (2226, 61) (336, 61) (231, 61)\n",
      "Window Train/Val/Test shape (2226, 2) (336, 2) (231, 2)\n",
      "[24.303529726164676, 31.691004061364634, 19.691110276064517, 6.545243853785161, 6.7461359174142865]\n",
      "mean mse: 17.795404766958658\n",
      "Train [2017-07-01 2017-08-15] Val [2017-08-16 2017-08-31] Test [2017-09-01 2017-09-16]\n",
      "Window Train/Val/Test shape (966, 61) (336, 61) (336, 61)\n",
      "Window Train/Val/Test shape (966, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-08-30] Val [2017-08-31 2017-09-15] Test [2017-09-16 2017-10-01]\n",
      "Window Train/Val/Test shape (1281, 61) (336, 61) (336, 61)\n",
      "Window Train/Val/Test shape (1281, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-09-14] Val [2017-09-15 2017-09-30] Test [2017-10-01 2017-10-16]\n",
      "Window Train/Val/Test shape (1596, 61) (336, 61) (336, 61)\n",
      "Window Train/Val/Test shape (1596, 2) (336, 2) (336, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-09-29] Val [2017-09-30 2017-10-15] Test [2017-10-16 2017-11-10]\n",
      "Window Train/Val/Test shape (1911, 61) (336, 61) (546, 61)\n",
      "Window Train/Val/Test shape (1911, 2) (336, 2) (546, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-10-14] Val [2017-10-15 2017-10-30] Test [2017-10-31 2017-11-10]\n",
      "Window Train/Val/Test shape (2226, 61) (336, 61) (231, 61)\n",
      "Window Train/Val/Test shape (2226, 2) (336, 2) (231, 2)\n",
      "[26.4602363556884, 40.73216202509369, 20.173707133744006, 12.293228692424629, 7.130070379613622]\n",
      "mean mse: 21.357880917312873\n",
      "Train [2017-07-01 2017-08-15] Val [2017-08-16 2017-08-31] Test [2017-09-01 2017-09-16]\n",
      "Window Train/Val/Test shape (966, 61) (336, 61) (336, 61)\n",
      "Window Train/Val/Test shape (966, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-08-30] Val [2017-08-31 2017-09-15] Test [2017-09-16 2017-10-01]\n",
      "Window Train/Val/Test shape (1281, 61) (336, 61) (336, 61)\n",
      "Window Train/Val/Test shape (1281, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-09-14] Val [2017-09-15 2017-09-30] Test [2017-10-01 2017-10-16]\n",
      "Window Train/Val/Test shape (1596, 61) (336, 61) (336, 61)\n",
      "Window Train/Val/Test shape (1596, 2) (336, 2) (336, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-09-29] Val [2017-09-30 2017-10-15] Test [2017-10-16 2017-11-10]\n",
      "Window Train/Val/Test shape (1911, 61) (336, 61) (546, 61)\n",
      "Window Train/Val/Test shape (1911, 2) (336, 2) (546, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-10-14] Val [2017-10-15 2017-10-30] Test [2017-10-31 2017-11-10]\n",
      "Window Train/Val/Test shape (2226, 61) (336, 61) (231, 61)\n",
      "Window Train/Val/Test shape (2226, 2) (336, 2) (231, 2)\n",
      "[26.22524030385032, 48.750006784522945, 14.20869772372219, 10.147233528156622, 5.969416542731558]\n",
      "mean mse: 21.060118976596726\n",
      "Train [2017-07-01 2017-08-15] Val [2017-08-16 2017-08-31] Test [2017-09-01 2017-09-16]\n",
      "Window Train/Val/Test shape (966, 61) (336, 61) (336, 61)\n",
      "Window Train/Val/Test shape (966, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-08-30] Val [2017-08-31 2017-09-15] Test [2017-09-16 2017-10-01]\n",
      "Window Train/Val/Test shape (1281, 61) (336, 61) (336, 61)\n",
      "Window Train/Val/Test shape (1281, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-09-14] Val [2017-09-15 2017-09-30] Test [2017-10-01 2017-10-16]\n",
      "Window Train/Val/Test shape (1596, 61) (336, 61) (336, 61)\n",
      "Window Train/Val/Test shape (1596, 2) (336, 2) (336, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-09-29] Val [2017-09-30 2017-10-15] Test [2017-10-16 2017-11-10]\n",
      "Window Train/Val/Test shape (1911, 61) (336, 61) (546, 61)\n",
      "Window Train/Val/Test shape (1911, 2) (336, 2) (546, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-10-14] Val [2017-10-15 2017-10-30] Test [2017-10-31 2017-11-10]\n",
      "Window Train/Val/Test shape (2226, 61) (336, 61) (231, 61)\n",
      "Window Train/Val/Test shape (2226, 2) (336, 2) (231, 2)\n",
      "[31.95732823596964, 41.75804620547421, 22.288010231264604, 12.132094733307017, 8.315299667088377]\n",
      "mean mse: 23.29015581462077\n",
      "Train [2017-07-01 2017-08-15] Val [2017-08-16 2017-08-31] Test [2017-09-01 2017-09-16]\n",
      "Window Train/Val/Test shape (966, 61) (336, 61) (336, 61)\n",
      "Window Train/Val/Test shape (966, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-08-30] Val [2017-08-31 2017-09-15] Test [2017-09-16 2017-10-01]\n",
      "Window Train/Val/Test shape (1281, 61) (336, 61) (336, 61)\n",
      "Window Train/Val/Test shape (1281, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-09-14] Val [2017-09-15 2017-09-30] Test [2017-10-01 2017-10-16]\n",
      "Window Train/Val/Test shape (1596, 61) (336, 61) (336, 61)\n",
      "Window Train/Val/Test shape (1596, 2) (336, 2) (336, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-09-29] Val [2017-09-30 2017-10-15] Test [2017-10-16 2017-11-10]\n",
      "Window Train/Val/Test shape (1911, 61) (336, 61) (546, 61)\n",
      "Window Train/Val/Test shape (1911, 2) (336, 2) (546, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-10-14] Val [2017-10-15 2017-10-30] Test [2017-10-31 2017-11-10]\n",
      "Window Train/Val/Test shape (2226, 61) (336, 61) (231, 61)\n",
      "Window Train/Val/Test shape (2226, 2) (336, 2) (231, 2)\n",
      "[24.193147058938926, 30.10672302056384, 16.94655938090279, 6.516587965063248, 6.767255423097156]\n",
      "mean mse: 16.90605456971319\n",
      "Train [2017-07-01 2017-08-15] Val [2017-08-16 2017-08-31] Test [2017-09-01 2017-09-16]\n",
      "Window Train/Val/Test shape (966, 61) (336, 61) (336, 61)\n",
      "Window Train/Val/Test shape (966, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-08-30] Val [2017-08-31 2017-09-15] Test [2017-09-16 2017-10-01]\n",
      "Window Train/Val/Test shape (1281, 61) (336, 61) (336, 61)\n",
      "Window Train/Val/Test shape (1281, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-09-14] Val [2017-09-15 2017-09-30] Test [2017-10-01 2017-10-16]\n",
      "Window Train/Val/Test shape (1596, 61) (336, 61) (336, 61)\n",
      "Window Train/Val/Test shape (1596, 2) (336, 2) (336, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-09-29] Val [2017-09-30 2017-10-15] Test [2017-10-16 2017-11-10]\n",
      "Window Train/Val/Test shape (1911, 61) (336, 61) (546, 61)\n",
      "Window Train/Val/Test shape (1911, 2) (336, 2) (546, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-10-14] Val [2017-10-15 2017-10-30] Test [2017-10-31 2017-11-10]\n",
      "Window Train/Val/Test shape (2226, 61) (336, 61) (231, 61)\n",
      "Window Train/Val/Test shape (2226, 2) (336, 2) (231, 2)\n",
      "[24.12021665115684, 41.87680814436556, 15.855544381097493, 8.793527573382908, 7.094034475051021]\n",
      "mean mse: 19.548026245010764\n",
      "Train [2017-07-01 2017-08-15] Val [2017-08-16 2017-08-31] Test [2017-09-01 2017-09-16]\n",
      "Window Train/Val/Test shape (966, 61) (336, 61) (336, 61)\n",
      "Window Train/Val/Test shape (966, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-08-30] Val [2017-08-31 2017-09-15] Test [2017-09-16 2017-10-01]\n",
      "Window Train/Val/Test shape (1281, 61) (336, 61) (336, 61)\n",
      "Window Train/Val/Test shape (1281, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-09-14] Val [2017-09-15 2017-09-30] Test [2017-10-01 2017-10-16]\n",
      "Window Train/Val/Test shape (1596, 61) (336, 61) (336, 61)\n",
      "Window Train/Val/Test shape (1596, 2) (336, 2) (336, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-09-29] Val [2017-09-30 2017-10-15] Test [2017-10-16 2017-11-10]\n",
      "Window Train/Val/Test shape (1911, 61) (336, 61) (546, 61)\n",
      "Window Train/Val/Test shape (1911, 2) (336, 2) (546, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-10-14] Val [2017-10-15 2017-10-30] Test [2017-10-31 2017-11-10]\n",
      "Window Train/Val/Test shape (2226, 61) (336, 61) (231, 61)\n",
      "Window Train/Val/Test shape (2226, 2) (336, 2) (231, 2)\n",
      "[23.83639972711628, 31.529782297351534, 12.496244337221718, 9.086840866792688, 7.747258019261014]\n",
      "mean mse: 16.939305049548643\n",
      "Train [2017-07-01 2017-08-15] Val [2017-08-16 2017-08-31] Test [2017-09-01 2017-09-16]\n",
      "Window Train/Val/Test shape (966, 61) (336, 61) (336, 61)\n",
      "Window Train/Val/Test shape (966, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-08-30] Val [2017-08-31 2017-09-15] Test [2017-09-16 2017-10-01]\n",
      "Window Train/Val/Test shape (1281, 61) (336, 61) (336, 61)\n",
      "Window Train/Val/Test shape (1281, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-09-14] Val [2017-09-15 2017-09-30] Test [2017-10-01 2017-10-16]\n",
      "Window Train/Val/Test shape (1596, 61) (336, 61) (336, 61)\n",
      "Window Train/Val/Test shape (1596, 2) (336, 2) (336, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-09-29] Val [2017-09-30 2017-10-15] Test [2017-10-16 2017-11-10]\n",
      "Window Train/Val/Test shape (1911, 61) (336, 61) (546, 61)\n",
      "Window Train/Val/Test shape (1911, 2) (336, 2) (546, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-10-14] Val [2017-10-15 2017-10-30] Test [2017-10-31 2017-11-10]\n",
      "Window Train/Val/Test shape (2226, 61) (336, 61) (231, 61)\n",
      "Window Train/Val/Test shape (2226, 2) (336, 2) (231, 2)\n",
      "[25.24842920778777, 37.839626990548446, 18.040454932950286, 9.727840333053528, 7.173263891453482]\n",
      "mean mse: 19.605923071158706\n",
      "Train [2017-07-01 2017-08-15] Val [2017-08-16 2017-08-31] Test [2017-09-01 2017-09-16]\n",
      "Window Train/Val/Test shape (966, 61) (336, 61) (336, 61)\n",
      "Window Train/Val/Test shape (966, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-08-30] Val [2017-08-31 2017-09-15] Test [2017-09-16 2017-10-01]\n",
      "Window Train/Val/Test shape (1281, 61) (336, 61) (336, 61)\n",
      "Window Train/Val/Test shape (1281, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-09-14] Val [2017-09-15 2017-09-30] Test [2017-10-01 2017-10-16]\n",
      "Window Train/Val/Test shape (1596, 61) (336, 61) (336, 61)\n",
      "Window Train/Val/Test shape (1596, 2) (336, 2) (336, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-09-29] Val [2017-09-30 2017-10-15] Test [2017-10-16 2017-11-10]\n",
      "Window Train/Val/Test shape (1911, 61) (336, 61) (546, 61)\n",
      "Window Train/Val/Test shape (1911, 2) (336, 2) (546, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-10-14] Val [2017-10-15 2017-10-30] Test [2017-10-31 2017-11-10]\n",
      "Window Train/Val/Test shape (2226, 61) (336, 61) (231, 61)\n",
      "Window Train/Val/Test shape (2226, 2) (336, 2) (231, 2)\n",
      "[360.76569886710104, 290.06820967564323, 146.42957835001957, 108.64448525904967, 112.28648832670142]\n",
      "mean mse: 203.63889209570297\n",
      "Train [2017-07-01 2017-08-15] Val [2017-08-16 2017-08-31] Test [2017-09-01 2017-09-16]\n",
      "Window Train/Val/Test shape (966, 61) (336, 61) (336, 61)\n",
      "Window Train/Val/Test shape (966, 2) (336, 2) (336, 2)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train [2017-07-01 2017-08-30] Val [2017-08-31 2017-09-15] Test [2017-09-16 2017-10-01]\n",
      "Window Train/Val/Test shape (1281, 61) (336, 61) (336, 61)\n",
      "Window Train/Val/Test shape (1281, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-09-14] Val [2017-09-15 2017-09-30] Test [2017-10-01 2017-10-16]\n",
      "Window Train/Val/Test shape (1596, 61) (336, 61) (336, 61)\n",
      "Window Train/Val/Test shape (1596, 2) (336, 2) (336, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-09-29] Val [2017-09-30 2017-10-15] Test [2017-10-16 2017-11-10]\n",
      "Window Train/Val/Test shape (1911, 61) (336, 61) (546, 61)\n",
      "Window Train/Val/Test shape (1911, 2) (336, 2) (546, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-10-14] Val [2017-10-15 2017-10-30] Test [2017-10-31 2017-11-10]\n",
      "Window Train/Val/Test shape (2226, 61) (336, 61) (231, 61)\n",
      "Window Train/Val/Test shape (2226, 2) (336, 2) (231, 2)\n",
      "[360.76569886710104, 290.06820967564323, 146.42957835001957, 108.64448525904967, 112.28648832670142]\n",
      "mean mse: 203.63889209570297\n",
      "Train [2017-07-01 2017-08-15] Val [2017-08-16 2017-08-31] Test [2017-09-01 2017-09-16]\n",
      "Window Train/Val/Test shape (966, 61) (336, 61) (336, 61)\n",
      "Window Train/Val/Test shape (966, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-08-30] Val [2017-08-31 2017-09-15] Test [2017-09-16 2017-10-01]\n",
      "Window Train/Val/Test shape (1281, 61) (336, 61) (336, 61)\n",
      "Window Train/Val/Test shape (1281, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-09-14] Val [2017-09-15 2017-09-30] Test [2017-10-01 2017-10-16]\n",
      "Window Train/Val/Test shape (1596, 61) (336, 61) (336, 61)\n",
      "Window Train/Val/Test shape (1596, 2) (336, 2) (336, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-09-29] Val [2017-09-30 2017-10-15] Test [2017-10-16 2017-11-10]\n",
      "Window Train/Val/Test shape (1911, 61) (336, 61) (546, 61)\n",
      "Window Train/Val/Test shape (1911, 2) (336, 2) (546, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-10-14] Val [2017-10-15 2017-10-30] Test [2017-10-31 2017-11-10]\n",
      "Window Train/Val/Test shape (2226, 61) (336, 61) (231, 61)\n",
      "Window Train/Val/Test shape (2226, 2) (336, 2) (231, 2)\n",
      "[360.76569886710104, 290.06820967564323, 146.42957835001957, 108.64448525904967, 112.28648832670142]\n",
      "mean mse: 203.63889209570297\n",
      "Train [2017-07-01 2017-08-15] Val [2017-08-16 2017-08-31] Test [2017-09-01 2017-09-16]\n",
      "Window Train/Val/Test shape (966, 61) (336, 61) (336, 61)\n",
      "Window Train/Val/Test shape (966, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-08-30] Val [2017-08-31 2017-09-15] Test [2017-09-16 2017-10-01]\n",
      "Window Train/Val/Test shape (1281, 61) (336, 61) (336, 61)\n",
      "Window Train/Val/Test shape (1281, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-09-14] Val [2017-09-15 2017-09-30] Test [2017-10-01 2017-10-16]\n",
      "Window Train/Val/Test shape (1596, 61) (336, 61) (336, 61)\n",
      "Window Train/Val/Test shape (1596, 2) (336, 2) (336, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-09-29] Val [2017-09-30 2017-10-15] Test [2017-10-16 2017-11-10]\n",
      "Window Train/Val/Test shape (1911, 61) (336, 61) (546, 61)\n",
      "Window Train/Val/Test shape (1911, 2) (336, 2) (546, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-10-14] Val [2017-10-15 2017-10-30] Test [2017-10-31 2017-11-10]\n",
      "Window Train/Val/Test shape (2226, 61) (336, 61) (231, 61)\n",
      "Window Train/Val/Test shape (2226, 2) (336, 2) (231, 2)\n",
      "[23.194804054512794, 37.60014636891545, 13.12919954169914, 6.619228198919065, 7.148029663954154]\n",
      "mean mse: 17.53828156560012\n",
      "Train [2017-07-01 2017-08-15] Val [2017-08-16 2017-08-31] Test [2017-09-01 2017-09-16]\n",
      "Window Train/Val/Test shape (966, 61) (336, 61) (336, 61)\n",
      "Window Train/Val/Test shape (966, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-08-30] Val [2017-08-31 2017-09-15] Test [2017-09-16 2017-10-01]\n",
      "Window Train/Val/Test shape (1281, 61) (336, 61) (336, 61)\n",
      "Window Train/Val/Test shape (1281, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-09-14] Val [2017-09-15 2017-09-30] Test [2017-10-01 2017-10-16]\n",
      "Window Train/Val/Test shape (1596, 61) (336, 61) (336, 61)\n",
      "Window Train/Val/Test shape (1596, 2) (336, 2) (336, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-09-29] Val [2017-09-30 2017-10-15] Test [2017-10-16 2017-11-10]\n",
      "Window Train/Val/Test shape (1911, 61) (336, 61) (546, 61)\n",
      "Window Train/Val/Test shape (1911, 2) (336, 2) (546, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-10-14] Val [2017-10-15 2017-10-30] Test [2017-10-31 2017-11-10]\n",
      "Window Train/Val/Test shape (2226, 61) (336, 61) (231, 61)\n",
      "Window Train/Val/Test shape (2226, 2) (336, 2) (231, 2)\n",
      "[29.433057212252066, 34.20509464848591, 16.411493971561235, 7.814535470870791, 6.368674659913948]\n",
      "mean mse: 18.84657119261679\n",
      "Train [2017-07-01 2017-08-15] Val [2017-08-16 2017-08-31] Test [2017-09-01 2017-09-16]\n",
      "Window Train/Val/Test shape (966, 61) (336, 61) (336, 61)\n",
      "Window Train/Val/Test shape (966, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-08-30] Val [2017-08-31 2017-09-15] Test [2017-09-16 2017-10-01]\n",
      "Window Train/Val/Test shape (1281, 61) (336, 61) (336, 61)\n",
      "Window Train/Val/Test shape (1281, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-09-14] Val [2017-09-15 2017-09-30] Test [2017-10-01 2017-10-16]\n",
      "Window Train/Val/Test shape (1596, 61) (336, 61) (336, 61)\n",
      "Window Train/Val/Test shape (1596, 2) (336, 2) (336, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-09-29] Val [2017-09-30 2017-10-15] Test [2017-10-16 2017-11-10]\n",
      "Window Train/Val/Test shape (1911, 61) (336, 61) (546, 61)\n",
      "Window Train/Val/Test shape (1911, 2) (336, 2) (546, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-10-14] Val [2017-10-15 2017-10-30] Test [2017-10-31 2017-11-10]\n",
      "Window Train/Val/Test shape (2226, 61) (336, 61) (231, 61)\n",
      "Window Train/Val/Test shape (2226, 2) (336, 2) (231, 2)\n",
      "[29.353379828890215, 38.23993887083479, 13.780235197019389, 7.083337810424604, 7.608799929258549]\n",
      "mean mse: 19.213138327285513\n",
      "Train [2017-07-01 2017-08-15] Val [2017-08-16 2017-08-31] Test [2017-09-01 2017-09-16]\n",
      "Window Train/Val/Test shape (966, 61) (336, 61) (336, 61)\n",
      "Window Train/Val/Test shape (966, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-08-30] Val [2017-08-31 2017-09-15] Test [2017-09-16 2017-10-01]\n",
      "Window Train/Val/Test shape (1281, 61) (336, 61) (336, 61)\n",
      "Window Train/Val/Test shape (1281, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-09-14] Val [2017-09-15 2017-09-30] Test [2017-10-01 2017-10-16]\n",
      "Window Train/Val/Test shape (1596, 61) (336, 61) (336, 61)\n",
      "Window Train/Val/Test shape (1596, 2) (336, 2) (336, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-09-29] Val [2017-09-30 2017-10-15] Test [2017-10-16 2017-11-10]\n",
      "Window Train/Val/Test shape (1911, 61) (336, 61) (546, 61)\n",
      "Window Train/Val/Test shape (1911, 2) (336, 2) (546, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-10-14] Val [2017-10-15 2017-10-30] Test [2017-10-31 2017-11-10]\n",
      "Window Train/Val/Test shape (2226, 61) (336, 61) (231, 61)\n",
      "Window Train/Val/Test shape (2226, 2) (336, 2) (231, 2)\n",
      "[25.01834790002982, 38.20522412099636, 16.892954917356484, 8.78296739535464, 7.279268138903223]\n",
      "mean mse: 19.235752494528107\n",
      "Train [2017-07-01 2017-08-15] Val [2017-08-16 2017-08-31] Test [2017-09-01 2017-09-16]\n",
      "Window Train/Val/Test shape (966, 61) (336, 61) (336, 61)\n",
      "Window Train/Val/Test shape (966, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-08-30] Val [2017-08-31 2017-09-15] Test [2017-09-16 2017-10-01]\n",
      "Window Train/Val/Test shape (1281, 61) (336, 61) (336, 61)\n",
      "Window Train/Val/Test shape (1281, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-09-14] Val [2017-09-15 2017-09-30] Test [2017-10-01 2017-10-16]\n",
      "Window Train/Val/Test shape (1596, 61) (336, 61) (336, 61)\n",
      "Window Train/Val/Test shape (1596, 2) (336, 2) (336, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-09-29] Val [2017-09-30 2017-10-15] Test [2017-10-16 2017-11-10]\n",
      "Window Train/Val/Test shape (1911, 61) (336, 61) (546, 61)\n",
      "Window Train/Val/Test shape (1911, 2) (336, 2) (546, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-10-14] Val [2017-10-15 2017-10-30] Test [2017-10-31 2017-11-10]\n",
      "Window Train/Val/Test shape (2226, 61) (336, 61) (231, 61)\n",
      "Window Train/Val/Test shape (2226, 2) (336, 2) (231, 2)\n",
      "[17.703022445851154, 44.28030223240887, 21.10432621557778, 12.57490157294826, 8.273719343633243]\n",
      "mean mse: 20.787254362083864\n",
      "Train [2017-07-01 2017-08-15] Val [2017-08-16 2017-08-31] Test [2017-09-01 2017-09-16]\n",
      "Window Train/Val/Test shape (966, 61) (336, 61) (336, 61)\n",
      "Window Train/Val/Test shape (966, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-08-30] Val [2017-08-31 2017-09-15] Test [2017-09-16 2017-10-01]\n",
      "Window Train/Val/Test shape (1281, 61) (336, 61) (336, 61)\n",
      "Window Train/Val/Test shape (1281, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-09-14] Val [2017-09-15 2017-09-30] Test [2017-10-01 2017-10-16]\n",
      "Window Train/Val/Test shape (1596, 61) (336, 61) (336, 61)\n",
      "Window Train/Val/Test shape (1596, 2) (336, 2) (336, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-09-29] Val [2017-09-30 2017-10-15] Test [2017-10-16 2017-11-10]\n",
      "Window Train/Val/Test shape (1911, 61) (336, 61) (546, 61)\n",
      "Window Train/Val/Test shape (1911, 2) (336, 2) (546, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-10-14] Val [2017-10-15 2017-10-30] Test [2017-10-31 2017-11-10]\n",
      "Window Train/Val/Test shape (2226, 61) (336, 61) (231, 61)\n",
      "Window Train/Val/Test shape (2226, 2) (336, 2) (231, 2)\n",
      "[360.76569886710104, 290.06820967564323, 146.42957835001957, 108.64448525904967, 112.28648832670142]\n",
      "mean mse: 203.63889209570297\n",
      "Train [2017-07-01 2017-08-15] Val [2017-08-16 2017-08-31] Test [2017-09-01 2017-09-16]\n",
      "Window Train/Val/Test shape (966, 61) (336, 61) (336, 61)\n",
      "Window Train/Val/Test shape (966, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-08-30] Val [2017-08-31 2017-09-15] Test [2017-09-16 2017-10-01]\n",
      "Window Train/Val/Test shape (1281, 61) (336, 61) (336, 61)\n",
      "Window Train/Val/Test shape (1281, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-09-14] Val [2017-09-15 2017-09-30] Test [2017-10-01 2017-10-16]\n",
      "Window Train/Val/Test shape (1596, 61) (336, 61) (336, 61)\n",
      "Window Train/Val/Test shape (1596, 2) (336, 2) (336, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-09-29] Val [2017-09-30 2017-10-15] Test [2017-10-16 2017-11-10]\n",
      "Window Train/Val/Test shape (1911, 61) (336, 61) (546, 61)\n",
      "Window Train/Val/Test shape (1911, 2) (336, 2) (546, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-10-14] Val [2017-10-15 2017-10-30] Test [2017-10-31 2017-11-10]\n",
      "Window Train/Val/Test shape (2226, 61) (336, 61) (231, 61)\n",
      "Window Train/Val/Test shape (2226, 2) (336, 2) (231, 2)\n",
      "[22.410503691881292, 45.04502382548372, 17.477295009140033, 9.445965297016354, 7.997116662491846]\n",
      "mean mse: 20.47518089720265\n",
      "Train [2017-07-01 2017-08-15] Val [2017-08-16 2017-08-31] Test [2017-09-01 2017-09-16]\n",
      "Window Train/Val/Test shape (966, 61) (336, 61) (336, 61)\n",
      "Window Train/Val/Test shape (966, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-08-30] Val [2017-08-31 2017-09-15] Test [2017-09-16 2017-10-01]\n",
      "Window Train/Val/Test shape (1281, 61) (336, 61) (336, 61)\n",
      "Window Train/Val/Test shape (1281, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-09-14] Val [2017-09-15 2017-09-30] Test [2017-10-01 2017-10-16]\n",
      "Window Train/Val/Test shape (1596, 61) (336, 61) (336, 61)\n",
      "Window Train/Val/Test shape (1596, 2) (336, 2) (336, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-09-29] Val [2017-09-30 2017-10-15] Test [2017-10-16 2017-11-10]\n",
      "Window Train/Val/Test shape (1911, 61) (336, 61) (546, 61)\n",
      "Window Train/Val/Test shape (1911, 2) (336, 2) (546, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-10-14] Val [2017-10-15 2017-10-30] Test [2017-10-31 2017-11-10]\n",
      "Window Train/Val/Test shape (2226, 61) (336, 61) (231, 61)\n",
      "Window Train/Val/Test shape (2226, 2) (336, 2) (231, 2)\n",
      "[360.76569886710104, 290.06820967564323, 146.42957835001957, 108.64448525904967, 112.28648832670142]\n",
      "mean mse: 203.63889209570297\n",
      "Train [2017-07-01 2017-08-15] Val [2017-08-16 2017-08-31] Test [2017-09-01 2017-09-16]\n",
      "Window Train/Val/Test shape (966, 61) (336, 61) (336, 61)\n",
      "Window Train/Val/Test shape (966, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-08-30] Val [2017-08-31 2017-09-15] Test [2017-09-16 2017-10-01]\n",
      "Window Train/Val/Test shape (1281, 61) (336, 61) (336, 61)\n",
      "Window Train/Val/Test shape (1281, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-09-14] Val [2017-09-15 2017-09-30] Test [2017-10-01 2017-10-16]\n",
      "Window Train/Val/Test shape (1596, 61) (336, 61) (336, 61)\n",
      "Window Train/Val/Test shape (1596, 2) (336, 2) (336, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-09-29] Val [2017-09-30 2017-10-15] Test [2017-10-16 2017-11-10]\n",
      "Window Train/Val/Test shape (1911, 61) (336, 61) (546, 61)\n",
      "Window Train/Val/Test shape (1911, 2) (336, 2) (546, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-10-14] Val [2017-10-15 2017-10-30] Test [2017-10-31 2017-11-10]\n",
      "Window Train/Val/Test shape (2226, 61) (336, 61) (231, 61)\n",
      "Window Train/Val/Test shape (2226, 2) (336, 2) (231, 2)\n",
      "[35.593189925322896, 22.129397170704287, 22.363468745484212, 6.133061695534337, 6.965070763391614]\n",
      "mean mse: 18.63683766008747\n",
      "Train [2017-07-01 2017-08-15] Val [2017-08-16 2017-08-31] Test [2017-09-01 2017-09-16]\n",
      "Window Train/Val/Test shape (966, 61) (336, 61) (336, 61)\n",
      "Window Train/Val/Test shape (966, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-08-30] Val [2017-08-31 2017-09-15] Test [2017-09-16 2017-10-01]\n",
      "Window Train/Val/Test shape (1281, 61) (336, 61) (336, 61)\n",
      "Window Train/Val/Test shape (1281, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-09-14] Val [2017-09-15 2017-09-30] Test [2017-10-01 2017-10-16]\n",
      "Window Train/Val/Test shape (1596, 61) (336, 61) (336, 61)\n",
      "Window Train/Val/Test shape (1596, 2) (336, 2) (336, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-09-29] Val [2017-09-30 2017-10-15] Test [2017-10-16 2017-11-10]\n",
      "Window Train/Val/Test shape (1911, 61) (336, 61) (546, 61)\n",
      "Window Train/Val/Test shape (1911, 2) (336, 2) (546, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-10-14] Val [2017-10-15 2017-10-30] Test [2017-10-31 2017-11-10]\n",
      "Window Train/Val/Test shape (2226, 61) (336, 61) (231, 61)\n",
      "Window Train/Val/Test shape (2226, 2) (336, 2) (231, 2)\n",
      "[360.76569886710104, 290.06820967564323, 146.42957835001957, 108.64448525904967, 112.28648832670142]\n",
      "mean mse: 203.63889209570297\n",
      "Train [2017-07-01 2017-08-15] Val [2017-08-16 2017-08-31] Test [2017-09-01 2017-09-16]\n",
      "Window Train/Val/Test shape (966, 61) (336, 61) (336, 61)\n",
      "Window Train/Val/Test shape (966, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-08-30] Val [2017-08-31 2017-09-15] Test [2017-09-16 2017-10-01]\n",
      "Window Train/Val/Test shape (1281, 61) (336, 61) (336, 61)\n",
      "Window Train/Val/Test shape (1281, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-09-14] Val [2017-09-15 2017-09-30] Test [2017-10-01 2017-10-16]\n",
      "Window Train/Val/Test shape (1596, 61) (336, 61) (336, 61)\n",
      "Window Train/Val/Test shape (1596, 2) (336, 2) (336, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-09-29] Val [2017-09-30 2017-10-15] Test [2017-10-16 2017-11-10]\n",
      "Window Train/Val/Test shape (1911, 61) (336, 61) (546, 61)\n",
      "Window Train/Val/Test shape (1911, 2) (336, 2) (546, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-10-14] Val [2017-10-15 2017-10-30] Test [2017-10-31 2017-11-10]\n",
      "Window Train/Val/Test shape (2226, 61) (336, 61) (231, 61)\n",
      "Window Train/Val/Test shape (2226, 2) (336, 2) (231, 2)\n",
      "[31.407062961891125, 42.28640895012846, 20.109466032553623, 11.068653092258728, 8.123729532974025]\n",
      "mean mse: 22.599064113961195\n",
      "Train [2017-07-01 2017-08-15] Val [2017-08-16 2017-08-31] Test [2017-09-01 2017-09-16]\n",
      "Window Train/Val/Test shape (966, 61) (336, 61) (336, 61)\n",
      "Window Train/Val/Test shape (966, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-08-30] Val [2017-08-31 2017-09-15] Test [2017-09-16 2017-10-01]\n",
      "Window Train/Val/Test shape (1281, 61) (336, 61) (336, 61)\n",
      "Window Train/Val/Test shape (1281, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-09-14] Val [2017-09-15 2017-09-30] Test [2017-10-01 2017-10-16]\n",
      "Window Train/Val/Test shape (1596, 61) (336, 61) (336, 61)\n",
      "Window Train/Val/Test shape (1596, 2) (336, 2) (336, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-09-29] Val [2017-09-30 2017-10-15] Test [2017-10-16 2017-11-10]\n",
      "Window Train/Val/Test shape (1911, 61) (336, 61) (546, 61)\n",
      "Window Train/Val/Test shape (1911, 2) (336, 2) (546, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-10-14] Val [2017-10-15 2017-10-30] Test [2017-10-31 2017-11-10]\n",
      "Window Train/Val/Test shape (2226, 61) (336, 61) (231, 61)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Window Train/Val/Test shape (2226, 2) (336, 2) (231, 2)\n",
      "[20.782740832563004, 43.19466377800971, 18.51107320650865, 8.711009414109387, 6.663112849634252]\n",
      "mean mse: 19.572520016165\n",
      "Train [2017-07-01 2017-08-15] Val [2017-08-16 2017-08-31] Test [2017-09-01 2017-09-16]\n",
      "Window Train/Val/Test shape (966, 61) (336, 61) (336, 61)\n",
      "Window Train/Val/Test shape (966, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-08-30] Val [2017-08-31 2017-09-15] Test [2017-09-16 2017-10-01]\n",
      "Window Train/Val/Test shape (1281, 61) (336, 61) (336, 61)\n",
      "Window Train/Val/Test shape (1281, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-09-14] Val [2017-09-15 2017-09-30] Test [2017-10-01 2017-10-16]\n",
      "Window Train/Val/Test shape (1596, 61) (336, 61) (336, 61)\n",
      "Window Train/Val/Test shape (1596, 2) (336, 2) (336, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-09-29] Val [2017-09-30 2017-10-15] Test [2017-10-16 2017-11-10]\n",
      "Window Train/Val/Test shape (1911, 61) (336, 61) (546, 61)\n",
      "Window Train/Val/Test shape (1911, 2) (336, 2) (546, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-10-14] Val [2017-10-15 2017-10-30] Test [2017-10-31 2017-11-10]\n",
      "Window Train/Val/Test shape (2226, 61) (336, 61) (231, 61)\n",
      "Window Train/Val/Test shape (2226, 2) (336, 2) (231, 2)\n",
      "[23.359367850020885, 38.14738796681053, 19.468194134543854, 10.618968091950178, 8.837808748841322]\n",
      "mean mse: 20.086345358433356\n",
      "Train [2017-07-01 2017-08-15] Val [2017-08-16 2017-08-31] Test [2017-09-01 2017-09-16]\n",
      "Window Train/Val/Test shape (966, 61) (336, 61) (336, 61)\n",
      "Window Train/Val/Test shape (966, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-08-30] Val [2017-08-31 2017-09-15] Test [2017-09-16 2017-10-01]\n",
      "Window Train/Val/Test shape (1281, 61) (336, 61) (336, 61)\n",
      "Window Train/Val/Test shape (1281, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-09-14] Val [2017-09-15 2017-09-30] Test [2017-10-01 2017-10-16]\n",
      "Window Train/Val/Test shape (1596, 61) (336, 61) (336, 61)\n",
      "Window Train/Val/Test shape (1596, 2) (336, 2) (336, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-09-29] Val [2017-09-30 2017-10-15] Test [2017-10-16 2017-11-10]\n",
      "Window Train/Val/Test shape (1911, 61) (336, 61) (546, 61)\n",
      "Window Train/Val/Test shape (1911, 2) (336, 2) (546, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-10-14] Val [2017-10-15 2017-10-30] Test [2017-10-31 2017-11-10]\n",
      "Window Train/Val/Test shape (2226, 61) (336, 61) (231, 61)\n",
      "Window Train/Val/Test shape (2226, 2) (336, 2) (231, 2)\n",
      "[30.39701266526809, 46.10264021840463, 13.892035275631006, 5.530244101472499, 7.409709135476383]\n",
      "mean mse: 20.666328279250525\n",
      "Train [2017-07-01 2017-08-15] Val [2017-08-16 2017-08-31] Test [2017-09-01 2017-09-16]\n",
      "Window Train/Val/Test shape (966, 61) (336, 61) (336, 61)\n",
      "Window Train/Val/Test shape (966, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-08-30] Val [2017-08-31 2017-09-15] Test [2017-09-16 2017-10-01]\n",
      "Window Train/Val/Test shape (1281, 61) (336, 61) (336, 61)\n",
      "Window Train/Val/Test shape (1281, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-09-14] Val [2017-09-15 2017-09-30] Test [2017-10-01 2017-10-16]\n",
      "Window Train/Val/Test shape (1596, 61) (336, 61) (336, 61)\n",
      "Window Train/Val/Test shape (1596, 2) (336, 2) (336, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-09-29] Val [2017-09-30 2017-10-15] Test [2017-10-16 2017-11-10]\n",
      "Window Train/Val/Test shape (1911, 61) (336, 61) (546, 61)\n",
      "Window Train/Val/Test shape (1911, 2) (336, 2) (546, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-10-14] Val [2017-10-15 2017-10-30] Test [2017-10-31 2017-11-10]\n",
      "Window Train/Val/Test shape (2226, 61) (336, 61) (231, 61)\n",
      "Window Train/Val/Test shape (2226, 2) (336, 2) (231, 2)\n",
      "[23.463310290801346, 38.3318572524651, 16.611250467357063, 9.04338448306208, 6.89121183024381]\n",
      "mean mse: 18.86820286478588\n",
      "Train [2017-07-01 2017-08-15] Val [2017-08-16 2017-08-31] Test [2017-09-01 2017-09-16]\n",
      "Window Train/Val/Test shape (966, 61) (336, 61) (336, 61)\n",
      "Window Train/Val/Test shape (966, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-08-30] Val [2017-08-31 2017-09-15] Test [2017-09-16 2017-10-01]\n",
      "Window Train/Val/Test shape (1281, 61) (336, 61) (336, 61)\n",
      "Window Train/Val/Test shape (1281, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-09-14] Val [2017-09-15 2017-09-30] Test [2017-10-01 2017-10-16]\n",
      "Window Train/Val/Test shape (1596, 61) (336, 61) (336, 61)\n",
      "Window Train/Val/Test shape (1596, 2) (336, 2) (336, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-09-29] Val [2017-09-30 2017-10-15] Test [2017-10-16 2017-11-10]\n",
      "Window Train/Val/Test shape (1911, 61) (336, 61) (546, 61)\n",
      "Window Train/Val/Test shape (1911, 2) (336, 2) (546, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-10-14] Val [2017-10-15 2017-10-30] Test [2017-10-31 2017-11-10]\n",
      "Window Train/Val/Test shape (2226, 61) (336, 61) (231, 61)\n",
      "Window Train/Val/Test shape (2226, 2) (336, 2) (231, 2)\n",
      "[28.30266852271552, 42.34067091225579, 15.48677212887541, 8.258840567624517, 6.866881516328551]\n",
      "mean mse: 20.25116672955996\n",
      "Train [2017-07-01 2017-08-15] Val [2017-08-16 2017-08-31] Test [2017-09-01 2017-09-16]\n",
      "Window Train/Val/Test shape (966, 61) (336, 61) (336, 61)\n",
      "Window Train/Val/Test shape (966, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-08-30] Val [2017-08-31 2017-09-15] Test [2017-09-16 2017-10-01]\n",
      "Window Train/Val/Test shape (1281, 61) (336, 61) (336, 61)\n",
      "Window Train/Val/Test shape (1281, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-09-14] Val [2017-09-15 2017-09-30] Test [2017-10-01 2017-10-16]\n",
      "Window Train/Val/Test shape (1596, 61) (336, 61) (336, 61)\n",
      "Window Train/Val/Test shape (1596, 2) (336, 2) (336, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-09-29] Val [2017-09-30 2017-10-15] Test [2017-10-16 2017-11-10]\n",
      "Window Train/Val/Test shape (1911, 61) (336, 61) (546, 61)\n",
      "Window Train/Val/Test shape (1911, 2) (336, 2) (546, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-10-14] Val [2017-10-15 2017-10-30] Test [2017-10-31 2017-11-10]\n",
      "Window Train/Val/Test shape (2226, 61) (336, 61) (231, 61)\n",
      "Window Train/Val/Test shape (2226, 2) (336, 2) (231, 2)\n",
      "[360.76569886710104, 290.06820967564323, 146.42957835001957, 108.64448525904967, 112.28648832670142]\n",
      "mean mse: 203.63889209570297\n",
      "Train [2017-07-01 2017-08-15] Val [2017-08-16 2017-08-31] Test [2017-09-01 2017-09-16]\n",
      "Window Train/Val/Test shape (966, 61) (336, 61) (336, 61)\n",
      "Window Train/Val/Test shape (966, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-08-30] Val [2017-08-31 2017-09-15] Test [2017-09-16 2017-10-01]\n",
      "Window Train/Val/Test shape (1281, 61) (336, 61) (336, 61)\n",
      "Window Train/Val/Test shape (1281, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-09-14] Val [2017-09-15 2017-09-30] Test [2017-10-01 2017-10-16]\n",
      "Window Train/Val/Test shape (1596, 61) (336, 61) (336, 61)\n",
      "Window Train/Val/Test shape (1596, 2) (336, 2) (336, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-09-29] Val [2017-09-30 2017-10-15] Test [2017-10-16 2017-11-10]\n",
      "Window Train/Val/Test shape (1911, 61) (336, 61) (546, 61)\n",
      "Window Train/Val/Test shape (1911, 2) (336, 2) (546, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-10-14] Val [2017-10-15 2017-10-30] Test [2017-10-31 2017-11-10]\n",
      "Window Train/Val/Test shape (2226, 61) (336, 61) (231, 61)\n",
      "Window Train/Val/Test shape (2226, 2) (336, 2) (231, 2)\n",
      "[22.957265914616546, 43.88366213249991, 15.970122306636524, 8.672995118315217, 6.601567515376985]\n",
      "mean mse: 19.617122597489036\n",
      "Train [2017-07-01 2017-08-15] Val [2017-08-16 2017-08-31] Test [2017-09-01 2017-09-16]\n",
      "Window Train/Val/Test shape (966, 61) (336, 61) (336, 61)\n",
      "Window Train/Val/Test shape (966, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-08-30] Val [2017-08-31 2017-09-15] Test [2017-09-16 2017-10-01]\n",
      "Window Train/Val/Test shape (1281, 61) (336, 61) (336, 61)\n",
      "Window Train/Val/Test shape (1281, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-09-14] Val [2017-09-15 2017-09-30] Test [2017-10-01 2017-10-16]\n",
      "Window Train/Val/Test shape (1596, 61) (336, 61) (336, 61)\n",
      "Window Train/Val/Test shape (1596, 2) (336, 2) (336, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-09-29] Val [2017-09-30 2017-10-15] Test [2017-10-16 2017-11-10]\n",
      "Window Train/Val/Test shape (1911, 61) (336, 61) (546, 61)\n",
      "Window Train/Val/Test shape (1911, 2) (336, 2) (546, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-10-14] Val [2017-10-15 2017-10-30] Test [2017-10-31 2017-11-10]\n",
      "Window Train/Val/Test shape (2226, 61) (336, 61) (231, 61)\n",
      "Window Train/Val/Test shape (2226, 2) (336, 2) (231, 2)\n",
      "[360.76569886710104, 290.06820967564323, 146.42957835001957, 108.64448525904967, 112.28648832670142]\n",
      "mean mse: 203.63889209570297\n",
      "Train [2017-07-01 2017-08-15] Val [2017-08-16 2017-08-31] Test [2017-09-01 2017-09-16]\n",
      "Window Train/Val/Test shape (966, 61) (336, 61) (336, 61)\n",
      "Window Train/Val/Test shape (966, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-08-30] Val [2017-08-31 2017-09-15] Test [2017-09-16 2017-10-01]\n",
      "Window Train/Val/Test shape (1281, 61) (336, 61) (336, 61)\n",
      "Window Train/Val/Test shape (1281, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-09-14] Val [2017-09-15 2017-09-30] Test [2017-10-01 2017-10-16]\n",
      "Window Train/Val/Test shape (1596, 61) (336, 61) (336, 61)\n",
      "Window Train/Val/Test shape (1596, 2) (336, 2) (336, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-09-29] Val [2017-09-30 2017-10-15] Test [2017-10-16 2017-11-10]\n",
      "Window Train/Val/Test shape (1911, 61) (336, 61) (546, 61)\n",
      "Window Train/Val/Test shape (1911, 2) (336, 2) (546, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-10-14] Val [2017-10-15 2017-10-30] Test [2017-10-31 2017-11-10]\n",
      "Window Train/Val/Test shape (2226, 61) (336, 61) (231, 61)\n",
      "Window Train/Val/Test shape (2226, 2) (336, 2) (231, 2)\n",
      "[27.560833679986235, 32.11943377226441, 13.377551972932151, 7.096895408464066, 6.693198268651316]\n",
      "mean mse: 17.369582620459635\n",
      "Train [2017-07-01 2017-08-15] Val [2017-08-16 2017-08-31] Test [2017-09-01 2017-09-16]\n",
      "Window Train/Val/Test shape (966, 61) (336, 61) (336, 61)\n",
      "Window Train/Val/Test shape (966, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-08-30] Val [2017-08-31 2017-09-15] Test [2017-09-16 2017-10-01]\n",
      "Window Train/Val/Test shape (1281, 61) (336, 61) (336, 61)\n",
      "Window Train/Val/Test shape (1281, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-09-14] Val [2017-09-15 2017-09-30] Test [2017-10-01 2017-10-16]\n",
      "Window Train/Val/Test shape (1596, 61) (336, 61) (336, 61)\n",
      "Window Train/Val/Test shape (1596, 2) (336, 2) (336, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-09-29] Val [2017-09-30 2017-10-15] Test [2017-10-16 2017-11-10]\n",
      "Window Train/Val/Test shape (1911, 61) (336, 61) (546, 61)\n",
      "Window Train/Val/Test shape (1911, 2) (336, 2) (546, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-10-14] Val [2017-10-15 2017-10-30] Test [2017-10-31 2017-11-10]\n",
      "Window Train/Val/Test shape (2226, 61) (336, 61) (231, 61)\n",
      "Window Train/Val/Test shape (2226, 2) (336, 2) (231, 2)\n",
      "[23.479756211749617, 33.17614525021786, 18.86085070553057, 6.619350738439699, 6.6348471649034115]\n",
      "mean mse: 17.754190014168227\n",
      "Train [2017-07-01 2017-08-15] Val [2017-08-16 2017-08-31] Test [2017-09-01 2017-09-16]\n",
      "Window Train/Val/Test shape (966, 61) (336, 61) (336, 61)\n",
      "Window Train/Val/Test shape (966, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-08-30] Val [2017-08-31 2017-09-15] Test [2017-09-16 2017-10-01]\n",
      "Window Train/Val/Test shape (1281, 61) (336, 61) (336, 61)\n",
      "Window Train/Val/Test shape (1281, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-09-14] Val [2017-09-15 2017-09-30] Test [2017-10-01 2017-10-16]\n",
      "Window Train/Val/Test shape (1596, 61) (336, 61) (336, 61)\n",
      "Window Train/Val/Test shape (1596, 2) (336, 2) (336, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-09-29] Val [2017-09-30 2017-10-15] Test [2017-10-16 2017-11-10]\n",
      "Window Train/Val/Test shape (1911, 61) (336, 61) (546, 61)\n",
      "Window Train/Val/Test shape (1911, 2) (336, 2) (546, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-10-14] Val [2017-10-15 2017-10-30] Test [2017-10-31 2017-11-10]\n",
      "Window Train/Val/Test shape (2226, 61) (336, 61) (231, 61)\n",
      "Window Train/Val/Test shape (2226, 2) (336, 2) (231, 2)\n",
      "[29.693537273278178, 42.06625710393886, 12.473915766140852, 9.545614382754046, 7.27892201477797]\n",
      "mean mse: 20.211649308177982\n",
      "Train [2017-07-01 2017-08-15] Val [2017-08-16 2017-08-31] Test [2017-09-01 2017-09-16]\n",
      "Window Train/Val/Test shape (966, 61) (336, 61) (336, 61)\n",
      "Window Train/Val/Test shape (966, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-08-30] Val [2017-08-31 2017-09-15] Test [2017-09-16 2017-10-01]\n",
      "Window Train/Val/Test shape (1281, 61) (336, 61) (336, 61)\n",
      "Window Train/Val/Test shape (1281, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-09-14] Val [2017-09-15 2017-09-30] Test [2017-10-01 2017-10-16]\n",
      "Window Train/Val/Test shape (1596, 61) (336, 61) (336, 61)\n",
      "Window Train/Val/Test shape (1596, 2) (336, 2) (336, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-09-29] Val [2017-09-30 2017-10-15] Test [2017-10-16 2017-11-10]\n",
      "Window Train/Val/Test shape (1911, 61) (336, 61) (546, 61)\n",
      "Window Train/Val/Test shape (1911, 2) (336, 2) (546, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-10-14] Val [2017-10-15 2017-10-30] Test [2017-10-31 2017-11-10]\n",
      "Window Train/Val/Test shape (2226, 61) (336, 61) (231, 61)\n",
      "Window Train/Val/Test shape (2226, 2) (336, 2) (231, 2)\n",
      "[26.76634656433368, 33.06480195557937, 14.958549875082978, 9.263859064159194, 6.370440039920687]\n",
      "mean mse: 18.08479949981518\n",
      "Train [2017-07-01 2017-08-15] Val [2017-08-16 2017-08-31] Test [2017-09-01 2017-09-16]\n",
      "Window Train/Val/Test shape (966, 61) (336, 61) (336, 61)\n",
      "Window Train/Val/Test shape (966, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-08-30] Val [2017-08-31 2017-09-15] Test [2017-09-16 2017-10-01]\n",
      "Window Train/Val/Test shape (1281, 61) (336, 61) (336, 61)\n",
      "Window Train/Val/Test shape (1281, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-09-14] Val [2017-09-15 2017-09-30] Test [2017-10-01 2017-10-16]\n",
      "Window Train/Val/Test shape (1596, 61) (336, 61) (336, 61)\n",
      "Window Train/Val/Test shape (1596, 2) (336, 2) (336, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-09-29] Val [2017-09-30 2017-10-15] Test [2017-10-16 2017-11-10]\n",
      "Window Train/Val/Test shape (1911, 61) (336, 61) (546, 61)\n",
      "Window Train/Val/Test shape (1911, 2) (336, 2) (546, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-10-14] Val [2017-10-15 2017-10-30] Test [2017-10-31 2017-11-10]\n",
      "Window Train/Val/Test shape (2226, 61) (336, 61) (231, 61)\n",
      "Window Train/Val/Test shape (2226, 2) (336, 2) (231, 2)\n",
      "[24.710823436872435, 39.319865259649966, 14.386347803551246, 9.919269629532124, 7.15950798379638]\n",
      "mean mse: 19.09916282268043\n",
      "Train [2017-07-01 2017-08-15] Val [2017-08-16 2017-08-31] Test [2017-09-01 2017-09-16]\n",
      "Window Train/Val/Test shape (966, 61) (336, 61) (336, 61)\n",
      "Window Train/Val/Test shape (966, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-08-30] Val [2017-08-31 2017-09-15] Test [2017-09-16 2017-10-01]\n",
      "Window Train/Val/Test shape (1281, 61) (336, 61) (336, 61)\n",
      "Window Train/Val/Test shape (1281, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-09-14] Val [2017-09-15 2017-09-30] Test [2017-10-01 2017-10-16]\n",
      "Window Train/Val/Test shape (1596, 61) (336, 61) (336, 61)\n",
      "Window Train/Val/Test shape (1596, 2) (336, 2) (336, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-09-29] Val [2017-09-30 2017-10-15] Test [2017-10-16 2017-11-10]\n",
      "Window Train/Val/Test shape (1911, 61) (336, 61) (546, 61)\n",
      "Window Train/Val/Test shape (1911, 2) (336, 2) (546, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-10-14] Val [2017-10-15 2017-10-30] Test [2017-10-31 2017-11-10]\n",
      "Window Train/Val/Test shape (2226, 61) (336, 61) (231, 61)\n",
      "Window Train/Val/Test shape (2226, 2) (336, 2) (231, 2)\n",
      "[22.14739295268729, 26.092548039062898, 15.042752231955118, 7.086586383792108, 6.6610869721356]\n",
      "mean mse: 15.406073315926601\n",
      "Train [2017-07-01 2017-08-15] Val [2017-08-16 2017-08-31] Test [2017-09-01 2017-09-16]\n",
      "Window Train/Val/Test shape (966, 61) (336, 61) (336, 61)\n",
      "Window Train/Val/Test shape (966, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-08-30] Val [2017-08-31 2017-09-15] Test [2017-09-16 2017-10-01]\n",
      "Window Train/Val/Test shape (1281, 61) (336, 61) (336, 61)\n",
      "Window Train/Val/Test shape (1281, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-09-14] Val [2017-09-15 2017-09-30] Test [2017-10-01 2017-10-16]\n",
      "Window Train/Val/Test shape (1596, 61) (336, 61) (336, 61)\n",
      "Window Train/Val/Test shape (1596, 2) (336, 2) (336, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-09-29] Val [2017-09-30 2017-10-15] Test [2017-10-16 2017-11-10]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Window Train/Val/Test shape (1911, 61) (336, 61) (546, 61)\n",
      "Window Train/Val/Test shape (1911, 2) (336, 2) (546, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-10-14] Val [2017-10-15 2017-10-30] Test [2017-10-31 2017-11-10]\n",
      "Window Train/Val/Test shape (2226, 61) (336, 61) (231, 61)\n",
      "Window Train/Val/Test shape (2226, 2) (336, 2) (231, 2)\n",
      "[26.962115944084818, 32.75104772960063, 14.741363175055076, 6.416328568456695, 6.445332518132318]\n",
      "mean mse: 17.463237587065905\n",
      "Train [2017-07-01 2017-08-15] Val [2017-08-16 2017-08-31] Test [2017-09-01 2017-09-16]\n",
      "Window Train/Val/Test shape (966, 61) (336, 61) (336, 61)\n",
      "Window Train/Val/Test shape (966, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-08-30] Val [2017-08-31 2017-09-15] Test [2017-09-16 2017-10-01]\n",
      "Window Train/Val/Test shape (1281, 61) (336, 61) (336, 61)\n",
      "Window Train/Val/Test shape (1281, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-09-14] Val [2017-09-15 2017-09-30] Test [2017-10-01 2017-10-16]\n",
      "Window Train/Val/Test shape (1596, 61) (336, 61) (336, 61)\n",
      "Window Train/Val/Test shape (1596, 2) (336, 2) (336, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-09-29] Val [2017-09-30 2017-10-15] Test [2017-10-16 2017-11-10]\n",
      "Window Train/Val/Test shape (1911, 61) (336, 61) (546, 61)\n",
      "Window Train/Val/Test shape (1911, 2) (336, 2) (546, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-10-14] Val [2017-10-15 2017-10-30] Test [2017-10-31 2017-11-10]\n",
      "Window Train/Val/Test shape (2226, 61) (336, 61) (231, 61)\n",
      "Window Train/Val/Test shape (2226, 2) (336, 2) (231, 2)\n",
      "[25.57790178203474, 31.02931272171637, 15.760116632935539, 7.333184158828672, 6.218072414922307]\n",
      "mean mse: 17.183717542087525\n",
      "Train [2017-07-01 2017-08-15] Val [2017-08-16 2017-08-31] Test [2017-09-01 2017-09-16]\n",
      "Window Train/Val/Test shape (966, 61) (336, 61) (336, 61)\n",
      "Window Train/Val/Test shape (966, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-08-30] Val [2017-08-31 2017-09-15] Test [2017-09-16 2017-10-01]\n",
      "Window Train/Val/Test shape (1281, 61) (336, 61) (336, 61)\n",
      "Window Train/Val/Test shape (1281, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-09-14] Val [2017-09-15 2017-09-30] Test [2017-10-01 2017-10-16]\n",
      "Window Train/Val/Test shape (1596, 61) (336, 61) (336, 61)\n",
      "Window Train/Val/Test shape (1596, 2) (336, 2) (336, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-09-29] Val [2017-09-30 2017-10-15] Test [2017-10-16 2017-11-10]\n",
      "Window Train/Val/Test shape (1911, 61) (336, 61) (546, 61)\n",
      "Window Train/Val/Test shape (1911, 2) (336, 2) (546, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-10-14] Val [2017-10-15 2017-10-30] Test [2017-10-31 2017-11-10]\n",
      "Window Train/Val/Test shape (2226, 61) (336, 61) (231, 61)\n",
      "Window Train/Val/Test shape (2226, 2) (336, 2) (231, 2)\n",
      "[27.957763126726995, 35.0109029581276, 16.092398161046347, 8.481504088789517, 6.2110395784054]\n",
      "mean mse: 18.750721582619175\n",
      "Train [2017-07-01 2017-08-15] Val [2017-08-16 2017-08-31] Test [2017-09-01 2017-09-16]\n",
      "Window Train/Val/Test shape (966, 61) (336, 61) (336, 61)\n",
      "Window Train/Val/Test shape (966, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-08-30] Val [2017-08-31 2017-09-15] Test [2017-09-16 2017-10-01]\n",
      "Window Train/Val/Test shape (1281, 61) (336, 61) (336, 61)\n",
      "Window Train/Val/Test shape (1281, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-09-14] Val [2017-09-15 2017-09-30] Test [2017-10-01 2017-10-16]\n",
      "Window Train/Val/Test shape (1596, 61) (336, 61) (336, 61)\n",
      "Window Train/Val/Test shape (1596, 2) (336, 2) (336, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-09-29] Val [2017-09-30 2017-10-15] Test [2017-10-16 2017-11-10]\n",
      "Window Train/Val/Test shape (1911, 61) (336, 61) (546, 61)\n",
      "Window Train/Val/Test shape (1911, 2) (336, 2) (546, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-10-14] Val [2017-10-15 2017-10-30] Test [2017-10-31 2017-11-10]\n",
      "Window Train/Val/Test shape (2226, 61) (336, 61) (231, 61)\n",
      "Window Train/Val/Test shape (2226, 2) (336, 2) (231, 2)\n",
      "[23.009984639753753, 34.238490919379814, 22.13647807652215, 7.718010175568682, 7.870313948484226]\n",
      "mean mse: 18.994655551941726\n",
      "Train [2017-07-01 2017-08-15] Val [2017-08-16 2017-08-31] Test [2017-09-01 2017-09-16]\n",
      "Window Train/Val/Test shape (966, 61) (336, 61) (336, 61)\n",
      "Window Train/Val/Test shape (966, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-08-30] Val [2017-08-31 2017-09-15] Test [2017-09-16 2017-10-01]\n",
      "Window Train/Val/Test shape (1281, 61) (336, 61) (336, 61)\n",
      "Window Train/Val/Test shape (1281, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-09-14] Val [2017-09-15 2017-09-30] Test [2017-10-01 2017-10-16]\n",
      "Window Train/Val/Test shape (1596, 61) (336, 61) (336, 61)\n",
      "Window Train/Val/Test shape (1596, 2) (336, 2) (336, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-09-29] Val [2017-09-30 2017-10-15] Test [2017-10-16 2017-11-10]\n",
      "Window Train/Val/Test shape (1911, 61) (336, 61) (546, 61)\n",
      "Window Train/Val/Test shape (1911, 2) (336, 2) (546, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-10-14] Val [2017-10-15 2017-10-30] Test [2017-10-31 2017-11-10]\n",
      "Window Train/Val/Test shape (2226, 61) (336, 61) (231, 61)\n",
      "Window Train/Val/Test shape (2226, 2) (336, 2) (231, 2)\n",
      "[31.271300322458913, 49.32851840184057, 17.099541359240085, 6.781685178885916, 6.40727485790386]\n",
      "mean mse: 22.17766402406587\n",
      "Train [2017-07-01 2017-08-15] Val [2017-08-16 2017-08-31] Test [2017-09-01 2017-09-16]\n",
      "Window Train/Val/Test shape (966, 61) (336, 61) (336, 61)\n",
      "Window Train/Val/Test shape (966, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-08-30] Val [2017-08-31 2017-09-15] Test [2017-09-16 2017-10-01]\n",
      "Window Train/Val/Test shape (1281, 61) (336, 61) (336, 61)\n",
      "Window Train/Val/Test shape (1281, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-09-14] Val [2017-09-15 2017-09-30] Test [2017-10-01 2017-10-16]\n",
      "Window Train/Val/Test shape (1596, 61) (336, 61) (336, 61)\n",
      "Window Train/Val/Test shape (1596, 2) (336, 2) (336, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-09-29] Val [2017-09-30 2017-10-15] Test [2017-10-16 2017-11-10]\n",
      "Window Train/Val/Test shape (1911, 61) (336, 61) (546, 61)\n",
      "Window Train/Val/Test shape (1911, 2) (336, 2) (546, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-10-14] Val [2017-10-15 2017-10-30] Test [2017-10-31 2017-11-10]\n",
      "Window Train/Val/Test shape (2226, 61) (336, 61) (231, 61)\n",
      "Window Train/Val/Test shape (2226, 2) (336, 2) (231, 2)\n",
      "[26.77060490449496, 33.43394851947659, 17.297798194459727, 7.837404527703023, 6.180709699804938]\n",
      "mean mse: 18.304093169187848\n",
      "Train [2017-07-01 2017-08-15] Val [2017-08-16 2017-08-31] Test [2017-09-01 2017-09-16]\n",
      "Window Train/Val/Test shape (966, 61) (336, 61) (336, 61)\n",
      "Window Train/Val/Test shape (966, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-08-30] Val [2017-08-31 2017-09-15] Test [2017-09-16 2017-10-01]\n",
      "Window Train/Val/Test shape (1281, 61) (336, 61) (336, 61)\n",
      "Window Train/Val/Test shape (1281, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-09-14] Val [2017-09-15 2017-09-30] Test [2017-10-01 2017-10-16]\n",
      "Window Train/Val/Test shape (1596, 61) (336, 61) (336, 61)\n",
      "Window Train/Val/Test shape (1596, 2) (336, 2) (336, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-09-29] Val [2017-09-30 2017-10-15] Test [2017-10-16 2017-11-10]\n",
      "Window Train/Val/Test shape (1911, 61) (336, 61) (546, 61)\n",
      "Window Train/Val/Test shape (1911, 2) (336, 2) (546, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-10-14] Val [2017-10-15 2017-10-30] Test [2017-10-31 2017-11-10]\n",
      "Window Train/Val/Test shape (2226, 61) (336, 61) (231, 61)\n",
      "Window Train/Val/Test shape (2226, 2) (336, 2) (231, 2)\n",
      "[360.76569886710104, 290.06820967564323, 146.42957835001957, 108.64448525904967, 112.28648832670142]\n",
      "mean mse: 203.63889209570297\n",
      "Train [2017-07-01 2017-08-15] Val [2017-08-16 2017-08-31] Test [2017-09-01 2017-09-16]\n",
      "Window Train/Val/Test shape (966, 61) (336, 61) (336, 61)\n",
      "Window Train/Val/Test shape (966, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-08-30] Val [2017-08-31 2017-09-15] Test [2017-09-16 2017-10-01]\n",
      "Window Train/Val/Test shape (1281, 61) (336, 61) (336, 61)\n",
      "Window Train/Val/Test shape (1281, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-09-14] Val [2017-09-15 2017-09-30] Test [2017-10-01 2017-10-16]\n",
      "Window Train/Val/Test shape (1596, 61) (336, 61) (336, 61)\n",
      "Window Train/Val/Test shape (1596, 2) (336, 2) (336, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-09-29] Val [2017-09-30 2017-10-15] Test [2017-10-16 2017-11-10]\n",
      "Window Train/Val/Test shape (1911, 61) (336, 61) (546, 61)\n",
      "Window Train/Val/Test shape (1911, 2) (336, 2) (546, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-10-14] Val [2017-10-15 2017-10-30] Test [2017-10-31 2017-11-10]\n",
      "Window Train/Val/Test shape (2226, 61) (336, 61) (231, 61)\n",
      "Window Train/Val/Test shape (2226, 2) (336, 2) (231, 2)\n",
      "[24.680134346494295, 32.72105629856419, 15.544885264173818, 6.252121932943271, 6.329085793798471]\n",
      "mean mse: 17.10545672719481\n",
      "Train [2017-07-01 2017-08-15] Val [2017-08-16 2017-08-31] Test [2017-09-01 2017-09-16]\n",
      "Window Train/Val/Test shape (966, 61) (336, 61) (336, 61)\n",
      "Window Train/Val/Test shape (966, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-08-30] Val [2017-08-31 2017-09-15] Test [2017-09-16 2017-10-01]\n",
      "Window Train/Val/Test shape (1281, 61) (336, 61) (336, 61)\n",
      "Window Train/Val/Test shape (1281, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-09-14] Val [2017-09-15 2017-09-30] Test [2017-10-01 2017-10-16]\n",
      "Window Train/Val/Test shape (1596, 61) (336, 61) (336, 61)\n",
      "Window Train/Val/Test shape (1596, 2) (336, 2) (336, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-09-29] Val [2017-09-30 2017-10-15] Test [2017-10-16 2017-11-10]\n",
      "Window Train/Val/Test shape (1911, 61) (336, 61) (546, 61)\n",
      "Window Train/Val/Test shape (1911, 2) (336, 2) (546, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-10-14] Val [2017-10-15 2017-10-30] Test [2017-10-31 2017-11-10]\n",
      "Window Train/Val/Test shape (2226, 61) (336, 61) (231, 61)\n",
      "Window Train/Val/Test shape (2226, 2) (336, 2) (231, 2)\n",
      "[23.159292440275756, 37.905113549200806, 16.124750925431172, 10.506285334224904, 8.95566725943587]\n",
      "mean mse: 19.3302219017137\n",
      "Train [2017-07-01 2017-08-15] Val [2017-08-16 2017-08-31] Test [2017-09-01 2017-09-16]\n",
      "Window Train/Val/Test shape (966, 61) (336, 61) (336, 61)\n",
      "Window Train/Val/Test shape (966, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-08-30] Val [2017-08-31 2017-09-15] Test [2017-09-16 2017-10-01]\n",
      "Window Train/Val/Test shape (1281, 61) (336, 61) (336, 61)\n",
      "Window Train/Val/Test shape (1281, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-09-14] Val [2017-09-15 2017-09-30] Test [2017-10-01 2017-10-16]\n",
      "Window Train/Val/Test shape (1596, 61) (336, 61) (336, 61)\n",
      "Window Train/Val/Test shape (1596, 2) (336, 2) (336, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-09-29] Val [2017-09-30 2017-10-15] Test [2017-10-16 2017-11-10]\n",
      "Window Train/Val/Test shape (1911, 61) (336, 61) (546, 61)\n",
      "Window Train/Val/Test shape (1911, 2) (336, 2) (546, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-10-14] Val [2017-10-15 2017-10-30] Test [2017-10-31 2017-11-10]\n",
      "Window Train/Val/Test shape (2226, 61) (336, 61) (231, 61)\n",
      "Window Train/Val/Test shape (2226, 2) (336, 2) (231, 2)\n",
      "[23.31395540261253, 36.82609824070924, 18.839422500750224, 8.291840728416805, 7.617645561310165]\n",
      "mean mse: 18.977792486759796\n",
      "Train [2017-07-01 2017-08-15] Val [2017-08-16 2017-08-31] Test [2017-09-01 2017-09-16]\n",
      "Window Train/Val/Test shape (966, 61) (336, 61) (336, 61)\n",
      "Window Train/Val/Test shape (966, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-08-30] Val [2017-08-31 2017-09-15] Test [2017-09-16 2017-10-01]\n",
      "Window Train/Val/Test shape (1281, 61) (336, 61) (336, 61)\n",
      "Window Train/Val/Test shape (1281, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-09-14] Val [2017-09-15 2017-09-30] Test [2017-10-01 2017-10-16]\n",
      "Window Train/Val/Test shape (1596, 61) (336, 61) (336, 61)\n",
      "Window Train/Val/Test shape (1596, 2) (336, 2) (336, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-09-29] Val [2017-09-30 2017-10-15] Test [2017-10-16 2017-11-10]\n",
      "Window Train/Val/Test shape (1911, 61) (336, 61) (546, 61)\n",
      "Window Train/Val/Test shape (1911, 2) (336, 2) (546, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-10-14] Val [2017-10-15 2017-10-30] Test [2017-10-31 2017-11-10]\n",
      "Window Train/Val/Test shape (2226, 61) (336, 61) (231, 61)\n",
      "Window Train/Val/Test shape (2226, 2) (336, 2) (231, 2)\n",
      "[25.509460522908086, 29.21294011599302, 14.791836317689492, 8.88657500159215, 7.409016300731143]\n",
      "mean mse: 17.161965651782776\n",
      "Train [2017-07-01 2017-08-15] Val [2017-08-16 2017-08-31] Test [2017-09-01 2017-09-16]\n",
      "Window Train/Val/Test shape (966, 61) (336, 61) (336, 61)\n",
      "Window Train/Val/Test shape (966, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-08-30] Val [2017-08-31 2017-09-15] Test [2017-09-16 2017-10-01]\n",
      "Window Train/Val/Test shape (1281, 61) (336, 61) (336, 61)\n",
      "Window Train/Val/Test shape (1281, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-09-14] Val [2017-09-15 2017-09-30] Test [2017-10-01 2017-10-16]\n",
      "Window Train/Val/Test shape (1596, 61) (336, 61) (336, 61)\n",
      "Window Train/Val/Test shape (1596, 2) (336, 2) (336, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-09-29] Val [2017-09-30 2017-10-15] Test [2017-10-16 2017-11-10]\n",
      "Window Train/Val/Test shape (1911, 61) (336, 61) (546, 61)\n",
      "Window Train/Val/Test shape (1911, 2) (336, 2) (546, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-10-14] Val [2017-10-15 2017-10-30] Test [2017-10-31 2017-11-10]\n",
      "Window Train/Val/Test shape (2226, 61) (336, 61) (231, 61)\n",
      "Window Train/Val/Test shape (2226, 2) (336, 2) (231, 2)\n",
      "[360.76569886710104, 290.06820967564323, 146.42957835001957, 108.64448525904967, 112.28648832670142]\n",
      "mean mse: 203.63889209570297\n",
      "Train [2017-07-01 2017-08-15] Val [2017-08-16 2017-08-31] Test [2017-09-01 2017-09-16]\n",
      "Window Train/Val/Test shape (966, 61) (336, 61) (336, 61)\n",
      "Window Train/Val/Test shape (966, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-08-30] Val [2017-08-31 2017-09-15] Test [2017-09-16 2017-10-01]\n",
      "Window Train/Val/Test shape (1281, 61) (336, 61) (336, 61)\n",
      "Window Train/Val/Test shape (1281, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-09-14] Val [2017-09-15 2017-09-30] Test [2017-10-01 2017-10-16]\n",
      "Window Train/Val/Test shape (1596, 61) (336, 61) (336, 61)\n",
      "Window Train/Val/Test shape (1596, 2) (336, 2) (336, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-09-29] Val [2017-09-30 2017-10-15] Test [2017-10-16 2017-11-10]\n",
      "Window Train/Val/Test shape (1911, 61) (336, 61) (546, 61)\n",
      "Window Train/Val/Test shape (1911, 2) (336, 2) (546, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-10-14] Val [2017-10-15 2017-10-30] Test [2017-10-31 2017-11-10]\n",
      "Window Train/Val/Test shape (2226, 61) (336, 61) (231, 61)\n",
      "Window Train/Val/Test shape (2226, 2) (336, 2) (231, 2)\n",
      "[27.41797170289224, 34.3170270329396, 14.369652426128498, 8.920972302195755, 8.290287635264665]\n",
      "mean mse: 18.66318221988415\n",
      "Train [2017-07-01 2017-08-15] Val [2017-08-16 2017-08-31] Test [2017-09-01 2017-09-16]\n",
      "Window Train/Val/Test shape (966, 61) (336, 61) (336, 61)\n",
      "Window Train/Val/Test shape (966, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-08-30] Val [2017-08-31 2017-09-15] Test [2017-09-16 2017-10-01]\n",
      "Window Train/Val/Test shape (1281, 61) (336, 61) (336, 61)\n",
      "Window Train/Val/Test shape (1281, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-09-14] Val [2017-09-15 2017-09-30] Test [2017-10-01 2017-10-16]\n",
      "Window Train/Val/Test shape (1596, 61) (336, 61) (336, 61)\n",
      "Window Train/Val/Test shape (1596, 2) (336, 2) (336, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-09-29] Val [2017-09-30 2017-10-15] Test [2017-10-16 2017-11-10]\n",
      "Window Train/Val/Test shape (1911, 61) (336, 61) (546, 61)\n",
      "Window Train/Val/Test shape (1911, 2) (336, 2) (546, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-10-14] Val [2017-10-15 2017-10-30] Test [2017-10-31 2017-11-10]\n",
      "Window Train/Val/Test shape (2226, 61) (336, 61) (231, 61)\n",
      "Window Train/Val/Test shape (2226, 2) (336, 2) (231, 2)\n",
      "[21.82721037528645, 32.47628190050028, 14.974665752844041, 8.080361618666128, 6.242162455012955]\n",
      "mean mse: 16.720136420461976\n",
      "Train [2017-07-01 2017-08-15] Val [2017-08-16 2017-08-31] Test [2017-09-01 2017-09-16]\n",
      "Window Train/Val/Test shape (966, 61) (336, 61) (336, 61)\n",
      "Window Train/Val/Test shape (966, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-08-30] Val [2017-08-31 2017-09-15] Test [2017-09-16 2017-10-01]\n",
      "Window Train/Val/Test shape (1281, 61) (336, 61) (336, 61)\n",
      "Window Train/Val/Test shape (1281, 2) (336, 2) (336, 2)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train [2017-07-01 2017-09-14] Val [2017-09-15 2017-09-30] Test [2017-10-01 2017-10-16]\n",
      "Window Train/Val/Test shape (1596, 61) (336, 61) (336, 61)\n",
      "Window Train/Val/Test shape (1596, 2) (336, 2) (336, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-09-29] Val [2017-09-30 2017-10-15] Test [2017-10-16 2017-11-10]\n",
      "Window Train/Val/Test shape (1911, 61) (336, 61) (546, 61)\n",
      "Window Train/Val/Test shape (1911, 2) (336, 2) (546, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-10-14] Val [2017-10-15 2017-10-30] Test [2017-10-31 2017-11-10]\n",
      "Window Train/Val/Test shape (2226, 61) (336, 61) (231, 61)\n",
      "Window Train/Val/Test shape (2226, 2) (336, 2) (231, 2)\n",
      "[27.004738005961215, 40.123602729945105, 15.50910960770598, 9.708950629280642, 8.617981441267714]\n",
      "mean mse: 20.19287648283213\n",
      "Train [2017-07-01 2017-08-15] Val [2017-08-16 2017-08-31] Test [2017-09-01 2017-09-16]\n",
      "Window Train/Val/Test shape (966, 61) (336, 61) (336, 61)\n",
      "Window Train/Val/Test shape (966, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-08-30] Val [2017-08-31 2017-09-15] Test [2017-09-16 2017-10-01]\n",
      "Window Train/Val/Test shape (1281, 61) (336, 61) (336, 61)\n",
      "Window Train/Val/Test shape (1281, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-09-14] Val [2017-09-15 2017-09-30] Test [2017-10-01 2017-10-16]\n",
      "Window Train/Val/Test shape (1596, 61) (336, 61) (336, 61)\n",
      "Window Train/Val/Test shape (1596, 2) (336, 2) (336, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-09-29] Val [2017-09-30 2017-10-15] Test [2017-10-16 2017-11-10]\n",
      "Window Train/Val/Test shape (1911, 61) (336, 61) (546, 61)\n",
      "Window Train/Val/Test shape (1911, 2) (336, 2) (546, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-10-14] Val [2017-10-15 2017-10-30] Test [2017-10-31 2017-11-10]\n",
      "Window Train/Val/Test shape (2226, 61) (336, 61) (231, 61)\n",
      "Window Train/Val/Test shape (2226, 2) (336, 2) (231, 2)\n",
      "[26.054056446860685, 37.02123418470073, 15.300801104348082, 5.489175853888543, 7.800245382065234]\n",
      "mean mse: 18.333102594372654\n",
      "Train [2017-07-01 2017-08-15] Val [2017-08-16 2017-08-31] Test [2017-09-01 2017-09-16]\n",
      "Window Train/Val/Test shape (966, 61) (336, 61) (336, 61)\n",
      "Window Train/Val/Test shape (966, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-08-30] Val [2017-08-31 2017-09-15] Test [2017-09-16 2017-10-01]\n",
      "Window Train/Val/Test shape (1281, 61) (336, 61) (336, 61)\n",
      "Window Train/Val/Test shape (1281, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-09-14] Val [2017-09-15 2017-09-30] Test [2017-10-01 2017-10-16]\n",
      "Window Train/Val/Test shape (1596, 61) (336, 61) (336, 61)\n",
      "Window Train/Val/Test shape (1596, 2) (336, 2) (336, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-09-29] Val [2017-09-30 2017-10-15] Test [2017-10-16 2017-11-10]\n",
      "Window Train/Val/Test shape (1911, 61) (336, 61) (546, 61)\n",
      "Window Train/Val/Test shape (1911, 2) (336, 2) (546, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-10-14] Val [2017-10-15 2017-10-30] Test [2017-10-31 2017-11-10]\n",
      "Window Train/Val/Test shape (2226, 61) (336, 61) (231, 61)\n",
      "Window Train/Val/Test shape (2226, 2) (336, 2) (231, 2)\n",
      "[360.76569886710104, 290.06820967564323, 146.42957835001957, 108.64448525904967, 112.28648832670142]\n",
      "mean mse: 203.63889209570297\n",
      "Train [2017-07-01 2017-08-15] Val [2017-08-16 2017-08-31] Test [2017-09-01 2017-09-16]\n",
      "Window Train/Val/Test shape (966, 61) (336, 61) (336, 61)\n",
      "Window Train/Val/Test shape (966, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-08-30] Val [2017-08-31 2017-09-15] Test [2017-09-16 2017-10-01]\n",
      "Window Train/Val/Test shape (1281, 61) (336, 61) (336, 61)\n",
      "Window Train/Val/Test shape (1281, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-09-14] Val [2017-09-15 2017-09-30] Test [2017-10-01 2017-10-16]\n",
      "Window Train/Val/Test shape (1596, 61) (336, 61) (336, 61)\n",
      "Window Train/Val/Test shape (1596, 2) (336, 2) (336, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-09-29] Val [2017-09-30 2017-10-15] Test [2017-10-16 2017-11-10]\n",
      "Window Train/Val/Test shape (1911, 61) (336, 61) (546, 61)\n",
      "Window Train/Val/Test shape (1911, 2) (336, 2) (546, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-10-14] Val [2017-10-15 2017-10-30] Test [2017-10-31 2017-11-10]\n",
      "Window Train/Val/Test shape (2226, 61) (336, 61) (231, 61)\n",
      "Window Train/Val/Test shape (2226, 2) (336, 2) (231, 2)\n",
      "[22.511978024668213, 32.83318393047098, 15.287316992905078, 9.365066526533335, 5.582987318021314]\n",
      "mean mse: 17.116106558519782\n",
      "Train [2017-07-01 2017-08-15] Val [2017-08-16 2017-08-31] Test [2017-09-01 2017-09-16]\n",
      "Window Train/Val/Test shape (966, 61) (336, 61) (336, 61)\n",
      "Window Train/Val/Test shape (966, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-08-30] Val [2017-08-31 2017-09-15] Test [2017-09-16 2017-10-01]\n",
      "Window Train/Val/Test shape (1281, 61) (336, 61) (336, 61)\n",
      "Window Train/Val/Test shape (1281, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-09-14] Val [2017-09-15 2017-09-30] Test [2017-10-01 2017-10-16]\n",
      "Window Train/Val/Test shape (1596, 61) (336, 61) (336, 61)\n",
      "Window Train/Val/Test shape (1596, 2) (336, 2) (336, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-09-29] Val [2017-09-30 2017-10-15] Test [2017-10-16 2017-11-10]\n",
      "Window Train/Val/Test shape (1911, 61) (336, 61) (546, 61)\n",
      "Window Train/Val/Test shape (1911, 2) (336, 2) (546, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-10-14] Val [2017-10-15 2017-10-30] Test [2017-10-31 2017-11-10]\n",
      "Window Train/Val/Test shape (2226, 61) (336, 61) (231, 61)\n",
      "Window Train/Val/Test shape (2226, 2) (336, 2) (231, 2)\n",
      "[360.76569886710104, 290.06820967564323, 146.42957835001957, 108.64448525904967, 112.28648832670142]\n",
      "mean mse: 203.63889209570297\n",
      "Train [2017-07-01 2017-08-15] Val [2017-08-16 2017-08-31] Test [2017-09-01 2017-09-16]\n",
      "Window Train/Val/Test shape (966, 61) (336, 61) (336, 61)\n",
      "Window Train/Val/Test shape (966, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-08-30] Val [2017-08-31 2017-09-15] Test [2017-09-16 2017-10-01]\n",
      "Window Train/Val/Test shape (1281, 61) (336, 61) (336, 61)\n",
      "Window Train/Val/Test shape (1281, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-09-14] Val [2017-09-15 2017-09-30] Test [2017-10-01 2017-10-16]\n",
      "Window Train/Val/Test shape (1596, 61) (336, 61) (336, 61)\n",
      "Window Train/Val/Test shape (1596, 2) (336, 2) (336, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-09-29] Val [2017-09-30 2017-10-15] Test [2017-10-16 2017-11-10]\n",
      "Window Train/Val/Test shape (1911, 61) (336, 61) (546, 61)\n",
      "Window Train/Val/Test shape (1911, 2) (336, 2) (546, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-10-14] Val [2017-10-15 2017-10-30] Test [2017-10-31 2017-11-10]\n",
      "Window Train/Val/Test shape (2226, 61) (336, 61) (231, 61)\n",
      "Window Train/Val/Test shape (2226, 2) (336, 2) (231, 2)\n",
      "[22.31688306744514, 30.449554773369794, 15.571880688198277, 5.927041065850392, 6.9223262925043185]\n",
      "mean mse: 16.237537177473584\n",
      "Train [2017-07-01 2017-08-15] Val [2017-08-16 2017-08-31] Test [2017-09-01 2017-09-16]\n",
      "Window Train/Val/Test shape (966, 61) (336, 61) (336, 61)\n",
      "Window Train/Val/Test shape (966, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-08-30] Val [2017-08-31 2017-09-15] Test [2017-09-16 2017-10-01]\n",
      "Window Train/Val/Test shape (1281, 61) (336, 61) (336, 61)\n",
      "Window Train/Val/Test shape (1281, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-09-14] Val [2017-09-15 2017-09-30] Test [2017-10-01 2017-10-16]\n",
      "Window Train/Val/Test shape (1596, 61) (336, 61) (336, 61)\n",
      "Window Train/Val/Test shape (1596, 2) (336, 2) (336, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-09-29] Val [2017-09-30 2017-10-15] Test [2017-10-16 2017-11-10]\n",
      "Window Train/Val/Test shape (1911, 61) (336, 61) (546, 61)\n",
      "Window Train/Val/Test shape (1911, 2) (336, 2) (546, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-10-14] Val [2017-10-15 2017-10-30] Test [2017-10-31 2017-11-10]\n",
      "Window Train/Val/Test shape (2226, 61) (336, 61) (231, 61)\n",
      "Window Train/Val/Test shape (2226, 2) (336, 2) (231, 2)\n",
      "[24.020751289998575, 33.15027281428152, 14.218051741370026, 6.407309983060179, 6.698862831342477]\n",
      "mean mse: 16.899049732010557\n",
      "Train [2017-07-01 2017-08-15] Val [2017-08-16 2017-08-31] Test [2017-09-01 2017-09-16]\n",
      "Window Train/Val/Test shape (966, 61) (336, 61) (336, 61)\n",
      "Window Train/Val/Test shape (966, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-08-30] Val [2017-08-31 2017-09-15] Test [2017-09-16 2017-10-01]\n",
      "Window Train/Val/Test shape (1281, 61) (336, 61) (336, 61)\n",
      "Window Train/Val/Test shape (1281, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-09-14] Val [2017-09-15 2017-09-30] Test [2017-10-01 2017-10-16]\n",
      "Window Train/Val/Test shape (1596, 61) (336, 61) (336, 61)\n",
      "Window Train/Val/Test shape (1596, 2) (336, 2) (336, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-09-29] Val [2017-09-30 2017-10-15] Test [2017-10-16 2017-11-10]\n",
      "Window Train/Val/Test shape (1911, 61) (336, 61) (546, 61)\n",
      "Window Train/Val/Test shape (1911, 2) (336, 2) (546, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-10-14] Val [2017-10-15 2017-10-30] Test [2017-10-31 2017-11-10]\n",
      "Window Train/Val/Test shape (2226, 61) (336, 61) (231, 61)\n",
      "Window Train/Val/Test shape (2226, 2) (336, 2) (231, 2)\n",
      "[25.521031593251468, 33.632292433724544, 14.24743261962241, 6.144926758300799, 6.917154513234714]\n",
      "mean mse: 17.292567583626788\n",
      "Train [2017-07-01 2017-08-15] Val [2017-08-16 2017-08-31] Test [2017-09-01 2017-09-16]\n",
      "Window Train/Val/Test shape (966, 61) (336, 61) (336, 61)\n",
      "Window Train/Val/Test shape (966, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-08-30] Val [2017-08-31 2017-09-15] Test [2017-09-16 2017-10-01]\n",
      "Window Train/Val/Test shape (1281, 61) (336, 61) (336, 61)\n",
      "Window Train/Val/Test shape (1281, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-09-14] Val [2017-09-15 2017-09-30] Test [2017-10-01 2017-10-16]\n",
      "Window Train/Val/Test shape (1596, 61) (336, 61) (336, 61)\n",
      "Window Train/Val/Test shape (1596, 2) (336, 2) (336, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-09-29] Val [2017-09-30 2017-10-15] Test [2017-10-16 2017-11-10]\n",
      "Window Train/Val/Test shape (1911, 61) (336, 61) (546, 61)\n",
      "Window Train/Val/Test shape (1911, 2) (336, 2) (546, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-10-14] Val [2017-10-15 2017-10-30] Test [2017-10-31 2017-11-10]\n",
      "Window Train/Val/Test shape (2226, 61) (336, 61) (231, 61)\n",
      "Window Train/Val/Test shape (2226, 2) (336, 2) (231, 2)\n",
      "[24.067659710830664, 31.88586219807102, 15.380400432755515, 7.801104285750867, 5.750672300561433]\n",
      "mean mse: 16.9771397855939\n",
      "Train [2017-07-01 2017-08-15] Val [2017-08-16 2017-08-31] Test [2017-09-01 2017-09-16]\n",
      "Window Train/Val/Test shape (966, 61) (336, 61) (336, 61)\n",
      "Window Train/Val/Test shape (966, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-08-30] Val [2017-08-31 2017-09-15] Test [2017-09-16 2017-10-01]\n",
      "Window Train/Val/Test shape (1281, 61) (336, 61) (336, 61)\n",
      "Window Train/Val/Test shape (1281, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-09-14] Val [2017-09-15 2017-09-30] Test [2017-10-01 2017-10-16]\n",
      "Window Train/Val/Test shape (1596, 61) (336, 61) (336, 61)\n",
      "Window Train/Val/Test shape (1596, 2) (336, 2) (336, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-09-29] Val [2017-09-30 2017-10-15] Test [2017-10-16 2017-11-10]\n",
      "Window Train/Val/Test shape (1911, 61) (336, 61) (546, 61)\n",
      "Window Train/Val/Test shape (1911, 2) (336, 2) (546, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-10-14] Val [2017-10-15 2017-10-30] Test [2017-10-31 2017-11-10]\n",
      "Window Train/Val/Test shape (2226, 61) (336, 61) (231, 61)\n",
      "Window Train/Val/Test shape (2226, 2) (336, 2) (231, 2)\n",
      "[360.76569886710104, 290.06820967564323, 146.42957835001957, 108.64448525904967, 112.28648832670142]\n",
      "mean mse: 203.63889209570297\n",
      "Train [2017-07-01 2017-08-15] Val [2017-08-16 2017-08-31] Test [2017-09-01 2017-09-16]\n",
      "Window Train/Val/Test shape (966, 61) (336, 61) (336, 61)\n",
      "Window Train/Val/Test shape (966, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-08-30] Val [2017-08-31 2017-09-15] Test [2017-09-16 2017-10-01]\n",
      "Window Train/Val/Test shape (1281, 61) (336, 61) (336, 61)\n",
      "Window Train/Val/Test shape (1281, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-09-14] Val [2017-09-15 2017-09-30] Test [2017-10-01 2017-10-16]\n",
      "Window Train/Val/Test shape (1596, 61) (336, 61) (336, 61)\n",
      "Window Train/Val/Test shape (1596, 2) (336, 2) (336, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-09-29] Val [2017-09-30 2017-10-15] Test [2017-10-16 2017-11-10]\n",
      "Window Train/Val/Test shape (1911, 61) (336, 61) (546, 61)\n",
      "Window Train/Val/Test shape (1911, 2) (336, 2) (546, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-10-14] Val [2017-10-15 2017-10-30] Test [2017-10-31 2017-11-10]\n",
      "Window Train/Val/Test shape (2226, 61) (336, 61) (231, 61)\n",
      "Window Train/Val/Test shape (2226, 2) (336, 2) (231, 2)\n",
      "[26.911011257522006, 41.5866741570106, 14.87954663565413, 7.412314624325479, 6.1688249359562795]\n",
      "mean mse: 19.3916743220937\n",
      "Train [2017-07-01 2017-08-15] Val [2017-08-16 2017-08-31] Test [2017-09-01 2017-09-16]\n",
      "Window Train/Val/Test shape (966, 61) (336, 61) (336, 61)\n",
      "Window Train/Val/Test shape (966, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-08-30] Val [2017-08-31 2017-09-15] Test [2017-09-16 2017-10-01]\n",
      "Window Train/Val/Test shape (1281, 61) (336, 61) (336, 61)\n",
      "Window Train/Val/Test shape (1281, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-09-14] Val [2017-09-15 2017-09-30] Test [2017-10-01 2017-10-16]\n",
      "Window Train/Val/Test shape (1596, 61) (336, 61) (336, 61)\n",
      "Window Train/Val/Test shape (1596, 2) (336, 2) (336, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-09-29] Val [2017-09-30 2017-10-15] Test [2017-10-16 2017-11-10]\n",
      "Window Train/Val/Test shape (1911, 61) (336, 61) (546, 61)\n",
      "Window Train/Val/Test shape (1911, 2) (336, 2) (546, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-10-14] Val [2017-10-15 2017-10-30] Test [2017-10-31 2017-11-10]\n",
      "Window Train/Val/Test shape (2226, 61) (336, 61) (231, 61)\n",
      "Window Train/Val/Test shape (2226, 2) (336, 2) (231, 2)\n",
      "[360.76569886710104, 290.06820967564323, 146.42957835001957, 108.64448525904967, 112.28648832670142]\n",
      "mean mse: 203.63889209570297\n",
      "Train [2017-07-01 2017-08-15] Val [2017-08-16 2017-08-31] Test [2017-09-01 2017-09-16]\n",
      "Window Train/Val/Test shape (966, 61) (336, 61) (336, 61)\n",
      "Window Train/Val/Test shape (966, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-08-30] Val [2017-08-31 2017-09-15] Test [2017-09-16 2017-10-01]\n",
      "Window Train/Val/Test shape (1281, 61) (336, 61) (336, 61)\n",
      "Window Train/Val/Test shape (1281, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-09-14] Val [2017-09-15 2017-09-30] Test [2017-10-01 2017-10-16]\n",
      "Window Train/Val/Test shape (1596, 61) (336, 61) (336, 61)\n",
      "Window Train/Val/Test shape (1596, 2) (336, 2) (336, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-09-29] Val [2017-09-30 2017-10-15] Test [2017-10-16 2017-11-10]\n",
      "Window Train/Val/Test shape (1911, 61) (336, 61) (546, 61)\n",
      "Window Train/Val/Test shape (1911, 2) (336, 2) (546, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-10-14] Val [2017-10-15 2017-10-30] Test [2017-10-31 2017-11-10]\n",
      "Window Train/Val/Test shape (2226, 61) (336, 61) (231, 61)\n",
      "Window Train/Val/Test shape (2226, 2) (336, 2) (231, 2)\n",
      "[25.97808157786695, 29.15011205523581, 16.280987162442702, 6.089243738843185, 7.182110799369818]\n",
      "mean mse: 16.936107066751692\n",
      "Train [2017-07-01 2017-08-15] Val [2017-08-16 2017-08-31] Test [2017-09-01 2017-09-16]\n",
      "Window Train/Val/Test shape (966, 61) (336, 61) (336, 61)\n",
      "Window Train/Val/Test shape (966, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-08-30] Val [2017-08-31 2017-09-15] Test [2017-09-16 2017-10-01]\n",
      "Window Train/Val/Test shape (1281, 61) (336, 61) (336, 61)\n",
      "Window Train/Val/Test shape (1281, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-09-14] Val [2017-09-15 2017-09-30] Test [2017-10-01 2017-10-16]\n",
      "Window Train/Val/Test shape (1596, 61) (336, 61) (336, 61)\n",
      "Window Train/Val/Test shape (1596, 2) (336, 2) (336, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-09-29] Val [2017-09-30 2017-10-15] Test [2017-10-16 2017-11-10]\n",
      "Window Train/Val/Test shape (1911, 61) (336, 61) (546, 61)\n",
      "Window Train/Val/Test shape (1911, 2) (336, 2) (546, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-10-14] Val [2017-10-15 2017-10-30] Test [2017-10-31 2017-11-10]\n",
      "Window Train/Val/Test shape (2226, 61) (336, 61) (231, 61)\n",
      "Window Train/Val/Test shape (2226, 2) (336, 2) (231, 2)\n",
      "[360.76569886710104, 290.06820967564323, 146.42957835001957, 108.64448525904967, 112.28648832670142]\n",
      "mean mse: 203.63889209570297\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train [2017-07-01 2017-08-15] Val [2017-08-16 2017-08-31] Test [2017-09-01 2017-09-16]\n",
      "Window Train/Val/Test shape (966, 61) (336, 61) (336, 61)\n",
      "Window Train/Val/Test shape (966, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-08-30] Val [2017-08-31 2017-09-15] Test [2017-09-16 2017-10-01]\n",
      "Window Train/Val/Test shape (1281, 61) (336, 61) (336, 61)\n",
      "Window Train/Val/Test shape (1281, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-09-14] Val [2017-09-15 2017-09-30] Test [2017-10-01 2017-10-16]\n",
      "Window Train/Val/Test shape (1596, 61) (336, 61) (336, 61)\n",
      "Window Train/Val/Test shape (1596, 2) (336, 2) (336, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-09-29] Val [2017-09-30 2017-10-15] Test [2017-10-16 2017-11-10]\n",
      "Window Train/Val/Test shape (1911, 61) (336, 61) (546, 61)\n",
      "Window Train/Val/Test shape (1911, 2) (336, 2) (546, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-10-14] Val [2017-10-15 2017-10-30] Test [2017-10-31 2017-11-10]\n",
      "Window Train/Val/Test shape (2226, 61) (336, 61) (231, 61)\n",
      "Window Train/Val/Test shape (2226, 2) (336, 2) (231, 2)\n",
      "[22.91450802084062, 34.21279622247461, 21.092236184680285, 11.550544486963007, 6.370478945545254]\n",
      "mean mse: 19.228112772100754\n",
      "Train [2017-07-01 2017-08-15] Val [2017-08-16 2017-08-31] Test [2017-09-01 2017-09-16]\n",
      "Window Train/Val/Test shape (966, 61) (336, 61) (336, 61)\n",
      "Window Train/Val/Test shape (966, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-08-30] Val [2017-08-31 2017-09-15] Test [2017-09-16 2017-10-01]\n",
      "Window Train/Val/Test shape (1281, 61) (336, 61) (336, 61)\n",
      "Window Train/Val/Test shape (1281, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-09-14] Val [2017-09-15 2017-09-30] Test [2017-10-01 2017-10-16]\n",
      "Window Train/Val/Test shape (1596, 61) (336, 61) (336, 61)\n",
      "Window Train/Val/Test shape (1596, 2) (336, 2) (336, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-09-29] Val [2017-09-30 2017-10-15] Test [2017-10-16 2017-11-10]\n",
      "Window Train/Val/Test shape (1911, 61) (336, 61) (546, 61)\n",
      "Window Train/Val/Test shape (1911, 2) (336, 2) (546, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-10-14] Val [2017-10-15 2017-10-30] Test [2017-10-31 2017-11-10]\n",
      "Window Train/Val/Test shape (2226, 61) (336, 61) (231, 61)\n",
      "Window Train/Val/Test shape (2226, 2) (336, 2) (231, 2)\n",
      "[20.787551598413835, 47.84984279743934, 17.252363129752098, 13.001954847920423, 6.756163442963167]\n",
      "mean mse: 21.12957516329777\n",
      "Train [2017-07-01 2017-08-15] Val [2017-08-16 2017-08-31] Test [2017-09-01 2017-09-16]\n",
      "Window Train/Val/Test shape (966, 61) (336, 61) (336, 61)\n",
      "Window Train/Val/Test shape (966, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-08-30] Val [2017-08-31 2017-09-15] Test [2017-09-16 2017-10-01]\n",
      "Window Train/Val/Test shape (1281, 61) (336, 61) (336, 61)\n",
      "Window Train/Val/Test shape (1281, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-09-14] Val [2017-09-15 2017-09-30] Test [2017-10-01 2017-10-16]\n",
      "Window Train/Val/Test shape (1596, 61) (336, 61) (336, 61)\n",
      "Window Train/Val/Test shape (1596, 2) (336, 2) (336, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-09-29] Val [2017-09-30 2017-10-15] Test [2017-10-16 2017-11-10]\n",
      "Window Train/Val/Test shape (1911, 61) (336, 61) (546, 61)\n",
      "Window Train/Val/Test shape (1911, 2) (336, 2) (546, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-10-14] Val [2017-10-15 2017-10-30] Test [2017-10-31 2017-11-10]\n",
      "Window Train/Val/Test shape (2226, 61) (336, 61) (231, 61)\n",
      "Window Train/Val/Test shape (2226, 2) (336, 2) (231, 2)\n",
      "[24.604557633889666, 31.587161862608998, 15.13436099508177, 6.435811471016717, 6.482601294216404]\n",
      "mean mse: 16.84889865136271\n",
      "Train [2017-07-01 2017-08-15] Val [2017-08-16 2017-08-31] Test [2017-09-01 2017-09-16]\n",
      "Window Train/Val/Test shape (966, 61) (336, 61) (336, 61)\n",
      "Window Train/Val/Test shape (966, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-08-30] Val [2017-08-31 2017-09-15] Test [2017-09-16 2017-10-01]\n",
      "Window Train/Val/Test shape (1281, 61) (336, 61) (336, 61)\n",
      "Window Train/Val/Test shape (1281, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-09-14] Val [2017-09-15 2017-09-30] Test [2017-10-01 2017-10-16]\n",
      "Window Train/Val/Test shape (1596, 61) (336, 61) (336, 61)\n",
      "Window Train/Val/Test shape (1596, 2) (336, 2) (336, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-09-29] Val [2017-09-30 2017-10-15] Test [2017-10-16 2017-11-10]\n",
      "Window Train/Val/Test shape (1911, 61) (336, 61) (546, 61)\n",
      "Window Train/Val/Test shape (1911, 2) (336, 2) (546, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-10-14] Val [2017-10-15 2017-10-30] Test [2017-10-31 2017-11-10]\n",
      "Window Train/Val/Test shape (2226, 61) (336, 61) (231, 61)\n",
      "Window Train/Val/Test shape (2226, 2) (336, 2) (231, 2)\n",
      "[360.76569886710104, 290.06820967564323, 146.42957835001957, 108.64448525904967, 112.28648832670142]\n",
      "mean mse: 203.63889209570297\n"
     ]
    }
   ],
   "source": [
    "orchestrator68.performHyperparameterOptimization(X_68_preselect[selected_features68], 100)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 115,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'colsample_bytree': 0.5,\n",
       " 'gamma': 1.5,\n",
       " 'learning_rate': 0.07500000000000001,\n",
       " 'max_depth': 7.0,\n",
       " 'min_child_weight': 2.0,\n",
       " 'n_estimators': 550.0,\n",
       " 'subsample': 0.7000000000000001}"
      ]
     },
     "execution_count": 115,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "orchestrator68.bo.best"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [],
   "source": [
    "orchestrator68.corr_selected_features_bi_week[selected_features68].to_csv('/Users/Rohil/Documents/iGEM/yemen/intermediary_files/X_6_8.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "orchestrator68.corr_selected_features_bi_week_y.to_csv('/Users/Rohil/Documents/iGEM/yemen/intermediary_files/y_6_8.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 95,
   "metadata": {},
   "outputs": [],
   "source": [
    "postselect_params_68 = {'colsample_bytree': 0.4,\n",
    "                         'gamma': 0.0,\n",
    "                         'learning_rate': 0.025,\n",
    "                         'max_depth': 8,\n",
    "                         'min_child_weight': 1.0,\n",
    "                         'n_estimators': 150,\n",
    "                         'subsample': 0.9}\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 165,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\Rohil\\Anaconda3\\lib\\site-packages\\ipykernel_launcher.py:78: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/indexing.html#indexing-view-versus-copy\n",
      "C:\\Users\\Rohil\\Anaconda3\\lib\\site-packages\\pandas\\core\\frame.py:3694: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame\n",
      "\n",
      "See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/indexing.html#indexing-view-versus-copy\n",
      "  errors=errors)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train [2017-07-01 2017-08-15] Val [2017-08-16 2017-08-31] Test [2017-09-01 2017-09-16]\n",
      "Window Train/Val/Test shape (966, 61) (336, 61) (336, 61)\n",
      "Window Train/Val/Test shape (966, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-08-30] Val [2017-08-31 2017-09-15] Test [2017-09-16 2017-10-01]\n",
      "Window Train/Val/Test shape (1281, 61) (336, 61) (336, 61)\n",
      "Window Train/Val/Test shape (1281, 2) (336, 2) (336, 2)\n",
      "Train [2017-07-01 2017-09-14] Val [2017-09-15 2017-09-30] Test [2017-10-01 2017-10-16]\n",
      "Window Train/Val/Test shape (1596, 61) (336, 61) (336, 61)\n",
      "Window Train/Val/Test shape (1596, 2) (336, 2) (336, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-09-29] Val [2017-09-30 2017-10-15] Test [2017-10-16 2017-11-10]\n",
      "Window Train/Val/Test shape (1911, 61) (336, 61) (546, 61)\n",
      "Window Train/Val/Test shape (1911, 2) (336, 2) (546, 2)\n",
      "Rolling window to end date\n",
      "Train [2017-07-01 2017-10-14] Val [2017-10-15 2017-10-30] Test [2017-10-31 2017-11-10]\n",
      "Window Train/Val/Test shape (2226, 61) (336, 61) (231, 61)\n",
      "Window Train/Val/Test shape (2226, 2) (336, 2) (231, 2)\n",
      "y-test mean 7.651104805708142, y-test std 7.819730710483535\n",
      "cv mse 16.557052173500796, holdout mse28.672515754478656\n"
     ]
    }
   ],
   "source": [
    "orchestrator68.deploy(0.001815, postselect_params_68, None)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}