GB.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "import utils, sklearn_ext\n",
    "utils=reload(utils)\n",
    "sklearn_ext=reload(sklearn_ext)\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "np.random.seed(0)\n",
    "import sklearn\n",
    "import itertools, functools, re, pickle, gzip, pprint,os\n",
    "from collections import defaultdict\n",
    "from functools import partial\n",
    "from io import StringIO\n",
    "import joblib"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# MODEL='RF'\n",
    "MODEL='XGBT'\n",
    "DYNAMIC_PREDICT=True\n",
    "IMPUTE=False\n",
    "\n",
    "WEATHER=True\n",
    "MULTIOUTPUT=False\n",
    "OHE=False\n",
    "DROP_CLOSE=False \n",
    "LOG_TRANS=False    # log transform to Y , 反而變差 , 難道要power transform?\n",
    "VALUE_IMPUTE_MISSING=-999\n",
    "COL_Y='Sales'\n",
    "COL_Y2='Customers'\n",
    "if MULTIOUTPUT: Y_COLS=[COL_Y,COL_Y2]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Load"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "bucket='gs://rossmann'\n",
    "%storage read -o {bucket}/data/train.csv -v train_data\n",
    "%storage read -o {bucket}/data/test.csv -v test_data\n",
    "%storage read -o {bucket}/data/store.csv -v store_data\n",
    "%storage read -o {bucket}/data/state.csv -v state_data\n",
    "%storage read -o {bucket}/data/holiday.csv -v holiday_data\n",
    "%storage read -o {bucket}/data/weather.csv -v weather_data\n",
    "%storage read -o {bucket}/data/weather.csv -v weather_data\n",
    "\n",
    "train_df=utils.load_train(train_data)\n",
    "if IMPUTE:\n",
    "  %storage read -o {bucket}/data/Yimpute.csv -v Yimpute\n",
    "  Yimpute=pd.read_csv(StringIO(unicode(Yimpute)))\n",
    "  Yimpute['Date']=Yimpute.Date.astype(np.datetime64)\n",
    "  Yimpute=Yimpute.set_index(['Store','Date'])\n",
    "  train_df=train_df.set_index(['Store','Date'])\n",
    "  train_df.loc[Yimpute.index,Yimpute.columns]=Yimpute\n",
    "  train_df=train_df.reset_index()\n",
    "\n",
    "test_df=utils.load_test(test_data)\n",
    "store_df=utils.load_store(store_data,state_data)\n",
    "holiday_df=utils.load_holiday(holiday_data)\n",
    "weather_df=pd.read_csv(StringIO(unicode(weather_data)))\n",
    "weather_df['Date']=pd.to_datetime(weather_df.Date)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(33121, 8)"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_df[train_df.isnull().any(1)].shape"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Preprocess\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "def preprocess(train,test,store,holiday,weather,\n",
    "               ohe=OHE,dynamic_predict=DYNAMIC_PREDICT,drop_na_y=True,\n",
    "               multioutput=MULTIOUTPUT,drop_close=DROP_CLOSE):\n",
    "    Y_cols = [COL_Y,COL_Y2]  if multioutput else [COL_Y]\n",
    "    to_be_dropped=['Id','Date']\n",
    "    if not multioutput: to_be_dropped+=[COL_Y2] # \n",
    "\n",
    "    df=pd.concat([train,test],axis=0)\n",
    "    df=df.sort_values(by=['Store','Date']).reset_index(drop=True)\n",
    "    assert len(df)==train.Store.nunique()*((train.Date.max()-train.Date.min()).days+1)+\\\n",
    "                    test.Store.nunique()*((test.Date.max()-test.Date.min()).days+1)\n",
    "    # =======================================\n",
    "#     if LOG_TRANS:\n",
    "  #     df[COL_Y]=np.log1p(df[COL_Y])\n",
    "  #     df[COL_Y2]=np.log1p(df[COL_Y2])\n",
    "    # extract date features and store feaures\n",
    "    df,_to_be_dropped=utils.basic_features(df,store)\n",
    "    to_be_dropped+=_to_be_dropped\n",
    "    \n",
    "    # \n",
    "    # padding , so that duration or time-series features do not end with nan\n",
    "    df=utils.padding(df) \n",
    "    df = df.sort_values(by=['Store', 'Date']).reset_index(drop=True)\n",
    "    \n",
    "    #=====join holiday and weather data ======\n",
    "    _len_df=len(df)\n",
    "    df=utils.holiday_features(df,holiday)\n",
    "    to_be_dropped+=['StateHoliday','HolidayStates']\n",
    "    if WEATHER:\n",
    "      assert df.Date.dtype==weather.Date.dtype\n",
    "      df=df.merge(weather,on=['Date','State'],how='left')\n",
    "      assert _len_df==len(df)\n",
    "    \n",
    "    cols=['Sales','Customers','StateSales_mean','StateSales_std','GroupSales_mean','GroupSales_std']\n",
    "    MA_order=[63,]\n",
    "    for order in MA_order:\n",
    "        df,_to_be_dropped=utils.window_statistics(df,order,cols)\n",
    "        to_be_dropped+=_to_be_dropped\n",
    "        df['S/C_EMA'+str(order)] = df['Sales_EMA'+str(order)] / df['Customers_EMA'+str(order)]\n",
    "        df['S/C_EMSTD'+str(order)] = df['Sales_EMSTD'+str(order)] / df['Customers_EMSTD'+str(order)]\n",
    "        to_be_dropped+=['S/C_EMA'+str(order), 'S/C_EMSTD'+str(order),]\n",
    "    weekday_MA_order=[21,]\n",
    "    for order in weekday_MA_order:\n",
    "        df,_to_be_dropped=utils.window_statistics_by_weekday(df,order,cols)\n",
    "        to_be_dropped+=_to_be_dropped\n",
    "        df['S/CByWeekDay_EMA'+str(order)] = df['SalesByWeekDay_EMA'+str(order)] / df['CustomersByWeekDay_EMA'+str(order)]\n",
    "        df['S/CByWeekDay_EMSTD'+str(order)] = df['SalesByWeekDay_EMSTD'+str(order)] / df['CustomersByWeekDay_EMSTD'+str(order)]\n",
    "        to_be_dropped+=['S/CByWeekDay_EMA'+str(order),'S/CByWeekDay_EMSTD'+str(order)]\n",
    "\n",
    "    df,_to_be_dropped=utils.weekday_diff(df,cols)\n",
    "    to_be_dropped+=_to_be_dropped\n",
    "    \n",
    "    df,_to_be_dropped=utils.window_statistics(df,14,['Open'])\n",
    "    to_be_dropped+=_to_be_dropped\n",
    "    \n",
    "    \n",
    "    # ========== dynamic features =========\n",
    "    #\"\"\"set dynamic features\"\"\"# * need complete date time index, so don't drop anything\n",
    "    df=utils.dynamic_features(df,state_columns=['Promo','Open','SchoolHoliday','IsPromo2Month'],)#  state_offsets=[],#binary state 不需要\n",
    "    if WEATHER:\n",
    "        df=utils.dynamic_features(df,state_columns=['Fog','Hail','Rain','Snow','Thunderstorm'])\n",
    "        df=utils.dynamic_features(df,conti_columns=['Fog','Hail','Rain','Snow','Thunderstorm',\n",
    "                                                    'MeanDew_PointC','CloudCover','Precipitationmm','Mean_VisibilityKm'],\n",
    "                                  time_offsets=[-3,-2,-1,+1,+2,+3])\n",
    "    df=utils.dynamic_features(df,event_columns=['HolidayType','HolidayName','IsHoliday'],\n",
    "                              event_offsets=[-1,1],)#offset效果不好\n",
    "    df=utils.dynamic_features(df,conti_columns=['Promo','Open_EMA14'],time_offsets=[-7,+7],)\n",
    "\n",
    "    cols=['Customers','Sales','S/C','StateSales_mean','StateSales_std','GroupSales_mean','GroupSales_std',]\n",
    "    for order in MA_order:\n",
    "        df=utils.dynamic_features(df,conti_columns=[name+'_EMA'+str(order) for name in cols]+[name+'_EMSTD'+str(order) for name in cols],\n",
    "                            time_offsets=[-52*i for i in (1,2,3)])# min is -49, test data has 48 days\n",
    "    for order in weekday_MA_order:\n",
    "        df=utils.dynamic_features(df,conti_columns=\n",
    "                                  [name+'ByWeekDay'+'_EMA'+str(order) for name in cols]+\n",
    "                                  [name+'ByWeekDay'+'_EMSTD'+str(order) for name in cols],\n",
    "                          time_offsets=[-49,-50,-51,-52,-53,-54,-55]+[-49* i for i in (2,3)])\n",
    "\n",
    "    df=utils.dynamic_features(df,conti_columns=\n",
    "                            [name+'ByWeekDay'+'_Diff' for name in cols if name != 'S/C'] ,\n",
    "                      time_offsets=[-49,-50,-51,-52,-53,-54,-55]+[-49* i for i in (2,3)])\n",
    "  \n",
    "    if dynamic_predict:\n",
    "#         df=utils.dynamic_features(df,conti_columns=['Customers_MA14','Sales_MA14',],#'S/C_14',\n",
    "#                                      time_offsets=[])          \n",
    "        df=utils.dynamic_features(df,conti_columns=Y_cols,\n",
    "                                 time_offsets=[-1,-2,\n",
    "                                               -7,-14,])          \n",
    "    \n",
    "    # ! depadding !\n",
    "    df=utils.depadding(df)\n",
    "    assert len(df)==len(train)+len(test)\n",
    "    # ===================encoding====================\n",
    "    # set one hot features \n",
    "    if ohe:\n",
    "        ohe_cats=['StoreType','Assortment','PromoInterval','WeekDay','StateHoliday',\n",
    "                  'State','HolidayType','HolidayName','Events']\n",
    "        for col in df.columns:\n",
    "            for ohe_col in ohe_cats:\n",
    "              if re.match(r'{}.+'.format(ohe_col),col): ohe_cats+=[col]\n",
    "        df=pd.get_dummies(df,columns=ohe_cats,dummy_na=True)\n",
    "\n",
    "    # random encoding the rest obj type\n",
    "    obj_cols=set(filter(lambda x : df[x].dtype.kind == \"O\",df.columns)).difference(['Date'])\n",
    "    df = sklearn_ext.ObjIndexer(obj_cols).fit(df).transform(df) # random encoding / StringIndexer\n",
    "\n",
    "    if drop_close:\n",
    "        df=df[df.Open==1]\n",
    "        df=df[(df.Sales>=10)|(df.Sales.isnull())]\n",
    "    # ====================    \n",
    "    df=df.sort_values(by=['Store','Date']).reset_index(drop=True)\n",
    "    train_df,test_df=df[df.Date<=train.Date.max()],df[df.Date>train.Date.max()]\n",
    "\n",
    "    # drop NaN  ** dorp rows if nan y **\n",
    "    if drop_na_y:\n",
    "        train_df=train_df[~train_df[Y_cols].isnull().any(1)]\n",
    "    # nan encoding for numerical X columns -- for training tree models.\n",
    "    X_col=train_df.columns.difference(Y_cols)\n",
    "    train_df[X_col]=train_df[X_col].fillna(VALUE_IMPUTE_MISSING)        \n",
    "    \n",
    "    # ======= keep meta-data =====\n",
    "    train_df=train_df.set_index(['Store','Date'],drop=False)\n",
    "    test_df=test_df.set_index(['Store','Date'],drop=False)\n",
    "    id=test_df['Id']  #will be used for submission\n",
    "    # ========drop useless and non-feature columns=========\n",
    "    train_df=train_df.drop(to_be_dropped,axis=1)\n",
    "    test_df=test_df.drop(to_be_dropped,axis=1)\n",
    "    # take out Y and X\n",
    "    Xtrain, Ytrain = train_df[train_df.columns.difference(Y_cols)], train_df[Y_cols]\n",
    "    Xtest = test_df[test_df.columns.difference(Y_cols)]\n",
    "    return Xtrain,Ytrain,Xtest,id#,index_train,index_test"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "ename": "IOError",
     "evalue": "[Errno 2] No such file or directory: 'data/preprocessed_data_dynamic.pkl'",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mIOError\u001b[0m                                   Traceback (most recent call last)",
      "\u001b[1;32m<ipython-input-15-8339155476b3>\u001b[0m in \u001b[0;36m<module>\u001b[1;34m()\u001b[0m\n\u001b[1;32m----> 1\u001b[1;33m \u001b[0mget_ipython\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mrun_cell_magic\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;34mu'time'\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;34mu''\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;34mu'FROM_LAST=False\\npostfix=\\'_dynamic\\'\\n# postfix=\\'\\'\\n# postfix=\\'_complete\\'\\nif FROM_LAST: \\n    X,Y,Xtest,id=joblib.load( \"data/preprocessed_data{}.pkl\".format(postfix))\\nelse:\\n    X,Y,Xtest,id = preprocess(train_df,test_df,store_df,holiday_df,weather_df)\\n    joblib.dump((X,Y,Xtest,id), \"data/preprocessed_data{}.pkl\".format(postfix));'\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[1;32m/usr/local/lib/python2.7/dist-packages/gcp/datalab/__init__.pyc\u001b[0m in \u001b[0;36m_run_cell_magic\u001b[1;34m(self, magic_name, line, cell)\u001b[0m\n\u001b[0;32m     81\u001b[0m     \u001b[1;31m# IPython will complain if cell is empty string but not if it is None\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     82\u001b[0m     \u001b[0mcell\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mNone\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m---> 83\u001b[1;33m   \u001b[1;32mreturn\u001b[0m \u001b[0m_orig_run_cell_magic\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mmagic_name\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mline\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mcell\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m     84\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     85\u001b[0m \u001b[0m_shell\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mInteractiveShell\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mrun_cell_magic\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0m_run_cell_magic\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m/usr/local/lib/python2.7/dist-packages/IPython/core/interactiveshell.pyc\u001b[0m in \u001b[0;36mrun_cell_magic\u001b[1;34m(self, magic_name, line, cell)\u001b[0m\n\u001b[0;32m   2291\u001b[0m             \u001b[0mmagic_arg_s\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mvar_expand\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mline\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mstack_depth\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   2292\u001b[0m             \u001b[1;32mwith\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mbuiltin_trap\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m-> 2293\u001b[1;33m                 \u001b[0mresult\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mfn\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mmagic_arg_s\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mcell\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m   2294\u001b[0m             \u001b[1;32mreturn\u001b[0m \u001b[0mresult\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   2295\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m/usr/local/lib/python2.7/dist-packages/IPython/core/magics/execution.pyc\u001b[0m in \u001b[0;36mtime\u001b[1;34m(self, line, cell, local_ns)\u001b[0m\n",
      "\u001b[1;32m/usr/local/lib/python2.7/dist-packages/IPython/core/magic.pyc\u001b[0m in \u001b[0;36m<lambda>\u001b[1;34m(f, *a, **k)\u001b[0m\n\u001b[0;32m    191\u001b[0m     \u001b[1;31m# but it's overkill for just that one bit of state.\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    192\u001b[0m     \u001b[1;32mdef\u001b[0m \u001b[0mmagic_deco\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0marg\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 193\u001b[1;33m         \u001b[0mcall\u001b[0m \u001b[1;33m=\u001b[0m \u001b[1;32mlambda\u001b[0m \u001b[0mf\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;33m*\u001b[0m\u001b[0ma\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;33m**\u001b[0m\u001b[0mk\u001b[0m\u001b[1;33m:\u001b[0m \u001b[0mf\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m*\u001b[0m\u001b[0ma\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;33m**\u001b[0m\u001b[0mk\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    194\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    195\u001b[0m         \u001b[1;32mif\u001b[0m \u001b[0mcallable\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0marg\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m/usr/local/lib/python2.7/dist-packages/IPython/core/magics/execution.pyc\u001b[0m in \u001b[0;36mtime\u001b[1;34m(self, line, cell, local_ns)\u001b[0m\n\u001b[0;32m   1165\u001b[0m         \u001b[1;32melse\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   1166\u001b[0m             \u001b[0mst\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mclock2\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m-> 1167\u001b[1;33m             \u001b[1;32mexec\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mcode\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mglob\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mlocal_ns\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m   1168\u001b[0m             \u001b[0mend\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mclock2\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   1169\u001b[0m             \u001b[0mout\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mNone\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m<timed exec>\u001b[0m in \u001b[0;36m<module>\u001b[1;34m()\u001b[0m\n",
      "\u001b[1;32m/usr/local/lib/python2.7/dist-packages/joblib/numpy_pickle.pyc\u001b[0m in \u001b[0;36mdump\u001b[1;34m(value, filename, compress, cache_size, protocol)\u001b[0m\n\u001b[0;32m    403\u001b[0m     \u001b[1;32mtry\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    404\u001b[0m         pickler = NumpyPickler(filename, compress=compress,\n\u001b[1;32m--> 405\u001b[1;33m                                cache_size=cache_size, protocol=protocol)\n\u001b[0m\u001b[0;32m    406\u001b[0m         \u001b[0mpickler\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mdump\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mvalue\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    407\u001b[0m         \u001b[0mpickler\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mclose\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m/usr/local/lib/python2.7/dist-packages/joblib/numpy_pickle.pyc\u001b[0m in \u001b[0;36m__init__\u001b[1;34m(self, filename, compress, cache_size, protocol)\u001b[0m\n\u001b[0;32m    212\u001b[0m         \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mcompress\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mcompress\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    213\u001b[0m         \u001b[1;32mif\u001b[0m \u001b[1;32mnot\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mcompress\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 214\u001b[1;33m             \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mfile\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mopen\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mfilename\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;34m'wb'\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    215\u001b[0m         \u001b[1;32melse\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    216\u001b[0m             \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mfile\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mBytesIO\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;31mIOError\u001b[0m: [Errno 2] No such file or directory: 'data/preprocessed_data_dynamic.pkl'"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "FROM_LAST=False\n",
    "postfix='_dynamic'\n",
    "# postfix=''\n",
    "# postfix='_complete'\n",
    "if FROM_LAST: \n",
    "    X,Y,Xtest,id=joblib.load( \"data/preprocessed_data{}.pkl\".format(postfix))\n",
    "else:\n",
    "    X,Y,Xtest,id = preprocess(train_df,test_df,store_df,holiday_df,weather_df)\n",
    "    joblib.dump((X,Y,Xtest,id), \"data/preprocessed_data{}.pkl\".format(postfix));"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(1017209, 421)"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X.shape"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "from sklearn.ensemble import ExtraTreesRegressor\n",
    "import xgboost as xgb\n",
    "from sklearn.grid_search import GridSearchCV\n",
    "# sample_weight=~(date_X.between('2013-11-13','2014-01-15') | date_X.between('2014-11-13','2015-01-15'))+1\n",
    "class DynamicXGBRegressor(xgb.XGBRegressor):\n",
    "    def predict(self, X):\n",
    "        Yhat = dynamic_predict(super(DynamicXGBRegressor, self),\n",
    "                               X, time='Date', subject='Store', Y_column_names=['Sales'], force=True)\n",
    "        return Yhat\n",
    "\n",
    "if MODEL =='RF':\n",
    "    param_grid={'n_estimators':[48],#'max_depth':[1],\n",
    "    #        'max_features':[1.0,0.8],\n",
    "    #         \"min_samples_split\":[2,5,10],\n",
    "               }\n",
    "    fix_params={\"criterion\":'mse', 'random_state':0,'n_jobs': -1,}#         'oob_score':False, 'bootstrap':True, 'warm_start':True\n",
    "    fit_params=lambda X,Y,Xv,Yv:{}\n",
    "    if DYNAMIC_PREDICT:\n",
    "        Model=partial(sklearn_ext.DynamicExtraTreesRegressor,Y_column_names=Y_COLS,**fix_params)\n",
    "    else:\n",
    "        Model=partial(ExtraTreesRegressor,**fix_params)\n",
    "    \n",
    "elif MODEL=='XGBT':\n",
    "    param_grid={ 'n_estimators': [3000], #3000,\n",
    "                'max_depth': [10], #12\n",
    "                \"learning_rate\": [0.02], #0.03\n",
    "                \"subsample\": [0.9], #0.9\n",
    "                \"colsample_bytree\": [0.7], #0.7 \n",
    "               }\n",
    "    fix_params={ \"objective\": \"reg:linear\",\"seed\": 19,'nthread':-1}\n",
    "    fit_params=lambda X,Y,Xv,Yv:{\n",
    "                      'eval_set':[(Xv,Yv)],\n",
    "                      'eval_metric': lambda Y,Yhat:('rmspe',utils.loss(Yhat.get_label(),Y,sales_idx=0)if not LOG_TRANS else None) ,\n",
    "                      'early_stopping_rounds':50, }\n",
    "#                     maximize            = FALSE,\n",
    "    predict_params=lambda model:{'ntree_limit':model.best_ntree_limit}\n",
    "    if DYNAMIC_PREDICT:\n",
    "        Model=partial(DynamicXGBRegressor,**fix_params)\n",
    "    else:\n",
    "        Model=partial(xgb.XGBRegressor,**fix_params)\n",
    "        \n",
    "\n",
    "# from sklearn.pipeline import Pipeline\n",
    "# clf = Pipeline([\n",
    "#   ('selector', RFE( n_features_to_select=None, step=2, verbose=1),\n",
    "#   ('model', RandomForestClassifier())\n",
    "# ])\n",
    "# clf.fit(X, y)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "from sklearn.cross_validation import KFold, train_test_split,ShuffleSplit\n",
    "n_folds=6  \n",
    "# 用n_folds=6  vali_len=7*9  step=7*9  去cover last year\n",
    "vali_len=step=7*9 #2 month\n",
    "date_X=X.index.get_level_values('Date')\n",
    "ts_cv_split = sklearn_ext.TimeSeriesCVSplit((date_X-date_X.min()).days,n_folds,vali_len,step) #return posiiton indices        \n",
    "ts_cv_split = list(x for i,x in enumerate(ts_cv_split) if i in [0,1,4,5]) #去掉過年 ,剩下4 folds\n",
    "# simple_cv_split = ShuffleSplit(Y.shape[0], n_iter=1, test_size=.9)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# RFE"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# %%time\n",
    "# _ts_cv_split =(x for i,x in enumerate(ts_cv_split) if i in [3] ) #4folds算太久 ，取2 folds\n",
    "# from sklearn.metrics import  make_scorer\n",
    "# mylosser = make_scorer(utils.loss, greater_is_better=False)\n",
    "\n",
    "# min_n_features=X.shape[1]//3\n",
    "# step=10\n",
    "# scores,rankings,rfes = sklearn_ext.recursive_selection_CV(\n",
    "#     estimator=Model(n_estimators=2,max_depth=2,colsample_bytree=0.7,learning_rate=0.03), X=X, Y=Y,\n",
    "#     n_features_to_select=min_n_features,\n",
    "#     step=step,\n",
    "#     scorer=mylosser,\n",
    "#     cv_spliter=_ts_cv_split  )\n",
    "# joblib.dump((scores,rfes), \"models/RFECV.pkl\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# n_features=X.shape[1]\n",
    "# # print len(scores)== np.ceil(float(n_features-min_n_features)/int(step*n_features)) +1\n",
    "# best_n_features = n_features-step*np.argmax(scores)\n",
    "# print best_n_features\n",
    "# ranking=np.min(np.array(rankings),0)\n",
    "# support=ranking<=(sorted(ranking)[:best_n_features][-1])\n",
    "# print ranking\n",
    "# print scores\n",
    "# sorted({k:v for k,v in zip(X.columns ,ranking)}.items(),key=lambda x:x[1])\n",
    "# # print rfes[0].support_*1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# pd.Series(scores).plot()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# import _RFE\n",
    "# selector = _RFE.RFE(Model(n_estimators=63), best_n_features, step=step,verbose=1).fit(X, Y)\n",
    "# print selector.support_ \n",
    "# print selector.ranking_\n",
    "# joblib.dump(selector, \"models/RFE.pkl\")\n",
    "# X=selector.transofrm(X) "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# _X=X.copy()\n",
    "# X=X.loc[:,support]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<module 'pickle' from '/usr/lib/python2.7/pickle.pyc'>"
      ]
     },
     "execution_count": 38,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# CrossValidation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "_ts_cv_split =(x for i,x in enumerate(ts_cv_split) if i in [0] ) #4folds算太久 "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Will train until validation_0 error hasn't decreased in 50 rounds.\n",
      "[0]\tvalidation_0-rmspe:0.950097\n",
      "[1]\tvalidation_0-rmspe:0.903200\n",
      "[2]\tvalidation_0-rmspe:0.857491\n",
      "[3]\tvalidation_0-rmspe:0.814805\n",
      "[4]\tvalidation_0-rmspe:0.774311\n",
      "[5]\tvalidation_0-rmspe:0.736271\n",
      "[6]\tvalidation_0-rmspe:0.700329\n",
      "[7]\tvalidation_0-rmspe:0.665777\n",
      "[8]\tvalidation_0-rmspe:0.633123\n",
      "[9]\tvalidation_0-rmspe:0.601719\n",
      "[10]\tvalidation_0-rmspe:0.572919\n",
      "[11]\tvalidation_0-rmspe:0.544972\n",
      "[12]\tvalidation_0-rmspe:0.518968\n",
      "[13]\tvalidation_0-rmspe:0.493938\n",
      "[14]\tvalidation_0-rmspe:0.470529\n",
      "[15]\tvalidation_0-rmspe:0.448353\n",
      "[16]\tvalidation_0-rmspe:0.427438\n",
      "[17]\tvalidation_0-rmspe:0.407443\n",
      "[18]\tvalidation_0-rmspe:0.388517\n",
      "[19]\tvalidation_0-rmspe:0.371002\n",
      "[20]\tvalidation_0-rmspe:0.354062\n",
      "[21]\tvalidation_0-rmspe:0.338238\n",
      "[22]\tvalidation_0-rmspe:0.322956\n",
      "[23]\tvalidation_0-rmspe:0.308786\n",
      "[24]\tvalidation_0-rmspe:0.295574\n",
      "[25]\tvalidation_0-rmspe:0.283280\n",
      "[26]\tvalidation_0-rmspe:0.271177\n",
      "[27]\tvalidation_0-rmspe:0.260132\n",
      "[28]\tvalidation_0-rmspe:0.249809\n",
      "[29]\tvalidation_0-rmspe:0.239893\n",
      "[30]\tvalidation_0-rmspe:0.230709\n",
      "[31]\tvalidation_0-rmspe:0.221958\n",
      "[32]\tvalidation_0-rmspe:0.213823\n",
      "[33]\tvalidation_0-rmspe:0.206326\n",
      "[34]\tvalidation_0-rmspe:0.199397\n",
      "[35]\tvalidation_0-rmspe:0.193050\n",
      "[36]\tvalidation_0-rmspe:0.187103\n",
      "[37]\tvalidation_0-rmspe:0.181472\n",
      "[38]\tvalidation_0-rmspe:0.176141\n",
      "[39]\tvalidation_0-rmspe:0.171265\n",
      "[40]\tvalidation_0-rmspe:0.166717\n",
      "[41]\tvalidation_0-rmspe:0.162561\n",
      "[42]\tvalidation_0-rmspe:0.158733\n",
      "[43]\tvalidation_0-rmspe:0.155119\n",
      "[44]\tvalidation_0-rmspe:0.151802\n",
      "[45]\tvalidation_0-rmspe:0.148765\n",
      "[46]\tvalidation_0-rmspe:0.145947\n",
      "[47]\tvalidation_0-rmspe:0.143395\n",
      "[48]\tvalidation_0-rmspe:0.141108\n",
      "[49]\tvalidation_0-rmspe:0.139110\n",
      "[50]\tvalidation_0-rmspe:0.137247\n",
      "[51]\tvalidation_0-rmspe:0.135490\n",
      "[52]\tvalidation_0-rmspe:0.133842\n",
      "[53]\tvalidation_0-rmspe:0.132418\n",
      "[54]\tvalidation_0-rmspe:0.131001\n",
      "[55]\tvalidation_0-rmspe:0.129864\n",
      "[56]\tvalidation_0-rmspe:0.128767\n",
      "[57]\tvalidation_0-rmspe:0.128023\n",
      "[58]\tvalidation_0-rmspe:0.127097\n",
      "[59]\tvalidation_0-rmspe:0.126345\n",
      "[60]\tvalidation_0-rmspe:0.125625\n",
      "[61]\tvalidation_0-rmspe:0.124900\n",
      "[62]\tvalidation_0-rmspe:0.124356\n",
      "[63]\tvalidation_0-rmspe:0.123820\n",
      "[64]\tvalidation_0-rmspe:0.123435\n",
      "[65]\tvalidation_0-rmspe:0.122926\n",
      "[66]\tvalidation_0-rmspe:0.122474\n",
      "[67]\tvalidation_0-rmspe:0.122123\n",
      "[68]\tvalidation_0-rmspe:0.121800\n",
      "[69]\tvalidation_0-rmspe:0.121485\n",
      "[70]\tvalidation_0-rmspe:0.121271\n",
      "[71]\tvalidation_0-rmspe:0.121435\n",
      "[72]\tvalidation_0-rmspe:0.121122\n",
      "[73]\tvalidation_0-rmspe:0.120908\n",
      "[74]\tvalidation_0-rmspe:0.120703\n",
      "[75]\tvalidation_0-rmspe:0.120471\n",
      "[76]\tvalidation_0-rmspe:0.120400\n",
      "[77]\tvalidation_0-rmspe:0.120243\n",
      "[78]\tvalidation_0-rmspe:0.120098\n",
      "[79]\tvalidation_0-rmspe:0.119999\n",
      "[80]\tvalidation_0-rmspe:0.119848\n",
      "[81]\tvalidation_0-rmspe:0.119722\n",
      "[82]\tvalidation_0-rmspe:0.119609\n",
      "[83]\tvalidation_0-rmspe:0.119523\n",
      "[84]\tvalidation_0-rmspe:0.119330\n",
      "[85]\tvalidation_0-rmspe:0.119246\n",
      "[86]\tvalidation_0-rmspe:0.119194\n",
      "[87]\tvalidation_0-rmspe:0.119062\n",
      "[88]\tvalidation_0-rmspe:0.118953\n",
      "[89]\tvalidation_0-rmspe:0.118851\n",
      "[90]\tvalidation_0-rmspe:0.118709\n",
      "[91]\tvalidation_0-rmspe:0.118757\n",
      "[92]\tvalidation_0-rmspe:0.118676\n",
      "[93]\tvalidation_0-rmspe:0.118572\n",
      "[94]\tvalidation_0-rmspe:0.118450\n",
      "[95]\tvalidation_0-rmspe:0.118343\n",
      "[96]\tvalidation_0-rmspe:0.118253\n",
      "[97]\tvalidation_0-rmspe:0.118172\n",
      "[98]\tvalidation_0-rmspe:0.117990\n",
      "[99]\tvalidation_0-rmspe:0.117863\n",
      "[100]\tvalidation_0-rmspe:0.117893\n",
      "[101]\tvalidation_0-rmspe:0.117787\n",
      "[102]\tvalidation_0-rmspe:0.117714\n",
      "[103]\tvalidation_0-rmspe:0.117611\n",
      "[104]\tvalidation_0-rmspe:0.117423\n",
      "[105]\tvalidation_0-rmspe:0.117322\n",
      "[106]\tvalidation_0-rmspe:0.117220\n",
      "[107]\tvalidation_0-rmspe:0.117079\n",
      "[108]\tvalidation_0-rmspe:0.117065\n",
      "[109]\tvalidation_0-rmspe:0.116976\n",
      "[110]\tvalidation_0-rmspe:0.116868\n",
      "[111]\tvalidation_0-rmspe:0.116787\n",
      "[112]\tvalidation_0-rmspe:0.116771\n",
      "[113]\tvalidation_0-rmspe:0.116655\n",
      "[114]\tvalidation_0-rmspe:0.116538\n",
      "[115]\tvalidation_0-rmspe:0.116487\n",
      "[116]\tvalidation_0-rmspe:0.116400\n",
      "[117]\tvalidation_0-rmspe:0.116215\n",
      "[118]\tvalidation_0-rmspe:0.116298\n",
      "[119]\tvalidation_0-rmspe:0.116171\n",
      "[120]\tvalidation_0-rmspe:0.116178\n",
      "[121]\tvalidation_0-rmspe:0.116205\n",
      "[122]\tvalidation_0-rmspe:0.116144\n",
      "[123]\tvalidation_0-rmspe:0.116080\n",
      "[124]\tvalidation_0-rmspe:0.115926\n",
      "[125]\tvalidation_0-rmspe:0.115883\n",
      "[126]\tvalidation_0-rmspe:0.115806\n",
      "[127]\tvalidation_0-rmspe:0.115784\n",
      "[128]\tvalidation_0-rmspe:0.115810\n",
      "[129]\tvalidation_0-rmspe:0.115771\n",
      "[130]\tvalidation_0-rmspe:0.115801\n",
      "[131]\tvalidation_0-rmspe:0.115703\n",
      "[132]\tvalidation_0-rmspe:0.115610\n",
      "[133]\tvalidation_0-rmspe:0.115552\n",
      "[134]\tvalidation_0-rmspe:0.115522\n",
      "[135]\tvalidation_0-rmspe:0.115434\n",
      "[136]\tvalidation_0-rmspe:0.115388\n",
      "[137]\tvalidation_0-rmspe:0.115287\n",
      "[138]\tvalidation_0-rmspe:0.115250\n",
      "[139]\tvalidation_0-rmspe:0.115191\n",
      "[140]\tvalidation_0-rmspe:0.115142\n",
      "[141]\tvalidation_0-rmspe:0.115062\n",
      "[142]\tvalidation_0-rmspe:0.115030\n",
      "[143]\tvalidation_0-rmspe:0.115002\n",
      "[144]\tvalidation_0-rmspe:0.114931\n",
      "[145]\tvalidation_0-rmspe:0.114863\n",
      "[146]\tvalidation_0-rmspe:0.114954\n",
      "[147]\tvalidation_0-rmspe:0.114903\n",
      "[148]\tvalidation_0-rmspe:0.114953\n",
      "[149]\tvalidation_0-rmspe:0.114903\n",
      "[150]\tvalidation_0-rmspe:0.114824\n",
      "[151]\tvalidation_0-rmspe:0.114769\n",
      "[152]\tvalidation_0-rmspe:0.114804\n",
      "[153]\tvalidation_0-rmspe:0.114752\n",
      "[154]\tvalidation_0-rmspe:0.114724\n",
      "[155]\tvalidation_0-rmspe:0.114648\n",
      "[156]\tvalidation_0-rmspe:0.114592\n",
      "[157]\tvalidation_0-rmspe:0.114572\n",
      "[158]\tvalidation_0-rmspe:0.114576\n",
      "[159]\tvalidation_0-rmspe:0.114584\n",
      "[160]\tvalidation_0-rmspe:0.114528\n",
      "[161]\tvalidation_0-rmspe:0.114469\n",
      "[162]\tvalidation_0-rmspe:0.114460\n",
      "[163]\tvalidation_0-rmspe:0.114437\n",
      "[164]\tvalidation_0-rmspe:0.114405\n",
      "[165]\tvalidation_0-rmspe:0.114401\n",
      "[166]\tvalidation_0-rmspe:0.114331\n",
      "[167]\tvalidation_0-rmspe:0.114253\n",
      "[168]\tvalidation_0-rmspe:0.114176\n",
      "[169]\tvalidation_0-rmspe:0.114161\n",
      "[170]\tvalidation_0-rmspe:0.114147\n",
      "[171]\tvalidation_0-rmspe:0.114100\n",
      "[172]\tvalidation_0-rmspe:0.114093\n",
      "[173]\tvalidation_0-rmspe:0.113997\n",
      "[174]\tvalidation_0-rmspe:0.113954\n",
      "[175]\tvalidation_0-rmspe:0.113893\n",
      "[176]\tvalidation_0-rmspe:0.113878\n",
      "[177]\tvalidation_0-rmspe:0.113866\n",
      "[178]\tvalidation_0-rmspe:0.113818\n",
      "[179]\tvalidation_0-rmspe:0.113695\n",
      "[180]\tvalidation_0-rmspe:0.113652\n",
      "[181]\tvalidation_0-rmspe:0.113598\n",
      "[182]\tvalidation_0-rmspe:0.113525\n",
      "[183]\tvalidation_0-rmspe:0.113508\n",
      "[184]\tvalidation_0-rmspe:0.113432\n",
      "[185]\tvalidation_0-rmspe:0.113430\n",
      "[186]\tvalidation_0-rmspe:0.113416\n",
      "[187]\tvalidation_0-rmspe:0.113361\n",
      "[188]\tvalidation_0-rmspe:0.113317\n",
      "[189]\tvalidation_0-rmspe:0.113247\n",
      "[190]\tvalidation_0-rmspe:0.113209\n",
      "[191]\tvalidation_0-rmspe:0.113177\n",
      "[192]\tvalidation_0-rmspe:0.113356\n",
      "[193]\tvalidation_0-rmspe:0.113337\n",
      "[194]\tvalidation_0-rmspe:0.113335\n",
      "[195]\tvalidation_0-rmspe:0.113277\n",
      "[196]\tvalidation_0-rmspe:0.113272\n",
      "[197]\tvalidation_0-rmspe:0.113208\n",
      "[198]\tvalidation_0-rmspe:0.113150\n",
      "[199]\tvalidation_0-rmspe:0.113125\n",
      "[200]\tvalidation_0-rmspe:0.113096\n",
      "[201]\tvalidation_0-rmspe:0.113052\n",
      "[202]\tvalidation_0-rmspe:0.113002\n",
      "[203]\tvalidation_0-rmspe:0.112972\n",
      "[204]\tvalidation_0-rmspe:0.112960\n",
      "[205]\tvalidation_0-rmspe:0.112917\n",
      "[206]\tvalidation_0-rmspe:0.112882\n",
      "[207]\tvalidation_0-rmspe:0.112854\n",
      "[208]\tvalidation_0-rmspe:0.112804\n",
      "[209]\tvalidation_0-rmspe:0.112786\n",
      "[210]\tvalidation_0-rmspe:0.112782\n",
      "[211]\tvalidation_0-rmspe:0.112773\n",
      "[212]\tvalidation_0-rmspe:0.112757\n",
      "[213]\tvalidation_0-rmspe:0.112746\n",
      "[214]\tvalidation_0-rmspe:0.112741\n",
      "[215]\tvalidation_0-rmspe:0.112705\n",
      "[216]\tvalidation_0-rmspe:0.112682\n",
      "[217]\tvalidation_0-rmspe:0.112658\n",
      "[218]\tvalidation_0-rmspe:0.112630\n",
      "[219]\tvalidation_0-rmspe:0.112576\n",
      "[220]\tvalidation_0-rmspe:0.112097\n",
      "[221]\tvalidation_0-rmspe:0.112150\n",
      "[222]\tvalidation_0-rmspe:0.112127\n",
      "[223]\tvalidation_0-rmspe:0.112121\n",
      "[224]\tvalidation_0-rmspe:0.112108\n",
      "[225]\tvalidation_0-rmspe:0.112085\n",
      "[226]\tvalidation_0-rmspe:0.112056\n",
      "[227]\tvalidation_0-rmspe:0.112084\n",
      "[228]\tvalidation_0-rmspe:0.112056\n",
      "[229]\tvalidation_0-rmspe:0.112047\n",
      "[230]\tvalidation_0-rmspe:0.112037\n",
      "[231]\tvalidation_0-rmspe:0.112022\n",
      "[232]\tvalidation_0-rmspe:0.112010\n",
      "[233]\tvalidation_0-rmspe:0.111993\n",
      "[234]\tvalidation_0-rmspe:0.111962\n",
      "[235]\tvalidation_0-rmspe:0.111934\n",
      "[236]\tvalidation_0-rmspe:0.111919\n",
      "[237]\tvalidation_0-rmspe:0.111875\n",
      "[238]\tvalidation_0-rmspe:0.111851\n",
      "[239]\tvalidation_0-rmspe:0.111825\n",
      "[240]\tvalidation_0-rmspe:0.111812\n",
      "[241]\tvalidation_0-rmspe:0.111788\n",
      "[242]\tvalidation_0-rmspe:0.111786\n",
      "[243]\tvalidation_0-rmspe:0.111816\n",
      "[244]\tvalidation_0-rmspe:0.111768\n",
      "[245]\tvalidation_0-rmspe:0.111771\n",
      "[246]\tvalidation_0-rmspe:0.111749\n",
      "[247]\tvalidation_0-rmspe:0.111748\n",
      "[248]\tvalidation_0-rmspe:0.111749\n",
      "[249]\tvalidation_0-rmspe:0.111697\n",
      "[250]\tvalidation_0-rmspe:0.111728\n",
      "[251]\tvalidation_0-rmspe:0.111757\n",
      "[252]\tvalidation_0-rmspe:0.111754\n",
      "[253]\tvalidation_0-rmspe:0.111721\n",
      "[254]\tvalidation_0-rmspe:0.111732\n",
      "[255]\tvalidation_0-rmspe:0.111679\n",
      "[256]\tvalidation_0-rmspe:0.111672\n",
      "[257]\tvalidation_0-rmspe:0.111662\n",
      "[258]\tvalidation_0-rmspe:0.111696\n",
      "[259]\tvalidation_0-rmspe:0.111688\n",
      "[260]\tvalidation_0-rmspe:0.111655\n",
      "[261]\tvalidation_0-rmspe:0.111642\n",
      "[262]\tvalidation_0-rmspe:0.111632\n",
      "[263]\tvalidation_0-rmspe:0.111619\n",
      "[264]\tvalidation_0-rmspe:0.111613\n",
      "[265]\tvalidation_0-rmspe:0.111616\n",
      "[266]\tvalidation_0-rmspe:0.111593\n",
      "[267]\tvalidation_0-rmspe:0.111620\n",
      "[268]\tvalidation_0-rmspe:0.111581\n",
      "[269]\tvalidation_0-rmspe:0.111571\n",
      "[270]\tvalidation_0-rmspe:0.111627\n",
      "[271]\tvalidation_0-rmspe:0.111611\n",
      "[272]\tvalidation_0-rmspe:0.111586\n",
      "[273]\tvalidation_0-rmspe:0.111504\n",
      "[274]\tvalidation_0-rmspe:0.111492\n",
      "[275]\tvalidation_0-rmspe:0.111455\n",
      "[276]\tvalidation_0-rmspe:0.111431\n",
      "[277]\tvalidation_0-rmspe:0.111459\n",
      "[278]\tvalidation_0-rmspe:0.111448\n",
      "[279]\tvalidation_0-rmspe:0.111433\n",
      "[280]\tvalidation_0-rmspe:0.111441\n",
      "[281]\tvalidation_0-rmspe:0.111386\n",
      "[282]\tvalidation_0-rmspe:0.111358\n",
      "[283]\tvalidation_0-rmspe:0.111340\n",
      "[284]\tvalidation_0-rmspe:0.111340\n",
      "[285]\tvalidation_0-rmspe:0.111324\n",
      "[286]\tvalidation_0-rmspe:0.111268\n",
      "[287]\tvalidation_0-rmspe:0.111272\n",
      "[288]\tvalidation_0-rmspe:0.111245\n",
      "[289]\tvalidation_0-rmspe:0.111225\n",
      "[290]\tvalidation_0-rmspe:0.111241\n",
      "[291]\tvalidation_0-rmspe:0.111238\n",
      "[292]\tvalidation_0-rmspe:0.111233\n",
      "[293]\tvalidation_0-rmspe:0.111266\n",
      "[294]\tvalidation_0-rmspe:0.111232\n",
      "[295]\tvalidation_0-rmspe:0.111217\n",
      "[296]\tvalidation_0-rmspe:0.111196\n",
      "[297]\tvalidation_0-rmspe:0.111302\n",
      "[298]\tvalidation_0-rmspe:0.111296\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 4h 59min 13s, sys: 7.29 s, total: 4h 59min 20s\n",
      "Wall time: 20min\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[299]\tvalidation_0-rmspe:0.111286\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "res ,models= sklearn_ext.grid_search_CV(Model,param_grid,X,Y, fit_params=fit_params,#predict_params,\n",
    "                                 cv_spliter=_ts_cv_split,\n",
    "                                 losser=lambda Y,Yhat: utils.loss(Y,Yhat,sales_idx=0) if not LOG_TRANS else None,                                 )\n",
    "joblib.dump(res, \"models/cv_errors.pkl\")\n",
    "# pickle.dump(models, gzip.open(\"model/\"+cv_splitter+\"_errors.pkl\", 'wb'), protocol=pickle.HIGHEST_PROTOCOL)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[((('n_estimators', 1100),\n",
      "   ('subsample', 0.9),\n",
      "   ('learning_rate', 0.02),\n",
      "   ('colsample_bytree', 0.7),\n",
      "   ('max_depth', 10)),\n",
      "  [0.35772909447066048])]\n"
     ]
    }
   ],
   "source": [
    "files=['cv_errors.pkl']#,'random_kfolds_cv_split_errors.pkl']\n",
    "d={file:joblib.load(\"models/\"+file) for file in files}\n",
    "for file,v1 in d.items() :\n",
    "  d = {k2:np.mean(v2) for k2,v2 in v1.items()}  \n",
    "  t = sorted(v1.items(),key=lambda x: np.mean(x[1]))\n",
    "#   print('{}: {}'.format(file,np.mean(t[0][1])))\n",
    "  pprint.pprint(t)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# select some features"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {
    "collapsed": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[('CompetitionDistance', 2104),\n",
       " ('MeanSales', 1500),\n",
       " ('MaxSales', 1453),\n",
       " ('SalesByWeekDay_EMA21_lag49', 1302),\n",
       " ('Open_EMA14_lag-7', 1093),\n",
       " ('CustomersByWeekDay_Diff_lag49', 1026),\n",
       " ('MaxCustomers', 964),\n",
       " ('Open_EMA14_lag7', 959),\n",
       " ('CloudCover', 935),\n",
       " ('CustomersByWeekDay_Diff_lag147', 934),\n",
       " ('Assortment', 854),\n",
       " ('maxS/C', 815),\n",
       " ('StateSpan_Open', 770),\n",
       " ('TimeToEnd_Open', 765),\n",
       " ('SalesByWeekDay_Diff_lag49', 728),\n",
       " ('Store', 715),\n",
       " ('YearDay', 709),\n",
       " ('CustomersByWeekDay_EMA21_lag49', 695),\n",
       " ('MonthDay', 680),\n",
       " ('TimeSincePromo2Open', 678),\n",
       " ('CloudCover_lag-1', 667),\n",
       " ('TimeSinceCompetitionOpen', 663),\n",
       " ('MeanCustomers', 656),\n",
       " ('CustomersByWeekDay_Diff_lag51', 613),\n",
       " ('CustomersByWeekDay_Diff_lag50', 613),\n",
       " ('TimeFromStart_Open', 612),\n",
       " ('Promo', 583),\n",
       " ('CustomersByWeekDay_Diff_lag98', 574),\n",
       " ('meanS/C', 570),\n",
       " ('Sales_EMA63_lag52', 545),\n",
       " ('Open', 541),\n",
       " ('CustomersByWeekDay_Diff_lag54', 537),\n",
       " ('SalesByWeekDay_EMSTD21_lag49', 536),\n",
       " ('CustomersByWeekDay_Diff_lag52', 535),\n",
       " ('SalesByWeekDay_EMA21_lag98', 533),\n",
       " ('SalesByWeekDay_Diff_lag147', 520),\n",
       " ('SalesByWeekDay_Diff_lag98', 491),\n",
       " ('CustomersByWeekDay_Diff_lag53', 489),\n",
       " ('CloudCover_lag-2', 488),\n",
       " ('CustomersByWeekDay_EMA21_lag147', 486),\n",
       " ('CustomersByWeekDay_Diff_lag55', 481),\n",
       " ('Sales_EMSTD63_lag52', 476),\n",
       " ('SalesByWeekDay_Diff_lag55', 458),\n",
       " ('StoreType', 453),\n",
       " ('SalesByWeekDay_EMA21_lag147', 436),\n",
       " ('TimeToEnd_Hail', 435),\n",
       " ('SalesByWeekDay_Diff_lag54', 434),\n",
       " ('WeekDay', 427),\n",
       " ('S/CByWeekDay_EMSTD21_lag49', 423),\n",
       " ('TimeToEnd_Promo', 413),\n",
       " ('CustomersByWeekDay_EMSTD21_lag49', 389),\n",
       " ('SalesByWeekDay_Diff_lag51', 381),\n",
       " ('TimeFromStart_Promo', 372),\n",
       " ('StateSpan_Promo', 366),\n",
       " ('SalesByWeekDay_Diff_lag53', 364),\n",
       " ('Customers_EMSTD63_lag52', 363),\n",
       " ('SalesByWeekDay_Diff_lag52', 348),\n",
       " ('HolidayName_lag-1', 346),\n",
       " ('SalesByWeekDay_EMSTD21_lag98', 344),\n",
       " ('sine8', 342),\n",
       " ('Customers_EMA63_lag52', 340),\n",
       " ('Sales_EMA63_lag104', 340),\n",
       " ('CloudCover_lag-3', 323),\n",
       " ('sine3', 322),\n",
       " ('HolidayName', 322),\n",
       " ('CloudCover_lag1', 316),\n",
       " ('sine2', 314),\n",
       " ('CustomersByWeekDay_EMA21_lag51', 313),\n",
       " ('sine4', 312),\n",
       " ('StateSales_meanByWeekDay_Diff_lag49', 310),\n",
       " ('cosine6', 306),\n",
       " ('SalesByWeekDay_Diff_lag50', 304),\n",
       " ('cosine3', 301),\n",
       " ('TimeFromStart_Snow', 301),\n",
       " ('CustomersByWeekDay_EMSTD21_lag51', 301),\n",
       " ('State', 296),\n",
       " ('CustomersByWeekDay_EMA21_lag52', 295),\n",
       " ('cosine1', 290),\n",
       " ('sine6', 288),\n",
       " ('StateSpan_SchoolHoliday', 288),\n",
       " ('StateSales_meanByWeekDay_EMA21_lag49', 284),\n",
       " ('SalesByWeekDay_EMA21_lag55', 283),\n",
       " ('SalesByWeekDay_EMSTD21_lag147', 283),\n",
       " ('GeoGroupIndex', 279),\n",
       " ('CustomersByWeekDay_EMA21_lag98', 279),\n",
       " ('Customers_EMA63_lag156', 278),\n",
       " ('TimeFromStart_IsPromo2Month', 278),\n",
       " ('Customers_EMA63_lag104', 277),\n",
       " ('CloudCover_lag2', 277),\n",
       " ('TimeToEnd_Snow', 276),\n",
       " ('SalesByWeekDay_EMSTD21_lag50', 275),\n",
       " ('CustomersByWeekDay_EMSTD21_lag147', 274),\n",
       " ('YearWeek', 269),\n",
       " ('Mean_TemperatureC', 269),\n",
       " ('S/CByWeekDay_EMA21_lag49', 266),\n",
       " ('Sales_EMSTD63_lag104', 265),\n",
       " ('TimeFromStart_SchoolHoliday', 263),\n",
       " ('Customers_EMSTD63_lag104', 259),\n",
       " ('S/C_EMA63_lag52', 255),\n",
       " ('Promo_lag7', 254),\n",
       " ('TimeToEnd_SchoolHoliday', 253),\n",
       " ('S/C_EMSTD63_lag52', 250),\n",
       " ('Sales_EMA63_lag156', 249),\n",
       " ('SalesByWeekDay_EMA21_lag52', 248),\n",
       " ('SalesByWeekDay_EMSTD21_lag55', 248),\n",
       " ('StateSales_meanByWeekDay_EMSTD21_lag49', 245),\n",
       " ('SalesByWeekDay_EMSTD21_lag54', 244),\n",
       " ('TimeToEnd_Thunderstorm', 243),\n",
       " ('cosine2', 243),\n",
       " ('SalesByWeekDay_EMA21_lag53', 242),\n",
       " ('CustomersByWeekDay_EMA21_lag53', 236),\n",
       " ('CustomersByWeekDay_EMSTD21_lag98', 234),\n",
       " ('CustomersByWeekDay_EMA21_lag50', 233),\n",
       " ('CustomersByWeekDay_EMA21_lag54', 233),\n",
       " ('TimeFromLast_HolidayName', 233),\n",
       " ('CustomersByWeekDay_EMSTD21_lag52', 233),\n",
       " ('SalesByWeekDay_EMA21_lag54', 232),\n",
       " ('SalesByWeekDay_EMSTD21_lag51', 232),\n",
       " ('S/CByWeekDay_EMSTD21_lag55', 231),\n",
       " ('S/CByWeekDay_EMSTD21_lag54', 230),\n",
       " ('StateSales_meanByWeekDay_EMSTD21_lag55', 230),\n",
       " ('S/CByWeekDay_EMSTD21_lag98', 228),\n",
       " ('CloudCover_lag3', 228),\n",
       " ('CustomersByWeekDay_EMSTD21_lag50', 227),\n",
       " ('HistoryMeanSalesOfGroupIndex', 224),\n",
       " ('CustomersByWeekDay_EMSTD21_lag53', 224),\n",
       " ('S/CByWeekDay_EMSTD21_lag147', 224),\n",
       " ('sine1', 223),\n",
       " ('S/CByWeekDay_EMA21_lag147', 222),\n",
       " ('Promo_lag-7', 221),\n",
       " ('StateSales_meanByWeekDay_EMSTD21_lag54', 221),\n",
       " ('S/CByWeekDay_EMSTD21_lag51', 220),\n",
       " ('SalesByWeekDay_EMSTD21_lag52', 216),\n",
       " ('sine7', 215),\n",
       " ('SalesByWeekDay_EMA21_lag51', 212),\n",
       " ('MeanDew_PointC_lag3', 212),\n",
       " ('HistoryMeanCustomersOfGroupIndex', 212),\n",
       " ('Sales_EMSTD63_lag156', 211),\n",
       " ('CustomersByWeekDay_EMSTD21_lag55', 209),\n",
       " ('MeanDew_PointC_lag1', 208),\n",
       " ('TimeToNext_HolidayName', 208),\n",
       " ('SalesByWeekDay_EMSTD21_lag53', 208),\n",
       " ('S/C_EMSTD63_lag156', 208),\n",
       " ('Mean_Wind_SpeedKm_h', 206),\n",
       " ('HolidayName_lag1', 206),\n",
       " ('CustomersByWeekDay_EMA21_lag55', 206),\n",
       " ('StateSales_std_EMSTD63_lag52', 205),\n",
       " ('StateSales_meanByWeekDay_Diff_lag147', 205),\n",
       " ('S/CByWeekDay_EMSTD21_lag53', 204),\n",
       " ('HolidayType_lag-1', 204),\n",
       " ('CustomersByWeekDay_EMSTD21_lag54', 204),\n",
       " ('HistoryCustomersOfState', 204),\n",
       " ('S/CByWeekDay_EMSTD21_lag52', 203),\n",
       " ('TimeToEnd_IsPromo2Month', 203),\n",
       " ('StateSales_meanByWeekDay_Diff_lag98', 202),\n",
       " ('Customers_EMSTD63_lag156', 199),\n",
       " ('SalesByWeekDay_EMA21_lag50', 199),\n",
       " ('cosine7', 198),\n",
       " ('Month', 196),\n",
       " ('Precipitationmm', 196),\n",
       " ('sine5', 195),\n",
       " ('StateSales_meanByWeekDay_EMA21_lag55', 192),\n",
       " ('Mean_VisibilityKm', 192),\n",
       " ('S/CByWeekDay_EMSTD21_lag50', 190),\n",
       " ('StateSales_meanByWeekDay_Diff_lag53', 189),\n",
       " ('HolidayType', 188),\n",
       " ('MeanDew_PointC', 186),\n",
       " ('TimeToNext_IsHoliday', 184),\n",
       " ('StateSpan_Snow', 183),\n",
       " ('cosine4', 178),\n",
       " ('GroupSales_meanByWeekDay_Diff_lag49', 176),\n",
       " ('S/CByWeekDay_EMA21_lag55', 176),\n",
       " ('S/CByWeekDay_EMA21_lag54', 176),\n",
       " ('StateSales_meanByWeekDay_EMSTD21_lag98', 176),\n",
       " ('StateSales_std_EMA63_lag52', 176),\n",
       " ('cosine8', 174),\n",
       " ('MeanDew_PointC_lag-3', 173),\n",
       " ('S/C_EMSTD63_lag104', 173),\n",
       " ('StateSales_meanByWeekDay_Diff_lag50', 172),\n",
       " ('StateSales_mean_EMA63_lag52', 171),\n",
       " ('GeoGroupSize', 170),\n",
       " ('StateSales_stdByWeekDay_EMSTD21_lag55', 170),\n",
       " ('StateSales_meanByWeekDay_Diff_lag52', 168),\n",
       " ('MeanDew_PointC_lag-1', 166),\n",
       " ('TimeFromStart_Hail', 166),\n",
       " ('StateSales_std_EMSTD63_lag156', 166),\n",
       " ('cosine5', 166),\n",
       " ('StateSales_meanByWeekDay_EMSTD21_lag147', 164),\n",
       " ('StateSales_meanByWeekDay_Diff_lag54', 163),\n",
       " ('S/CByWeekDay_EMA21_lag51', 163),\n",
       " ('StateSales_std_EMSTD63_lag104', 161),\n",
       " ('StateSales_meanByWeekDay_EMSTD21_lag53', 161),\n",
       " ('S/C_EMA63_lag156', 160),\n",
       " ('StateSales_meanByWeekDay_Diff_lag55', 159),\n",
       " ('StateSales_meanByWeekDay_EMA21_lag147', 156),\n",
       " ('StateSales_mean_EMSTD63_lag104', 156),\n",
       " ('S/C_EMA63_lag104', 155),\n",
       " ('StateSales_stdByWeekDay_EMSTD21_lag49', 155),\n",
       " ('MeanDew_PointC_lag2', 154),\n",
       " ('StateSales_stdByWeekDay_Diff_lag147', 154)]"
      ]
     },
     "execution_count": 29,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "fea_imp0=models[0].booster().get_fscore()\n",
    "fea_imp1=models[1].booster().get_fscore()\n",
    "fea_imp={ k: fea_imp0.get(k,0) + fea_imp1.get(k,0) for k in set(fea_imp0) }\n",
    "# len(fea_imp0)\n",
    "# len(fea_imp1)\n",
    "col_ranking=sorted(fea_imp.items(),key=lambda x:x[1],reverse=True)\n",
    "col_support= [k for k,v in col_ranking[:200]]\n",
    "# col_support\n",
    "col_ranking=col_ranking[:200]\n",
    "pickle.dump(col_ranking,open('GB_feature_rank200.pkl','wb'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "col_ranking=pickle.load(open('GB_feature_rank200.pkl','rb'))\n",
    "col_support= [k for k,v in col_ranking[:200]]\n",
    "X_backup=X.copy()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(1017209, 200)"
      ]
     },
     "execution_count": 58,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X.loc[:,col_support]\n",
    "X.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "X.loc[:,col_support+['abc']];"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "78"
      ]
     },
     "execution_count": 51,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(set(k for k,v in RFs)^set(k for k,v, in GBs))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "True"
      ]
     },
     "execution_count": 49,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Full Fit"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# full fit\n",
    "file='GB1200'\n",
    "!rm \"models/\"$file\".pkl\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# param={ 'n_estimators': [900], #3000,\n",
    "#             'max_depth': [10], #12\n",
    "#             \"learning_rate\": [0.02], #0.03\n",
    "#             \"subsample\": [0.9], #0.9\n",
    "#             \"colsample_bytree\": [0.7], #0.7 \n",
    "#            }\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "XGBRegressor(base_score=0.5, colsample_bylevel=1, colsample_bytree=0.7,\n",
       "       gamma=0, learning_rate=0.02, max_delta_step=0, max_depth=10,\n",
       "       min_child_weight=1, missing=None, n_estimators=1100, nthread=-1,\n",
       "       objective='reg:linear', reg_alpha=0, reg_lambda=1,\n",
       "       scale_pos_weight=1, seed=0, silent=True, subsample=0.9)"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "Model(n_estimators=1100,max_depth= 10,learning_rate= 0.02,subsample=0.9,colsample_bytree=0.7)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 11h 47min 39s, sys: 10.5 s, total: 11h 47min 50s\n",
      "Wall time: 46min 59s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "import os\n",
    "if not os.path.exists(\"models/\"+file+\".pkl\"):\n",
    "  model = Model(n_estimators=900,max_depth= 10,learning_rate= 0.03,subsample=0.9,colsample_bytree=0.7)\n",
    "  model.fit(X,Y)\n",
    "  joblib.dump(model, \"models/\"+file+\".pkl\")\n",
    "else:\n",
    "  model = joblib.load( \"models/\"+file+\".pkl\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# predict"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(41088, 200)"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "Xtest=Xtest.loc[:,col_support]\n",
    "Xtest.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# fill NaN for Xtest\n",
    "import re\n",
    "# cols=filter(lambda x: not re.match(r'Customers_lag|Sales_lag',x) , Xtest.columns)\n",
    "_Xtest=Xtest.copy()\n",
    "# _Xtest[cols]=_Xtest[cols].fillna(VALUE_IMPUTE_MISSING)\n",
    "_Xtest=_Xtest.fillna(VALUE_IMPUTE_MISSING)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Id,Sales\n",
      "1,4323.79394531\n",
      "2,7374.86328125\n",
      "3,9038.84277344\n",
      "4,7232.20605469\n",
      "5,7526.80517578\n",
      "6,5586.7114\n",
      "CPU times: user 10 s, sys: 116 ms, total: 10.1 s\n",
      "Wall time: 2.78 s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "#predict\n",
    "# Yhat= np.expm1(model.predict(_Xtest))\n",
    "Yhat= model.predict(_Xtest)\n",
    "\n",
    "#submit format\n",
    "assert len(id)==len(Yhat)\n",
    "try:  # numpy structure\n",
    "  sales_hat = Yhat[:,0] if Yhat.ndim==2 else Yhat\n",
    "except: # pandas structure\n",
    "  sales_hat = Yhat.values[:,0] if Yhat.ndim==2 else Yhat.values\n",
    "  \n",
    "anwser_df = pd.DataFrame({\"Id\": id.astype(np.int).values, 'Sales': sales_hat}).sort_values(by='Id')\n",
    "full_id=pd.DataFrame({'Id':range(test_df.Id.min(),test_df.Id.max()+1)})\n",
    "anwser_df=full_id.merge(anwser_df,on='Id',how='left').fillna(0)\n",
    "assert ~anwser_df.isnull().any().any()\n",
    "anwser_str = anwser_df.to_csv(index=False)\n",
    "%storage write -o {bucket}/submit.csv -v anwser_str\n",
    "print(anwser_str[:100])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>valid</th>\n",
       "      <th>test</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>GroupCustomers_mean</th>\n",
       "      <td>25056</td>\n",
       "      <td>41088</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>GroupCustomers_std</th>\n",
       "      <td>25056</td>\n",
       "      <td>41088</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>GroupCustomers_sum</th>\n",
       "      <td>25056</td>\n",
       "      <td>41088</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>GroupSales_mean</th>\n",
       "      <td>25056</td>\n",
       "      <td>41088</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>GroupSales_std</th>\n",
       "      <td>25056</td>\n",
       "      <td>41088</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>GroupSales_sum</th>\n",
       "      <td>25056</td>\n",
       "      <td>41088</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>IsHoliday</th>\n",
       "      <td>40232</td>\n",
       "      <td>40728</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>IsHoliday_lag-1</th>\n",
       "      <td>40232</td>\n",
       "      <td>40728</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>IsHoliday_lag1</th>\n",
       "      <td>40232</td>\n",
       "      <td>40728</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>StateCustomers_mean</th>\n",
       "      <td>0</td>\n",
       "      <td>41088</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>StateCustomers_std</th>\n",
       "      <td>0</td>\n",
       "      <td>41088</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>StateCustomers_sum</th>\n",
       "      <td>0</td>\n",
       "      <td>41088</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>StateSales_mean</th>\n",
       "      <td>0</td>\n",
       "      <td>41088</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>StateSales_std</th>\n",
       "      <td>0</td>\n",
       "      <td>41088</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>StateSales_sum</th>\n",
       "      <td>0</td>\n",
       "      <td>41088</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>StateSpan_Open</th>\n",
       "      <td>528</td>\n",
       "      <td>529</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>TimeFromStart_Open</th>\n",
       "      <td>288</td>\n",
       "      <td>284</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>TimeToEnd_Open</th>\n",
       "      <td>480</td>\n",
       "      <td>485</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                     valid   test\n",
       "GroupCustomers_mean  25056  41088\n",
       "GroupCustomers_std   25056  41088\n",
       "GroupCustomers_sum   25056  41088\n",
       "GroupSales_mean      25056  41088\n",
       "GroupSales_std       25056  41088\n",
       "GroupSales_sum       25056  41088\n",
       "IsHoliday            40232  40728\n",
       "IsHoliday_lag-1      40232  40728\n",
       "IsHoliday_lag1       40232  40728\n",
       "StateCustomers_mean      0  41088\n",
       "StateCustomers_std       0  41088\n",
       "StateCustomers_sum       0  41088\n",
       "StateSales_mean          0  41088\n",
       "StateSales_std           0  41088\n",
       "StateSales_sum           0  41088\n",
       "StateSpan_Open         528    529\n",
       "TimeFromStart_Open     288    284\n",
       "TimeToEnd_Open         480    485"
      ]
     },
     "execution_count": 61,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "_df=pd.DataFrame()\n",
    "_df['valid']=(X[(X.index.get_level_values('Date')>'2015-06-13')& X.Store.isin(Xtest.Store) ]==-999).sum(0)\n",
    "_df['test']=(_Xtest==-999).sum(0)\n",
    "_df[(_df.valid-_df.test)!=0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "df = train_df.merge(store_df, on='Store', how='left')\n",
    "df=df.merge(weather_df,on=['Date','State'],how='left')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "Int64Index: 16064 entries, 0 to 16063\n",
      "Data columns (total 13 columns):\n",
      "Date                   16064 non-null object\n",
      "State                  16064 non-null object\n",
      "Mean_TemperatureC      16064 non-null int64\n",
      "MeanDew_PointC         16064 non-null int64\n",
      "Mean_VisibilityKm      15683 non-null float64\n",
      "Mean_Wind_SpeedKm_h    16064 non-null int64\n",
      "Precipitationmm        16064 non-null float64\n",
      "CloudCover             14891 non-null float64\n",
      "Fog                    16064 non-null int64\n",
      "Hail                   16064 non-null int64\n",
      "Rain                   16064 non-null int64\n",
      "Snow                   16064 non-null int64\n",
      "Thunderstorm           16064 non-null int64\n",
      "dtypes: float64(3), int64(8), object(2)\n",
      "memory usage: 1.7+ MB\n"
     ]
    }
   ],
   "source": [
    "weather_df.info()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1115"
      ]
     },
     "execution_count": 61,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "\n",
    "df=store_df[store_df.CompetitionDistance<10000].groupby(\n",
    "  ['CompetitionOpenSinceYear','CompetitionOpenSinceMonth','State']).size().to_frame('Size')\n",
    "df=df[df.Size>1].reset_index()\n",
    "df['GeoGroupIndex']=np.arange(len(df))\n",
    "df=store_df.merge(df,on=['CompetitionOpenSinceYear','CompetitionOpenSinceMonth','State'],how='left')\n",
    "len(df)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<matplotlib.axes._subplots.AxesSubplot at 0x7f1ad2a093d0>"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAgQAAAFXCAYAAADd8FZSAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAIABJREFUeJzt3XtwVPX9//HX5gJfICGYZLPbIcSZhkuxTpxWRjC1Zhpi\nADdZEjTVWBxZbGOYKkVSqEAdqnYo1dGB8ocmMhaRXqxQiJ2kzTSJhsmooB3HVIszTX9tCC17MSEk\nBCGXPb8/gBUkIeRydrOb5+MvcvZ8zr7ffkby4pzPOcdiGIYhAAAwoUWFugAAABB6BAIAAEAgAAAA\nBAIAACACAQAAEIEAAAAoCIGgv79fBQUFKi0tlSR1dHTI5XJpyZIlWr16tTo7OwP7lpeXKzc3V0uX\nLlVjY6PZpQEAgItMDwR79+5Venp64OeKigplZmaqpqZGixYtUkVFhSSpublZ1dXVqqqq0u7du/XU\nU0/J7/ebXR4AAJDJgcDtdquhoUFFRUWBbfX19SosLJQkFRYWqra2VpJUV1cnh8Oh2NhYpaamKi0t\nTU1NTWaWBwAALjI1EGzbtk0bN25UVNQXX9PW1qbk5GRJUnJystra2iRJXq9Xdrs9sJ/dbpfH4zGz\nPAAAcJFpgeCtt95SUlKSbrrpJg32dGSLxSKLxTLoMa71GQAAGDsxZh34ww8/VH19vRoaGtTT06Mz\nZ85ow4YNSkpKks/nk9VqldfrVWJioiTJZrPJ7XYHxrvdbtlstmt+h2EYhAYAAMaAJRgvNzp69Khe\neeUVvfTSS3r22Wc1Y8YMlZSUqKKiQp2dnfrxj3+s5uZmlZWV6Y033pDH45HL5dJf//rXa/7C3/T0\nDn3WOfzyjZ7T+sWTZaNpKSis1nj5fF2hLsM09Be+Irk3if7C3UTozwymnSEYTElJidatW6cDBw5o\n5syZ2rFjhyRp9uzZWrZsmRwOh6Kjo7V169Yh//XvOW3Ip68Ou4bos/8YUe0AAESqoASC2267Tbfd\ndpskacaMGdqzZ8+A+5WWlgaeVwAAAIKHJxUCAAACAQAAIBAAAAARCAAAgAgEAABABAIAACACAQAA\nEIEAAACIQAAAAEQgAAAAIhAAAAARCAAAgAgEAABABAIAACACAQAAEIEAAACIQAAAAEQgAAAAIhAA\nAAARCAAAgAgEAABABAIAACACAQAAEIEAAACIQAAAAEQgAAAAIhAAAAARCAAAgAgEAABABAIAACAp\nxqwDnz9/XitXrlRPT496e3u1ePFilZWVadeuXXrjjTeUmJgoSXr88ceVlZUlSSovL9eBAwcUFRWl\nn/70p7rjjjvMKg8AAFzGtEAwefJk7d27V1OmTFFfX58eeOABffDBB7JYLHK5XHK5XFfs39zcrOrq\nalVVVcnj8cjlcqmmpkZRUZzEAADAbKb+tp0yZYokqbe3V/39/UpISJAkGYZx1b51dXVyOByKjY1V\namqq0tLS1NTUZGZ5AADgIlMDgd/v1/Lly5WZmamFCxdqzpw5kqR9+/bJ6XRq8+bN6uzslCR5vV7Z\n7fbAWLvdLo/HY2Z5AADgItMuGUhSVFSUKisr1dXVpYcfflhHjhxRcXGxfvjDH0qSduzYoe3bt2vb\ntm0DjrdYLKbUFRMTJas13pRjj7VwqXOk6C98RXJvEv2Fu0jvzwymBoJL4uPjlZWVpY8//lgLFy4M\nbC8qKtKaNWskSTabTW63O/CZ2+2WzWYzpZ6+Pr98vi5Tjj2WrNb4sKhzpOgvfEVybxL9hbuJ0J8Z\nTLtk0N7eHrgccO7cOb3zzju66aab5PP5AvvU1tZq7ty5kqTs7GxVVVWpp6dHra2tamlpUUZGhlnl\nAQCAy5h2hsDn8+mJJ56Q3+8PrCW4/fbbtXHjRh07dkwWi0Wpqal6+umnJUmzZ8/WsmXL5HA4FB0d\nra1bt5p2yQAAAFzJtEAwb948HTx48Krtzz777KBjSktLVVpaalZJAABgENzkDwAACAQAAIBAAAAA\nRCAAAAAiEAAAABEIAACACAQAAEAEAgAAIAIBAAAQgQAAAIhAAAAARCAAAAAiEAAAABEIAACACAQA\nAEAEAgAAIAIBAAAQgQAAAIhAAAAARCAAAAAiEAAAABEIAACACAQAAEAEAgAAIAIBAAAQgQAAAIhA\nAAAARCAAAAAiEAAAAJkYCM6fP6+ioiItX75cd999t55//nlJUkdHh1wul5YsWaLVq1ers7MzMKa8\nvFy5ublaunSpGhsbzSoNAAB8iWmBYPLkydq7d68qKyv15ptv6siRI/rggw9UUVGhzMxM1dTUaNGi\nRaqoqJAkNTc3q7q6WlVVVdq9e7eeeuop+f1+s8oDAACXMfWSwZQpUyRJvb296u/vV0JCgurr61VY\nWChJKiwsVG1trSSprq5ODodDsbGxSk1NVVpampqamswsDwAAXGRqIPD7/Vq+fLkyMzO1cOFCzZkz\nR21tbUpOTpYkJScnq62tTZLk9Xplt9sDY+12uzwej5nlAQCAi2LMPHhUVJQqKyvV1dWlhx9+WO+9\n994Vn1ssFlkslkHHX+szAAAwdkwNBJfEx8crKytLn3zyiZKSkuTz+WS1WuX1epWYmChJstlscrvd\ngTFut1s2m82UemJiomS1xpty7LEWLnWOFP2Fr0juTaK/cBfp/ZnBtEDQ3t6umJgYTZ8+XefOndM7\n77yjRx99VNnZ2Tp48KBKSkp06NAh5eTkSJKys7NVVlamVatWyePxqKWlRRkZGabU1tfnl8/XZcqx\nx5LVGh8WdY4U/YWvSO5Nor9wNxH6M4NpgcDn8+mJJ56Q3+8PrCW4/fbbNX/+fK1bt04HDhzQzJkz\ntWPHDknS7NmztWzZMjkcDkVHR2vr1q1cMgAAIEhMCwTz5s3TwYMHr9o+Y8YM7dmzZ8AxpaWlKi0t\nNaskAAAwCJ5UCAAACAQAAIBAAAAARCAAAAAiEAAAABEIAACACAQAAEAEAgAAIAIBAAAQgQAAAIhA\nAAAARCAAAAAiEAAAABEIAACACAQAAEAEAgAAIAIBAAAQgQAAAIhAAAAARCAAAAAiEAAAABEIAACA\nCAQAAEAEAgAAIAIBAAAQgQAAAIhAAAAARCAAAAAiEAAAAJkcCE6ePKkHH3xQDodDeXl52rt3ryRp\n165duvPOO1VQUKCCggI1NDQExpSXlys3N1dLly5VY2OjmeUBAICLYkw9eEyMNm/erPnz56u7u1sr\nVqzQt771LVksFrlcLrlcriv2b25uVnV1taqqquTxeORyuVRTU6OoKE5kAABgJlN/01qtVs2fP1+S\nNG3aNKWnp8vj8UiSDMO4av+6ujo5HA7FxsYqNTVVaWlpampqMrNEAACgIK4hOHHihI4dO6ZbbrlF\nkrRv3z45nU5t3rxZnZ2dkiSv1yu73R4YY7fbAwECAACYJyiBoLu7W2vXrtWWLVs0bdo0FRcXq66u\nTpWVlbJardq+ffugYy0WSzBKBABgQjN1DYEk9fb2au3atXI6ncrJyZEkJSUlBT4vKirSmjVrJEk2\nm01utzvwmdvtls1mG/OaYmKiZLXGj/lxzRAudY4U/YWvSO5Nor9wF+n9mcHUQGAYhrZs2aL09HSt\nWrUqsN3r9SolJUWSVFtbq7lz50qSsrOzVVZWplWrVsnj8ailpUUZGRljXldfn18+X9eYH3esWa3x\nYVHnSNFf+Irk3iT6C3cToT8zmBoI/va3v+nNN9/UvHnzVFBQIEl6/PHHVVVVpWPHjslisSg1NVVP\nP/20JGn27NlatmyZHA6HoqOjtXXrVi4ZAAAQBKYGggULFujTTz+9antWVtagY0pLS1VaWmpmWQAA\n4Eu4wR8AABAIAAAAgQAAAIhAAAAARCAAAAAiEAAAABEIAACACAQAAEAEAgAAoOsIBA8//LDq6+tl\nGEYw6gEAACEwZCC477779Oqrr2rx4sWqqKjQqVOnglEXAAAIoiEDQW5url599VW9/PLL8nq9ysvL\n08aNG/Xxxx8Hoz4AABAEw1pDYBiGYmJiNHnyZP3kJz/RL37xC7PqAgAAQTTk2w7/8pe/6Le//a18\nPp9Wrlyp6upqTZs2TX19fcrNzdWmTZuCUScAADDRkIHgj3/8o37wgx/ojjvukMVi+WJgTIy2bNli\nanEAACA4hgwE5eXlVwQB6cKlA4vFosWLF5tWGAAACJ4h1xA88MADOn36dODnU6dOaeXKlaYWBQAA\ngmvIQHD27FklJCQEfr7hhhvU3d1talEAACC4hgwEfr9fZ8+eDfzc3d2tvr4+U4sCAADBNeQagry8\nPK1evVrFxcUyDEO///3vlZ+fH4zaAABAkAwZCB555BGlpKSorq5OFotF999/vwoKCoJRGwAACJIh\nA4EkFRYWqrCw0OxaAABAiAwZCD777DO99tpram1tDawdsFgs2rlzp+nFAQCA4BgyEDz22GOaPXu2\nMjMzFRV1YQ3il59LAAAAwtuQgaCrq0vPPPNMMGoBAAAhMuRth3PmzJHH4wlGLQAAIESGPENw+vRp\n5efn65vf/KYmTZokiTUEAABEmut6DkFeXt4V21hDAABAZBkyEKxYsSIYdQAAgBAacg3Bv//9bxUX\nFys7O1uS9Mknn2jXrl2mFwYAAIJnyEDws5/9TKWlpYqPj5ckfe1rX9Of//zn6zr4yZMn9eCDD8rh\ncCgvL0979+6VJHV0dMjlcmnJkiVavXq1Ojs7A2PKy8uVm5urpUuXqrGxcSQ9AQCAYRoyEHR1dSkr\nKyuwbiA6OlqxsbHXdfCYmBht3rxZVVVVev311/Wb3/xG//rXv1RRUaHMzEzV1NRo0aJFqqiokCQ1\nNzerurpaVVVV2r17t5566in5/f5RtAcAAK7HkIEgJiZGPT09gZ89Ho+io6Ov6+BWq1Xz58+XJE2b\nNk3p6enyeDyqr68PPAq5sLBQtbW1kqS6ujo5HA7FxsYqNTVVaWlpampqGnZTAABgeIYMBMXFxXrs\nscd06tQp/epXv1JxcbFcLtewv+jEiRM6duyYMjIy1NbWpuTkZElScnKy2traJEler1d2uz0wxm63\n8wwEAACCYMi7DAoLCzVr1izV19fr3LlzevbZZ7VgwYJhfUl3d7fWrl2rLVu2KC4u7orPLBbLNW9j\nNOMWx5iYKFmt8WN+XDOES50jRX/hK5J7k+gv3EV6f2a4rrcdLliwYNgh4JLe3l6tXbtWTqdTOTk5\nkqSkpCT5fD5ZrVZ5vV4lJiZKkmw2m9xud2Cs2+2WzWYb0fdeS1+fXz5f15gfd6xZrfFhUedI0V/4\niuTeJPoLdxOhPzMMGQjuueeeq7ZZLBbt379/yIMbhqEtW7YoPT1dq1atCmzPzs7WwYMHVVJSokOH\nDgWCQnZ2tsrKyrRq1Sp5PB61tLQoIyNjGO0AAICRGDIQbNy4MfDn8+fPq6qqSikpKdd18L/97W96\n8803NW/ePBUUFEiS1q9fr5KSEq1bt04HDhzQzJkztWPHDknS7NmztWzZMjkcDkVHR2vr1q08FREA\ngCAYMhAsXLjwip+//e1vq7i4+LoOvmDBAn366acDfrZnz54Bt5eWlqq0tPS6jg8AAMbGkHcZfFlX\nV5c+++wzM2oBAAAhMqw1BIZhqLW1dUS3HQIAgPFrWGsIoqOjNWvWLFNW/gMAgNAZ9hoCAAAQeYYM\nBIsWLZLFYpFhGFd9ZrFY9O6775pSGAAACJ4hA8H999+v06dP67777pNhGNq/f7+mT58+4PMJAABA\neBoyEBw+fFh//OMfAz8/+eSTWrFihX70ox+ZWpiZ/P19+te//jmisbNm3ahJkyaNcUUAAITWkIHg\nzJkzam9vDzxeuL29Xd3d3aYXZqazZzr0o+fe1NSE63vAUmDcaa92bnAqPX2OSZUBABAaQwaChx56\nSMuXL9d3vvMdGYahhoYGPfLII8GozVRTE1IUd8PMUJcBAMC4MGQg+N73vqdbb71VR48elcVi0cqV\nKzVv3rxg1AYAAILkut52mJqaqr6+Pt18882SLjygiHcMAAAQOYZ8dHFDQ4McDocee+wxSVJTU5PW\nrFljemEAACB4hgwEO3fu1BtvvKGEhARJUkZGho4fP256YQAAIHiu6+VGX37dcWxsrCnFAACA0Bgy\nEMTFxcnn8wV+PnLkiKZPn25qUQAAILiGXFRYVlamkpIS/fe//9XKlSv1n//8Ry+++GIwagMAAEFy\nzUDg9/s1efJkvfrqq/rwww8lSd/4xjc4QwAAQIS5ZiCIiorShg0b9Kc//UlZWVnBqgkAAATZkGsI\nbrzxRrW2tgajFgAAECLX9S4Dp9OpW2+9VVOnTpV04bXHO3fuNL04AAAQHIMGgu3bt+uJJ56Q0+nU\nsmXLrrjVkKcUAgAQWQYNBO+9954kacWKFSooKNChQ4eCVhQAAAiu63owEQAAiGyDniE4f/68mpub\nZRhG4M+Xmz17tunFAQCA4LhmICgpKQn8fPmfJam+vt68qgAAQFANGgj4hQ8AwMTBGgIAAEAgAAAA\nBAIAACCTA8GmTZuUmZmp/Pz8wLZdu3bpzjvvVEFBgQoKCtTQ0BD4rLy8XLm5uVq6dKkaGxvNLA0A\nAFxmyEcXj8Y999yjBx98UD/5yU8C2ywWi1wul1wu1xX7Njc3q7q6WlVVVfJ4PHK5XKqpqVFUFCcx\nAAAwm6m/bRcsWDDgq5INw7hqW11dnRwOh2JjY5Wamqq0tDQ1NTWZWR4AALgoJP/83rdvn5xOpzZv\n3qzOzk5Jktfrld1uD+xjt9vl8XhCUR4AABNO0ANBcXGx6urqVFlZKavVqu3btw+6Ly9RAgAgOExd\nQzCQpKSkwJ+Lioq0Zs0aSZLNZpPb7Q585na7ZbPZTKkhOtqi/hGOTUyMk9UaP6b1XEswvysU6C98\nRXJvEv2Fu0jvzwxBDwRer1cpKSmSpNraWs2dO1eSlJ2drbKyMq1atUoej0ctLS3KyMgwpYb+/qvX\nMFyv9vYz8vm6xrCawVmt8UH7rlCgv/AVyb1J9BfuJkJ/ZjA1EKxfv15Hjx5VR0eHsrKy9Nhjj+no\n0aM6duyYLBaLUlNT9fTTT0u68LKkZcuWyeFwKDo6Wlu3buWSAQAAQWJqIHjhhReu2nbvvfcOun9p\naalKS0vNLAkAAAyAm/wBAACBAAAAEAgAAIAIBAAAQAQCAAAgAgEAABCBAAAAiEAAAABEIAAAACIQ\nAAAAEQgAAIAIBAAAQAQCAAAgAgEAABCBAAAAiEAAAABEIAAAACIQAAAAEQgAAIAIBAAAQAQCAAAg\nAgEAAJAUE+oCwom/v0/Hj7eMePysWTdq0qRJY1gRAABjg0AwDOfOtOn519s1NeHksMeePe3Vzg1O\npafPMaEyAABGh0AwTFMTUhR3w8xQlwEAwJhiDQEAACAQAAAAAgEAABCBAAAAyORAsGnTJmVmZio/\nPz+wraOjQy6XS0uWLNHq1avV2dkZ+Ky8vFy5ublaunSpGhsbzSwNAABcxtRAcM8992j37t1XbKuo\nqFBmZqZqamq0aNEiVVRUSJKam5tVXV2tqqoq7d69W0899ZT8fr+Z5QEAgItMDQQLFizQ9OnTr9hW\nX1+vwsJCSVJhYaFqa2slSXV1dXI4HIqNjVVqaqrS0tLU1NRkZnkAAOCioK8haGtrU3JysiQpOTlZ\nbW1tkiSv1yu73R7Yz263y+PxBLs8AAAmpJAuKrRYLLJYLNf8HAAAmC/oTypMSkqSz+eT1WqV1+tV\nYmKiJMlms8ntdgf2c7vdstlsptQQHW1RvylHvrbExDhZrfHDGjPc/cMN/YWvSO5Nor9wF+n9mSHo\ngSA7O1sHDx5USUmJDh06pJycnMD2srIyrVq1Sh6PRy0tLcrIyDClhv5+w5TjDqW9/Yx8vq7r3t9q\njR/W/uGG/sJXJPcm0V+4mwj9mcHUQLB+/XodPXpUHR0dysrK0tq1a1VSUqJ169bpwIEDmjlzpnbs\n2CFJmj17tpYtWyaHw6Ho6Ght3bqVSwYAAASJqYHghRdeGHD7nj17BtxeWlqq0tJSEysCAAAD4UmF\nAACAQAAAAAgEAABABAIAACACAQAAEIEAAACIQAAAAEQgAAAAIhAAAAARCAAAgAgEAABABAIAACAC\nAQAAkMlvO8QX/P19On68ZVhjTp2KU3v7Gc2adaMmTZpkUmUAABAIgubcmTY9/3q7piacHNa4s6e9\n2rnBqfT0OSZVBgAAgSCopiakKO6GmaEuAwCAq7CGAAAAEAgAAACBAAAAiEAAAABEIAAAACIQAAAA\nEQgAAIAIBAAAQAQCAAAgAgEAABCBAAAAiEAAAABEIAAAACIQAAAAhfD1x9nZ2Zo2bZqio6MVExOj\n/fv3q6OjQ48//rj+97//aebMmdqxY4emT58eqhIBAJgwQnqG4LXXXtOhQ4e0f/9+SVJFRYUyMzNV\nU1OjRYsWqaKiIpTlAQAwYYQ0EBiGccXP9fX1KiwslCQVFhaqtrY2FGUBADDhhCwQWCwWuVwurVix\nQn/4wx8kSW1tbUpOTpYkJScnq62tLVTlAQAwoYRsDcHvfvc7paSkqL29XS6XS1/96lev+Nxischi\nsZjy3dHRFvWbcmRzJCbGyWqND3UZpojUvi6J5P4iuTeJ/sJdpPdnhpAFgpSUFElSYmKi7rrrLjU1\nNSkpKUk+n09Wq1Ver1eJiYmmfHd/vzH0TuNIe/sZ+XxdoS5jzFmt8RHZ1yWR3F8k9ybRX7ibCP2Z\nISSB4PPPP1d/f7/i4uJ09uxZNTY26tFHH1V2drYOHjyokpISHTp0SDk5OaEob1zx9/fp+PGWEY2d\nNetGTZo0aYwrAgBEopAEgs8++0yPPvqoJKm/v1/5+fm64447dPPNN2vdunU6cOBA4LbDie7cmTY9\n/3q7piacHNa4s6e92rnBqfT0OSZVBgCIJCEJBLNmzVJlZeVV22fMmKE9e/YEv6BxbmpCiuJumBnq\nMgAAESxkawgwfvX09Ki1lcsUADCREAhwldbWFv3ouTc1NSFlWOO4TAEA4YtAgAFxmQIAJhZebgQA\nADhDEKlGc7viSMcBAMIXgSBCjfR2RUlqO3FMSanzTagKADBeEQgi2EjXAZw97TGhGgDAeMYaAgAA\nQCAAAAAEAgAAIAIBAAAQiwoxhoZ7q+OpU3Fqbz8T+JnHHgNA6BAIMGZGc6sjjz0GgNAiEGBM8chj\nAAhPrCEAAAAEAgAAQCAAAABiDQEmqJ6eHrW2jvwlTtwRASDSEAgwIbW2tuhHz72pqQkpwx575tRJ\nbSj+ptLSbrzmfl++rVIaWZAgvAAIBgIBJqzRvPzp+dc/GvbtlSO9tXI04YXbOQFcLwIBxoXhPtTo\ncqH4F3Cwb6/kdk4AZiMQYFwY6UONrvf0/ZeNNHyMxkhDTyhqBTDxEAgwbozkX8EjPX3fduKYklLn\nD2vMaI009Iym1nA78wIgdAgECHsjDRKhEOxazTrzMtCCycsRJoDwQyAAIlwwz7xcGMtCRiAcEQgA\nDIiFjMDEwpMKAQAAZwgAjK2RLmTs7e2VJMXGxg57LGsWgNEjEAAYU6O5m2JKfNKwH8DEmgVgbIy7\nQHD48GFt27ZNfr9f9957r0pKSkJdEoBhGulCRtYtAKEzrgJBf3+/nnnmGf3617+WzWbTvffeq8WL\nFys9PT3UpQGIQIO9J2Ko2ypHc3kj2JdGBupxqP5G+50IT+MqEDQ1NSktLU2pqamSJIfDobq6OgIB\ngEGN5uFLx4+3XLy9cniXKUZ6eWM0Y0fzVM6R9Dia7xxN6JHCJ4SM5sVj47HHcRUIPB6PvvKVrwR+\nttlsampqCmFFAMa7ka5ZkL54CmQwL2+MdOxon8oZzBd5jSYwjUUIGc4ZkNGEl5GGrfG67mVcBQKL\nxTKs/XvP+OQ/3z3s7+k949E5f/ywx33e1S5peDWOdmywx02U76TWyPnOz7vaNSU+adjjLjl72jui\n7wzFf5+R9jmSHkf7nSN1vrtDP3/5r/q/uMRhjTvt+X+aPG1G0MZdGjvjK3OHPW68GleBwGaz6eTJ\nL5Ko2+2WzWYbdP/XyrcFoywAACLeuHow0c0336yWlhadOHFCPT09qq6u1uLFi0NdFgAAEW9cnSGI\niYnRk08+qYcffjhw2yELCgEAMJ/FMAwj1EUAAIDQGleXDAAAQGgQCAAAAIEAAACEcSA4fPiwli5d\nqtzcXFVUVIS6nOuWnZ2t/Px8FRQU6N5775UkdXR0yOVyacmSJVq9erU6OzsD+5eXlys3N1dLly5V\nY2NjYPvHH3+s/Px85ebm6uc//3nQ+7hk06ZNyszMVH5+fmDbWPbT09OjdevWKTc3V9/97nf13//+\nNziNXTRQf7t27dKdd96pgoICFRQUqKGhIfBZOPV38uRJPfjgg3I4HMrLy9PevXslRc78DdZfpMzf\n+fPnVVRUpOXLl+vuu+/W888/Lyly5m+w/iJl/qQLj+svKChQaWmppHEwd0YY6uvrM3JycozW1laj\np6fHcDqdRnNzc6jLui7f+c53jFOnTl2x7Ze//KVRUVFhGIZhlJeXG88995xhGIbxz3/+03A6nUZP\nT4/R2tpq5OTkGH6/3zAMw7jnnnuMjz76yDAMw/j+979vNDQ0BLGLL7z//vvGJ598YuTl5QW2jWU/\n+/btM7Zu3WoYhmFUVVUZ69atC1ZrhmEM3N+uXbuMV1555ap9w60/r9dr/OMf/zAMwzDOnDlj5Obm\nGs3NzREzf4P1FynzZxiGcfbsWcMwDKO3t9coKioy3n///YiZP8MYuL9Imr9XXnnFWL9+vfHII48Y\nhhH6vzvD8gzB5e88iI2NDbzzIFwYX7qxo76+XoWFhZKkwsJC1dbWSpLq6urkcDgUGxur1NRUpaWl\n6aOPPpLX61V3d7cyMjIkSQUFBYExwbZgwQJNnz79im1j2c/lx8rNzdW7774brNYkDdyfdPUcSuHX\nn9Vq1fz58yVJ06ZNU3p6ujweT8TM32D9SZExf5I0ZcoUSRcev9vf36+EhISImb/B+pMiY/7cbrca\nGhpUVFQU2BbquQvLQDDQOw8u/Y8+3lksFrlcLq1YsUJ/+MMfJEltbW1KTk6WJCUnJ6utrU2S5PV6\nZbfbA2NA9U4HAAADpUlEQVTtdrs8Hs9V2202m7zekT2a1Axj2c/ln8XExCg+Pl4dHR3BamVQ+/bt\nk9Pp1ObNmwOn9cK5vxMnTujYsWPKyMiIyPm71N8tt9wiKXLmz+/3a/ny5crMzNTChQs1Z86ciJq/\ngfqTImP+tm3bpo0bNyoq6otfw6Geu7AMBMN958F48rvf/U6HDh3S7t279Zvf/EYffPDBFZ9bLJaw\n7u/LIq0fSSouLlZdXZ0qKytltVq1ffv2UJc0Kt3d3Vq7dq22bNmiuLi4Kz6LhPm7vL9p06ZF1PxF\nRUWpsrJShw8f1gcffKD33nvvis/Dff6+3N+RI0ciYv7eeustJSUl6aabbhrwbIcUmrkLy0Aw3Hce\njCcpKRfeipWYmKi77rpLTU1NSkpKks/nk3Qh1SUmXnjJhs1mk9vtDox1u92y2+0Dbr903PFgLPq5\nNJ8pKSmBue7r61NXV5dmzJgRrFYGlJSUFPiftaioSH//+98lhWd/vb29Wrt2rZxOp3JyciRF1vwN\n1l+kzN8l8fHxysrK0ieffBJR83fJpf4+/vjjiJi/Dz/8UPX19crOzlZZWZnee+89bdiwIeRzF5aB\nIFzfefD555/rzJkLr+Q8e/asGhsbNXfuXGVnZ+vgwYOSpEOHDgX+4srOzlZVVdXFd263qqWlRRkZ\nGbJarYqLi9NHH30kwzBUWVkZGDMejEU/l+bz8mPV1NTo9ttvD01Tl7n88kxtba3mzr3wtrNw688w\nDG3ZskXp6elatWpVYHukzN9g/UXK/LW3twdOl587d07vvPOObrrppoiZv8H6u/QLUwrf+Vu/fr0a\nGhpUX1+vF154QYsWLdJzzz0X+rkb3RrJ0Hn77beN3NxcIycnx3jppZdCXc51OX78uOF0Og2n02k4\nHI5A3adOnTIeeughIzc313C5XMbp06cDY1588UUjJyfHWLJkiXH48OHA9r///e9GXl6ekZOTYzzz\nzDNB7+WSxx9/3PjWt75lfP3rXzfuvPNOY//+/WPaz/nz5421a9cad911l1FUVGS0traGtL833njD\n2LBhg5GXl2fk5+cba9asMXw+X1j29/777xvz5s0znE6nsXz5cmP58uVGQ0NDxMzfQP29/fbbETN/\nn376qVFQUGA4nU4jLy/PePnllw3DGNu/T8Zjf5Eyf5ccOXIkcJdBqOeOdxkAAIDwvGQAAADGFoEA\nAAAQCAAAAIEAAACIQAAAAEQgAAAAIhAAAAARCAAAgKT/D850kVJBw25pAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x7f1ad289bb50>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# store_df.groupby(['CompetitionOpenSinceYear','CompetitionOpenSinceMonth']).count()\n",
    "store_df[store_df.CompetitionDistance<40000].CompetitionDistance.plot.hist(bins=30)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "False"
      ]
     },
     "execution_count": 29,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "weather_df.MeanDew_PointC.isnull().any()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Store  Date      \n",
       "1      2013-01-01   -999\n",
       "       2013-01-02   -999\n",
       "       2013-01-03   -999\n",
       "       2013-01-04   -999\n",
       "       2013-01-05   -999\n",
       "       2013-01-06   -999\n",
       "       2013-01-07   -999\n",
       "       2013-01-08   -999\n",
       "       2013-01-09   -999\n",
       "       2013-01-10   -999\n",
       "       2013-01-11   -999\n",
       "       2013-01-12   -999\n",
       "       2013-01-13   -999\n",
       "       2013-01-14   -999\n",
       "       2013-01-15   -999\n",
       "       2013-01-16   -999\n",
       "       2013-01-17   -999\n",
       "       2013-01-18   -999\n",
       "       2013-01-19   -999\n",
       "       2013-01-20   -999\n",
       "       2013-01-21   -999\n",
       "       2013-01-22   -999\n",
       "       2013-01-23   -999\n",
       "       2013-01-24   -999\n",
       "       2013-01-25   -999\n",
       "       2013-01-26   -999\n",
       "       2013-01-27   -999\n",
       "       2013-01-28   -999\n",
       "       2013-01-29   -999\n",
       "       2013-01-30   -999\n",
       "                    ... \n",
       "1115   2015-07-02   -999\n",
       "       2015-07-03   -999\n",
       "       2015-07-04   -999\n",
       "       2015-07-05   -999\n",
       "       2015-07-06   -999\n",
       "       2015-07-07   -999\n",
       "       2015-07-08   -999\n",
       "       2015-07-09   -999\n",
       "       2015-07-10   -999\n",
       "       2015-07-11   -999\n",
       "       2015-07-12   -999\n",
       "       2015-07-13   -999\n",
       "       2015-07-14   -999\n",
       "       2015-07-15   -999\n",
       "       2015-07-16   -999\n",
       "       2015-07-17   -999\n",
       "       2015-07-18   -999\n",
       "       2015-07-19   -999\n",
       "       2015-07-20   -999\n",
       "       2015-07-21   -999\n",
       "       2015-07-22   -999\n",
       "       2015-07-23   -999\n",
       "       2015-07-24   -999\n",
       "       2015-07-25   -999\n",
       "       2015-07-26   -999\n",
       "       2015-07-27   -999\n",
       "       2015-07-28   -999\n",
       "       2015-07-29   -999\n",
       "       2015-07-30   -999\n",
       "       2015-07-31   -999\n",
       "Name: Snow, dtype: float64"
      ]
     },
     "execution_count": 31,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X.Snow"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.9"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}