Ch6 Linear Model Selection and Regularization.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Ch6 Linear Model Selection and Regularization"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "#load modules\n",
    "import pandas as pd\n",
    "from sklearn.linear_model import LinearRegression\n",
    "from mlxtend.feature_selection import ExhaustiveFeatureSelector as EFS\n",
    "import numpy as np\n",
    "import statsmodels.api as sm\n",
    "import matplotlib.pyplot as plt\n",
    "from mlxtend.feature_selection import SequentialFeatureSelector as sfs\n",
    "from sklearn.model_selection import train_test_split"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Caucasian           199\n",
       "Asian               102\n",
       "African American     99\n",
       "Name: Ethnicity, dtype: int64"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#load credit card dataset\n",
    "credit = pd.read_csv('Dataset/Credit.csv', index_col =0)\n",
    "credit.head()\n",
    "credit.Ethnicity.value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "#convert the categorical variables\n",
    "credit_dummies = pd.get_dummies(credit[['Gender', 'Student', 'Married', 'Ethnicity']])\n",
    "credit =credit.drop(credit[['Gender', 'Student', 'Married', 'Ethnicity']],axis=1)\n",
    "credit_dummies =credit_dummies.drop(['Gender_Female', 'Student_No', 'Married_No', 'Ethnicity_African American'],axis=1)\n",
    "credit_df= pd.concat([credit, credit_dummies], axis=1)\n",
    "credit_df.head()\n",
    "\n",
    "#mark the predictors and respond variables\n",
    "X= credit_df[['Income', 'Limit', 'Rating', 'Cards', 'Age', 'Education', 'Gender_Male', 'Student_Yes', 'Married_Yes', 'Ethnicity_Asian', 'Ethnicity_Caucasian']]\n",
    "y= credit_df.Balance\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Best Subset Selection"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.preprocessing import StandardScaler\n",
    "from sklearn.linear_model import LinearRegression\n",
    "from sklearn.metrics import mean_squared_error\n",
    "\n",
    "def fit_ols(X,y):\n",
    "    ols = LinearRegression()\n",
    "    ols_model =ols.fit(X,y)\n",
    "    RSS= round(mean_squared_error(y,ols_model.predict(X)) * len(y),2)\n",
    "    R_square = round(ols_model.score(X,y),2)\n",
    "    return RSS, R_square"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(3786730.19, 0.96)"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "fit_ols(X,y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "import itertools\n",
    "from tqdm import tnrange, tqdm_notebook\n",
    "k = 11\n",
    "RSS_list, R_squared_list, feature_list = [],[], []\n",
    "numb_features = []\n",
    "\n",
    "#Looping over k = 1 to k = 11 features in X\n",
    "for k in range(1,len(X.columns) + 1):\n",
    "    #Looping over all possible combinations: from 11 choose k\n",
    "    for combo in itertools.combinations(X.columns,k):\n",
    "        tmp_result = fit_ols(X[list(combo)],y)   #Store temp result b\n",
    "        RSS_list.append(tmp_result[0])                  #Append lists\n",
    "        R_squared_list.append(tmp_result[1])\n",
    "        feature_list.append(combo)\n",
    "        numb_features.append(len(combo))   \n",
    "\n",
    "#Store in DataFrame\n",
    "df = pd.DataFrame({'numb_features': numb_features,'RSS': RSS_list, 'R_squared':R_squared_list,'features':feature_list})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "df['min_RSS'] = df.groupby('numb_features')['RSS'].transform(min)\n",
    "df['max_R_squared'] = df.groupby('numb_features')['R_squared'].transform(max)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Figure 6.1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAA6oAAAGDCAYAAAAiQO3JAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4zLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvIxREBQAAIABJREFUeJzs3Xl4VOX9///nO5NJJpnsCcimbKZWixJlUayttWpFUQGrVj/utYiobeXrUrR1Rau1VtSfilaxVrRqRUilKm4orgXZ3dCwSiAs2ZPJJLPdvz/OJAwhYTHLPQnvx3XlIrln5pxXhsmceZ97OWKMQSmllFJKKaWUihcJtgMopZRSSimllFKxtFBVSimllFJKKRVXtFBVSimllFJKKRVXtFBVSimllFJKKRVXtFBVSimllFJKKRVXtFBVSimllFJKKRVXtFBVqgsRkfUicpLtHHtDRAaIiBGRxHbe7kEiUisirvbcrlJKqfjUlY597UFE3heR39jOsbeix/qDO2C7tSIyqL23q7oOLVRVp4oebPzRN58tIvKMiKTF3N5PRF4RkVIRqRKRL0Tk0pjbLxeRVSJSIyJbReR1EUn/nlluF5FgNEutiHwtIr9sh9/xdhF5rq3b6Szxnrf5BxRjzHfGmDRjTNhmLqWUsmVPx1K1g4hcKiLhmGP9WhGZ1E7b/ag9MnaGeM/bUnEePdavtZVJ2aeFqrLhDGNMGlAAHAncFHPbTGAj0B/IBS4CtgKIyPHAn4HzjTHpwKHAS23M8lL0jTANuBZ4TkQOaOM2lVJKqY62u2Np3GrvUTZ76dOYY/0vgftE5EgLOZRS+0ALVWWNMWYL8CbOQbbRCOAZY4zPGBMyxiwzxrwRc9unxphl0ceXG2P+aYypaac8bwI1wODGNhE5XUSWi0iliHwiIkfE3PYHEdkU7d39RkROFJHRwM3Ar6Jnble0tK+WHhttf0ZE7oq5389EpLjZw0eIyFciUiEi/xART/S+eSLy32jWchH5UEQSorf1ifZUbxeRdSLyu2h7W/MmiMgUEVkjImUi8m8RyWllG5kiMkNESqLbuit2+K6ITIj2atdEf7+jRGQmcBAwN5rvRmk2pDj6u70a/Z1Xi8iEmG3eHs30bHS7X4rI8JbyKaVUV9TKsbRFInJa9P21Jvo+fH3MbTdE3583i8ivJWY4Z/Perua9cyLykIhsFJFqEVkiIj+Jue12EZklIs+JSDVw6Z6OHSJykYhsiN72xzY/STGinyG+xjnZ3bi/Y6LH+EoRWSEiP2v2u66NPmfrROQCETkUeBwYFT02Vba0r5YeG/OcPBdzv5amygwWkUXR5/Q/jc+PiHiiz2VZNO9nEj3B3tpxtq15o7f9OnqMrhCRN0WkfyvbSBaR+0XkO3FGvj0uIikxt48V53NVdfT/f7SI3A38BHgkmu+R6H1jX4OZ0WP59uhr40+y4zPOpSLyUXS/FdHsp7aUT3UtWqgqa0SkH3AqsDqm+X/AoyJynogc1OwhC4FTROQOEfmxiCS3YxYRkTFAEvBVtO1I4GlgIk7v7hPAq9E34UOAa4AR0d7dU4D1xph5OL2+jT21Q1vYV4uP3Ye4F0QfMxj4AfCnaPt1QDHQAzgApwA10TfyucAKoC9wInCtiJzSDnl/C4wDjgf6ABXAo63kfgYIAQfjnP3/BfCb6D7OAW4HLgYygDOBMmPMRcB3RHsOjDH3tbDdF6O/dx/gbODPIvLzmNvPjN4nC3gVeKSVfEop1eW0cixtzQxgYvS9fAgwP7qN0cD1wMlAPrCv80E/wymUc4B/AS9L9CRq1FhgFs778PPs5tghIocB03FGVPXBOf7228c8rRKRETjHzsXRn/sCrwF3RfNfD7wiIj1ExAs8DJwafc6OBZYbY74GrmRHT21WC/tp8bH7EPVi4NdAb5xj58PR9kuATOBAnOfmSsAfve0ZWjjOtjWviIzF+UxxFs5njA+BF1rJfS/O81sQzdEXuDW6nZHAs8ANOK+Fn+J8dvpjdJvXRPNd08J2/7/o7z0I53VzMXBZzO1HA98AecB9wAwRkVYyqi4i7gpVEXlaRLaJyBd7cd9p0bMyy0Xk29bOEKm4UygiNThDfLcBt8Xcdg7Om9UtwLro/+0IAGPMhzhvkkfhHFTKROQBaduiOudGXze1OEXMn40xja+jK4AnjDELjTFhY8w/gQbgGCAMJAOHiYjbGLPeGLNmL/fZlscCPGKM2WiMKQfuBs6PtgdxDmj9jTFBY8yHxhiD0xPdwxhzpzEmEJ3v8SRwXjvkvRL4ozGm2BjTgFNsnt3srDDRs72nAddGe8u3AdNiMvwGuM8Y85lxrDbGbNhTMBE5EPgx8AdjTL0xZjnwFM4BrNFHxpjXo3NaZwK7FONKKdUF7e5Y2pogznt5hjGmwhizNNp+LvAPY8wXxhgfznv5XjPGPGeMKYuOhPobzjHjkJi7fGqMKTTGRIwxfnZ/7Dgb+K8x5oPobbcAkX3J04Jjor2PNcAinGNBUfS2C4HXo8eJiDHmbZwi9rTo7RFgiIikGGNKjDFf7sN+2/LYmTH/H7fgfF5x4fwf5gIHRz+bLDHGVO/FcbYtea8E7jHGfG2MCeGc4C5o3qsaLQyvACZHR73VRO/bmOFy4GljzNvR53qTMWbVnkJFf+/zgJuMMTXGmPXA33BOZjTaYIx5Mnqs/yfO5yGdytXFxV2hinM2aPTe3NEYM9kYU2CMKcA50zK7I4OpdjMuerbuZ8APcc5+ARA9cE4xxvwI5w1mOc7BWKK3v2GMOQPnrOdY4FKivXKxROQnsmPhhN0dGP5tjMkyxnhxeigvFpGJ0dv6A9dFD26V0YL2QKCPMWY1zpzW24FtIvKiiPTZm1++LY+N2hjz/QacM84Af8U5o/5WdOjOlJjfo0+z3+Nm9vINfA95+wNzYrb7NU5h23zb/QE3UBJz3yeAntHbDwT2pVhv1AdoPBg22oBzBrfRlpjv6wBP80JaKaW6oFaPpbvxS5xiZoOILBCRUdH2Pux6bNlrInK9OMNCq6Lv75nN8mxs9pDdHTt2yhIt1Mpa2W/jKvC1IlK7m4j/ix7r04FewI9wCqjGLOc0O0YeB/SO7vtXOIVaiYi8JiI/3JvnpC2PjWr+/+HGeU5n4gz1flGcYdr3iYibPR9n25K3P/BQzHbLAWHnYy04va2pwJKY+86LtsP3P9bnRX+32Ndlq8d6Y0xd9FtdYKyLi7tC1RjzAc4fQBMRGSwi88SZ9/BhK3/o59P6MAQVh4wxC3BOTNzfyu2l0dv64BSmsbdFjDHv4gxbGtLCYz+MDh9Jixa9e5NnPfAGcEa0aSNwd/Tg1viVaox5IXr/fxljjsN5AzfAXxo3tRf7au2xPpw3+Ua9Wnj4gTHfHwRsjm6zxhhznTFmEM5w1/8nzlzSjcC6Zr9HujGm8WxxW/JuxBkmFLttjzFmU7NNbMTpjc6LuV9GzP/NRmLmBjff/W6ibQZyZOeVnw8Cmu9fKaW6pT0dS5vd9zNjzFic4qUQ+Hf0phJ2PbbEavXYJM581BtxemWzjTOstAqnkGnadbPt7e7YsVMWEUnF6UFs6fdpXAW+caGkPTLGbAVeYedj/cxmWbzGmHuj93/TGHMyTg/dKpwRSS39Ti3tq7XHfp9jfRAoNc6IqTuMMYfhDM89HWcU0Z6Os23JuxFnyHjsc5RijPmk2SZKcYYh/yjmfpkx/zff91hfGv39Y3tw9Vi/H4i7QrUVfwd+a4wZhjN34LHYG6NDDwYSnWuhupQHgZNFZCiAiPxFRIaISGK0+JgErDbGlIkzAf88EckWx0iceQr/a48g4szzGQ009sA+CVwpIkdH9+cVkTEiki4ih4jIz8WZJ1uP88bcODRpKzBAopP8W9jP7h67HDhNRHJEpBdOT2ZzV4tzGZ8c4I9EVz4WZ+Gng6O9z1U4Z6cjOMOcasRZEClFnIUVhkh0SHUb8z4O3N04/EecOT1jm2/DGFMCvAX8TUQyxFlIY7A4KzmDM1z3ehEZFn2uD5YdQ4q24sxJ2YUxZiPwCXCPOAtMHIEztChuL7ejlFIdYKdjaUtEJEmchYAyjTFBoJod7+X/xlnk6LBoYdh8GPFy4CwRSRVncZvLY25Lx5kXuR1IFJFbcdYa2J3dHTtmAaeLyHEikgTcSTt+XhWRXGA8O471zwFniMgp0eOjR5yFDPuJyAHRzx5enCKwlp2P9f2iGVvaz+4euxz4abRHOJOWV2y+MOb/405gljEmLCIniMjh4gyHrcYp4CJ7cZxtS97HgZtE5EfR+2aKs7bETowxEZzPTtNEpGf0vn1F5JToXWYAl4mz+GRC9LbGzqfdHevDOK/Ru6OfwfoD/w891nd7cV+oinNdsGNxJuYvxxnG0LvZ3c4j+gfc2flU2xhjtuNMrL812pQKzAEqgbU4Z8/OjN5WAUzAmVdSjfMG9VdjzPNtiNC42m0tzmIQHwN3RLMtju7vkei+V+MMNQZn/s29OGf5tuCcnW480Lwc/bdMRBrn/8Ta3WNn4ix6tB7ngNPS5Xf+Fb1tLc4QmsZVgvOBd3AOLp8Cjxlj3ov+XZyOs7DBuuh+n8IZmtXWvA/hzO19S5y5P//DWdCgJRezY7GqCpwPI70BjDEv48y3/RfOysuF7OhFvwf4kzjDiK5vvlGc0RQDcHpX5wC3GWPeaSWDUkp1Oy0cS1tzEbBenNV3r8RZnA/jrK7/IM4J/9XseuJ/GhDAKSb+ibMgUqM3cYZ3foszHLOeXYf6NtfqscM48yKvxjkelOAcL5qvfr+vRsUc67/GKap/G93fRpypRDdH2zfiLPaTEP36fzjHl3Kck+ON12Cdj1PsbhGR0hb22epjjTMP9iVgJbAE+G8Lj5+J01O+BfAAv4u298I5flZHf5cF0fvCbo6zbcw7B2ck1YvR184XOAt4teQPOK+h/0Xv+w7R+crGmEU4CyBNwzmhvoAdvaQP4cxTrhCRh3fZqvP/5cP57PMRzuvj6VYyqG5CjNnjSIBOJyIDcCbSDxGRDOAbY0zz4jT2/suAq1sYgqCUUkoppfaRiBggP7pOgVJKdbq471E1xlTjrP56DjRdRqRpaEt0yEA2Tg+SUkoppZRSSqkuLu4KVRF5AafoPEREikXkcpyhKZeLyAqcYQuxc+DOA1408dg1rJRSSinViUTkS4lZCTfm6wLb2ZRSal/E5dBfpZRSSimllFL7r7jrUVVKKaWUUkoptX/TQlUppZRSSimlVFxJtB0gVl5enhkwYIDtGEoppbqBJUuWlBpjetjO0dXpsVkppVR72Zdjc1wVqgMGDGDx4sW2YyillOoGRGSD7QzdgR6blVJKtZd9OTbr0F+llFKqGxCRp0Vkm4h80crtIiIPi8hqEVkpIkd1dkallFJqb2mhqpRSSnUPzwCjd3P7qUB+9OsKYHonZFJKKaW+Fy1UlVJKqW7AGPMBUL6bu4wFnjWO/wFZItK7c9IppZRS+yau5qgqpVRXFwwGKS4upr6+3naU/YbH46Ffv3643W7bUeJdX2BjzM/F0baS5ncUkStwel056KCDOiWcUkopFUsLVaWUakfFxcWkp6czYMAARMR2nG7PGENZWRnFxcUMHDjQdpxuwxjzd+DvAMOHDzeW4yillNoP6dBfpZRqR/X19eTm5mqR2klEhNzcXO3B3jubgANjfu4XbVNKKaXijhaqSinVzrRI7Vz6fO+1V4GLo6v/HgNUGWN2GfarlFJKxQMtVJVSqptxuVwUFBQwdOhQjjrqKD755JPvtZ0HH3yQurq6Nud5//33Of3009u8ncLCQr766qs2b6e7EpEXgE+BQ0SkWEQuF5ErReTK6F1eB9YCq4EngassRVVKKaX2SOeoKqVUN5OSksLy5csBePPNN7nppptYsGDBPm/nwQcf5MILLyQ1NbW9I34vhYWFnH766Rx22GG2o8QlY8z5e7jdAFd3UhyllFKqTbpVj2phYRFjxsziyCOfYcyYWRQWFtmOxPTpyzj88H/Qu/ejHH74P5g+fZntSMyfv4FLL32dk09+iUsvfZ358zfYjkR5uZ+FC0t48811LFxYQnm533YkpbqF6upqsrOzm37+61//yogRIzjiiCO47bbbAPD5fIwZM4ahQ4cyZMgQXnrpJR5++GE2b97MCSecwAknnLDLdqdMmcJhhx3GEUccwfXXXw/ApZdeyqxZs5ruk5aWtlOOMWPGcMghh3DllVcSiUQIh8NceumlDBkyhMMPP5xp06YBsGbNGkaPHs2wYcP4yU9+wqpVq/jkk0949dVXueGGGygoKGDNmjUd8nwppZRSKj50mx7VwsIibr75A5KTE0hOTmTz5hpuvvkDAMaNy7eSafr0ZVx33XvU10cwBrZu9XPdde8BMGnSkVYyzZ+/gbPOmkNVVaiprbDwW2bPHs/Pf97fSqbycj/jxs3mk09KCIfB5YJjj+1NYeFZ5OSkWMkEMHPml0yfvpytW30ccICXSZMKuOiiH1nLU17up6ioksrKerKyPOTnZ1l9ftReuPZaiPZstpuCAnjwwd3exe/3U1BQQH19PSUlJcyfPx+At956i6KiIhYtWoQxhjPPPJMPPviA7du306dPH1577TUAqqqqyMzM5IEHHuC9994jLy9vp+2XlZUxZ84cVq1ahYhQWVm5x9iLFi3iq6++on///owePZrZs2czcOBANm3axBdffAHQtJ0rrriCxx9/nPz8fBYuXMhVV13F/PnzOfPMMzn99NM5++yz9/lpU0oppVTX0m16VB99dBkiQkZGCqmpbjIyUhARHn3UXg/mHXd8jN8fISHBKb4SEsDvj3DHHR9by3TRRf/dqUgFqKoKcdFF/7WUCM499z98+KFTpCYkQDgMH35Ywrnn/sdappkzv+SWWz7C5wvQr58Xny/ALbd8xMyZX1rJ09jj3NAQIjfXQ0NDSHueVasah/6uWrWKefPmcfHFF2OM4a233uKtt97iyCOP5KijjmLVqlUUFRVx+OGH8/bbb/OHP/yBDz/8kMzMzN1uPzMzE4/Hw+WXX87s2bP3amjwyJEjGTRoEC6Xi/PPP5+PPvqIQYMGsXbtWn77298yb948MjIyqK2t5ZNPPuGcc86hoKCAiRMnUlKi6/0opZRS+5tu06O6fn0VOTnJ+HwBQqEIiYkJpKUlsn59lbVMW7c6l0sIh1tut2Hz5pYLm9baO8P77xc3fR+JtNze2aZPX055eR0bNtQ0taWnu5g+fbmVXtWioko++mgTc+euobzcT05OCmecMZicnBSOPtper6r28u7BHno+O8OoUaMoLS1l+/btGGO46aabmDhx4i73W7p0Ka+//jp/+tOfOPHEE7n11ltb3WZiYiKLFi3i3XffZdasWTzyyCPMnz+fxMREItE/4kgkQiAQaHpM85V5RYTs7GxWrFjBm2++yeOPP86///1vHnzwQbKysprm2CqllFL7iyVLtjB7dhGbNtXQt286Z52Vz7BhvaxmsvlZr9v0qKanuykt9ROJgNudQCQCpaV+0tPdtqOpPWheyO+pvTMsX76VmpqdA9TUhFm+fKuVPLNmfcOMGSvx+4P07JmC3x9kxoyVzJr1jZU8oL28XcWqVasIh8Pk5uZyyimn8PTTT1NbWwvApk2b2LZtG5s3byY1NZULL7yQG264gaVLlwKQnp5OTU3NLtusra2lqqqK0047jWnTprFixQoABgwYwJIlSwB49dVXCQaDTY9ZtGgR69atIxKJ8NJLL3HcccdRWlpKJBLhl7/8JXfddRdLly4lIyODgQMH8vLLLwNgjGnafmt5lFJKqa5uyZItTJu2mNraAAMGpFNbG2DatMUsWbLFWibbn/W6TY/qMcf0pbDwWxITA3i9Lny+MH5/iFNOGWQ7muqC/P7IPrV3tHnz1uH1usnKSgYgKyuZSMQwb946/vrXn1nJVFRUidfrxut1TgY1/ltUVGm1l1ftmKMKTqH3z3/+E5fLxS9+8Qu+/vprRo0aBTiLHT333HOsXr2aG264gYSEBNxuN9OnTwecuaKjR4+mT58+vPfee03br6mpYezYsdTX12OM4YEHHgBgwoQJjB07lqFDhzJ69Gi8Xm/TY0aMGME111zD6tWrOeGEExg/fjyff/45l112WVMv7D333APA888/z6RJk7jrrrsIBoOcd955DB06lPPOO48JEybw8MMPM2vWLAYPHtzxT6ZSSinVEYyBykrYvh22b+fraQs4vaKUnlvrcIWdaXp1dUEqJ8+GE+2sI1NXXMOh4QhJbhdVx52IHOZ8tuisz3rirFYfH4YPH24WL178vR47a9Y3vPHGWhYuLKG6uoGMjGSOPro3p546iLPPPqSdk+6d7OyHqawM7NKelZVERcXvLCQCkftbvc2Y6zsxyQ6aac8GDHiCtLREkpJcJCQIkYghEAhTWxti/fpdh3F2hjffXEdurmenIZ3GGMrK6jnllIFWMsWDr7/+mkMPPdR2jP1OS8+7iCwxxgy3FKnbaMuxWe29NWsqWLCgmJKSWnr3TuP44/sxeHD2nh/YgcaPn83cuWubFjo844xBzJlzltVMLR2fbX1WgPjLA5ppb2VkTNtp9Fx6uovq6snff4ORCJSXNxWee/UVCu15u3Fiw5R7KT37kjZ/1tuXY3O36VE98MAMxo/P56c/7UdNTYD09CRyc1Po0cO75wd3kEMPzWXt2goqK+sJBsHthqwsD4MG2TvwuFwtD6l1uTo/SzyLt+epT580qqrqSUgQQiFDYqJgjNNuS1aWh7q6UFNPKkBdXYisLI+1TEop1RWtWVPB889/RXa2hwMPTKOqKsDzz3/FBRccZq1YHT9+NoWFa4EdCx0WFq5l/PjZ1orV1k4ii9xvpeiJtzyN+26tXTPt0LxIBWeKV0bGtB3FaigEZWW7LzS3bdvxfVnZzoutxMrMhB49nK8BA2DEiB0/9+gBPXty7d0rmbekBr83C5PsoaEhTF1dkBNOOIhXX/1lxz4hrWgc9mvrs163KVTz87MoL/czcGAWqamJ1NWF8PmC5OdnWcs0aVIBt9zyEYcc4iUrK4nKygBVVQEmTSqwlumMMwbtdOBp/Hs64wx7Q6RHjOjJZ59ta7Hdlnh7nhpfS4mJCU2vJZ8vxJQp9l5L+flZLFzorMYa+zc3ZEjeHh6plFIq1oIFxWRnJTNk/f/I3ORcI9hfF6R8ZSKDR9hZSOWH//mAG4FEV3TUjAtCYUPCf96Dv9i5Tv2NfND6jX/p/IUt4i0PaKa9NanGySQY0gjQg1p64KNHjQ9++IRTeFZUOMNzW5KTs6PIPOQQOO446Nlz5+Kz8SsvD5KS9php/g2bKUl0k5HsITERkkigviHM+g010GxBws6S/4Ns57OeiPNZzxfs1M963WboL8TnCqTxdi1OiM+hPCNHPrtTsTpiRE8WLbrYYqL4e57i8bUUj39ztunQXzt06G/H0aG/He+pa/7J6e8+Qq9V+jwr1dnCCGWksp00tpHGCecc1XLB2fiVmwuJ7d/XN3jw38nIcFNR0UBDQ5jkZBfZ2clUVwdZs+aKdt/f3mrvz3r7cmzuVoWqUkrZpoWqHVqodhw9NnegbdvgllswTz5JfWoGK8/+LWuPOxMSEqioqMfrTeKSS4ZYiZaW9iDhCCQn7bhAREMggisBamuvtZIpJbX1S3756zo/U7zlAc20t2IzBXARibkQiq3hyMce+zw+X4C8vB1FYGmpH683iU8+ucBKpo6wX85RVUoppZTqEgIBeOQRuOMOqKuj6pIr+HuvsaT06UlmUhJVVQEqfHDKuIMhxc4olZPPPITCwrXUB2KnwLgYd+Yga5nq2c0lBy1kirc8oJn2ljvds8scVXAWVLKlcZoXsNOUweuvH2ktk23d5jqqSimllFJxzRj473/h8MPhuuvgxz+Gzz8n6x+P88vfHIPXm8TGjbV4vUlWF1ICmDPnLMaNG4TL5RSpLheMG2d3CkxrPV22esDiLc/u9q2ZdlZdPXmXorTNq/620UUX/YipU4/D602iuNiH15vE1KnHWZ/mZZMO/VVKqXYUD0N/XS4Xhx9+OKFQiIEDBzJz5kyysrKIRCJce+21zJ8/HxHB4/Hw73//m4EDB/L0008zbdo0RIRIJMLdd9/N2LFjrf4e+0KH/nYcPTa3k6+/hsmT4c03ncVXHngATjvNdiqllOpUOvRXKaX2YykpKSxfvhyASy65hEcffZQ//vGPvPTSS2zevJmVK1eSkJBAcXExXq+X4uJi7r77bpYuXUpmZia1tbVs377d8m+hVDdRXu4M8X30UUhLcwrUq6/eq1VAlVJqf6aFqlJKWdTRKyePGjWKlStXAlBSUkLv3r1JSHBmffTr1w+AdevWkZ6eTlqac13etLS0pu+VUt9TKARPPAG33gqVlXDFFXDnnc6qoUoppfaoQ+eoishkEflSRL4QkRdEpHOuDquUUl1Aebm/6WLaubkeGhpCLFxYQnm5v122Hw6HeffddznzzDMBOPfcc5k7dy4FBQVcd911LFu2DIChQ4dywAEHMHDgQC677DLmzp3bLvtXar/1zjtQUADXXANDh8KyZTB9uhapSim1DzqsUBWRvsDvgOHGmCGACzivo/anlFJdTVFRJV6vG6/XjYg0fV9UVNmm7fr9fgoKCujVqxdbt27l5JNPBpwe1G+++YZ77rmHhIQETjzxRN59911cLhfz5s1j1qxZ/OAHP2Dy5Mncfvvt7fAbKrWfWb0axo2Dk0+Gujp45RV491044gjbyZRSqsvp6FV/E4EUEUkEUoHNHbw/pZTqMior60lN3XkGRmpqIpWV9W3abuMc1Q0bNmCM4dFHH226LTk5mVNPPZW//vWv3HzzzRQWFgIgIowcOZKbbrqJF198kVdeeaVNGZTar1RXwx/+AIcd5vSm3nMPfPUVnHUWiNhOp5RSXVKHFarGmE3A/cB3QAlQZYx5q/n9ROQKEVksIot18Q6l1P4kK8tDXV1op7a6uhBZWe0zSyI1NZWHH36Yv/3tb4RCIZYuXcrmzc75wkgkwsqVK+nfvz+bN29m6dKlTY9bvnw5/fv3b5cMSnVr4TDMmAE/+AHcdx9ccAEUFcGUKeDR2U6Eqo3GAAAgAElEQVRKKdUWHbaYkohkA2OBgUAl8LKIXGiMeS72fsaYvwN/B2cJ/I7Ko5RS8SY/P4uFC0sApye1ri6EzxdkyJC8dtvHkUceyRFHHMELL7xAjx49mDBhAg0NDQCMHDmSa665hq1bt3L99dezefNmPB4PPXr04PHHH2+3DEp1Sx99BL//PSxdCsceC3PnwogRtlMppVS30ZGr/p4ErDPGbAcQkdnAscBzu31UG3T06pnfR2FhEU8+uYLNm2vp0yeNCROGMm5cvtVMd975MU88sYKqqgCZmUlMnDiUW2/9sdVMM2d+yfTpy9m61ccBB3iZNKnA+gWO4/H1pLqXnJwUjj66N0VFlZSVOa+zIUPy2vw6q62t3enn2MWRRo8evcv9+/fvz/z589u0T6X2G999BzfeCC+9BP36wb/+Beedp0N8lVKqnXVkofodcIyIpAJ+4ESgw64Y3rh6ptfrJjfXGU63cGEJRx/d21pxUVhYxM03f0BycgLJyYls3lzDzTd/AGCtWL3zzo+5665PCYXAGGeY4V13fQpgrVidOfNLrrhiHvX1Tof62rXVLFu2BcBasVpe7mfcuNl88kkJ4TC4XHDssb0pLDzL2utJi/nuySlW9TlTKu75fM7w3vvuc4rS226DG24Ar9d2MqWU6pY6co7qQmAWsBT4PLqvv3fU/jpq9cy2ePTRZYgIGRkppKa6ychIQUR49NFl1jI98MBigkGn8EpKcv4NBp12W37/+3eaitRG9fWG3//+HUuJ4Nxz/8OHHzpFakKCMw3pww9LOPfc/1jJM3Pml9xyy0f4fAH69fPi8wW45ZaPmDnzSyt5oOMvraKUUnHBGHj+eTjkEOc6qOPGwapVcPvtWqQqpVQH6sgeVYwxtwG3deQ+GlVW1pObu/PCBampiZSVtW31zLZYv76KnJxkfL4AoVCExMQE0tISWb++ylqmqqog4FyHvKV2GyoqWt53a+2d4f33i5u+j0Rabu9M06cvp7y8jg0bapra0tNdTJ++3FqvalFRJR99tIm5c9dQXu4nJyeFM84YbL2HUHt5lVLt5rPPnHmon34Kw4bBiy/CccfZTqWUUvuFjr48Tafp6NUzv4/0dDelpX4iEXC7E4hEoLTUT3q621omtXfC4X1r72jLl2+lpmbnndfUhFm+fKudQMCsWd8wY8ZK/P4gPXum4PcHmTFjJbNmfWMtU7z08hqj68J1Jn2+VbvbvBkuvRRGjoS1a+Hpp2HRIi1SlVKqE3WbQjU/PwufL4jPF8QY0/R9fn6WtUyjRvXF7w9RVxcgHA5TVxfA7w8xalRfa5lcrn1rV/HB74/sU3tnmDdvHV6vm6ysZFyuBLKykvF63cybt85apniYAuDxeCgrK9PiqZMYYygrK8OjlwJR7aG+Hv78Z+dyMy+84Fwb9dtv4bLLnHkgSimlOk2HDv3tTB21emZbnHDCQdTXB1m4sITS0gYyMtyceupATjjhIGuZsrM9lJbuOhw6O1s/5Kl9U1MTIC0tkXA4QkKCEIkYUlJc1NQErGWKhykA/fr1o7i4GL0udOfxeDz069fPdgzVhewyReDgTHLefwOuvx7Wr3fmod5/Pwwe3GmZ7r9/UXSaRz05OR4mTSrg+utHdtr+W5Kd/RCVlTum4WRluamo+L3FRCBy/y5txlxvIYkj3vJAfGbKzHyQ6uodIx8zMhKpqrrWYqL4vAqGTl/aWbcpVCH+Vs888MAMxo//AT/96YHU1ARIT08iNzeFHj3sLb6Qn59NcnIV1dUNBAKGpCQhIyOZgw7KtJbJ5Wp5SK328u4Qj89Rnz5pVFXVk5AghEKGxETBGKfdlsYpAF7vjuH1nT0FwO12M3DgwE7bn1Jq3zS/SoCsXIm54jZYuRCGDIF33oETT+zUTPffv4ipUz/F63XTo4eH2toQU6c6K/LbKlabF6kAlZVBsrMfslastlSANbbbKMTiLU/jvltrt5WpeZEKUF0dIjPzQWvF6p13fsy99y4iJSWR7Owk6urC3HvvIsDeVTDi8Qomtuk4lg6Un5+Fy5XAwIFZHHNMHwYOdH62ORx50qQCEhNdDByYxdFHH8DAgVkkJrqYNKnAWqYzzhjU9H3syKrY9s42YkTPfWrvaPH4HE2aVIDPFyIYjJCZ6SYYjODzhay+luJxCoBSKr40ThHICFbT/54/cNSE08jY8C3rbrgHli3r9CIVnAXzGqdSJCa6mqZSTJ++vNOzNGpepO6pXanWNC9S99TeGZ54YgUpKYlkZCSRmOgiIyOJlJREnnhihbVM8TB9Kd5oodqBGocjJyc7Qw+TkxOtnxW56KIfMXXqcXi9SRQX+/B6k5g69Tir1+OcM+csxo0bhMvlrLDrcsG4cYOYM+csa5kWLbp4l6J0xIieLFp0sZU88fgcxeNrKR7/5pRS8aWysp6MhioO+c048v7zAtt+9Wu+mP0x3574K0i0M9CsvLyetLSd952Wlkh5ub0rFyjVnVVVBUhN3XlYWmqqi6oqu9OXUlN3fh9ITU2ksnL/fR/oVkN/41G8DUcGp8CwWUy0xGbB1RpbRWlr4vE5isfXUjz+zSml4kduYoABE8/DvWkjc65+jJqjRpFb5wy5tSUnx8OWLXU0NISbplIkJ7vo0SPVWialurPMTGe4b0bGjmK1ri5MZmaStUzxMH0p3miPqlJKKaX2D34/P7rp16SvW8Xrv/4LtcOOobq6gaVLt5KXZ+/D4BFH5FFTEyIQcFYLDwQMNTUhjjgiz1ompdpLenrLC2q01t4ZJk4cit8foro6QCgUprrauTLHxIlDrWXS6Uu70kJVKaWUUt1fIABnn41n8acsv/5vVIz6OdXVATIykjjqqANaXBG/s6xeXUVmZhJuNxgDbrfT47N6dZW1TCL71r4/isdL/o0a1Xuf2jvD22+fR2qzwQGpqU67Lbfe+mOmTBmJx+OioiKAx+NiypSRVlf91elLu9Khv0oppZTq3sJhuOQSeP11vvrd3ZhzzmVITMXlXI/XXqFaWlrHwIHpJCbuqHBCoTBbttRZy9R4KWi3e8fzFAwa4uES0bHFss084bCzwKHbvaPfJxiMtLhKf2fZutXHT3/ae5fXUnGxz1qmysp6Fiy4AImjvzlwilXbl6NpTqcv7Ux7VJVSSinVfRkDV18NL74If/kLtf93GXV1O682anseWF5eKtXVO6+mW10dJC/P3hxV7VHds8YFDmM1LnhoywEHeKms3HlBoMrKAAccYO/SiI1zL2PZ/ptTXYMWqkoppZTqvm66CZ54AqZMgRtvjMt5YFddVUBNTZDy8npCoTDl5fXU1AS56ip7l/s69dT+gNOL2vgV225D42r8xuz4im3vbI2XiGtoiBAMRmhoiOzUbsOkSQVUVQUoLfUTCoUpLfVTVRXQS8epLkkLVaWUUkp1T3/5i/N15ZXw5z8D8TkPbNKkI7njjmNJTXWzZUsdqalu7rjjWCZNOtJaptdeO4fTTuvf1IMqAqed1p/XXjvHWia9dNye6aXjVHciJh4mG0QNHz7cLF682HYMpZRS3YCILDHGDLedo6vrssfmJ55wCtTzz4fnnnMmEyqllLJqX47N+q6tlFJKqe7lhRdg0iQYMwb++U8tUpVSqgvSd26llFJKdR+vvQYXXww/+Qm8/LJzrRellFJdjhaqSimllOoePvgAzj4bhg6FuXMhRefAKaVUV6WFqlJKKaW6viVL4PTTYcAAmDcPMjJsJ1JKKdUGibYDKKWUUkq1yddfw+jRkJMDb78NeXm2E3UL5eV+iooqqaysJyvLQ35+lvWVWmfO/JLp05ezdauPAw7wMmlSgdUVbcePn83cuWsJh51Vf884w+6qvwAjRz7LZ59ta/rZ5srIjeLxeYq31xLE59+cTdqjqpRSSqmua/16OPlk59PvO+9Av362E3UL5eV+Fi4soaEhRG6uh4aGEAsXllBe7reWaebML7nllo/w+QL06+fF5wtwyy0fMXPml1byjB8/m8JCp/hKSIBwGAoL1zJ+/GwreWDXIhXgs8+2MXLks5YSxefzFG+vJYjPvznbtFBVSimlVNe0ZYtTpPp88NZbcPDBthN1G0VFlXi9brxeNyLS9H1RUaW1TNOnLyczM4m8vBQSE13k5aWQmZnE9OnLreSZO3ctAMnJCbjdCSQnJ+zUbkNjkSqy4yu23YZ4fJ7i7bUE8fk3Z5sO/VVKKaVU11NRAaecAps3Oz2pRxxhO1GbzJ+/gWef/ZJNm2ro2zediy/+ET//eX9reSor67nyyjdZv762qW3AgDQef/wUa5m2bvWxdm31Lu2DBtmZjxwOO/82NERabLfJGNsJdojH52nrVh8ZGYmsXl1JQ0OI5OREcnKS2LrVZy1TZWU9ubmendpSUxMpK6u3lMg+7VFVSimlVNfi8znXSF21CgoLYdQo24naZP78DUyd+im1tUEGDMigtjbI1KmfMn/+BmuZJk6ct1ORCrB+fS0TJ86zlIgWi9TdtSvVGq83kbVrawiFIng8LkKhCGvX1uD12uvDy8ryUFcX2qmtri5EVpanlUd0f1qoKqWUUqrraGiA8eNh4UJ44QVn6G8X9+yzX5Kbm0KPHs4wxB49UsjNTeHZZ+3Nl9uwoeWepdbalepK+vfPJBKJEA5HACEcjhCJROjfP9Napvz8LHy+ID5fEGNM0/f5+VnWMtmmQ3+VUkop1TWEQnDBBc7Kvv/4B5z1/VYNXbOmggULiikpqaV37zSOP74fgwdnt3PYvbdpUw3FxTWsXl1JKASJiXDwwVn065duLZNS3VlubgonndSfxYu3UVUVIC0tiZNO6kNmpr3ey5ycFI4+ujdFRZWUlTmr/g4Zkrdfr/qrhapSSiml4l8kAldcAa+8AtOmwaWXfq/NrFlTwfPPf0V2tocDD0yjqirA889/xQUXHGatWN24sZpvvqlCxFm8OByGVasqMfE00VCpNmpc2Ansz6Ht2zedzMxkjjqqV1NbaamftLQki6kai9X9tzBtTof+KqWUUt2AiIwWkW9EZLWITGnh9oNE5D0RWSYiK0XkNBs5vxdj4LrrnF7U226Da6/93ptasKCY7GwP2dkeEhISmr5fsKC4HQPvm40bawBwuaTpK7Zdqa5sxIiegPNn3PgV227DWWflU1bmp7TUTzgcprTUT1mZn7POyreWSe1KC1WllFKqixMRF/AocCpwGHC+iBzW7G5/Av5tjDkSOA94rHNTtsHUqfDgg/C73zmFahuUlNSSmblzr0lmZhIlJbWtPKLjBQIRUlMhIUGIRAwJCUJqqtOu1L4oKMjbp/bOsGjRxbsUpSNG9GTRoostJYJhw3oxefJw0tKSWL++hrS0JCZPHs6wYb32/GDVaTps6K+IHAK8FNM0CLjVGPNgR+1TKaWU2k+NBFYbY9YCiMiLwFjgq5j7GKDxOh6ZwOZOTfh9PfywU5xecokz5Dd2/OD30Lu3M9w3O3vHXLSqqgC9e6e1Nen35vW6CQTCZGTsKKB9vgBer8taJrX3EmK6fSKWzy306ZNGRkYyPXrsGD66fbuftDS3xVRYLUpbM2xYLy1M41yH9agaY74xxhQYYwqAYUAdMKej9qeUUkrtx/oCG2N+Lo62xboduFBEioHXgd92TrQ2ePZZ+P3vYdw4eOqpnSuC7+n44/tRUVFPRUU9kUik6fvjj+/XDoG/n0su+RGBQASfL0AwGMLnCxAIRLjkkh9ZyzR4cMsLObXW3hl69mx5oZvW2jvascc6RU4ksuMrtt2GCROGUlFRz/btfkKhMNu3+6moqGfChKHWMin1fXXW0N8TgTXGGHsXBFNKKaX2b+cDzxhj+gGnATNFpMXPASJyhYgsFpHF27dv79SQTQoL4de/hhNPdC5Dk9g+g8AGD87mggsOw+tNYuPGWrzeJKsLKQE89NBJXH11AUlJLny+EElJLq6+uoCHHjrJWqbVqyfuUpQOHpzO6tUTLSWCrVuv2aUo7dnTw9at11jJ8/HHF+5SlB57bC8+/vhCK3kAxo3L5847jyMtzc2GDTWkpbm5887jGDdO516qrkc6Y0U5EXkaWGqMeaSF264ArgA46KCDhm3YoLWsUkqpthORJcaY4bZzdAYRGQXcbow5JfrzTQDGmHti7vMlMNoYszH681rgGGPMtt1te/jw4Wbx4sUdlr1F774Lp50GRx4J77wDafaG5SqllGo/+3Js7vAeVRFJAs4EXm7pdmPM340xw40xw3v06NHRcZRSSqnu6DMgX0QGRo+75wGvNrvPdzgjnBCRQwEPYKm7dDcWLoSxY+EHP4DXX9ciVSml9lOdMfT3VJze1K2dsC+llFJqv2OMCQHXAG8CX+Os7vuliNwpImdG73YdMEFEVgAvAJeaeLtQ5+efw6mnQq9e8NZbkJNjO5FSSilLOmzV3xjn4xwQlVJKKdVBjDGv4yySFNt2a8z3XwE/7uxce23NGvjFLyAlBd5+G3r3tp1IKaWURR3aoyoiXuBkYHZH7kcppZRSXdimTXDSSRAMOkXqwIG2EymllLKsQ3tUjTE+ILcj96GUUkqpLqyszOlJLS2F+fPhsMNsJ1JKKRUHOmPor1JKKaXUrqqrYfRoZ9jvvHkwYoTtREoppeKEFqpKKaWU6nx+v7O677JlMGcO/OxnthMppZSKI1qoKqWUUqpzBYPwq1/BggXw3HNwxhm2E1m3ZMkWZs8uYtOmGvr2Teess/IZNqyX1UxjxrzMG29swBgQgVNP7c9rr51jNVNGxjRqasJNP6enu6iunmwtT0rK/dTX7/jZ4wG//3preQAOPvgJ1qypafp58OB0Vq+eaDER3H//IqZPX055eT05OR4mTSrg+utHWs1UXu6nqKiSysp6srI85OdnkZOTYjWT2llnXJ5GKaWUUsoRicBll8HcufDoo/B//2c7kXVLlmxh2rTF1NYGGDAgndraANOmLWbJki3WMo0Z8zKvv+4UqQDGwOuvb2DMmJetZWpepALU1ITJyJhmJU/zIhWgvt5pt6V5kQqwZk0NBx/8hKVETpE6deqn+P0hevTw4PeHmDr1U+6/f5G1TOXlfhYuLKGhIURuroeGhhALF5ZQXu63lkntSgtVpZRSSnUOY+C3v4Xnn4e774ZJk2wniguzZxeRm5tCXl4KLpeLvLwUcnNTmD27yFqmN97YAIDbLU1fse02NC9S99Te0ZoXqXtq7wzNi9Q9tXeG6dOX4/W6ycpKJjHRRVZWMl6vm+nTl1vLVFRUidfrxut1IyJN3xcVVVrLpHbVrYb+xmMXfmFhEU8+uYLNm2vp0yeNCROGMm5cvtVMd975MU88sYKqqgCZmUlMnDiUW2+1e2m96dOX8dhjyyktrSMvL5Wrripg0qQjrWZas6aCBQuKKSmppXfvNI4/vh+DB2dbzaSUUl1N7LF5yIsP0feZx+CGG+Cmm6xlmjLlfWbM+JyamiDp6W4uv/xw7r33Z9bybNpUw3PPfUU4pt5yueDCC+2tgNzYkxoMmhbbldpbznDfJPz+IOGw89pOSUmgvNxeRV9ZWU9urmenttTURMrKLJ5lULvoNoVqYxe+1+smN9dDXZ3ThX/00b2tFauFhUXceutHZGd76N8/nYqKALfe+hGAtWL1zjs/5t57F+J2J+DxCLW1Ae69dyGAtWJ1+vRl3HzzB5jo0W/jxipuvvkDAGvF6po1Fdx33yLWravC5wvg9SaxcOFmbrxxpLViNR5PxCil1O7EHpsPe/0f9H3mIb4b/SvS/nA7OSJWMk2Z8j4PPLAEj8dFRoab+vowDzywBMBasdq8SAUIh532Z545zUompdpLRkYSFRVBMjLcuFzOyQ7n5yRrmbKynFrB63U3tdXVhcjK8uzmUaqzdZuhv/HYhf/kkyvwepNISBBKS+tJSBC83iSefHKFtUyPPbaCSCSC3x+msjKI3x8mEonw2GP2Mv3tb59RUxOkqirU9FVTE+Rvf/vMWqZnnvmCd97ZwJIlW1m5cjtLlmzlnXc28MwzX1jJU17u5+23N7Bs2Va+/bacZcu28vbbG3QuhVIqrjUemw96+2UOfHgq5Sefyfob76VodZW1TDNmfI7H4yItLYnEROdfj8fFjBmfW8vUvEjdU7tSXcnYsQdTXx+itjZIOByhtjZIfX2IsWMPtpYpPz8Lny+IzxfEGNP0fX5+lrVMalfdpkc1Hrvwv/uuhuRkob4+RCRiSEgI4XIZvvsuYC1TWVkdoRCA03sZDpumdls2bKhu8Uzyhg3VdgIBr7zyLWvXxu4/THl5A6+88i1Tp/6k0/MsWbKFp55awWefbcHvD5GSksiIEb3IyUnm5JMHdnqeRjo8Wim1O5WV9SQmCl+Hcwkc9QsW/up2+kTAX2nv2FxTE8TthrKyesLhCC5XAklJUFMTsZZJqfaSnJxAQ8Our+XkZHt9U2PGDEYEZs36lsrKAOnpSVxwwaGcdtpga5lyclI4+ujeFBVVUlbmjFQbMiRPR6rFmW5TqMZjF35KSiLbttWRk5NCUpIQDkNpaT09e6Zay+QUqQ6RHXNNYts7W2v7tpnpm28q9qm9oz3wwGLee28jHo8LrzeRQCDCe+9tJDExwVqhumZNBc8//xXZ2R4OPDCNqqoAzz//FRdccJgWq0opABISYMmSrWQPLKD02mHU14dZsmQrRx7Z01qmpCShpiaM2y0kJiYQiRhqaw3p6S5rmZRqL+npbpKSwqSl7RhWW1sbIDnZ3us7K8vDL395CBdfPKSpzecLkpxstwxxilUtTONZtxn6G49d+EOG5BEMhqirCxAOh6mrCxAMhhgyJM9apli6IELrIq2cWG+tvaN9/PHm6KqLLsJhcLtduN3Cxx9vthMIWLCgmOxsD9nZHhISEpq+X7Cg2FompVR8MUaA5nNRJdpuR16e88E0FDKEQhFCIbNTu1J7K6WVl0xr7Z3h8ssPp74+TG1tgFDI+be+Pszllx9uLVM8fkZXXUO36VGNxy78oUN7kpycwCefbKasrIGsrGROOOFgfvhDe4VqamoCdXW7Vlupqd3mnEW31NAQJBSC+vodw8YTEiASCVrLVFJSy7ZtPhYsKKaiop7sbA/HH9+Pnj291jKBLjqlVDwxxjBsWE+Ki2uoqmogPT2JYcN6NhWHNmRmehg0CL77rpZQCBIT4aCD0sjIsDcCKx6Ha8ZjpoKCPJYvL22x3YY5c37J+PGv4I9ZLiIlxWm3pXFBsBkzPqe62lnV+qqrCqyuah2Pn9FV19BtClWIvy78Qw/NpaEhzFFH9cLjcVFfH6aiooFDD821lqlv33TWrasiKSkhWuhAIBChb990a5nUnons2psbiTjttmzeXMucOUVkZCSRl+fB5wvx8svfMn68vcsvxePq30rtz7KyPDQ0hBgypEdTm88XJC3N3sePPn3SyMhI5sgjezW1bd/uJy3NvZtHdax4HK4Zj5ka/+969Njxfm7z/y4ry8O8eb/aadpZPAxpvffen1ktTFsSb5/RVdeg3WgdaPjwAxgwIINwOEJlZQPhcIQBAzIYPvwAa5nOO++H5OamkJAghMNCQoKQm5vCeef90Fqmnj1bPovdWvv+qLVh2jaHb3/55XbcbhfJyQmIOGfZ3W4XX3653VqmeFz9W6n9WTwO+ZswYSgVFfVs3+4nFAqzfbufiop6JkwYai1TPA7XjMdM8fZ/F4+vb6W6k27VoxpvcnJSOOmk/nE1DHHMmINJTnbxxhvr2Latjp49Uzn11IGcdJK9lWN/+MMcEhMr2bq1rulC0AcckMrBB9t7o09NTcDvj+xUCIo4F6i2ISEhAY8nQn3MQpkeD9g811RVFSA/P4OKigb8/jAej4v8/AzKyhqsZYrH1b+V2p/F45C/xuuYP/nkCjZsqKFPnzQmTx5u7frmEJ/DNeMxU7z938Xj61up7kQL1Q4Wb0Md8vOzKC/vxU9+ciCpqYnU1YWsn/07/vgD8fmCDBmS15SprMzP8ccfaC1T795eSkrq8HrduFxCOOycKe3d286KzZmZyfh8QXr1cjet1lxbG9xpuFFn69MnjdraIAMH7njtbN/up0+fNGuZ4nH1b6X2d/F2HASn4LFZmLYkHodrxmOmePu/i8fXt1LdhQ793c80nv1LTnZ6mZKTE63P37vkkiEMG9aLSAS2bfMTicCwYb245JIhe35wB7nuupEkJQkNDWHq64M0NIRJShKuu26klTxXXVVAOBzB5wsSCDgnF8LhCFddVWAlD8TfECzQYVhKKaWUUt2F9qjuh+Lt7N/gwdnceONIFiwopqSklt690zj++H5Wr8U5adKRADz22HJKS+vIy0vlqqsKmto72623/hiAJ55YQVVVgMzMJCZOHNrUbkO8DcECHYallFJKKdVdiImji2kOHz7cLF682HYMpZRS3YCILDHGDLedo6vTY7NSSqn2si/HZh36q5RSSimllFIqrmihqpRSSimllFIqrmihqpRSSimllFIqrmihqpRSSimllFIqrmihqpRSSimllFIqrmihqpRSSimllFIqruh1VDtYebmfoqJKKiudazrm52dZv6ZjYWERTz65gs2ba+nTJ40JE4ZavfYlwJQp7zNjxufU1ARJT3dz+eWHc++9P9NMMeLx/y0eX99KKaWUUqrr00K1A5WX+1m4sASv101uroe6uhALF5Zw9NG9rX2YLyws4uabPyA5OYHk5EQ2b67h5ps/ALBW9EyZ8j7337+YcNj5uaGhgfvvd67ZZ6swnDLlfe67bzGNlxluaGjgvvvsZSosLOI3v5lHRUUDkQisXFnKwoUlgL3/t/JyP9OmLeHtt9dRXl5PTo6Hk08eyOTJw6wWq1o8K6WUUkp1fTr0twMVFVXi9brxet2ISNP3RUWV1jI99tgyRISMjBRSU91kZKQgIjz22DKrmcJhcLuF5OQE3G4hHMZqpgcfXNJUpDYyxmm3YfLk+ZSVNaLSS48AACAASURBVCDiPE8iUFbWwOTJ863kAXjqqZX8619fEQhE6NPHSyAQ4V//+oqnnlppLVPjyaGGhhC5uR4aGpyTQ+XlfmuZlFJKKaXUvuvQQlVEskRkloisEpGvRWRUR+4v3lRW1pOaunOndWpqIpWV9ZYSwbp1VaSlJeLzBSgvr8fnC5CWlsi6dVXWMtXUOF2pwaChoSFCMGh2arehocHsU3tH++67GgDCYed5aux9bmy34eWXvyEcNmzYUM2SJdvYsKGacNjw8svfWMtUVFRJOBxh3boq/ve/EtatqyIcjlg9OaSUUkoppfZdR/eoPgTMM8b8EBgKfN3B+4srWVnOcN9YdXUhsrI8lhJBerqb0lI/kQi43QlEIlBa6ic93W0tUywR2wniUySyb+2dYdOmWrZsqSEcjuDxuAiHI2zZUsOmTbXWMn33XRVr1lQSCITJzEwiEAizZk0l331n70SMUkoppZTadx02R1VEMoGfApcCGGMCQKCj9heP8vOzmuYRpqYmUlcXwucLMmRInrVMo0b1Zc6cb3G7A6SmuqirC+P3hxg9epC1TMnJ0NDgfB873DY52U4etXcaGkIEg9DQEGxqS0hw2m2pqQny+eelLF68hYqKerKzPQwf3ov0dLsvJp03q5RSSim1bzqyR3UgsB34h4gsE5GnRMTb/E4icoWILBaRxdu3b+/AOJ0vJyeFo4/uTXJyImVl9SQnJ1pdSAn+f/buPD6q+t7/+OubmawTyJAEAojIFq0UBCEsol7FqlAFBLXVFlxaixas2utVL72K1mpr/dXr0qqxuF1BulgVHm4FFZQKVjYFlUUjqC0QgSQEyD7L9/fHJCHsBDLzPYH38/HII3O+OZnzZpjJmc98lwPDh3dl5Mju+HyGkpJafD7DyJHdGT68q7NM7dvv9bQ4YLt4Q11dZK8e3Wg01u7KmjWlvP76OqqqQuTkpFJVFeL119exZk2ps0yaNysiIiLSfPFc9dcPDABusNYuNsY8AkwBpjbdyVo7DZgGUFBQ4GYCYBzFilXv9Jwcf3xbLr74RM4663h27qyjTZsUcnLSnRaFxx/flqQk2Lmzjro6S0qKoU2bFI47rq2zTD4fjfNA92yXmOrqfRek+2tPhEWLNpKdnUZ6up9w2NK2bQrJyUksWrTRWaami6oBjd+Liso99bdBRERExEvi2aO6AdhgrV1cv/0iscJVHMrPD+LzJdG9e5ChQzvTvXtsOz8/6CzTued2JSXFR7duWQwc2J5u3bJISfFx7rnuennbtYsNFTUmVpw2zJ1taJfYMO2Gx8fv3/U47blaciLt3FlL27YpBALJZGenEggk07ZtCjt31jrL5MVF1URERES8Lm6FqrX2G+DfxpiT6pu+A6yO1/Hk0HhxOPKFF/bi6qv7kJGRTHFxFRkZyVx9dR8uvLCXs0yDB3eic+cM/P5Yz6rfD507ZzB4cCcneXJzU/fqzfX5Yu2uZGenkpoam5dqbex7amqs3ZWuXdtiTBI+n6GuLorPZzAmia5d3fXOe3FRNRERERGvi+fQX4AbgJnGmBRgPfCjOB9PDoHXhiPn5wcpK+vImWcev9uiUy57eSdO7Meddy6kV692tGuXwrZtdWzbVsPEif2c5Lnmmr48+OByMjJ8pKX5qKmJUFMT4Zpr+jrJ0zRTWpp3MjX8vyUnp5Gbm8a2bXVUVNRx222DnWXy4qJqIiIiIl4X10LVWrsCKIjnMaT1a+jlLSoqp7Q0tipqnz65Tnt5x47NB+DJJ1fy9dc76dw5k//8z4LG9kT77W/PBuDppz9hx44QbdokM3ly/8Z2ZYrx2v8bePP5LSIiIuJ1xrqcULaHgoICu2zZMtcxRETkKGCMWW6t1YelR0jnZhERaSnNOTfHczElERERERERkWZToSoiIiIiIiKeokJVREREREREPEWFqoiIiIiIiHiKClURERERERHxFBWqIiIiIiIi4ikqVEVERERERMRTVKiKiIiIiIiIp6hQFREREREREU9RoSoiIiIiIiKeokJVREREREREPEWFqoiIiIiIiHiKClUREZGjgDFmpDHmM2PMF8aYKfvZ5/vGmNXGmFXGmD8lOqOIiMih8rsOICIiIkfGGOMDHgPOAzYAS40xr1hrVzfZJx/4BXC6tXabMaaDm7QiIiIHpx5VERGR1m8w8IW1dr21tg74C3DRHvtMBB6z1m4DsNZuSXBGERGRQ6ZCVUREpPU7Dvh3k+0N9W1NnQicaIxZZIz5wBgzcn93Zoy51hizzBizbOvWrXGIKyIicmAqVEVERI4NfiAfOBv4AfCkMSa4rx2ttdOstQXW2oL27dsnMKKIiEiMClUREZHWbyNwfJPtLvVtTW0AXrHWhqy1XwKfEytcRUREPEeFqoiISOu3FMg3xnQ3xqQAlwOv7LHPbGK9qRhjcokNBV6fyJAiIiKHSoWqiIhIK2etDQM/A+YCa4AXrLWrjDG/MsaMqd9tLlBqjFkNvAPcaq0tdZNYRETkwHR5GhERkaOAtfYN4I092u5sctsCN9d/iYiIeJp6VEVERERERMRTVKiKiIiIiIiIp6hQFREREREREU9RoSoiIiIiIiKeokJVREREREREPCWuq/4aY74CdgIRIGytLYjn8URERERERKT1S8TlaYZba0sScBwRERERERE5Cmjor4iIiIiIiHhKvHtULfCmMcYCf7TWTovz8URERFolY8wnxM6b+2StPSWBcURERJyKd6F6hrV2ozGmA/CWMWattfYfTXcwxlwLXAvQtWvXOMcRERHxrFH136+v/z6j/vt4B1lERESciuvQX2vtxvrvW4BZwOB97DPNWltgrS1o3759POOIiIh4lrX2a2vt18B51trbrLWf1H9NAc53nU9ERCSR4laoGmMCxpg2DbeJnWQ/jdfxREREjhLGGHN6k41haE0JERE5xsRz6G8eMMsY03CcP1lr58TxeCIiIkeDa4BnjDFZ9dvlwI8d5hEREUm4uBWq1tr1QL943b+IiMjRyFq7HOjXUKhaa7c7jiQiIpJwGkokIiLiIcaYPGPM08BfrLXbjTG9jTHXuM4lIiKSSCpURUREvOX/gLlA5/rtz4GfO0sjIiLigApVERERb8m11r4ARAGstWEg4jaSiIhIYqlQFRER8ZZKY0wOYAGMMUMBzVMVEZFjSjxX/RUREZHmuxl4BehpjFkEtAcudRtJREQksVSoioiIeIQxJglIA84CTgIM8Jm1NuQ0mIiISIKpUBUREfEIa23UGPOYtfZUYJXrPCIiIq5ojqqIiIi3zDPGXGKMMa6DiIiIuKJCVURExFuuA/4G1BpjdhhjdhpjdrgOJSIikkga+isiIuIh1to2rjOIiIi4dsBC1RgzGvjYWvt1/fadwCXA18BN1tov4x9RRETk2GKMaQfkE1tYCQBr7T/cJRIREUmsg/Wo/hoYCmCMGQVMAH4AnAo8AYyIazoREZFjjDHmJ8BNQBdgBbHz8D+Bc1zmEhERSaSDzVG11tqq+tsXA09ba5dba58idl03ERERaVk3AYOAr621w4l9OFzuNpKIiEhiHaxQNcaYzPrrun0HmNfkZ2n7+R0RERE5fDXW2hoAY0yqtXYtsWuqioiIHDMONvT3YWLDjnYAa6y1ywCMMacCxXHOJiIicizaYIwJArOBt4wx24itDSEiInLMOGChaq19xhgzF+gArGzyo2LgR/EMJiIiciyy1o6rv/lLY8w7QBYwx2EkERGRhDvYqr8nAOXW2o3128OBscQ+2X00/vFERESOLcaYrk02G1bX7wj8y0EcERERJw42R/UFIABgjOlP7ALk/wL6AY/HN5qIiMgx6XXgtfrv84D1wN+dJhIREUmwg81RTbfWbqq/PQF4xlr7v/WLK62IbzQREZFjj7W2b9NtY8wAYLKjOCIiIk4cdNXfJrfPoX7VX2ttNG6JREREpJG19kNgiOscIiIiiXSwHtX5xpgXiC2e1A6YD2CM6QTUxTmbiIjIMccYc3OTzSRgALBpP7uLiIgclQ5WqP4cuAzoBJxhrQ3Vt3cEbo9nMBERkWNUmya3w8Tmqr7kKIuIiIgTB7s8jQX+so8frQR+EJdEIiIixzBr7d2uM4iIiLh2sMvTtAWuB44DXgHeAn4G/BexYnVmvAOKiIgcS4wxrwJ2fz+31o5JYBwREREnDjb0dwawDfgn8BPgf4gtsDTWWqtVf0VERFreemJTbJ6v3/4BsBmY7SyRiIhIgh2sUO3RsEy+MeYpYosqdbXW1sQ9mYiIyLHpdGttQZPtV40xy6y1/+kskYiISIId7PI0DYsnYa2NABtUpIqIiMRVwBjTo2HDGNMdCDjMIyIiknAH61HtZ4zZUX/bAOn124bYWktt45pORETk2POfwLvGmPXEzrcnANe6jSQiIpJYB1v113ekBzDG+IBlwEZr7agjvT8REZGjmbV2jjEmH/hWfdNaa22ty0wiIiKJdrChvy3hJmBNAo4jIiLS6hljvgekWGtXAqOBPxtjBjiOJSIiklBxLVSNMV2AC4Gn4nkcERGRo8hUa+1OY8wZwHeAp4FCx5lEREQSKt49qg8DtwHROB9HRETkaBGp/34h8KS19nUgxWEeERGRhItboWqMGQVssdYuP8h+1xpjlhljlm3dujVecURERFqLjcaYPwKXAW8YY1JJzFQdERERz4jnie90YIwx5ivgL8A5xpjn99zJWjvNWltgrS1o3759HOOIiIi0Ct8H5gIjrLXlQDZwq9tIIiIiiXWwy9McNmvtL4BfABhjzgZusdZOiNfxREREjgbW2irg5SbbxUCxu0QiIiKJp6FEIiIiIiIi4ilx61Ftylr7LvBuIo4lIiIiIiIirZt6VEVERDzMGJNkjBnvOoeIiEgiqVAVERHxAGNMW2PML4wxjxpjzjcxNwDriS2wdLDfH2mM+cwY84UxZsoB9rvEGGONMQUtmV9ERKQlJWTor4iIiBzUDGAb8E/gJ8D/AAYYa61dcaBfNMb4gMeA84ANwFJjzCvW2tV77NcGuAlY3PLxRUREWo4KVREREW/oYa3tC2CMeYrYSr9drbU1h/C7g4EvrLXr63//L8BFwOo99rsHuB9d7kZERDxOQ39FRES8IdRww1obATYcYpEKcBzw7ybbG+rbGhljBgDHW2tfP9idGWOuNcYsM8Ys27p16yFGEBERaTnqURUREfGGfsaYHfW3DZBev20Aa61te7h3bIxJAh4Erj6U/a2104BpAAUFBfZwjysiInK4VKiKiIh4gLXWdwS/vhE4vsl2l/q2Bm2APsC7xhiAjsArxpgx1tplR3BcERGRuNDQXxERkdZvKZBvjOlujEkBLgdeafihtXa7tTbXWtvNWtsN+ABQkSoiIp6lQlVERKSVs9aGgZ8Bc4E1wAvW2lXGmF8ZY8a4TSciItJ8GvorIiJyFLDWvgG8sUfbnfvZ9+xEZBIRETlc6lEVERERERERT1GhKiIiIiIiIp6iQlVEREREREQ8RYWqiIiIiIiIeIoKVREREREREfEUFaoiIiIiIiLiKSpURURERERExFNUqIqIiIiIiIinqFAVERERERERT1GhKiIiIiIiIp6iQlVEREREREQ8RYWqiIiIiIiIeIoKVREREREREfEUFaoiIiIiIiLiKSpURURERERExFNUqIqIiIiIiIinqFAVERERERERT1GhKiIiIiIiIp4St0LVGJNmjFlijFlpjFlljLk7XscSERERERGRo4c/jvddC5xjra0wxiQDC40xf7fWfhDHY4qIiIiIiEgrF7dC1VprgYr6zeT6Lxuv44mIiIiIiMjRIa5zVI0xPmPMCmAL8Ja1dvE+9rnWGLPMGLNs69at8YwjIiIiIiIirUBcC1VrbcRa2x/oAgw2xvTZxz7TrLUF1tqC9u3bxzOOiIiIiIiItAIJWfXXWlsOvAOMTMTxREREREREpPWK2xxVY0x7IGStLTfGpAPnAffH63giIiIirVVZWTVFReWUl9cQDKaRnx8kOzvdaaYZM1ZRWLiCzZsrycsLMGlSf6644ttOM40b9zKvvrqeSAR8Phg9ugezZl3sLM/gwdNZunRL4/agQR1YsuRKZ3nAe48RePO55MXXnOwunj2qnYB3jDEfA0uJzVF9LY7HExEREWl1ysqqWby4mNraMDk5adTWhlm8uJiysmpnmWbMWMXUqQuprKyjS5cAlZV1TJ26kBkzVjnLNG7cy8yeHSvAkpIgEoHZs9czbtzLTvLsWaQCLF26hcGDpzvJA957jMCbzyUvvuZkb3ErVK21H1trT7XWnmKt7WOt/VW8jiUiIiLSWhUVlRMIJBMIJGOMabxdVFTuLFNh4QqyslLIzU3H7/eRm5tOVlYKhYUrnGV69dX1AKSmJpGcnERqatJu7YnWUKQas+urabsLXnuMwJvPJS++5mRv8byOqoiIiIjneG3IX3l5DR98sJGXXiqitLSanJx0Lrkkn6FDj3OWafPmSsrKqvj449LGtmDQT3Z2hrNMkUjse21tdJ/trlgPXXzRi49RbLhvOtu31xIOW/x+QyDgZ/PmSmeZystryMlJ260tI8NPaWmNo0SyLwlZTElERETEC7w45G/Roo0UFq6kpiZMhw7p1NSEKSxcyaJFG51lKimporw8vFtbeXmYkpIqR4mktcrOTqe4uIpoNFakRqOW4uIqpx8OBYNpVFXt/vyuqgoTDKbt5zfEBRWqIiIicszw4pC/OXO+JC3NT2ZmKn6/j8zMVNLS/MyZ86WzTDt2hJvVLt6QlZXcrPZEuOCC7lRVhdm5M0Q0atm5M0RVVZgLLujuLFN+fpDKyhCVlSGstY238/ODzjLJ3lSoioiIyDGjvLyGjIzdZz5lZPgpL3c35K+0tJrOnTNISoJQKEpSEnTunEFpqRZ2kebJyUknGNz9+R0M+snJcdd7OWzYcUye3J/09GS2bKkmPT2ZyZP7M2yYu6Ht2dnpDBnSidTU2HDf1FQ/Q4Z00qq/HqM5qiIiInLMaBjyFwjs6mFyPeQvLy+2Empu7q43ySUl1eTlBZxl8vli8xobFuOB2LxHn89ZpEYNixaB2/mhXnyM8vICZGYmM2DA7s+lQCDFWaZgMI3hw7syalTPxrbKyhCpqW7LkFixqsLUy9SjKiIiIscMLw75mzSpP9u311FSUk04HKGkpJrt2+uYNKm/s0yjR/cAYoVXKBRtXJynod2FQYM6ALHitOGraXuiefEx8uJzyYuvOWkdVKiKiIjIMcOLQ/6uuOLb3HPPGQQCKWzYUEkgkMI995zBFVd821mmWbMuZuzYHvh8EI3Geg/Hju3BrFkXO8u0ZMmVexWlgwZ1YMmSK53k8eJj5MXnkhdfc9I6GOuhNbULCgrssmXLXMcQEZGjgDFmubW2wHWO1k7nZhERaSnNOTerR1VEREREREQ8RYWqiIiIiIiIeIoKVREREREREfEUFaoiIiIiIiLiKSpURURERERExFNUqIqIiIiIiIinqFAVERERERERT/G7DiAiIiJyrCsrq6aoqJzy8hqCwTTy84NkZ6c7zTRjxioKC1eweXMleXkBJk3qzxVXfNtppquueo0XXviMmhpLWprh+98/ieeeG6U8TXjx/82Lz2/xPvWoioiIiDhUVlbN4sXF1NaGyclJo7Y2zOLFxZSVVTvLNGPGKqZOXUhlZR1dugSorKxj6tSFzJixylmmq656jenT1xIKWVJTIRSyTJ++lquuek156nnx/82Lz29pHVSoioiIiDhUVFROIJBMIJCMMabxdlFRubNMhYUryMpKITc3Hb/fR25uOllZKRQWrnCW6YUXPsPng/R0P8nJftLT/fh8sXblifHi/5sXn9/SOqhQFREREXGovLyGjIzdZ2NlZPgpL69xlAg2b64kGEzZrS0YTGHz5kpHiaCmxuLfY9Ka3x9rV54YL/6/efH5La2D5qiKiIjIMcVrc/iCwTQmTHiVzz7b0dh20kltee650c4y5eUF+Mc/ivdqP+20Tg7SxKSlGWpqLLW14b3alScmLy9ASUkN6el+wmGL32+org6TlxdwlikYTKOqKkwgkNzYVlUVJhhMc5ZJWgf1qIqIiMgxw4tz+CZNmrtbkQrw2Wc7mDRprqNE8OmnW5rVngjJyft+27q/9ng7+eScZrUnwhVX9Gbr1hq2bavFGMu2bbVs3VrDFVf0dpYpPz9IZWWIysoQ1trG2/n5QWeZpHVQoSoiIiLHDC/O4fvoo5JmtSfCzp2RZrUngtcybd9eR2rq7m2pqbF2VwYM6MjEiacQCKSwdWsNgUAKEyeewoABHZ1lys5OZ8iQTqSm+iktrSE11c+QIZ206q8c1FE19NeLS1/Pnl3Ek0+uZNOmCjp3zmTixH6MHZvvNNOUKe/y9NOfsHNniDZtkrnmmr789rdnO81UWPgRjz++gpKSKnJzM5g8uT+TJp3qNNO6ddtYsGADxcUVdOqUyVlndaFnz3ZOM4mItDZeOzfHhvums2NHLaGQJTnZEAj4nc7ha2CajBi17qY5yiEqK6uha9e2+P2+xrZwOMLWre7mXpaX1zBuXC8uvnjXe01rLaWlbueDxopVFabSPEdNoVpWVs3bb39NWVk1oVCU5OQkvv56O+eee4KzE+Ls2UVcf/1bbN9eSzRqKSraxooVsSEzrorVKVPe5f77lzVu19bWNm67KlYLCz/ixhvnEa6f4vHNN9XceOM8AGfF6rp12/jpT9/iww83U10dIj09mQED8njiifOcFasqnEWktWm4LEUgkExOTmye2uLFxU57U3Jy0igurqZduxSSkw2RiKWkpJr27TVfTponOzuNioowweCuQrWiIkx2trvnkuaDytHkqBn6u2zZZr76agc+XxLBrBR8viS++moHy5ZtdpbprrsWsnlzFbW1EWpro9TWRti8uYq77lroLNNDDy1rVnsi3HbbO41FaoNwONbuypQpC3j77X9RVlZLdXWUsrJa3n77X0yZssBJnnXrtnHaaTO55pq53HHHIq65Zi6nnTaTdeu2OcnTYPnyb7j99ve4+uo3uP3291i+/BuneUSOZcaYkcaYz4wxXxhjpuzj5zcbY1YbYz42xswzxpwQ70xevCzFBRf0oLo6REVFiEgkSkVFiOrqEBdc0MNZpkGDOgCxXtSGr6btLnTsuO8PEvbXngh9+2Y3qz3eJk3qT2VliPLyWsLhCOXltVRWhpg0qb+TPKD5oHJ0OWoK1TVrSgkGU+i1diHn/+ZHtKGWYDCFNWtKHWYqIxoFYww+n8EYQzQaa3elbj/TJvbXnggVFdFmtSfCiy9+0az2ePuP//jzXkOJtm6t4T/+489O8kCsSL3hhnk89dRKZs/+nKeeWskNN8xzXqw29ODMnfulLiguxwxjjA94DPgu0Bv4gTFmz9VTPgIKrLWnAC8C/y/eubx4WYrTTjuOyZP7k5bmZ8uWGtLS/Eye3J/TTjvOWaYlS67cqygdNKgDS5Zc6SgRFBdfv1dR2rFjOsXF1ztKBB9//OO9itK+fbP5+OMfO8lzyy2DmTr1NNLT/WzdGltpd+rU07jllsFO8oDmg8rR5agZ+ttU+6IVnPXwjbw++feA76D7x0skYrE2Vqga0/ApqSUS0cQTaZ5Nm6qa1Z4Iv/zlQpYuLSYSof55HqasrJhf/nIhr756qZNMXpwCIJIgg4EvrLXrAYwxfwEuAlY37GCtbTpM5QNgQrxDeXEYYjCYxtlnd+XCC3s2tlVWhkhNdfuWyGVRuj8ui9L9cVWU7s8ttwx2Wpjui+aDytHiqClUe/fO5qOPtvDFt84g5Se/4vRpt3PmE1P41++mOcuUmmqorrZYazHGYOvH8qSmuru+lkhLmT//37sN2bY2NmR7/vx/O8u0bNlm/vSnNSxdWkxFRR2ZmSkMGtSJYDCN88/v5iyX1xaTkaPScUDTF98GYMgB9r8G+HtcExEbhrh4cexanBkZfqqqwlRWhujTJzfeh25VmUREZG9xG/prjDneGPNO/XyYVcaYm+J1LICBAztywglZRCKWD/uM4P1LbuakT9/hzL/+1tnSeX37diA5OdazWlsbJRKxJCfH2kVau6qqfV8OYH/tiTBt2grmzfuaUChCMJhKKBRh3ryvmTbN3WUnGoYi19aGyclJo7Y2rCHJ4pQxZgJQAPzuAPtca4xZZoxZtnXr1sM+lheHIXoxk4iI7C2ePaph4L+stR8aY9oAy40xb1lrVx/sFw9HdnY65513QmOvhW/wLVR3SSL9kQegSye45554HPaAJk/uz/33L+GbbyobhyF27Bhg8mR3k+xFjmYLF24iJcVHIJACQCCQQiRSx8KFm5xlarqYTCxTcmO7hmZJC9oIHN9ku0t9226MMecCtwNnWWtr93dn1tppwDSAgoKCI/q014vDEL2YSUREdhe3QtVaWwwU19/eaYxZQ2xoUlwKVdjHiWfw/4PKcrj3XmjfHm68MV6H3qfRo3uxfXsdS5d+w7Zt1bRrl86gQR0ZPbpXQnM0NXx4Z955Z+837cOHd3aQRqRl1daGSE311Q+3jw2mSE421NaGnGUqL68hJ2f3+XgZGX7n17STo85SIN8Y051YgXo58MOmOxhjTgX+CIy01m5JfEQREZFDl5A5qsaYbsCpwOJ9/Oxa4FqArl27tvSBobAQSkvhppsgJwfGj2/ZYxxAdnY6Eyb0ZsiQzp6Zm3bHHaezefPblJTUUFcXISXFR25uGnfccbqzTCeckMnXX1fss12kOTp2DFBWFisAw2Hw+WLFaseOAWeZvLiYjBx9rLVhY8zPgLnEVhF8xlq7yhjzK2CZtfYVYkN9M4G/GWMA/mWtHeMstIiIyAHE/fI0xphM4CXg59baHXv+3Fo7zVpbYK0taN++fcsH8PvhT3+C4cPh6qvh73FfO2I3DXNhRozo7ok5MOeccwJ/+MO5fPe73SkoyOO7mN6WkQAAIABJREFU3+3OH/5wLuecE/fL6e3XvHmX0alTxm5tnTplMG/eZY4SwY037nt49v7a4+3kk/d9/bP9tSfCsGEdm9WeCDfeOJBIxBIKxeaDh0KxVbZvvHGgs0y6pp0kirX2DWvtidbantbaX9e33VlfpGKtPddam2et7V//pSJVREQ8y9g4LjRkjEkGXgPmWmsfPNj+BQUFdtmyZfEJs2NHrFhdswbefhuGDYvPceSwrFu3jQULNlBcXEGnTpmcdVYXevZs5zTTTTe9zXPPraKyMkQgkMxVV32bRx4511me3r2fYs2a8sbtk08Osnr1T5zlATj99Od5//1d100dNqwjixbF/YoXB1RY+BGPP76CkpIqcnMzmDy5P5Mmneo0k1b9dcMYs9xaW+A6R2sX13OziIgcU5pzbo5boWpi44qeA8qstT8/lN+J+8lwyxY44wzYuhXeew/69InfsURExCkVqi1DhaqIiLSU5pyb4zn093TgCuAcY8yK+q8L4ni8g+vQAd58EzIyYMQI+Oorp3FERERERERkb/Fc9XchYOJ1/4etWzeYOxf+4z/gvPNg0aJYASsiIiIiIiKeEPfFlDypTx947TXYuBFGjozNXxURERERERFPODYLVYgtpvTSS/DJJ3DRRVCjaxqKiIiIiIh4wbFbqAJ897vwf/8H774LP/xh7MKLIiIiIiIi4tSxXagCjB8PjzwCs2bBT38Kcbxcj4iIiIiIiBxc3BZTalVuvDF2yZp774X27eG++1wnEhEREREROWapUG3wq1/FitXf/jZWrN58s+tEIiIiIiIixyQVqg2Mgcceg9JS+K//gtxcuPJK16lERERERESOOSpUm/L54PnnobwcfvxjaNcORo8+ortct24bCxZsoLi4gk6dMjnrrC707NmuhQIfnptuepvnnltFZWWIQCCZq676No88cq7TTL16/ZF163Y2bvfs2YYvvrjOYSLIy3uULVt2rQbdoUMamzf/zFmewYOns3TplsbtQYM6sGSJ2w9TCgs/4vHHV1BSUkVubgaTJ/dn0qRTnWYqK6umqKic8vIagsE08vODZGenO80kIiIiIs2jxZT2lJoKL78MAwbA978P77132He1bt02Zs5cTWVlHccfn0llZR0zZ65m3bptLRi4eW666W0ee2wF1dUhUlIM1dUhHntsBTfd9LazTHsWqQDr1u2kV68/Okq0d5EKsGVLDXl5jzrJs2eRCrB06RYGD57uJA/EitQpUxZQVFRGWVkNRUVlTJmygMLCj5xlKiur5u23v+ajj77h88/L+Oijb3j77a8pK6t2lklEREREmk89qvvSpg288QaccUasR3XBAujXr9l3s2DBBtq1S6NduzSAxu8LFmxw1qv67LOr6m8lUVcXJSkpCYjy7LOrnPWq7lmkHqw9EfYsUg/WHm97FqkHa0+E3/zmA3bs2P2STrW1YX7zmw+c9aouW7aZ++//gI8/LiEcBr8fTjkll2AwjfPP7+YkE3hzZIWIiIiIl6lHdX9yc+HNNyEzE0aOhPXrm30XxcUVZGWl7NaWlZVCcXFFS6VstoqKEJEIWGvx+ZKw1hKJxNpFmmPDhspmtSfC7bcv4MMPSxoviRwOw4cflnD77QucZVq3bhtPPrmSVau2Ul5ew6pVW3nyyZVOR1aIiIiIeJ16VA+ka9dYsXrmmXDeebBoEXTseMi/3qlTJtu31zX2pAJs315Hp06Z8Uh7SJKSIBKBpCRTv20AS5I+spCjwLJlW5vVngivv76eJUu+4bPPtlFZWUcgkMJJJ7Wjc+c23HjjQGe5NJdXREREvEzlycH07g2vvw7ffBPrWS0vP+RfPeusLmzbVsO2bTVEo9HG22ed1SWOgQ/suOMCANTVRQmHo9TVRXdrF5GWNWtWEcuXFxOJRAgGU4lEIixfXsysWUXOMpWVVbN4cTG1tWFyctKorQ2zeHGx5vKKiIiIZ6hQPRRDh8YWWFq9GsaMgepDezPXs2c7xo/vTSCQwr//XUEgkML48b2dzk275JKT6NYt0KRnFbp1C3DJJSc5yyRyNFu7tpTkZD/p6ckYY0hPTyY52c/ataXOMhUVlRMIJBMIxDI13C4qOvQP4kRERETiSUN/D9WIETB9Ovzwh3D55fDSS7GVWg6iZ892nlo0Zfz43vz73zsYPNiQnJxEKBQlGrWMH9/bWab8/DYUFe29cFJ+fhsHaURaViRi8fkgHI7i9xvC4dh2JGKdZSovryEnJ223towMP6WlbhYLExEREdmTelSb4/LL4Q9/gFdegYkTwbp7o3m4Bg7syJQpQ+nVKxu/30evXtlMmTKUgQMPfe5tS3v00RH06rV7UdqrVxsefXSEo0SQn5/VrHaR/TnhhCz8fj9JSVBbGyEpCfx+Pyec4O65FAymUVW1+4rNVVVhgsG0/fyGiIiISGKpR7W5rr8etm6Fu++OrQz8u9+5TtRsAwd2dFqY7qmgII9f//osysqqCYWiJCcnkZ2dTkFBnrNMU6cO4+ab36G6OkQ0GhsinZ6ezNSpw5zkGTu2B7Nn773y9NixPRykiRk0qMM+L48zaFAHB2livPg43XjjAP7nf94jOTmJtm1TqK6OEI1GufHGAc4y5ecHWby4GIj1pFZVhamsDNGnT66zTCIiIiJNGeuhXsGCggK7bNky1zEOzlq44QZ47DG4/3647TbXiVo9L65AOmPGKgoLV7B5cyV5eQEmTerPFVd821meceNe5tVX1xOJgM8Ho0f3YNasi53lARg8ePpuxeqgQR1YsuRKh4m8+Th57bkE3nzNtTRjzHJrbYHrHK1dqzk3i4iI5zXn3KxC9XBFo7H5qn/9Kzz9NPz4x64TiYhIEypUW0arOjeLiIinNefcrKG/hyspKba40rZtsfmq2dkwdqzrVCIiIiIiIq2eFlM6EikpsdV/Bw2KLbT07ruuE4mIiIiIiLR6KlSPVGYmvP469OgRu8bqRx+5TiQiIiIiItKqqVBtCTk5MHcuBIMwciQUFblOJCIiIiIi0mqpUG0pxx8Pb74JkQicfz5s2uQ6kYiIiIiISKukxZRa0re+BX//OwwfDiNGwD/+Ae3auU4lIiIiTXjx8kxevIyVFy/35bXLonnxMfLic8mLrznxPvWotrRBg2D2bPj8cxg9mrINpSxeXMzcuV+yeHExZWXVrhMyY8Yqhg2bSc+e0xg2bCYzZqxyHYlx417G738AYx7A73+AceNedh2J009/HmMeaPw6/fTnnea57ro5ZGY+hM/3AJmZD3HddXOc5gFYvvwbbr/9Pa6++g1uv/09li//xnUkEZEDKiurZvHiYmprw+TkpFFbG3Z+fp4xYxVTpy6ksrKOLl0CVFbWMXXqQqfn53HjXmb27FgBlpQUGzA2e/Z6p+fnPYtUgKVLtzB48HQnebz4GHnxueTF15y0DrqOary8+CL2+9+nuP+ZvHbN76iN+khOTiI7O51zzz3B2adIM2as4vrr51JREcVaMAYyM5N47LERzj5ta/hDv6exY919Knn66c/z/vt7F13DhnVk0aIJCc9z3XVzmDbt073ar722D3/848iE54FYkXrDDfNYv76cmpoQaWnJ9OgR5A9/+A4DB3Z0kgn0qa3souuotoyj6twMjW+YA4HkxrbKyhCpqX6GDOnkJNOwYTOprKwjN3fX36qSkmoCgRTef3+8k0x+/wNEIpCauqtPo7Y2is8H4fAtTjIZ80D9911tDW9jrU18Ji8+Rl58LnnxNSfu6DqqXnDppay54V56//52hs/4FR9M+g01dZavvtrBsmWbOf/8bk5iTZmygJ07oxgTG6ISjcLOnVGmTFngrFDdV5F6oPZE2FeReqD2eNtXkdrQ7qpQvfvuRfzzn8WN29u3h9m8uZq7717EK69c4iRTWVk13boVsnNntLGtTZskvvpqktNidfbsIp58ciWbNlXQuXMmEyf2Y+zYfGd5RBLJax8elZfX8MEHG3nppSJKS6vJyUnnkkvyGTr0OGeZNm+upKysio8/Lm1sCwb9ZGdnOMsUicS+19ZG99nuklf6WLz4GMWG+6azfXst4bDF7zcEAn42b650lqm8vIacnLTd2jIy/JSW1jhKJK1F3ApVY8wzwChgi7W2T7yO42Vv9RhB5ZiNDHrlcWwgg2Xjp2CDKaxZU+qsUN20qQpjICVl16d/dXVRNm2qcpJHWq9XX/2yWe2J0L37E7sVqRD7IKZ79yfYvv0/nWSaPbuIm29+h4qKWsLhCBs37mTNmjIAp8Wq14oHOTo1DPkLBJLJyUmjqio25G/IkE7Onm+LFm1k2rSVtG2bQocO6VRWhiksXEkkAiNGdHeSqaSkih07wru1lZeHiUZ1bpbmyc5Op7i4inbtUvH7DdGopbi4itxcdx96BIOx137THtWqqjDBYNoBfkskvnNU/w9w09XjIR9d8BNWXfAjTpz/N0bdfjGd1y5xHWmvTyK98smkyJHasWPfH2Pvrz0R7rvvA0pKqvD7fWRlpeH3+ygpqeK++z5wlknzhSRRiorKCQSSCQSSMcY03i4qKneWac6cL0lL85OZmYrf7yMzM5W0ND9z5rj7kG3PIvVg7eINWVnJzWpPhAsu6E5VVZidO0NEo5adO0NUVYW54AI3H8IA5OcHqawMUVkZwlrbeDs/P+gsk7QOcetRtdb+wxjTLV733xr07p3NRx9t4f2xN7Gpz2kMfvYeRj10HZvOuwTGPwG5uQnPlJWVzPbtISKRKD6fIRKxje0i0vI+/3wbGRk+UlN9AKSm+ohGfXz++TZnmZoWD0Dj96KicoYMUa+qtBwvDvkrLa2mc+cM6uosoVAUvz+Jzp0z2LxZH9RI8+TkpGOMpbx81wcKsSHb7v6ODht2HElJhpdeKmLLlmqys9OZMKE3Q4d2dpYpOzudIUM6UVRUTmlpbBRPnz65CX2cQqEQGzZsoKZGw40TJS0tjS5dupCcfPg1hvM5qsaYa4FrAbp27eo4TcsaOLAjZWW1bNtWw9ouA/jqjhc4ff6znPz6M3DyAnjwQZgwYfdVAeLs5psLuPfefxIKQTgcK1KTk2PtItLyjNn3KIYEvuz34sXiQY5OXhzyl5cX2OdiM3l5AWeZfD72uyiPa/tauMgVrz1OeXkBMjOTGTBg74WLXAkG0xg+vCujRvVsbGtYuMilWLHqroDfsGEDbdq0oVu3bhiXJ+BjhLWW0tJSNmzYQPfuh9+b7/zyNNbaadbaAmttQfv27V3HaVHZ2emcd94JnHpqHieemM0pQ06g4zMPYz78EHr1giuvhPPPh3XrEpbpzjtP5447TqNz5wwCAT+dO2dwxx2nceedpycsw54uu2zf8/T2154I+/sbpr9t3paa2rz2ROjXrz3V1RGqq8NEo1Gqq8NUV0fo18/d37uG4qEp18WDHJ28OORv0qT+bN9eR0lJNeFwhJKSarZvr2PSpP7OMo0e3QOIFV2hULRxcZ6GdhcGDeoAxIrThq+m7S547XHy4nPJi685L6ipqSEnJ0dFaoIYY8jJyTniHmznherRrmG4w4gR3XctHtG3LyxaBI89BkuWQJ8+cN99EAolJNOdd57Oxo2Tqaj4ORs3TnZapAL85S8Xcdll+TSMDEhOjhWpf/nLRc4yRaO37FWUGhNrd2F/y+67WI6/QWnp9c1qT4RFiybg3+NDY78fJ5cUajB16jCOP74NEJsrBJbjj2/D1KnDnGXSGxlJlIZzYGpqrMe+4XIULodGXnHFt7nnnjMIBFLYsKGSQCCFe+45w9nK9wCzZl3M2LE9Glfj9/ncXqINYMmSK/cqSgcN6sCSJVc6SuS9x8mLzyUvvua8QkVqYrXE4x3X66jWz1F97VBX/T3artV2SDZtghtvhJdeihWs06bBaae5TiVySLy4cuzy5d/w8stFbNy4k+OOa8PFF+c7va4rwPz5XzN9+qrGTFde+W3OOecEp5m8+H/X0nQd1ZZxTJ6bReSosmbNGk4++WSnGXw+H3379sVai8/n49FHH2XYsOZ/aP3www9z7bXXkpFxZCs5v/vuuzzwwAO89tprR3Q/s2fP5sQTT6R37957/Wxfj7snrqNqjPkzcDaQa4zZANxlrX06XsdrtTp3hhdfhFdegeuvh9NPh0mT4De/gaws1+lEDsj1nJN9GTiwo/PCdE/nnHOC88J0T178vxMRETlapaens2LFCgDmzp3LL37xCxYsWNDs+3n44YeZMGHCEReqLWX27NmMGjVqn4XqkYrb0F9r7Q+stZ2stcnW2i4qUg9izBhYvTrWu1pYCL17x3pZXa9cICIiIiIiLWbHjh20a9eucft3v/sdgwYN4pRTTuGuu+4CoLKykgsvvJB+/frRp08f/vrXv/L73/+eTZs2MXz4cIYPH77X/U6ZMoXevXtzyimncMstselhV199NS+++GLjPpmZmbvluPDCCznppJP46U9/SjQaJRKJcPXVV9OnTx/69u3LQw89BMC6desYOXIkAwcO5Mwzz2Tt2rW8//77vPLKK9x6663079+fdS287o7zVX+liTZt4OGHYfx4uPZauPTSWAH76KNw/PGu04mIiIiItG4//znU92y2mP79Y+/hD6C6upr+/ftTU1NDcXEx8+fPB+DNN9+kqKiIJUuWYK1lzJgx/OMf/2Dr1q107tyZ119/HYDt27eTlZXFgw8+yDvvvEPuHpe5LC0tZdasWaxduxZjDOXlB79e9ZIlS1i9ejUnnHACI0eO5OWXX6Z79+5s3LiRTz/9FKDxfq699lqeeOIJ8vPzWbx4MZMnT2b+/PmMGTOGUaNGcemllzb7YTsYLabkRYMGwdKl8Lvfwdtvx3pXH3kktia7iIiIiIi0Kg1Df9euXcucOXO48sorsdby5ptv8uabb3LqqacyYMAA1q5dS1FREX379uWtt97iv//7v3nvvffIOsiUwKysLNLS0rjmmmt4+eWXD2lo8ODBg+nRowc+n48f/OAHLFy4kB49erB+/XpuuOEG5syZQ9u2bamoqOD999/ne9/7Hv379+e6666juLi4pR6a/VKPqlf5/XDLLXDJJTB5cuzTn5kzY4st9Xe37LmIiIiISKt1kJ7PRDjttNMoKSlh69atWGv5xS9+wXXXXbfXfh9++CFvvPEGd9xxB9/5zne4884793uffr+fJUuWMG/ePF588UUeffRR5s+fj9/vJxqNXcopGo1SV1fX+Dt7rsxrjKFdu3asXLmSuXPn8sQTT/DCCy/w8MMPEwwGG+fYJop6VL2ue3d44w3485/h66+hoABuvRUqK10nExERERGRZlq7di2RSIScnBxGjBjBM888Q0VFBQAbN25ky5YtbNq0iYyMDCZMmMCtt97Khx9+CECbNm3YuXPnXvdZUVHB9u3bueCCC3jooYdYuXIlAN26dWP58uUAvPLKK4SaXA5zyZIlfPnll0SjUf76179yxhlnUFJSQjQa5ZJLLuHee+/lww8/pG3btnTv3p2//e1vAFhrG+9/f3lagnpUWwNj4PLL4fzz4b//Gx54ILZScGEhjBzpOp2IiIiIiBxAwxxViBV6zz33HD6fj/PPP581a9ZwWv3lKTMzM3n++ef54osvuPXWW0lKSiI5OZnCwkIgNld05MiRdO7cmXfeeafx/nfu3MlFF11ETU0N1loefPBBACZOnMhFF11Ev379GDlyJIFAoPF3Bg0axM9+9jO++OILhg8fzrhx4/jkk0/40Y9+1NgLe9999wEwc+ZMJk2axL333ksoFOLyyy+nX79+XH755UycOJHf//73vPjii/Ts2bPFHrO4Xke1uXSttkP03nuxxZbWro0VsA8/DHl5rlOJiHiKrqPaMnRuFpHWzgvXUT0WHel1VDX0tzU688zYamW//CW8/DJ861vw1FNQ/8mHiIiIiIhIa6ZCtbVKTYW77oKVK6FfP5g4Ec4+G9ascZ1MRERERETkiKhQbe2+9S145x14+mn49NNY0frLX0JtretkIiIilJVVs3hxMXPnfsnixcWUlVW7jsSMGasYNmwmPXtOY9iwmcyYscp1JMaNexm//wGMeQC//wHGjXvZdSQGD56OMQ80fg0ePN11JM89Tl58LnnxNSdyOFSoHg2MgR//ODZn9Xvfg7vvjhWsCxa4TiYiIsewhjfMtbVhcnLSqK0NO3/jPGPGKqZOXUhlZR1dugSorKxj6tSFTguMceNeZvbs9UQikJQUu2z67NnrnRZhgwdPZ+nSLbu1LV26xWmx6rXHyYvPJS++5kQOlwrVo0mHDrFrrc6ZA3V1saHA11wDZWWuk4mIyDGoqKicQCCZQCAZY0zj7aKicmeZCgtXkJWVQm5uOn6/j9zcdLKyUigsTOz1AZt69dX1AKSmJpGcnERqatJu7S40FKnG7Ppq2u6C1x4nLz6XvPiaEzlcujzN0WjEiNgw4Lvvhv/9X3jtNXjoIfjBD3adaURE5KhijBkJPAL4gKestb/d4+epwHRgIFAKXGat/SqemcrLa/jgg4289FIRpaXV5OSkc8kl+Qwdelw8D3tAmzdXUlZWxccflza2BYN+srMznGWKRGLfa2uj+2x3yUMXh/Dc47R5cyV5eels315LOGzx+w2BgJ/Nm91d6768vIacnLTd2jIy/JSW1jhKJHL4VKgerTIy4P774Yc/jF3KZvx4mD4dHn+cdbYdCxZsoLi4gk6dMjnrrC707NnOadzrrpvDzJlrqK6OkJ7uY/z4k/njH91eIzYv71G2bNn1h71DhzQ2b/6ZszyZmf9LZeWudwyBgKGi4r+c5QHo3fsp1qzZ9SntyScHWb36Jw4TQWHhRzz++ApKSqrIzc1g8uT+TJp0qtNMZWXVFBWVU15eQzCYRn5+kOzsdKeZ5OhijPEBjwHnARuApcaYV6y1q5vsdg2wzVrbyxhzOXA/cFk8cy1atJFp01bStm0KHTqkU1kZprBwJZEIjBjRPZ6H3q+Skip27Ajv1lZeHiYarXKSR1qv7Ox0iouraNcuFb/fEI1aiotj5x5XgsE0qqrCBALJjW1VVWGCwbQD/JaIN2no79GuXz94/334/e9h0SKi3+7Dl5Nup3pHFccfn0llZR0zZ65m3bptziJed90cpk37lOrqCMZAdXWEadM+5brr5jjLtGeRCrBlSw15eY86ybNnkQpQWWnJzPxfJ3lg7yIVYM2acnr3fspRoliROnnyPD79tJRvvqnm009LmTx5HoWFHznLVFZWzfe+N5szz5zJyJEvceaZM/ne92Y7ny+0bt02nnnmE37963/yzDOfOP0bIC1iMPCFtXa9tbYO+Atw0R77XAQ8V3/7ReA7xsR3mM2cOV+SluYnMzMVv99HZmYqaWl+5sz5Mp6HPaA9i9SDtYt3ZGUlN6s93i64oDtVVWF27gwRjVp27gxRVRXmggvcfAgDkJ8fpLIyRGVlCGtt4+38/KCzTBLj8/no378/ffr0YfTo0ZSXt77h2O+++y6jRo1K2PHUo3os8Pnghhtg3Dj+PeYqzn2rkIFr5/HZeT8knJrOjjrDuj+spOeYb0Fa2v6/UlNj99XCpk+PLTrg9xusjY1OrquzTJ++ylmv6p5F6sHa423PIvVg7YmwZ5F6sPZEmDx53n7bXfWqTpjwGvPnb2zcDoVg/vyNTJjwGm+88T0nmdat28ZNN81n2bJvqKoKkZGRTEFBRx555BynoyvU83xEjgP+3WR7AzBkf/tYa8PGmO1ADlCy550ZY64FrgXo2rXrYYcqLa2mc+cM6uosoVAUvz+Jzp0z2LxZC7tI8+XkpGOMpbx814cKsWHbbv5ODBt2HElJhpdeKmLLlmqys9OZMKE3Q4d2dpIHYr28Q4Z0oqionNLS2N/SPn1y9be0meJxPkpPT2fFitj85auuuorHHnuM22+/vSXiHhFrLdZakpK813+pQvVY0qULz19yL8POWMyQ5+9j6DO/3P3njxzCfSQnH7iYPYyv79Z8TC1+IjaFCElYC3VYojUm1hvcdCUHY2JL/cW5rT0VWMBiGr9H62+zY0fz71OOWX//+9fNak+Eu+5axDvv/IuMjGTatYsNE3vnnX9x112LeP75xH1S2lTDSpWBQDI5ObFMixcXM2RIJ73BcsBaOw2YBlBQUHDYn4jl5cVWQs3N3fV/WFJSTV5e4MhDHiafLzansWEhHojNeYzD57DN1vR04XpuqBcfp7y8AJmZyQwYsPvzKRBIcZInGExj+PCujBrVs7GtsjJEaqrbt9exYlV/Nw9XIs5Hp512Gh9//PF+f15cXMxll13Gjh07CIfDFBYWcuaZZ/Lss89y3333EQwG6devH6mpqTz66KNcffXVjBo1iksvvRSAzMxMKioqqKio4KKLLmLbtm2EQiHuvfdeLrroIr766itGjBjBkCFDWL58OW+88QafffYZd911F7W1tfTs2ZNnn32WzMxM5syZw89//nMyMjI444wzWuTff6hUqB5jOnXK5NO2Z1L84NmklZfgC9VSWbqTTH+UMed3gZqaI/uqqICSkv3/fB9n3sZF5UP7CHz64/F8OPbrgGsaZt15eHd6BIXzFmr3KpwbdWoy/HdfRfH+CuUj3PdrdrKv91EWA932MUT6SLMdwn18zq7e3L2ynTitxY93KPt+sndn1a7/v77P7bV/IrLd9slWbrNgwkmwI9YWjUThrwbW/vLQj9eCmZMrQpwetST5DJvHX4c5dzQQW8FSb7gOyUbg+CbbXerb9rXPBmOMH8gitqhS3Eya1J+pUxcCEAymUF5ex/btddxyy+B4HvaARo/uwezZ66mtjZKUBNHornZXBg3qwNKlW/Y6RQ4a1MFNILz5OHnt+ZSfH2Tx4mIgtmBRVVWYysoQffrkOskjLaPpyslA4/eWOh9FIhHmzZvHNddcs999/vSnPzFixAhuv/12IpEIVVVVFBcXc9ddd7F8+XKysrIYPnw4p545x1yjAAAOZElEQVR64NFiaWlpzJo1i7Zt21JSUsLQoUMZM2ZM/b+niOeee46hQ4dSUlLCvffey9tvv00gEOD+++/nwQcf5LbbbmPixInMnz+fXr16cdllcV3WYC8qVI8xZ53VhZkzVwNpZGXnsX17Hdsygowf3xviPeTP2ti4xz2K14sv+BMb1pWRToiUJEs0GivHup+QyZN/HBH7vT2/otG4tk2e/BZJ9WWhgcbvSUR58IGzE57nhcc+rM9DQ79uo+vG9Nv1+O7rMd/f/8WR7AvMe/aTvdoasl19dp8WP96h3MeSL9bslqOpEwd+q8WPdyj7rl31+W7NTbP1PfHEFj/eobRv+LiO5P/f3t0HV1XfeRx/f0JiIk+tVh6sdAUpCspTKCgpHQplqR3ZqeDYsQpohkq3zLogYteHznbcTlntqKW1dTpFa1O3LFqfZqk4Sx0Q6DiIJlSrgEvRhRpqJeKuuIgG5Lt/3APykJAbyL3nJPm8Zu7k3N89955vfrnJN997fr/fKRUlJR8XjgcUfLT/ACP69i1aHIe3fdi4l7JTSvgIiNLcPwVeqbJVXgAGSRpAriD9OnDVUfssA64B1gGXA6siCnvebubMC4DcZTzq6/fQp083brzxwkPtaXjiicuYNu1xfvvb3PU4u3TJFV9PPHFZajE9//zVx1y3dMyY3jz//NWpxZTFfsra+8nDbDumQq2cvHfvXkaOHMmOHTsYMmQIkydPbnbfMWPGMGvWLPbt28fUqVMZOXIkK1euZMKECfTq1QuAK664gi1btjT7GpAb1nvrrbeydu1aSkpK2LFjB2+99RYAZ599NmPHjgXgueeeY9OmTYwbNw6AxsZGqqqqePXVVxkwYACDBg0CYMaMGSxevPik+qE1XKh2MgMHnsb06eezZk09b7yRW/X3kkvOKc68NAlOOSV369nzUPP0H1zJLbesZsPr7x1Khuec04M5t0+Ai88rfFxNWHLzh00urNGzZyk/XHB90eO57t67mn3s739+YxEj+disXzYfU3VNOjHN+PfmY7pqaToxfU3NxxSPpRPT9efex86d79Oz5ymHzpTs3t1I795d2fLk7FRiei25QL1XqjwxyZzT64AV5C5P80BEbJT0PaA2IpYBvwD+TdJW4B1yxWzBzZx5QaqFaVPSLLaak2ZR2pws9lPW3k8eZtvxFGrl5INzVN9//30uvvhi7r33XubOndvkvuPHj2ft2rUsX76c6upqbrjhBnoe9r/z0UpLSzmQDHs4cOAAjY2NACxZsoSGhgbq6uooKyujf//+fPBBruDu1u3jKRgRweTJk1m6dOkRr3twTm1asjdr1gpu4MDTmDVrGN/5ThWzZg1L/dI0Eyf+DTfdVMWCBWP41reGs2DBGG66qYqJE098AY+TdccdX6S8/Mi28vJcexoqK5seRtRcezH06NH0RKXm2oshizFl8Wc3f/5oSkpK2Lt3Px9++BF79+6npKSE+fNHpxaTV6o8eRHxVEScGxEDI2Jh0vbdpEglIj6IiK9FxGcj4sKIeD3diM3MsqfQ+ahr167cc8893H333ezf3/Rq49u3b6dPnz7Mnj2ba6+9lg0bNnDRRRexZs0adu3axb59+3jkkUcO7d+/f3/q6uoAWLZsGfv25ebTvfvuu/Tu3ZuysjKeeeYZtm9ven2MsWPH8uyzz7J161YA9uzZw5YtWxg8eDDbtm3jtddeAzimkC00F6qWutNPP5Vp0wZx2WXnMnVq7uu0aYNSHTozZ04lixZNYujQT9G376kMHfopFi2alNrKsRs2VB9T2FRWnsGGDdWpxAOwe/f8YwrAHj26sHv3/JQiymZMWfzZzZlTycKFX6Bv3+6A6Nu3OwsXfiHV680eHEJXXp4bXlVeXuqFlMzMrOiKkY8qKysZPnx4s4Xf6tWrGTFiBJWVlTz88MPMmzePM888k9tuu42qqirGjRvHkCFDDu0/e/Zs1qxZw4gRI1i3bt2hs6XTp0+ntraWYcOG8eCDDzJ4cNPToHr16kVNTQ1XXnklw4cPPzTst6KigsWLFzNlyhRGjRpF797FnTevAk9PaZXRo0dHbW1t2mGYmVkHIKkuItI7TdxBODebWXu3efPmIwq7jqCmpoba2lp++tMmFrDMiKb6vTW52WdUzczMzMzMLFO8mJKZmZmZmVmRvfzyy8ycOfOItvLyctavX9/ic6urq6muri5QZNngQtXMzMzMzKzIhg0blvrKulnmob9mZmZmZtahZWldns6gLfrbhaqZmZmZmXVYFRUV7Nq1y8VqkUQEu3btoqLi5K49W9Chv5K+AvyY3MXH74+IOwp5PDMzMzMzs8P169eP+vp6Ghoa0g6l06ioqKBfv34n9RoFK1QldQHuBSYD9cALkpZFxKZCHdPMzMzMzOxwZWVlDBgwIO0wrJUKOfT3QmBrRLweEY3AQ8ClBTyemZmZmZmZdQCFLFTPAt447H590mZmZmZmZmbWrNQXU5L0TUm1kmo9btzMzMzMzMwKuZjSDuAzh93vl7QdISIWA4sBJDVI2l7AmNJ0BvB22kG0A+6nlrmP8uN+yk9H7qez0w6gI6irq3vbubnTcz+1zH2UH/dTfjpyP+Wdm1WoZZollQJbgEnkCtQXgKsiYmNBDphxkmojYnTacWSd+6ll7qP8uJ/y436yzszv//y4n1rmPsqP+yk/7qecgp1RjYj9kq4DVpC7PM0DnbVINTMzMzMzs/wV9DqqEfEU8FQhj2FmZmZmZmYdS+qLKXUii9MOoJ1wP7XMfZQf91N+3E/Wmfn9nx/3U8vcR/lxP+XH/UQB56iamZmZmZmZnQifUTUzMzMzM7NMcaFaYJI+I+kZSZskbZQ0L+2YskpSF0l/kPRk2rFklaRPSnpU0quSNkuqSjumLJI0P/l9e0XSUkkVaceUBZIekLRT0iuHtZ0u6WlJf0q+npZmjGbF4NycP+fmljk358e5uWnOzc1zoVp4+4EFEXE+MBb4B0nnpxxTVs0DNqcdRMb9GPjPiBgMjMD9dQxJZwFzgdERMZTcquNfTzeqzKgBvnJU283AyogYBKxM7pt1dM7N+XNubplzcwucm4+rBufmJrlQLbCIeDMiNiTb75H743VWulFlj6R+wBTg/rRjySpJnwDGA78AiIjGiPjfdKPKrFLg1OR6zl2Bv6QcTyZExFrgnaOaLwV+lWz/Cpha1KDMUuDcnB/n5pY5N7eKc3MTnJub50K1iCT1ByqB9elGkkk/Av4JOJB2IBk2AGgAfpkMw7pfUre0g8qaiNgB3AX8GXgTeDcifpduVJnWJyLeTLb/CvRJMxizYnNuPi7n5pY5N+fBubnVnJtxoVo0kroDjwHXR8TutOPJEkl/B+yMiLq0Y8m4UmAU8LOIqAT20EmHghxPMo/jUnL/PHwa6CZpRrpRtQ+RWwbeS8Fbp+Hc3Dzn5rw5N+fBufnEdebc7EK1CCSVkUuESyLi8bTjyaBxwFclbQMeAr4k6dfphpRJ9UB9RBz81P9RcsnRjvS3wH9HRENE7AMeBz6fckxZ9pakMwGSrztTjsesKJybW+TcnB/n5vw4N7eOczMuVAtOksjNW9gcET9MO54siohbIqJfRPQnN7F+VUT4U7ajRMRfgTcknZc0TQI2pRhSVv0ZGCupa/L7NwkvbHE8y4Brku1rgP9IMRazonBubplzc36cm/Pm3Nw6zs24UC2GccBMcp9EvpjcLkk7KGu3/hFYIumPwEjgX1OOJ3OST7UfBTYAL5P7O7c41aAyQtJSYB1wnqR6Sd8A7gAmS/oTuU+870gzRrMicW62tuTc3ALn5uY5NzdPuWHPZmZmZmZmZtngM6pmZmZmZmaWKS5UzczMzMzMLFNcqJqZmZmZmVmmuFA1MzMzMzOzTHGhamZmZmZmZpniQtWsjUm6XdJESVMl3dLMPrdJ2nHYZRFOaNnx5Bjnn1zEZmZmHZtzs1n740LVrO1dBDwHfBFYe5z9FkXEyOR28wkeayrQqmQoqfQEj2VmZtZeOTebtTMuVM3aiKQ7k4t9jyF34eZrgZ9J+m4rXuNzktZIqpO0QtKZSftsSS9IeknSY5K6Svo88FXgzuST34GSVksanTznDEnbku1qScskrQJWJm3fTl7zj5L+JWnrJml5cpxXJF3Rdj1kZmZWXM7NZu2XP70xayMR8W1JvwGuBm4AVkfEuOM8Zb6kGcn2TcAq4CfApRHRkCSihcAs4PGIuA9A0veBb0TETyQtA56MiEeTx44X4ihgeES8I+nLwCDgQkDAMknjgV7AXyJiSvJ6n2h9T5iZmWWDc7NZ++VC1axtjQJeAgYDm1vYd1FE3HXwjqShwFDg6SSpdQHeTB4emiTBTwLdgRUnENvTEfFOsv3l5PaH5H53csnx98Ddkn5ALsn+/gSOY2ZmliXOzWbtkAtVszYgaSRQA/QD3ga65pr1IlAVEXvzeRlgY0RUNfFYDTA1Il6SVA1MaOY19vPxkP6Kox7bc9Sxbo+InzfxvYwCLgG+L2llRHwvj9jNzMwyxbnZrH3zHFWzNhARL0bESGALuQUUVgEXJ4sx5JMIAf4L6CWpCkBSmaQLksd6AG9KKgOmH/ac95LHDtoGfC7Zvvw4x1oBzJLUPTnWWZJ6S/o08H5E/Bq4k9yn0GZmZu2Oc7NZ++YzqmZtRFIv4H8i4oCkwRGxqTXPj4hGSZcD9yTzT0qBHwEbgX8G1gMNydeDCfAh4D5Jc8klv7uA30j6JrD8OMf6naQhwLpkKNP/ATOAz5JbAOIAsA+Y05rvwczMLEucm83aL0VE2jGYmZmZmZmZHeKhv2ZmZmZmZpYpLlTNzMzMzMwsU1yompmZmZmZWaa4UDUzMzMzM7NMcaFqZmZmZmZmmeJC1czMzMzMzDLFhaqZmZmZmZlligtVMzMzMzMzy5T/B01VCUNTjNENAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 1152x432 with 2 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "fig = plt.figure(figsize = (16,6))\n",
    "ax = fig.add_subplot(1, 2, 1)\n",
    "\n",
    "ax.scatter(df.numb_features,df.RSS, alpha = .2, color = 'darkblue' )\n",
    "ax.set_xlabel('# Features')\n",
    "ax.set_ylabel('RSS')\n",
    "ax.set_title('RSS - Best subset selection')\n",
    "ax.plot(df.numb_features,df.min_RSS,color = 'r', label = 'Best subset')\n",
    "ax.legend()\n",
    "\n",
    "ax = fig.add_subplot(1, 2, 2)\n",
    "ax.scatter(df.numb_features,df.R_squared, alpha = .2, color = 'darkblue' )\n",
    "ax.plot(df.numb_features,df.max_R_squared,color = 'r', label = 'Best subset')\n",
    "ax.set_xlabel('# Features')\n",
    "ax.set_ylabel('R squared')\n",
    "ax.set_title('R_squared - Best subset selection')\n",
    "ax.legend()\n",
    "\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Forward Stepwise"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done   1 out of   1 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  11 out of  11 | elapsed:    0.1s finished\n",
      "\n",
      "[2019-06-02 18:55:43] Features: 1/1 -- score: -54293.17658429352"
     ]
    },
    {
     "data": {
      "text/plain": [
       "SequentialFeatureSelector(clone_estimator=True, cv=5,\n",
       "             estimator=LinearRegression(copy_X=True, fit_intercept=True, n_jobs=None,\n",
       "         normalize=False),\n",
       "             floating=False, forward=True, k_features=1, n_jobs=1,\n",
       "             pre_dispatch='2*n_jobs', scoring='neg_mean_squared_error',\n",
       "             verbose=2)"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# one variable\n",
    "ols = LinearRegression()\n",
    "sfs1 = sfs(ols,\n",
    "           k_features=1,\n",
    "           forward=True,\n",
    "           floating=False,\n",
    "           verbose=2,\n",
    "           scoring='neg_mean_squared_error',\n",
    "           cv=5)\n",
    "sfs1.fit(X, y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done   1 out of   1 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  11 out of  11 | elapsed:    0.1s finished\n",
      "\n",
      "[2019-06-02 18:55:46] Features: 1/2 -- score: -54293.17658429352[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done   1 out of   1 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  10 out of  10 | elapsed:    0.0s finished\n",
      "\n",
      "[2019-06-02 18:55:46] Features: 2/2 -- score: -26775.95426714271"
     ]
    },
    {
     "data": {
      "text/plain": [
       "SequentialFeatureSelector(clone_estimator=True, cv=5,\n",
       "             estimator=LinearRegression(copy_X=True, fit_intercept=True, n_jobs=None,\n",
       "         normalize=False),\n",
       "             floating=False, forward=True, k_features=2, n_jobs=1,\n",
       "             pre_dispatch='2*n_jobs', scoring='neg_mean_squared_error',\n",
       "             verbose=2)"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# two variables\n",
    "sfs2 = sfs(ols,\n",
    "           k_features=2,\n",
    "           forward=True,\n",
    "           floating=False,\n",
    "           verbose=2,\n",
    "           scoring='neg_mean_squared_error',\n",
    "           cv=5)\n",
    "sfs2.fit(X, y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done   1 out of   1 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  11 out of  11 | elapsed:    0.1s finished\n",
      "\n",
      "[2019-06-02 18:55:47] Features: 1/3 -- score: -54293.17658429352[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done   1 out of   1 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  10 out of  10 | elapsed:    0.0s finished\n",
      "\n",
      "[2019-06-02 18:55:47] Features: 2/3 -- score: -26775.95426714271[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done   1 out of   1 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done   9 out of   9 | elapsed:    0.0s finished\n",
      "\n",
      "[2019-06-02 18:55:47] Features: 3/3 -- score: -10851.452873513992"
     ]
    },
    {
     "data": {
      "text/plain": [
       "SequentialFeatureSelector(clone_estimator=True, cv=5,\n",
       "             estimator=LinearRegression(copy_X=True, fit_intercept=True, n_jobs=None,\n",
       "         normalize=False),\n",
       "             floating=False, forward=True, k_features=3, n_jobs=1,\n",
       "             pre_dispatch='2*n_jobs', scoring='neg_mean_squared_error',\n",
       "             verbose=2)"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# three variable\n",
    "sfs3 = sfs(ols, k_features =3, forward =True, floating =False, verbose= 2, scoring = 'neg_mean_squared_error', cv=5)\n",
    "sfs3.fit(X,y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done   1 out of   1 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  11 out of  11 | elapsed:    0.1s finished\n",
      "\n",
      "[2019-06-02 18:55:47] Features: 1/4 -- score: -54293.17658429352[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done   1 out of   1 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  10 out of  10 | elapsed:    0.0s finished\n",
      "\n",
      "[2019-06-02 18:55:47] Features: 2/4 -- score: -26775.95426714271[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done   1 out of   1 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done   9 out of   9 | elapsed:    0.0s finished\n",
      "\n",
      "[2019-06-02 18:55:47] Features: 3/4 -- score: -10851.452873513992[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done   1 out of   1 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done   8 out of   8 | elapsed:    0.0s finished\n",
      "\n",
      "[2019-06-02 18:55:47] Features: 4/4 -- score: -10437.358402225083"
     ]
    },
    {
     "data": {
      "text/plain": [
       "SequentialFeatureSelector(clone_estimator=True, cv=5,\n",
       "             estimator=LinearRegression(copy_X=True, fit_intercept=True, n_jobs=None,\n",
       "         normalize=False),\n",
       "             floating=False, forward=True, k_features=4, n_jobs=1,\n",
       "             pre_dispatch='2*n_jobs', scoring='neg_mean_squared_error',\n",
       "             verbose=2)"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# four variables\n",
    "sfs4 = sfs(ols, k_features =4, forward =True, floating =False, verbose= 2, scoring = 'neg_mean_squared_error', cv=5)\n",
    "sfs4.fit(X,y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "dat= [['one variable', sfs1.k_feature_names_],['two variables', sfs2.k_feature_names_], ['three variables', sfs3.k_feature_names_], ['four variables', sfs4.k_feature_names_]]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "dat_df= pd.DataFrame(dat, columns=['Number of Variables','Forward stepwise Selection'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Number of Variables</th>\n",
       "      <th>Forward stepwise Selection</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>one variable</td>\n",
       "      <td>(Rating,)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>two variables</td>\n",
       "      <td>(Income, Rating)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>three variables</td>\n",
       "      <td>(Income, Rating, Student_Yes)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>four variables</td>\n",
       "      <td>(Income, Limit, Rating, Student_Yes)</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "  Number of Variables            Forward stepwise Selection\n",
       "0        one variable                             (Rating,)\n",
       "1       two variables                      (Income, Rating)\n",
       "2     three variables         (Income, Rating, Student_Yes)\n",
       "3      four variables  (Income, Limit, Rating, Student_Yes)"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dat_df"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Other techniques used to adjust the training erorr for the model size: Cp, AIC, BIC, adjusted R square\n",
    "\n",
    "Now I am going to use the following Mallow's Cp definition to create figure 6.2\n",
    "\n",
    "[This blog](https://xavierbourretsicotte.github.io/subset_selection.html) gives better explaination of Mallow's Cp and other common variable selection techniques"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "$$C_p = \\frac{1}{m} (RSS+ 2d\\sigma^2) $$"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "#define RSS\n",
    "def fit_linear_reg(X,Y):\n",
    "    #Fit linear regression model and return RSS and R squared values\n",
    "    model_k = LinearRegression(fit_intercept = True)\n",
    "    model_k.fit(X,Y)\n",
    "    RSS = mean_squared_error(Y,model_k.predict(X)) * len(Y)\n",
    "    #MSE = mean_squared_error(Y,model_k.predict(X))\n",
    "    return RSS"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Importing tqdm for the progress bar\n",
    "from tqdm import tnrange, tqdm_notebook\n",
    "\n",
    "#Initialization variables\n",
    "\n",
    "k = 11\n",
    "RSS_list, feature_list = [],[]\n",
    "numb_features = []\n",
    "\n",
    "#Looping over k = 1 to k = 11 features in X\n",
    "for k in range(1,len(X.columns) + 1):\n",
    "\n",
    "    #Looping over all possible combinations: from 11 choose k\n",
    "    for combo in itertools.combinations(X.columns,k):\n",
    "        tmp_result = fit_linear_reg(X[list(combo)],y)   #Store temp result \n",
    "        RSS_list.append(tmp_result) #Append lists\n",
    "        feature_list.append(combo)\n",
    "        numb_features.append(len(combo))   \n",
    "\n",
    "#Store in DataFrame\n",
    "df = pd.DataFrame({'numb_features': numb_features,'RSS': RSS_list,'features':feature_list})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>numb_features</th>\n",
       "      <th>RSS</th>\n",
       "      <th>features</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>6.620874e+07</td>\n",
       "      <td>(Income,)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>2.171566e+07</td>\n",
       "      <td>(Limit,)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1</td>\n",
       "      <td>2.143512e+07</td>\n",
       "      <td>(Rating,)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1</td>\n",
       "      <td>8.370950e+07</td>\n",
       "      <td>(Cards,)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>1</td>\n",
       "      <td>8.433963e+07</td>\n",
       "      <td>(Age,)</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   numb_features           RSS   features\n",
       "0              1  6.620874e+07  (Income,)\n",
       "1              1  2.171566e+07   (Limit,)\n",
       "2              1  2.143512e+07  (Rating,)\n",
       "3              1  8.370950e+07   (Cards,)\n",
       "4              1  8.433963e+07     (Age,)"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.head()\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>RSS</th>\n",
       "      <th>features</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>numb_features</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2.143512e+07</td>\n",
       "      <td>(Age,)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1.053254e+07</td>\n",
       "      <td>(Age, Education)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>4.227219e+06</td>\n",
       "      <td>(Age, Education, Ethnicity_Asian)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>3.915058e+06</td>\n",
       "      <td>(Age, Education, Ethnicity_Asian, Ethnicity_Ca...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>3.866091e+06</td>\n",
       "      <td>(Age, Education, Gender_Male, Ethnicity_Asian,...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>3.821620e+06</td>\n",
       "      <td>(Age, Education, Gender_Male, Married_Yes, Eth...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>3.810759e+06</td>\n",
       "      <td>(Age, Education, Gender_Male, Student_Yes, Mar...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>3.804746e+06</td>\n",
       "      <td>(Cards, Age, Education, Gender_Male, Student_Y...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>3.798367e+06</td>\n",
       "      <td>(Income, Cards, Age, Education, Gender_Male, S...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>3.791345e+06</td>\n",
       "      <td>(Income, Limit, Cards, Age, Education, Gender_...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>3.786730e+06</td>\n",
       "      <td>(Income, Limit, Rating, Cards, Age, Education,...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                        RSS                                           features\n",
       "numb_features                                                                 \n",
       "1              2.143512e+07                                             (Age,)\n",
       "2              1.053254e+07                                   (Age, Education)\n",
       "3              4.227219e+06                  (Age, Education, Ethnicity_Asian)\n",
       "4              3.915058e+06  (Age, Education, Ethnicity_Asian, Ethnicity_Ca...\n",
       "5              3.866091e+06  (Age, Education, Gender_Male, Ethnicity_Asian,...\n",
       "6              3.821620e+06  (Age, Education, Gender_Male, Married_Yes, Eth...\n",
       "7              3.810759e+06  (Age, Education, Gender_Male, Student_Yes, Mar...\n",
       "8              3.804746e+06  (Cards, Age, Education, Gender_Male, Student_Y...\n",
       "9              3.798367e+06  (Income, Cards, Age, Education, Gender_Male, S...\n",
       "10             3.791345e+06  (Income, Limit, Cards, Age, Education, Gender_...\n",
       "11             3.786730e+06  (Income, Limit, Rating, Cards, Age, Education,..."
      ]
     },
     "execution_count": 27,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "temp= df.groupby('numb_features').min()\n",
    "temp"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>RSS</th>\n",
       "      <th>features</th>\n",
       "      <th>Cp</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>numb_features</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2.143512e+07</td>\n",
       "      <td>(Age,)</td>\n",
       "      <td>53636.603151</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1.053254e+07</td>\n",
       "      <td>(Age, Education)</td>\n",
       "      <td>26380.151295</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>4.227219e+06</td>\n",
       "      <td>(Age, Education, Ethnicity_Asian)</td>\n",
       "      <td>10616.846346</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>3.915058e+06</td>\n",
       "      <td>(Age, Education, Ethnicity_Asian, Ethnicity_Ca...</td>\n",
       "      <td>9836.444257</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>3.866091e+06</td>\n",
       "      <td>(Age, Education, Gender_Male, Ethnicity_Asian,...</td>\n",
       "      <td>9714.026084</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>3.821620e+06</td>\n",
       "      <td>(Age, Education, Gender_Male, Married_Yes, Eth...</td>\n",
       "      <td>9602.847244</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>3.810759e+06</td>\n",
       "      <td>(Age, Education, Gender_Male, Student_Yes, Mar...</td>\n",
       "      <td>9575.695002</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>3.804746e+06</td>\n",
       "      <td>(Cards, Age, Education, Gender_Male, Student_Y...</td>\n",
       "      <td>9560.662476</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>3.798367e+06</td>\n",
       "      <td>(Income, Cards, Age, Education, Gender_Male, S...</td>\n",
       "      <td>9544.715859</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>3.791345e+06</td>\n",
       "      <td>(Income, Limit, Cards, Age, Education, Gender_...</td>\n",
       "      <td>9527.161442</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>3.786730e+06</td>\n",
       "      <td>(Income, Limit, Rating, Cards, Age, Education,...</td>\n",
       "      <td>9564.421616</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                        RSS  \\\n",
       "numb_features                 \n",
       "1              2.143512e+07   \n",
       "2              1.053254e+07   \n",
       "3              4.227219e+06   \n",
       "4              3.915058e+06   \n",
       "5              3.866091e+06   \n",
       "6              3.821620e+06   \n",
       "7              3.810759e+06   \n",
       "8              3.804746e+06   \n",
       "9              3.798367e+06   \n",
       "10             3.791345e+06   \n",
       "11             3.786730e+06   \n",
       "\n",
       "                                                        features            Cp  \n",
       "numb_features                                                                   \n",
       "1                                                         (Age,)  53636.603151  \n",
       "2                                               (Age, Education)  26380.151295  \n",
       "3                              (Age, Education, Ethnicity_Asian)  10616.846346  \n",
       "4              (Age, Education, Ethnicity_Asian, Ethnicity_Ca...   9836.444257  \n",
       "5              (Age, Education, Gender_Male, Ethnicity_Asian,...   9714.026084  \n",
       "6              (Age, Education, Gender_Male, Married_Yes, Eth...   9602.847244  \n",
       "7              (Age, Education, Gender_Male, Student_Yes, Mar...   9575.695002  \n",
       "8              (Cards, Age, Education, Gender_Male, Student_Y...   9560.662476  \n",
       "9              (Income, Cards, Age, Education, Gender_Male, S...   9544.715859  \n",
       "10             (Income, Limit, Cards, Age, Education, Gender_...   9527.161442  \n",
       "11             (Income, Limit, Rating, Cards, Age, Education,...   9564.421616  "
      ]
     },
     "execution_count": 29,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "m = len(y)\n",
    "p = 11\n",
    "hat_sigma_squared = (1/(m - p -1)) * min(df['RSS'])\n",
    "temp['Cp']= 1/m*(temp.RSS+ 2* df.numb_features *hat_sigma_squared)\n",
    "temp"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Text(0,0.5,'Mallow Cp')"
      ]
     },
     "execution_count": 37,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAZUAAAEWCAYAAACufwpNAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4zLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvIxREBQAAIABJREFUeJzt3Xl8XXWd//HXO0mXlC5p0lLatJAGCoggW6EBXBBRcAUdF3ChVUfGGRyZGUXh58youAwOM6g4ihvQ4gLyUDYBQWQVpKVlsexSSkubFlrapoXSNfn8/jjf296GbG3vzU1y38/H4z5yzvdsn5Pc3M/9fr/nfI8iAjMzs0KoKHUAZmY2cDipmJlZwTipmJlZwTipmJlZwTipmJlZwTipmJlZwTip2E6R1CApJFWl+bsk/X2p4xroJC2SdGKJjj1O0j2SXpb0vyU4/rZzl/T/JP28t2OwnnNSKSPpn3OzpDHtyh9OiaKhNJF1TdJ4SZdKWp4+2J6S9HVJe5Qonlxivbld+S8lfa0UMRXZmcBLwMiI+EL7hZJmpvfVK5JWS7pN0oHFCCQivh0R3X6JSTF9sxgxWNecVMrPc8DpuRlJhwDDShdO1yTVAvcD1cAxETECeDtQA+xbytiAaZKOLXEMOyVXw9xJ+wBPRNd3Sv93RAwHJgIrgJkFPH6vk1RZ6hj6KyeV8vML4Iy8+enAFfkrSHp3qr2sk7Skp9++JVVI+ndJiyWtkHSFpFFp2SxJX0jT9emb/llpft/0Dbej9+O/AS8DH4+IRQARsSQizo6I+Wn7kPR5SQslvSTpwo72JWmCpA0pUeXKDk/bDJK0n6S7Ja1NZb/p5pT/G/hWJ7+LGZLubVcWkvZL0zMl/UjSH9I3/Psk7SXpe5LWpNrY4e12e5SkJ9LyyyUNzdv3eyQ9IqlF0l8kvSFv2SJJX5Y0H1jf0Qe7pGMlzU3nPjeXLCXNJHuPfCnF2WUTXES8CvwaODht/zVJv021uHXAjPQ+OVfSs5JWSbq63d/kE+k9tErSV9rF+TVJv8ybf2M635b0Xp0h6UzgY3kx/z6t+zplzbUtkh6X9L68/cyUdImkmyWtB94q6V3p9/2ypGZJX+zq3C3jpFJ+ZgMj0z9YJXAa8Mt266wnSzw1wLuBf5R0ag/2PSO93go0AsOB/0vL7gaOT9NvARYCb86b/3NEtHWwzxOBazpZlu/9wFTgCOAU4FPtV4iIZWS1nr/LK/4o8NuI2AJ8A/gjMJrsG/cPujnmj4D9u/ug7cKHgX8HxgCbUmwPpfnfAhe1W/9jwElkNbT907ak5HMZ8A9AHfAT4AZJQ/K2PZ3sb1kTEVvzd5o+0G8CLk7bXwTcJKkuImYAvyLVRCLiT12dkKThKc6H84pPSedTk/b1z8CpZH/3CcAa4Idp+4OAS4BPpGV1ZH+Ljo61D/AHsr/TWOAw4JGI+Gm7mN8raRDwe7K/754phl9JOiBvlx8l+5IwArgXuBT4h1Q7Phi4o6tzt4yTSnnK1VbeDjwJNOcvjIi7IuLRiGhLtYEryT4AuvMx4KKIWBgRrwDnAaelb8Z3A29MNYg3k33LPy5t95a0vCN1wPIeHPs7EbE6Ip4HvkdeE187v84tkySypPrrtGwLWVPPhIjYGBH3dryLbTaQfQjtatv9tRHxYERsBK4FNkbEFRHRCvwGaF9T+b9US1udjps7xzOBn0TEnIhojYhZZEmqKW/bi9O2GzqI493AMxHxi4jYGhFXAk8B792Jc/mipBZgAdmXiRl5y+6PiOvS+2kD8FngKxGxNCI2AV8DPpjeJx8EboyIe9Ky/wA6+0LxUeBPEXFlRGyJiFUR8Ugn6zaluC6IiM0RcQdwIzu+T66PiPtSnBvJ3g8HSRoZEWsi4qGd+H2ULSeV8vQLsn/IGbRr+gKQNE3SnZJWSlpL9iEwpv16HZgALM6bXwxUAeMi4lmyGtBhwJvI/qGXpW+KXSWVVcD4Hhx7SbvjTuhkvd8Bx0gaT5bc2oA/p2VfAgQ8kJpHXlPb6cDPgXGSduYDOOfFvOkNHcwPb7d+Z+e4D/CF1KzTkj7cJ7Hj7yB/2/ba/91y+6/vOvwd/E9E1ETEXhHxvvT37uzY+wDX5sX6JNAKjEuxbFs/ItaTvQc6Mgl4tpNl7U0AlrSr8bY/x/Zx/h3wLmBxahY9pofHKmtOKmUoIhaTddi/C7img1V+DdwATIqIUcCPyT5su7OM7AMjZ29gK9s/LO8m+yY6OCKa0/x0suamzr5h/gl4f0d9JO1ManfcZR2tFBFryJpAPkKWWK/KdUBHxAsR8ZmImEDWlPSjXB9IZyJiM/B1sqaz/N/RevIugJC0Vzfx90Rn57gE+Fb6UM+9hqUax7ZQu9hv+79bbv/NHay7K9ofewnwznbxDk3vieXknaekYWS11Y4sofOLNdofcxkwqd37qP057rBNRMyNiFPImsuuA67u5FiWx0mlfH0aOCF9E2xvBLA6IjZKOprsw7cnrgT+VdLk1Lb+beA3eW34dwOfA+5J83el+XtTk09HLgJGArNSG3quo/+i/M5o4BxJoyVNAs4maz7qzK/Jmv8+yPamLyR9SFKu/X4N2YdMd305kNX8hgIn55X9FXi9pMNSh/rXerCf7pwlaWLqA/kK28/xZ8BnUw1TkvZQdrHFiB7u92ayvqGPSqqS9BHgILLaZDH8GPhW3t9zrKRT0rLfAu9JHfCDgfPp/HPqV8CJkj6c4q6TdFha9iJZv17OHOBVss77QZKOJ2veu6qjHUsaLOljkkal/rZ19Oy9UPacVMpURDwbEfM6WfxPwPmSXgb+k55/Q7uM7AP2HrKa0EayDtGcu8kSVi6p3Ev2bf4eOpH6D44la9+ek2K6HVhL1n6fcz3wIFmN5yayTtbO3ABMAV6IiL/mlR+VjvFKWufsiFjYxX5yMbaS/Z5q88r+RvaB+CfgmXSuu+vXZLWshWTNPt9Mx5oHfIbsoog1ZL+XGT3daUSsAt4DfIGsqelLwHsi4qUCxNyR75P9fv+Y/p6zgWkplseBs8jOdTnZ+SztJO7nyWrbXwBWk/3tD02LLyXrD2mRdF2qUb4XeCfZPTc/As6IiKe6iPMTwKJ01dpnyfoMrRvyQ7qsv5MUwJSIWNDtymZWVK6pmJlZwTipmJlZwbj5y8zMCsY1FTMzK5h+MbhbIY0ZMyYaGhpKHYaZWb/x4IMPvhQRY3uybtkllYaGBubN6+xKWjMza09S+xEXOuXmLzMzKxgnFTMzKxgnFTMzKxgnFTMzKxgnFTMzK5iyu/prV1z3cDMX3vo0y1o2MKGmmnNOOoBTD9+ZR02YmZUHJ5VuXPdwM+dd8ygbtmQjsze3bOC8ax4FcGIxM2vHzV/duPDWp7cllJwNW1q58NanSxSRmVnf5aTSjWUtHT3Su/NyM7Ny5qTSjQk11TtVbmZWzpxUunHOSQdQPahyh7LqQZWcc9IBJYrIzKzvckd9N3Kd8f99y1MsW7uR6kGV/NcHDnEnvZlZB1xT6YFTD6/nL+e9jfceOoGR1VWcctiEUodkZtYnOanshKbGWl5ct4lFq14tdShmZn2Sk8pOaGqsA2D2wlUljsTMrG9yUtkJjWP2YMzwIcxxUjEz65CTyk6QRFNjLbMXriYiSh2OmVmf46Syk5oa63hh3UYWu1/FzOw1nFR2UlNjLeB+FTOzjjip7KR9xw5nzPDBzHludalDMTPrc5xUdpIkpjXWMXvhKvermJm1U9SkImmRpEclPSJpXiqrlXSbpGfSz9GpXJIulrRA0nxJR+TtZ3pa/xlJ0/PKj0z7X5C2VTHPJ6dpci3L127k+dXuVzEzy9cbNZW3RsRhETE1zZ8L3B4RU4Db0zzAO4Ep6XUmcAlkSQj4KjANOBr4ai4RpXU+k7fdycU/ne33q8xZ6CYwM7N8pWj+OgWYlaZnAafmlV8RmdlAjaTxwEnAbRGxOiLWALcBJ6dlIyNidmTtUFfk7auo9ttzOHV7DHZnvZlZO8VOKgH8UdKDks5MZeMiYnmafgEYl6brgSV52y5NZV2VL+2gvOiyfpVa96uYmbVT7KTyxog4gqxp6yxJb85fmGoYRf9UlnSmpHmS5q1cubIg+2xqrGPZ2o0sXeOHdZmZ5RQ1qUREc/q5AriWrE/kxdR0Rfq5Iq3eDEzK23xiKuuqfGIH5R3F8dOImBoRU8eOHbu7pwVs71e5301gZmbbFC2pSNpD0ojcNPAO4DHgBiB3Bdd04Po0fQNwRroKrAlYm5rJbgXeIWl06qB/B3BrWrZOUlO66uuMvH0V3ZQ9h1PrfhUzsx0U8yFd44Br01W+VcCvI+IWSXOBqyV9GlgMfDitfzPwLmAB8CrwSYCIWC3pG8DctN75EZG77OqfgJlANfCH9OoVkpg2udZXgJmZ5SlaUomIhcChHZSvAt7WQXkAZ3Wyr8uAyzoonwccvNvB7qKmxjr+8NgLLFn9KpNqh5UqDDOzPsN31O+GaR4HzMxsB04qu2H/PUcwetggjwNmZpY4qeyGigoxbXKdaypmZomTym6a1ljL0jUbWLrG44CZmTmp7CaPA2Zmtp2Tym46YNwIaoYNchOYmRlOKrutokIc3VDL7OecVMzMnFQKoKmxjiWrN9Dc4nHAzKy8OakUwPZ+FddWzKy8OakUwIF7jWBUtftVzMycVAqgokIcPbnWN0GaWdlzUimQpsY6Fq96lWXuVzGzMuakUiBNaRywOb4KzMzKmJNKgRy410hGDq3yTZBmVtacVAqkskIc7XHAzKzMOakUUFNjLYtWvcoLazeWOhQzs5JwUimgbferuF/FzMqUk0oBvW78SEYMrXITmJmVLSeVAqqsyJ5bP9ud9WZWppxUCmza5Dqee2k9L65zv4qZlR8nlQLL9au4CczMypGTSoEdNGEkI4ZUuQnMzMqSk0qBVVaIoybX+gowMytLTipF0NRYy8KV61nhfhUzKzNOKkWwrV/FoxabWZlxUimCg8aPZPiQKj+0y8zKjpNKEVRVVnBUw2hfAWZmZcdJpUiaGut4duV6VrzsfhUzKx9OKkUyLfWrPOB+FTMrI04qRXLwhJHsMbjSTWBmVlacVIqkqrKCozwOmJmVGSeVIpo2uY4FK17hpVc2lToUM7Ne4aRSRNueW+/aipmVCSeVIjq4fpT7VcysrDipFNGgygqObPA4YGZWPpxUiqypsZa/veh+FTMrD0VPKpIqJT0s6cY0P1nSHEkLJP1G0uBUPiTNL0jLG/L2cV4qf1rSSXnlJ6eyBZLOLfa57Iom369iZmWkN2oqZwNP5s1/B/huROwHrAE+nco/DaxJ5d9N6yHpIOA04PXAycCPUqKqBH4IvBM4CDg9rdunHFI/imGDKz0OmJmVhaImFUkTgXcDP0/zAk4AfptWmQWcmqZPSfOk5W9L658CXBURmyLiOWABcHR6LYiIhRGxGbgqrdunDKqs4Mh9Rvt+FTMrC8WuqXwP+BLQlubrgJaI2JrmlwL1aboeWAKQlq9N628rb7dNZ+WvIelMSfMkzVu5cuXuntNOa2qs4+kXX2b1+s29fmwzs95UtKQi6T3Aioh4sFjH6KmI+GlETI2IqWPHju314+fuV3nAV4GZ2QBXzJrKccD7JC0ia5o6Afg+UCOpKq0zEWhO083AJIC0fBSwKr+83Tadlfc5h9TXUD2o0k1gZjbgFS2pRMR5ETExIhrIOtrviIiPAXcCH0yrTQeuT9M3pHnS8jsiIlL5aenqsMnAFOABYC4wJV1NNjgd44Zinc/uGFxVwVQ/X8XMykAp7lP5MvBvkhaQ9ZlcmsovBepS+b8B5wJExOPA1cATwC3AWRHRmvpdPgfcSnZ12dVp3T5p2uRannrhZda4X8XMBrCq7lfZfRFxF3BXml5IduVW+3U2Ah/qZPtvAd/qoPxm4OYChlo0uftV5jy3mpMP3qvE0ZiZFYfvqO8lb5hYw9BBFW4CM7MBzUmllwyuyu5XmeM7681sAHNS6UVNk+t46oV1tLzqfhUzG5icVHpR0751RODaipkNWE4qvegNE0cxpMr9KmY2cDmp9KIhVZVZv4pvgjSzAcpJpZc1NdbxpPtVzGyAclLpZdMm1xLh56uY2cDUo6QiaS9J75P0Xkm+c283HDqphiFVFe6sN7MBqdukIunvycba+gDZmFyzJX2q2IENVEMHVXLE3h4HzMwGpp4M03IOcHhErAKQVAf8BbismIENZE2NdXzv9r+x9tUtjBo2qNThmJkVTE+av1YBL+fNv5zKbBdNa8z6VeYuchOYmQ0sPampLADmSLoeCLJH9s6X9G8AEXFREeMbkA6bVMPgdL/KiQeNK3U4ZmYF05Ok8mx65eSefzKi8OGUh6xfpYbZfhKkmQ0wnSYVSUOBERHx9XblewLr0lD1toumTa7jB3c8w9oNWxhV7X4VMxsYuupTuRh4UwflxwHfLU445aOpsY62gHnuVzGzAaSrpHJkRFzTvjAirgXeXLyQysPhe2/vVzEzGyi6SirDdnE764Ghgyo5bFKNb4I0swGlq+SwQtJrHvsr6ShgZfFCKh9NjXU81ryWdRu3lDoUM7OC6CqpnANcLelraXiW90r6OnB1Wma7qamx1v0qZjagdJpUIuIB4GhAwIz0EjAtIub0RnAD3RF7j2ZwZYWHwjezAaPL+1QiYgXw1V6Kpezk+lXcWW9mA4U73EusqbGWR5vX8rL7VcxsAHBSKbFpuftVFq8pdShmZrutJ0Pf79sbgZSrI/YezaBKuQnMzAaEnoz9dZmkicBc4M/APRHxaHHDKh/Vg3P9Ku6sN7P+r9uaSkS8BXgd8AOgBrhJkj8BC2ja5Ox+lVc2bS11KGZmu6UnzV9vBL4AfAV4N3AjcFaR4yorTY11tLaF71cxs36vJ81fdwEPAv8F3BwRm4saURk6Yp+a1K+ymuMP2LPU4ZiZ7bKeJJUxZCMTvxn4vKQ24P6I+I+iRlZGhg2u4g0Ta5jj56uYWT/Xkz6VFmAh8BywHNgXj1JccE2Ntcxfupb17lcxs36sJ30qC4H/BWqBS4ADUue9FdC2fhXfr2Jm/VhPmr/2i4i2okdS5o7cZzRVFWLOwlW8Zf+xpQ7HzGyX9OSO+gmSrpW0Ir1+l+5bsQLK+lVG+SZIM+vXepJULgduACak1+9TmRVYU2Md85eu5dXN7lcxs/6pJ0llbERcHhFb02sm4PaZIpjWWMfWtuBB96uYWT/Vk6SyStLHJVWm18eBbttoJA2V9ICkv0p6PD3gC0mTJc2RtEDSbyQNTuVD0vyCtLwhb1/npfKnJZ2UV35yKlsg6dydPfm+Zuo+o6ms8DhgZtZ/9SSpfAr4MPAC2SXFHwQ+2YPtNgEnRMShwGHAyZKagO8A342I/YA1wKfT+p8G1qTy76b1kHQQcBrweuBk4Ee5BAf8EHgncBBwelq339pjSK5fxXfWm1n/1JP7VBZHxPsiYmxE7BkRp0bE8z3YLiLilTQ7KL0COAH4bSqfBZyapk9J86Tlb5OkVH5VRGyKiOeABWRPpDwaWBARC9Nd/leldfu1aZPrmL+0xf0qZtYvdXpJsaQfkCWBDkXE57vbeapNPAjsR1areBZoiYjcJ+ZSoD5N1wNL0r63SloL1KXy2Xm7zd9mSbvyaZ3EcSZwJsDee+/dXdgl1dRYy4/vfpaHFrfwxiljSh2OmdlO6eo+lXm7u/OIaAUOk1QDXAscuLv73MU4fgr8FGDq1KmdJsq+YGpD7bZ+FScVM+tvOk0qETGrs2U7KyJaJN0JHAPUSKpKtZWJQHNarRmYBCyVVAWMIrsgIFeek79NZ+X91vAhVRxcP8rjgJlZv9RV89fv6br5631d7VjSWGBLSijVwNvJOt/vJOvsvwqYDlyfNrkhzd+flt8RESHpBuDXki4iu09mCvAAIGCKpMlkyeQ04KPdnnE/0NRYy2X3PseGza1UD64sdThmZj3WVfPX/+zmvscDs1K/SgVwdUTcKOkJ4CpJ3wQeBi5N618K/ELSAmA1WZIgIh6XdDXwBLAVOCs1qyHpc8CtQCVwWUQ8vpsx9wlNjXX85O6FPPT8Go7bz01gZtZ/dNX8dffu7Dgi5gOHd1C+kOzKrfblG4EPdbKvbwHf6qD8ZuDm3YmzL5q6z2gqBHMWrnJSMbN+pdsBJSVNIXtA10HA0Fx5RDQWMa6yNmLoIA6p9/0qZtb/9HTsr0vImp7eClwB/LKYQVnWBPbIkhY2bmktdShmZj3Wk6RSHRG3A0o3Qn6N7Fn1VkTTGmvZ3NrGQ897HDAz6z96klQ2SaoAnpH0OUnvB4YXOa6yN7WhlgrhJjAz61d6klTOBoYBnweOBD5BdumvFdHIoYM4uN7PVzGz/qXbjvqImJsmX6FnA0lagUybXMus+xezcUsrQwf5fhUz6/u6uvnxhq427O7mR9t9TY11/OzPz/Hw8y0cs29dqcMxM+tWVzWVY8gGbLwSmEN2B7v1ohfXbQTg9J/Npr6mmnNOOoBTD6/vZiszs9LpKqnsRTa0yulkw5/cBFw5UO5a7+uue7iZb9z45Lb55pYNnHfNowBOLGbWZ3XaUR8RrRFxS0RMB5rInmNyVxoaxYrswlufZkO7e1Q2bGnlwlufLlFEZmbd67KjXtIQsntSTgcagIvJhrC3IlvWsmGnys3M+oKuOuqvAA4mG1vr6xHxWK9FZUyoqaa5gwQyoaa6BNGYmfVMV/epfJxsmPmzgb9IWpdeL0ta1zvhla9zTjqA6naXEQ+pquCckw4oUURmZt3rapTintwYaUWS64y/8Nant9VYjth7tDvpzaxP6/bmRyudUw+v35ZEzv3dfK57pJmWVzdTM2xwiSMzM+uYayP9xPRjG9i4pY3fzF1S6lDMzDrlpNJPvG78SJoaa7ni/sVsbW0rdThmZh1yUulHPnncZJpbNvCnJ18sdShmZh1yUulHTnzdOOprqrn8vkWlDsXMrENOKv1IZYWYfuw+zHluNU8s81XdZtb3OKn0Mx+ZujfVgyqZ9ZdFpQ7FzOw1nFT6mVHDBvH+I+q57pFmVq/fXOpwzMx24KTSD804toFNW9u4au7zpQ7FzGwHTir90P7jRnDcfnX8wpcXm1kf46TST804djLL127kj0/48mIz6zucVPqpEw7ck0m11Vx+33OlDsXMbBsnlX6qskJMP6aBuYvW8Fjz2lKHY2YGOKn0ax+aOolhgyuZ6cuLzayPcFLpx0ZVD+LvjpjIDY8s46VXNpU6HDMzJ5X+bvqx+7C5tY2rHvDlxWZWek4q/dx+e47gTVPG8IvZi9niy4vNrMScVAaATx7XwIvrNnHLYy+UOhQzK3NOKgPA8fvvyT51w9xhb2Yl56QyAFSky4sfXLyG+UtbSh2OmZUxJ5UB4oNTJ7LH4Epm+lkrZlZCTioDxMihg/jgkRP5/fxlrHh5Y6nDMbMyVbSkImmSpDslPSHpcUlnp/JaSbdJeib9HJ3KJeliSQskzZd0RN6+pqf1n5E0Pa/8SEmPpm0ulqRinU9/MP3YBra0BlfOWVLqUMysTBWzprIV+EJEHAQ0AWdJOgg4F7g9IqYAt6d5gHcCU9LrTOASyJIQ8FVgGnA08NVcIkrrfCZvu5OLeD59XuPY4Rx/wFh+OWcxm7f68mIz631FSyoRsTwiHkrTLwNPAvXAKcCstNos4NQ0fQpwRWRmAzWSxgMnAbdFxOqIWAPcBpyclo2MiNkREcAVefsqWzOObWDly5v4w2PLSx2KmZWhXulTkdQAHA7MAcZFRO4T7wVgXJquB/LbbZamsq7Kl3ZQ3tHxz5Q0T9K8lStX7ta59HVvnjKWxjF7cLk77M2sBIqeVCQNB34H/EtErMtflmoYUewYIuKnETE1IqaOHTu22IcrqYoKMf3YBh5Z0sLDz68pdThmVmaKmlQkDSJLKL+KiGtS8Yup6Yr0c0UqbwYm5W0+MZV1VT6xg/Ky93dHTmT4kCpm+WZIM+tlxbz6S8ClwJMRcVHeohuA3BVc04Hr88rPSFeBNQFrUzPZrcA7JI1OHfTvAG5Ny9ZJakrHOiNvX2Vt+JAqPjR1Ijc9upwV63x5sZn1nmLWVI4DPgGcIOmR9HoXcAHwdknPACemeYCbgYXAAuBnwD8BRMRq4BvA3PQ6P5WR1vl52uZZ4A9FPJ9+ZfoxDWxtC345x6MXm1nvUdatUT6mTp0a8+bNK3UYveJTM+cyf2kL9517AkOqKksdjpn1U5IejIipPVnXd9QPYJ88roGXXtnMTfN9ebGZ9Q4nlQHsjfuNYb89h3P5fYsotxqpmZWGk8oAJmWXFz/avJaHnvfoxWZWfE4qA9wHDq9nxNAqP2vFzHqFk8oAt8eQKj4ydRJ/eHQ5L6z15cVmVlxOKmXgjGMaaI3gV3MWlzoUMxvgnFTKwN51w3jbgeP49Zzn2biltdThmNkA5qRSJj55XAOr1m/m939dVupQzGwAc1IpE8fuW8eUPYcz8y++vNjMisdJpUxIYsZxDTy+bB3zFnv0YjMrDieVMvL+w+sZVT2ImX7WipkViZNKGRk2uIrTjprELY+/wLKWDaUOx8wGICeVMvPxpn2ICH4525cXm1nhOamUmUm1w3j7QeO48gFfXmxmheekUoZmHDuZNa9u4YZHfHmxmRWWk0oZamqs5cC9RnC5Ly82swJzUilDkphxbANPLl/HnOdWd7+BmVkPOamUqVMOq6dmmC8vNrPCclIpU9WDKzntqL354xMvsHTNq6UOx8wGCCeVMvaJY/ZBEr/w5cVmViBOKmWsvqaak14/jqseWMKGzb682Mx2n5NKmZtx7GTWbtjCdY80lzoUMxsAnFTK3FENozlo/Ehm3ufLi81s9zmplLnc6MVPv/gy9y9cVepwzKyfc1Ix3nfoBGr3GOzLi81stzmpGEMHVXL60ZO47ckXWbLalxeb2a5zUjEgG724QuKK+xeVOhQz68ecVAyA8aOqOfngvbhq7hLWb9pa6nDMrJ9yUrFtPnVcAy9v3Mq1D/vyYjPbNU4qts0Re4/mkPpRzPToxWa2i5xUbJvc6MULVrzCfQt8ebGZ7TwnFdvBew4dz5jhg5n5l+dKHYqZ9UNOKraDIVWVfPTovbn9qRUsXrUlJTP7AAANeUlEQVS+1OGYWT/jpGKv8bGmfRDwrov/zORzb+K4C+7gOnfem1kPVJU6AOt77n92FUKs35SNXNzcsoHzrnkUgFMPry9laGbWx7mmYq9x4a1P09ru6q8NW1o5/8YneHzZWtas3+yrw8ysQ0WrqUi6DHgPsCIiDk5ltcBvgAZgEfDhiFgjScD3gXcBrwIzIuKhtM104N/Tbr8ZEbNS+ZHATKAauBk4O/xJVxDLWjZ0WL56/WbeffG9AFQPqmR8zVDqa6oZP2ooE2qqmTCqmgk11YyvGcqEUdVUD67szbDNrA8oZvPXTOD/gCvyys4Fbo+ICySdm+a/DLwTmJJe04BLgGkpCX0VmAoE8KCkGyJiTVrnM8AcsqRyMvCHIp5P2ZhQU01zB4ll7PAhnH/K61m2diPLWjawfO0Gmls28vQLK1n5yibap/TRwwZlSWZUNfU1QxmfElB9TTXja6oZN2IIVZU7Vpave7iZC299mmUtG5hQU805Jx3Qa01upTy22UBRtKQSEfdIamhXfApwfJqeBdxFllROAa5INY3ZkmokjU/r3hYRqwEk3QacLOkuYGREzE7lVwCn4qRSEOecdADnXfMoG7Zsfxpk9aBKvvLu1/HOQ8Z3uM3mrW28uG4jzSnZLGvJJZ6NLF3zKg88t4p1G3cc/qVCMG7k0JR4hvLq5lb+/LeVbGnLslNzywa+/Lv5vLB2IycdvBdVFaKqUlRVVOw4XSmqKkRW4d011z3cvMM592Y/UqmSWTkmcJ9z8Y/b2x314yJieZp+ARiXpuuBJXnrLU1lXZUv7aC8Q5LOBM4E2HvvvXcj/PKQe8PtzBtxcFUFk2qHMal2WKfrvLJpK8tbNqTEkyWdXPJ5rHkti1a9doTkTVvbuOCWp7jglqe6jbuyIksugyorqKwQgyqVyiq2TeeWVVVWMKhie9ncRavZtLVth/1t2NLK/7v2UR5YtJqqinbbdzJflfa9Y9n2xJebz8V374KXuOSuZ7cde1siXbeRE183DglEdmNq9hOEyOVPqeNlAmg3n7/ezY8u5/zfP8HGvOOe+7v5bNrSynsPm5DtO9sL7XN17jj5y7RtmbbNb49xxx2UKoGX+otDuZyzitkNkWoqN+b1qbRERE3e8jURMVrSjcAFEXFvKr+drAZzPDA0Ir6Zyv8D2EBWw7kgIk5M5W8CvhwR7+kupqlTp8a8efMKdo5WOA3n3tTpsu9+5FC2tgZb29KrtW37fGtbKt+xbEtb0NoabGlro7Ut2NoabGnNpre0Ba1tbWxpzdZ96PmWTo89ZvgQtra10Zr23ZqO1eYevIIbVKltCSv/x/bktT2ZdZTI6CzJCdZt2NLh36xCULvHkHb77PpY28q7WG/bmoLnV73K1g4OXlkh6muqCYK29J2mLYIIsrIgm44gyH625eaDHcvS+kTaB9DayZu0vqaa+849ocNlHZH0YERM7cm6vV1TeVHS+IhYnpq3VqTyZmBS3noTU1kz25vLcuV3pfKJHaxv/Vh9J3059TXVvP/wiR1sUTjHXXBHp8fu7J+vrW3HJNPaFmxp7Xp+a0puuflPXPpApzF9/7TDALZ9wETuA4btHzLkL2u/btp4W3ne9Pk3PtHpcc9754HkPopy3znT0XboN8t9Id2+zvZt2q8f2zfi4jsWdHrsv39T42uP+5r9R4+OmR9nAFfcv7jDY7YFvP2gcdv2lvs9546ff6z2ce0Q0w6x7Fi+cGXHNxK3tgVH7F1DRcpIFXm1yQrpNTXR7cu1LaFVaPuy9vv44Z3Pdnjczi7GKYTeTio3ANOBC9LP6/PKPyfpKrKO+rUp8dwKfFvS6LTeO4DzImK1pHWSmsg66s8AftCbJ2KF11lfzjknHdAnj11RIQZX5L6P7tqVbl0l0lMOK16zyKX3Ptfpcf/hLfsW7bgAv3uoudNjf/nkA4t23NufXNHpcf/rA4cU7bgAjzzf+ZeW7512eNGOe93Dyzo87oSa6qIds2j3qUi6ErgfOEDSUkmfJksmb5f0DHBimofs6q2FwALgZ8A/AaQO+m8Ac9Pr/FynfVrn52mbZ3Enfb936uH1/NcHDqG+phqx/Z+9NzozS3Xsc046gOpBOyak3kikpTpuKY/tc+6d4xa1T6Uvcp+K9TXlclVQXzi2z3nXjrszfSpOKmZm1qWdSSoepsXMzArGScXMzArGScXMzArGScXMzArGScXMzAqm7K7+krQS6PjW2r5rDPBSqYPoZT7n8uBz7h/2iYixPVmx7JJKfyRpXk8v5xsofM7lwec88Lj5y8zMCsZJxczMCsZJpX/4aakDKAGfc3nwOQ8w7lMxM7OCcU3FzMwKxknFzMwKxkmlD5M0SdKdkp6Q9Liks0sdU2+RVCnp4fSo6QFPUo2k30p6StKTko4pdUzFJOlf03v6MUlXShpa6pgKTdJlklZIeiyvrFbSbZKeST9Hd7WP/shJpW/bCnwhIg4CmoCzJB1U4ph6y9nAk6UOohd9H7glIg4EDmUAn7ukeuDzwNSIOJjssZmnlTaqopgJnNyu7Fzg9oiYAtye5gcUJ5U+LCKWR8RDafplsg+a3nmiUAlJmgi8m+zJngOepFHAm4FLASJic0S0lDaqoqsCqiVVAcOAZSWOp+Ai4h5gdbviU4BZaXoWcGqvBtULnFT6CUkNwOHAnNJG0iu+B3wJaCt1IL1kMrASuDw1+f1c0h6lDqpYIqIZ+B/geWA5sDYi/ljaqHrNuIhYnqZfAMaVMphicFLpByQNB34H/EtErCt1PMUk6T3Aioh4sNSx9KIq4Ajgkog4HFjPAGwWyUn9CKeQJdMJwB6SPl7aqHpfZPdzDLh7OpxU+jhJg8gSyq8i4ppSx9MLjgPeJ2kRcBVwgqRfljakolsKLI2IXC30t2RJZqA6EXguIlZGxBbgGuDYEsfUW16UNB4g/VxR4ngKzkmlD5Mksnb2JyPiolLH0xsi4ryImBgRDWSdt3dExID+FhsRLwBLJB2Qit4GPFHCkIrteaBJ0rD0Hn8bA/jChHZuAKan6enA9SWMpSicVPq244BPkH1bfyS93lXqoKwo/hn4laT5wGHAt0scT9GkGtlvgYeAR8k+hwbc0CWSrgTuBw6QtFTSp4ELgLdLeoasxnZBKWMsBg/TYmZmBeOaipmZFYyTipmZFYyTipmZFYyTipmZFYyTipmZFYyTivVbkkLS/+bNf1HS1wq075mSPliIfXVznA+lUYnvbFfeIGlDuoz8CUk/lrTL/6+SZkj6vzT9WUlndLFug6SP7uqxrLw5qVh/tgn4gKQxpQ4kXxoksac+DXwmIt7awbJnI+Iw4A3AQbQbfHAnj7NNRPw4Iq7oYpUGYKeSyq7GYgOPk4r1Z1vJbpr71/YL2tc0JL2Sfh4v6W5J10taKOkCSR+T9ICkRyXtm7ebEyXNk/S3NCZZ7jkvF0qaK2m+pH/I2++fJd1AB3fDSzo97f8xSd9JZf8JvBG4VNKFnZ1kRGwF/gLs19FxJH08xf+IpJ9Iqkzln0yxP0B2I20ulq9J+mKa3k/SnyT9VdJD6fwvAN6U9vevkoZKujzF/7Ckt6ZtZ0i6QdIdwO2Sxku6J233mKQ3dfP3swHI3y6sv/shMF/Sf+/ENocCryMblnwh8POIOFrZQ9D+GfiXtF4DcDSwL3CnpP2AM8hG1T1K0hDgPkm5EXaPAA6OiOfyDyZpAvAd4EhgDfBHSadGxPmSTgC+GBHzOgtW0jCyoUz+s/1xJL0O+AhwXERskfQj4GOSbgO+no65FrgTeLiD3f8KuCAirlX2oKwKssEsvxgRuUT6BbLxDw+RdGCKf/+8WN4QEavTerdGxLdSYhvW2TnZwOWkYv1aRKyTdAXZQ5829HCzubnhxyU9C+SSwqNAfjPU1RHRBjwjaSFwIPAO4A15taBRwBRgM/BA+4SSHAXcFREr0zF/Rfb8lOu6iXNfSY+QjWR7fUT8QdLx7Y7zNrLEMTcbRotqskEKp7U75m+A/fN3LmkEUB8R1wJExMZU3j6ONwI/SOs8JWlx3r5ui4jcM0PmApcpGwT1uoh4pJvzswHIScUGgu+RjSN1eV7ZVlLzburgHpy3bFPedFvefBs7/k+0H8MoAAH/HBG35i9IH/brdy38TuX6VNrLP46AWRFxXrt4euvhT9tiiYh7JL2Z7AFrMyVd1E3fjQ1A7lOxfi99U76arNM7ZxHZN3iA9wGDdmHXH5JUkfoZGoGngVuBf0zfxpG0v7p/oNYDwFskjUnNQqcDd+9CPB25HfigpD1TPLWS9iF7mNtbJNWlWD/UfsP0NNGluQQkaUhqansZGJG36p+Bj6V19gf2Jvtd7CAd98WI+BnZUzsH8vD91gnXVGyg+F/gc3nzPwOul/RX4BZ2rRbxPFlCGAl8NiI2Svo5WV/LQ8raiVbSzSNhI2K5pHPJ+jUE3BQRBRnyPCKekPTvZP0cFcAW4KyImK3s8ur7gRags6aoTwA/kXR+2vZDwHygNf3uZgI/Ai6R9ChZDXBGRGzqoJnseOAcSVuAV8j6n6zMeJRiMzMrGDd/mZlZwTipmJlZwTipmJlZwTipmJlZwTipmJlZwTipmJlZwTipmJlZwfx/EACbx3Zr6kMAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.scatter(temp.index, temp.Cp)\n",
    "plt.plot(temp.index, temp.Cp)\n",
    "plt.title('Mallow Cp vs Number of Predictors')\n",
    "plt.xlabel('Number of Predictors')\n",
    "plt.ylabel('Mallow Cp')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## AIC (AKaike Information Criterion)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "AIC is similar to Mallow's Cp. It is defined for a large class of models fit by maximum likelihood. It is calculated using this following formula. This omited an additive constant. Hence for least squares models, Cp and AIC are proportional to each other"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "$$ AIC= \\frac{1}{n\\hat{\\sigma}^2 }(RSS+2d\\hat{\\sigma}^2 ) $$"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [],
   "source": [
    "temp['AIC'] = (1/m* hat_sigma_squared) *(temp.RSS+2*temp.index *hat_sigma_squared)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Text(0,0.5,'AIC')"
      ]
     },
     "execution_count": 45,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXwAAAERCAYAAAB4jRxOAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4zLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvIxREBQAAHXtJREFUeJzt3XtwHOWZ7/HvM5IsyTfJku8CWxIEgzHYBmMJnM0mkMWQEKCopQIJCU6lwuZUNktSgRxcRW2y2aoTNk52cw65bAgJlyxLNgsOSUiFywYTbsZGxmBzMzdfsGwsge+2ZMvSc/7oHnkkS0KS1dMz079PlUujnp5+n5FnftPz9ttvm7sjIiKFLxV3ASIikh0KfBGRhFDgi4gkhAJfRCQhFPgiIgmhwBcRSYicC3wz+6WZtZjZS4NYd4aZrTCztWa2zsw+kY0aRUTyUc4FPnAncNEg170Z+I27zweuAn4SVVEiIvku5wLf3Z8AdmYuM7OTzOwhM1tjZk+a2anp1YHx4e0KYFsWSxURySvFcRcwSLcBX3b3N8ysgWBP/nzg28AjZvZVYAzw8fhKFBHJbTkf+GY2FjgP+G8zSy8uDX9eDdzp7j8ws3OBX5nZHHfviqFUEZGclvOBT9DttNvd5/Vx3xcJ+/vdfaWZlQETgZYs1icikhdyrg+/N3ffC2w0sysBLDA3vHsLcEG4/DSgDGiNpVARkRxnuTZbppndC3yUYE99B/At4DHgp8A0oAT4tbt/x8xmAz8HxhIcwP2muz8SR90iIrku5wJfRESikfNdOiIiMjJy6qDtxIkTvba2Nu4yRETyxpo1a95z90mDWTenAr+2tpampqa4yxARyRtmtnmw66pLR0QkIRT4IiIJocAXEUkIBb6ISEIo8EVEEiKnRukMxwNrm1n28Aa27W5jemU5Ny6exeXza+IuS0Qk5+R14D+wtpmly9fT1tEJQPPuNpYuXw+g0BcR6SWvu3SWPbyhO+zT2jo6WfbwhpgqEhHJXXkd+Nt2tw1puYhIkuV14E+vLB/SchGRJMvrwL9x8SzKS4p6LCsvKeLGxbNiqkhEJHfl9UHb9IHZ7z30Gtv2tFNeUsR3rzhDB2xFRPqQ13v4EIT+M0sv4JNnTqOivITL5k2PuyQRkZyU94Gf1lhfzbt729my82DcpYiI5KTCCfy6KgCeffv9mCsREclNBRP4J08eS/WYUax6e2fcpYiI5KRID9qa2SZgH9AJHHH3BRG2RUN9Fas2KvBFRPqSjT38j7n7vCjDPq2hrprm3W28o358EZFjFEyXDkBDvfrxRUT6E3XgO/CIma0xs+v6WsHMrjOzJjNram1tPa7GTpk8jgmjS9StIyLSh6gD/8PufhZwMfAVM/tI7xXc/TZ3X+DuCyZNGtSF1/uVShkL66pYtVF7+CIivUUa+O7eHP5sAX4LLIyyPQj68d/Z2UazJlATEekhssA3szFmNi59G7gQeCmq9tIa66sBWKV+fBGRHqLcw58CPGVmLwKrgT+6+0MRtgfAqVPHUVFeovH4IiK9RDYO393fBuZGtf3+pFLGObVVPKt+fBGRHgpqWGZaY30Vm98/yLt72uMuRUQkZxRo4If9+NrLFxHpVpCBf9q08YwrK9YJWCIiGQoy8ItSxsLaKh24FRHJUJCBD8E0C2+/d4CWverHFxGBQg78uqAf/1lNsyAiAhRw4J8+fTxjS4t1ApaISKhgA7+4KMWC2gmaSE1EJFSwgQ9Bt86bLftp3Xco7lJERGJX0IHfGM6Pv1p7+SIihR34c2oqGD2qSCdgiYhQ4IFfUpTi7JkTdAKWiAgFHvgQTLPw+o797DxwOO5SRERilYDAT/fjay9fRJKt4AP/jJpKykpSPKtpFkQk4Qo+8EcVp1gws0r9+CKSeAUf+AANdVVs2LGP3QfVjy8iyZWMwK+vxh2ddSsiiZaIwJ97YgWlxSlNlywiiZaIwC8tLuKsGRN0ApaIJFoiAh+C+fFf2b6XPQc74i5FRCQWiQn8xrAf/7lN6tYRkWRKTODPO7GSUcUpdeuISGIlJvDLSoqYd2KlTsASkcRKTOBD0K3z8rY97G1XP76IJE+yAr+uii6HNZt2xV2KiEjWJSrw58+YQEmRaZoFEUmkRAV++aiwH19n3IpIAiUq8CG4zu1LzXvYf+hI3KWIiGRV8gK/vorOLqdJ4/FFJGESF/hnz5xAcco0kZqIJE7iAn/0qGLOPKGCVTpwKyIJk7jAh2C65HVb93DwsPrxRSQ5Ehn4jfXVHOly1mzWeHwRSY5EBv7ZMydQlDLNjy8iiZLIwB9bWsycmgpNpCYiiZLIwAdorK/ihXd203a4M+5SRESyIvLAN7MiM1trZg9G3dZQNNZV09HprN2ifnwRSYZs7OFfD7yahXaGZEHtBFKGplkQkcSINPDN7ATgk8DtUbYzHOPKSphTU6GJ1EQkMaLew/8h8E2gq78VzOw6M2sys6bW1taIy+mpoS7ox2/vUD++iBS+yALfzC4BWtx9zUDruftt7r7A3RdMmjQpqnL61FBXzeEjXbzwzu6stisiEoco9/AXAZea2Sbg18D5ZvYfEbY3ZOfUVWGGunVEJBEiC3x3X+ruJ7h7LXAV8Ji7XxNVe8NRUV7C7GnjdQKWiCRCYsfhpzXUVfP8ll0cOqJ+fBEpbFkJfHd/3N0vyUZbQ9VQX8WhI128+M6euEsREYmU9vDDfnxNlywihS7xgV85ehSzpozTBVFEpOAlPvAhmC65afNODh/p93QBEZG8p8AnmEitvaOL9c0ajy8ihUuBDyysqwbgWQ3PFJECpsAHqsaM4pQpY3UClogUNAV+qLG+mjWbd9HRqX58ESlMCvxQQ101Bw938lKzxuOLSGFS4IcW1lUB6scXkcKlwA9NGlfKyZPH6jq3IlKwFPgZGuqqaNq0iyPqxxeRAqTAz9BQX83+Q0d4edveuEsRERlxCvwMjWE/vrp1RKQQKfAzTB5fRv3EMZofX0QKkgK/l4b6KlZv3Elnl8ddiojIiFLg99JYX82+Q0d4dbv68UWksCjwe2nonldH/fgiUlgU+L1MrShjZvVonYAlIgVHgd+Hxrpqntu0ky7144tIAVHg96Ghvoo9bR289u6+uEsRERkxCvw+NNSrH19ECo8Cvw81leWcWFWuE7BEpKAo8PvRUFfN6o3qxxeRwqHA70dDXRW7Dnbweov68UWkMCjw+9EY9uNrmgURKRQK/H6cWDWamkr144tI4VDgD6ChropVb+/EXf34IpL/FPgDaKyv5v0Dh3mzZX/cpYiIHDcF/gAa6sPr3G5UP76I5D8F/gBmVI1m6vgynYAlIgWh38A3szIzm9TH8klmVhZtWbnBzGisVz++iBSGgfbw/x/wV30s/zDwb9GUk3sa6qt5b/8h3n7vQNyliIgcl4EC/2x3X957obv/FvhIdCXllobwOrfq1hGRfDdQ4I8e5uMKSt3EMUweV6oTsEQk7w0U3C1mtrD3QjM7B2iNrqTcYmY01FezauP76scXkbxWPMB9NwK/MbM7gTXhsgXA54GrIq4rpzTUVfGHF7ex+f2D1E4cE3c5IiLD0u8evruvBhoAA5aE/wxocPdV2SguVzRqfnwRKQAD7eHj7juAbw1nw+HQzSeA0rCd+9x9WNuK20mTxjBxbCmrNu7kqoUz4i5HRGRY+g18M1sP9NVpbUCXu8/9gG0fAs539/1mVgI8ZWZ/cvdnh19uPMwsnFcn6Mc3s7hLEhEZsoH28C/pY5kBJwJLP2jDHhzhTE9CUxL+y9ujno31Vfxx/Xbe2dnGjOqBBjCJiOSmfgPf3Tenb5vZfOAzwJXARuD+wWzczIoIDvieDPw4n/v+u69zu/F9Bb6I5KWBplY4xcy+ZWavAbcCWwBz94+5+48Gs3F373T3ecAJwEIzm9NHO9eZWZOZNbW25u5ozw9NHkvVmFEajy8ieWugcfivAecDl7j7h939VqBzOI24+25gBXBRH/fd5u4L3H3BpEnHTN2TM9L9+BqpIyL5aqDAvwLYDqwws5+b2QUEffiDEk6yVhneLgf+huBDJG811FXRvLuNrbsOxl2KiMiQDTQO/wF3vwo4lWDv/GvAZDP7qZldOIhtTyP4sFgHPAc86u4PjkTRcWnQdW5FJI994Jw47n7A3f/T3T9F0Be/Fvjfg3jcOnef7+5nuvscd//OCNQbq1lTxlE5ukTdOiKSl4Y0CZq77wr73C+IqqBclkoZC2urWKUrYIlIHkrMrJcjpaG+mi07D7Jtd1vcpYiIDIkCf4gaw+vcrtqobh0RyS8K/CE6dep4xpcV68CtiOQdBf4QFaWMhXXqxxeR/KPAH4bG+mo2vneAHXvb4y5FRGTQFPjD0FCn+fFFJP8o8Idh9vTxjCstVreOiOQVBf4wFKWMczSvjojkGQX+MDXUVfF26wFa9qkfX0TygwJ/mNLz6qxWt46I5AkF/jC91bIfA/7+P9ey6JbHeGBtc9wliYgMSIE/DA+sbebmB17qvl5j8+42li5fr9AXkZymwB+GZQ9voK2j57Vg2jo6WfbwhpgqEhH5YAr8Yehv4jRNqCYiuUyBPwzTK8uHtFxEJBco8IfhxsWzKC8p6rFsVFGKGxfPiqkiEZEPVhx3Afno8vk1QNCX37y7jSIzaieO7l4uIpKLFPjDdPn8mu6A//GKN1n28Abe2LGPD00ZF3NlIiJ9U5fOCLh64QxGFae485lNcZciItIvBf4IqBozisvnTWf5883sOdgRdzkiIn1S4I+Qa8+rpa2jk/9e807cpYiI9EmBP0JOn17Bwtoq7lq5ic4u/8D1RUSyTYE/gpYsquWdnW089lpL3KWIiBxDgT+CLpw9hWkVZdylg7cikoMU+COouCjFNY0zeerN93hjx764yxER6UGBP8I0RFNEcpUCf4RVjRnFZXPDIZptGqIpIrlDgR+B7iGaTRqiKSK5Q4EfgTk1FZxTO0FDNEUkpyjwI7LkvDre2dnGCg3RFJEcocCPyIWnB0M0dfBWRHKFAj8iJRqiKSI5RoEfoavOOZFRxSnuWrkp7lJERBT4UaoeW8plc6dz/xoN0RSR+CnwI6YhmiKSKxT4EUsP0bx75WYN0RSRWCnws2DJeXVs2XlQQzRFJFaRBb6ZnWhmK8zsFTN72cyuj6qtXHfh6VOYOr5MB29FJFZR7uEfAb7h7rOBRuArZjY7wvZyVklRis+dO5Mn39AQTRGJT2SB7+7b3f358PY+4FWgJqr2cp2GaIpI3LLSh29mtcB8YFUf911nZk1m1tTa2pqNcmJRPbaUSzWLpojEKPLAN7OxwP3A19x9b+/73f02d1/g7gsmTZoUdTmxWnJeLQcPa4imiMQj0sA3sxKCsL/H3ZdH2VY+mFNTwYKZGqIpIvGIcpSOAb8AXnX3f42qnXyzZFEtW3Ye5PENGqIpItkV5R7+IuBzwPlm9kL47xMRtpcXFp8+lanjNYumiGRfcVQbdvenAItq+/kqmEVzBt9/5HXebNnHyZPHxV2SiCSEzrSNQfpC53c9sznuUkQkQRT4MUgP0bz/+a0aoikiWaPAj4mGaIpItinwY6IhmiKSbQr8GGmIpohkkwI/RhqiKSLZpMCPUXqI5pNvvMebLZpFU0SipcCPmYZoiki2KPBjVj22lE+dGQzR3NuuIZoiEh0Ffg44OkRza9yliEgBU+DngDNOCIZo3vXMJg3RFJHIKPBzxLXnaYimiERLgZ8jLpozlSnjSzVEU0Qio8DPESVFKa5pmBkO0dwfdzkiUoAU+Dnk6oYZjCpKcffKTXGXIiIFSIGfQyaOLeVTc6dz3xoN0RSRkafAzzEaoikiUVHg55gzTqjg7JkTuHvlJro0RFNERpACPwctOa+Wze8f5PHXNURTREaOAj8HpYdo3vH0prhLEZECosDPQRqiKSJRUODnKA3RFJGRpsDPURPHlnLJ3GkaoikiI0aBn8O+cF4dBw93cp+GaIrICFDg57D0EM27VmqIpogcPwV+jrtWQzRFZIQo8HPcxd2zaOoSiCJyfBT4Oa6kKMVnG2byxOutGqIpIsdFgZ8Hrl6oIZoicvwU+Hlg0rhgiOb9GqIpIsdBgZ8nlpxXywEN0RSR46DAzxNnnlDJWTMqNYumiAybAj+PLFlUx6b3D/KX11vjLkVE8pACP49cPGcqk8eVcocudC4iw6DAzyMlRSmuaQyGaL7VqiGaIjI0Cvw8c/XCGRSljEtvfYq6m/7Iolse44G1zXGXJSJ5oDjuAmRonn7zPXA4cLgTgObdbSxdvh6Ay+fXxFmaiOS4yPbwzeyXZtZiZi9F1UYSLXt4A53ec5ROW0cn//SHl1n51vtseu8A7R2dMVUnIrksyj38O4EfAXdH2EbibNvd1ufyXQc7uPrnz3b/PmF0CVMrypk6vpSpFeVMqyhjakUZ08J/U8aXMa6sZEhtP7C2mWUPb2Db7jamV5Zz4+JZ+lYhBSGu13a2240s8N39CTOrjWr7STW9spzmPkJ/8rhS/u3T89i+p50de9vZvqeNd/e0s31PO+u27uH9A4ePeczY0uLuD4Gp48OfFeVMrShl6vjgQ6JydAlmxgNrm1m6fD1tHfF0JSXlDRl3u3G2HWe7cby242jX3KM7iScM/Afdfc4A61wHXAcwY8aMszdv1qyQA+n9IgEoLyniu1ecMeCL5NCRTlr2HmL7np4fBsGHQzvv7mmnZV87vc/pKi1OMa2ijG172jl8pOuY7VaUl/DNi2ZRZEYqZaTMKEoR/jSKzLD07YzlKbOj64TLM+9LLzczHnu1he8/soFDGe2XFae4+ZLZXHLmtO7tp+zodlIGRamg7Wz/rY/XSLTr7rhDlzsOuIMTLEvf7vJwvfT97vxx3Xb++cFXaO/1t77p4lNZPGdq+Fi6T/4baFuZ7XZ1HW0fwrrCbbk7j29o5aePv8XhzqPtjipK8aWP1LHo5IngBG1w9HFd4Y3M55X5fCH9N+j5OO9uO/j5zw++wq6Dx05ZUllewg2LZ/V4TO+6e7fZFT7Brq6ey/pa/1crN7P/0JFj2q2pLOfpm84f1P8zgJmtcfcFg1o37sDPtGDBAm9qaoqsnkIR1Z7Qkc4uWvcf4t3wA2D7nnbeDT8Q/vDithGoPB6Z4Z8ygg8nMyxcnur+sMr4wEjBtt3tdPZxVnNxyjixajRwbLCl305Hf2YGUO/gSW+xZ4jsbuugr7elAeWjinqEBr0Dpcd2JZdY+Pqy8HZHZ9//UQZsvOWTQ9ju4ANfo3Ty0OXzayLZwywuSjGtopxpFeXH3Pf85l19diVNHV/G7/9+EZ3udHYFwdXZ5XS609X9Mwil9HJ3p7MrWC+9vMvTt+n1u3P9r1/ot+ZvfWp2995l8Ligra7w9tH2vO/1Mtrtvd7ynX0Pdz3S5cypqSD93cGM8E0cvJnTdxiWcV/we/f64YrWve7RdX71bN/fch34zMIZpFJH28kMECP40CJclv5Q676/+/dgvfTt7vuA7zz4Sr9/6+9ecUb3dsl4zv1tq3db6efbo+Zw/S/c8Vy/7d77pcZwG9bzb93r755+vkf/T4IP7p7/D8c+7tM/W8mOfYeOaTf92k4/pmfdxy5LhY1n/t27b/fxTXPRLY/1+Z6aXnns+2+kKPBlUG5cPKvPboabLj6VyePLIm37ew9t6PONUVNZzhcW1UXW7qqNO/tt99ar50fW7mOvtfTb7s2XzI6sXYBfPLWx37avXjgjsnZr+jk2VVNZzrknVUfWLsDST5wWy2u7v/fUjYtnRdZmlMMy7wVWArPMbKuZfTGqtiR6l8+v4btXnEFNZTlG8EaMui877cbFsygvKeqxLOo3RhLbjbPtOJ9zXK/tONqNtA9/qNSHL/1J4sgRjdLRsN/ByJmDtkOlwBcRGZqhBL7m0hERSQgFvohIQijwRUQSQoEvIpIQCnwRkYTIqVE6ZtYK5NtkOhOB9+IuIsv0nJNBzzk/zHT3SYNZMacCPx+ZWdNgh0QVCj3nZNBzLjzq0hERSQgFvohIQijwj99tcRcQAz3nZNBzLjDqwxcRSQjt4YuIJIQCX0QkIRT4w2RmJ5rZCjN7xcxeNrPr464pW8ysyMzWmtmDcdeSDWZWaWb3mdlrZvaqmZ0bd01RMrOvh6/pl8zsXjOL9go3MTCzX5pZi5m9lLGsysweNbM3wp8T4qwxCgr84TsCfMPdZwONwFfMLNrLEeWO64FX4y4ii/4v8JC7nwrMpYCfu5nVAP8ALAivRV0EXBVvVZG4E7io17KbgD+7+4eAP4e/FxQF/jC5+3Z3fz68vY8gBAr+ag1mdgLwSeD2uGvJBjOrAD4C/ALA3Q+7++54q4pcMVBuZsXAaCB/r2DfD3d/AtjZa/FlwF3h7buAy7NaVBYo8EeAmdUC84FV8VaSFT8Evgl0xV1IltQBrcAdYTfW7WY2Ju6iouLuzcD3gS3AdmCPuz8Sb1VZM8Xdt4e33wWmxFlMFBT4x8nMxgL3A19z971x1xMlM7sEaHH3NXHXkkXFwFnAT919PnCAAvyqnxb2W19G8EE3HRhjZtfEW1X2eTBeveDGrCvwj4OZlRCE/T3uvjzuerJgEXCpmW0Cfg2cb2b/EW9JkdsKbHX39Le3+wg+AArVx4GN7t7q7h3AcuC8mGvKlh1mNg0g/NkScz0jToE/TGZmBP26r7r7v8ZdTza4+1J3P8HdawkO5D3m7gW99+fu7wLvmNmscNEFwCsxlhS1LUCjmY0OX+MXUMAHqXv5PXBtePta4Hcx1hIJBf7wLQI+R7CX+0L47xNxFyWR+Cpwj5mtA+YB/yfmeiITfpO5D3geWE+QEQU33YCZ3QusBGaZ2VYz+yJwC/A3ZvYGwTedW+KsMQqaWkFEJCG0hy8ikhAKfBGRhFDgi4gkhAJfRCQhFPgiIgmhwJfImZmb2Q8yfr/BzL49Qtu+08z+diS29QHtXBnOlLmi1/JaM2sLh+W+Ymb/bmbDfl+Z2RIz+1F4+8tm9vkB1q01s88Mty1JHgW+ZMMh4Aozmxh3IZnCycEG64vAl9z9Y33c95a7zwPOBGbTa9KtIbbTzd3/3d3vHmCVWmBIgT/cWqQwKPAlG44QnLzz9d539N5DN7P94c+PmtlfzOx3Zva2md1iZp81s9Vmtt7MTsrYzMfNrMnMXg/n+0nP2b/MzJ4zs3Vm9ncZ233SzH5PH2fMmtnV4fZfMrN/CZf9I/Bh4Bdmtqy/J+nuR4BngJP7asfMrgnrf8HMfmZmReHyL4S1ryY4oS9dy7fN7Ibw9slm9j9m9qKZPR8+/1uAvwq393UzKzOzO8L615rZx8LHLjGz35vZYwTT/kpC6dNesuXHwDoz+94QHjMXOI1gGtu3gdvdfaEFF5v5KvC1cL1aYCFwErDCzE4GPk8w0+M5ZlYKPG1m6VkfzwLmuPvGzMbMbDrwL8DZwC7gETO73N2/Y2bnAze4e1N/xZrZaIKpCP6xdztmdhrwaWCRu3eY2U+Az5rZo8A/hW3uAVYAa/vY/D3ALe7+WwsuSJIimMTtBndPf8h9g2DerzPM7NSw/lMyajnT3XtPCSwJosCXrHD3vWZ2N8HFNdoG+bDn0tPVmtlbQDqw1wOZXSu/cfcu4A0zexs4FbgQODPj20MF8CHgMLC6d9iHzgEed/fWsM17CObCf+AD6jzJzF4gmF3xd+7+JzP7aK92LiAI9eeCKWooJ5icq6FXm/8FnJK5cTMbB9S4+28B3L09XN67jg8Dt4brvGZmmzO29ajCXhT4kk0/JJij5Y6MZUcIuxbDg52jMu47lHG7K+P3Lnq+dnvPD+KAAV9194cz7wiD+MDwyu9Xug+/t8x2DLjL3Zf2qidbF9kY6ecseUh9+JI14R7mbwgOgKZtItjzBbgUKBnGpq80s1TYr10PbAAeBv6XBVNYY2an2AdfuGQ18NdmNjHsX78a+Msw6unLn4G/NbPJYT1VZjaT4KI5f21m1WGtV/Z+YHhFta3pDwczKw27j/YB4zJWfRL4bLjOKcAMgr+FCKDAl+z7AZA5WufnBIH3InAuw9sT3UIQ1n8Cvhx2edxOcLD0eQsuVP0zPuAbbdh9dBNBP/qLwBp3H5Epct39FeBmgn71dcCjwLSwzW8TzNz4NP1PRfw54B/Cxz4DTAXWAZ3hgdyvAz8BUma2HvgvYIm7H+pne5JAmi1TRCQhtIcvIpIQCnwRkYRQ4IuIJIQCX0QkIRT4IiIJocAXEUkIBb6ISEL8fzfGzxZ+R3KfAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.scatter(temp.index, temp.AIC)\n",
    "plt.plot(temp.index, temp.AIC)\n",
    "plt.xlabel('Number of Predictor')\n",
    "plt.ylabel('AIC')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## BIC (Bayesian Information Criterion)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "BIC was derived from a Bayesian point of view, but ends up looking similar to Cp and AIC. For the least squares model with d predictors, the BIC is, up to irrelevant constants, given by"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "$$ BIC= \\frac{1}{n}(RSS+ log(n)d\\hat{\\sigma}^2) $$"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Text(0,0.5,'BIC')"
      ]
     },
     "execution_count": 52,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAZUAAAEKCAYAAADaa8itAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4zLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvIxREBQAAIABJREFUeJzt3XuYVfV97/H3d++ZgeE67AFRhssMiniNIiMbMbGJpkouDTRPkqMnRujx1JMTk5OmjYn26alt0jSmtk1j2lxsTMTGxljjrYmGEC9JT1RgEAWEEJD7gDAyDNcB5vI9f+zfhs0wMDDsvde+fF7PM8+s/Vu/tdZ3K7M+e63f2muZuyMiIpINsagLEBGR0qFQERGRrFGoiIhI1ihUREQkaxQqIiKSNQoVERHJGoWKiIhkjUJFRESyRqEiIiJZUxF1Afk2cuRIr6+vj7oMEZGisWTJkrfdfdSp9C27UKmvr6epqSnqMkREioaZbTzVvjr9JSIiWaNQERGRrFGoiIhI1ihUREQkaxQqIiKSNWV39Vd/PLm0mXvnr2ZrWztjaqq544bJzJ5SF3VZIiIFR6HShyeXNnPX48tp7+gCoLmtnbseXw6gYBER6UGnv/pw7/zVRwIlrb2ji3vnr46oIhGRwqVQ6cPWtvbTahcRKWcKlT6Mqak+rXYRkXKmUOnDHTdMproyfkxbdWWcO26YHFFFIiKFSwP1fUgPxv/dz3/L1t0Hqa6M89UPX6pBehGRXuhI5RTMnlLHS3ddx/svPZsRgyqZdfmYqEsSESlICpXTkGyoZevug2zZpUF6EZHeKFROQ3JiAoBX1u2MuBIRkcKkUDkN5581lJpBlSxc3xp1KSIiBUmhchpiMWNafYKF63WkIiLSG4XKaUpOrGVza7u+/Cgi0guFymlKNqTGVXS0IiJyPIXKabrwnGEMHVjBIo2riIgcR6FymuLpcZV1ChURkZ4UKv2QnJhg3dv72bHnYNSliIgUlJyGipltMLPlZvaamTWFtoSZLTCzNeH3iNBuZnafma01s2VmdkXGeuaE/mvMbE5G+9Sw/rVhWcvl+0lLNtQC6NJiEZEe8nGk8h53v9zdG8PrO4Hn3H0S8Fx4DfA+YFL4uQ34NqRCCLgbSALTgLvTQRT6/HHGcjNz/3bg4jHDGDKgQoP1IiI9RHH6axYwL0zPA2ZntD/kKa8ANWZ2DnADsMDdW919F7AAmBnmDXP3V9zdgYcy1pVTFfEYUyeM0LiKiEgPuQ4VB35hZkvM7LbQNtrdt4Xpt4DRYboO2Jyx7JbQdrL2Lb20H8fMbjOzJjNramlpOZP3c0RyYoI1O/axc9+hrKxPRKQU5DpU3unuV5A6tXW7mV2TOTMcYXiOa8Dd73f3RndvHDVqVFbWmR5X0aXFIiJH5TRU3L05/N4BPEFqTGR7OHVF+L0jdG8GxmUsPja0nax9bC/teXFp3XAGVsY0WC8ikiFnoWJmg81saHoauB5YATwNpK/gmgM8FaafBm4JV4FNB3aH02TzgevNbEQYoL8emB/m7TGz6eGqr1sy1pVzVRWpcRXdsVhE5KhcPvlxNPBEuMq3Avh3d/+5mS0GHjWzW4GNwMdC/2eA9wNrgQPAHwG4e6uZfRlYHPp9yd3ThwefAh4EqoFnw0/eJBtq+fovf0fbgcPUDKrK56ZFRApSzkLF3dcBl/XSvhO4rpd2B24/wbq+D3y/l/Ym4JIzLrafkg0J3FPjKtdffHZUZYiIFAx9o/4MXDauhqoKjauIiKQpVM7AwMo4U8bV6EuQIiKBQuUMJSfWsnLrHvYc7Ii6FBGRyClUztD0hgTdDk0bdApMREShcoamjB9BZdw0riIigkLljFVXxblsbI3uAyYigkIlK5ITEyxv3s3+Q51RlyIiEimFShYkG2rp6naWbNwVdSkiIpFSqGTB1AkjiMdMlxaLSNlTqGTB4AEVXFo3XOMqIlL2FCpZkpyY4PUtbbQf7oq6FBGRyChUsmR6Qy0dXc7STRpXEZHypVDJkqn1I4gZvKLvq4hIGVOoZMmwgZVcNGYYC/V8FREpYwqVLEo21LJ0cxsHOzSuIiLlSaGSRcmGBIc7u3l9c1vUpYiIREKhkkXTGhKYofuAiUjZUqhkUc2gKiaPHqovQYpI2VKoZNn0ibUs2biLw53dUZciIpJ3CpUsSzYkONjRzfLm3VGXIiKSdwqVLJvWkADQKTARKUsKlSyrHTKASWcN0X3ARKQsKVRyIDkxQdOGVjq7NK4iIuVFoZIDyYZa9h/u4o2te6IuRUQkrxQqOZCcqHEVESlPCpUcOGvoQCaOHKxxFREpOwqVHElOTLBoQytd3R51KSIieaNQyZFpDQn2Huxk1TaNq4hI+VCo5EiyoRbQfcBEpLwoVHJkTE014xLVer6KiJQVhUoOJRtqWbShlW6Nq4hImVCo5FCyIUHbgQ5+t2Nv1KWIiOSFQiWHpk8M4yq6tFhEyoRCJYfGjqhmzPCB+hKkiJQNhUoOmRnJibUsWt+Ku8ZVRKT0KVRyLNmQ4O19h3mzZX/UpYiI5FzOQ8XM4ma21Mx+Gl43mNlCM1trZj82s6rQPiC8Xhvm12es467QvtrMbshonxna1prZnbl+L/2RTI+r6BSYiJSBfBypfBZYlfH6a8DX3f08YBdwa2i/FdgV2r8e+mFmFwE3AhcDM4FvhaCKA/8CvA+4CLgp9C0o9bWDOGvoAA3Wi0hZyGmomNlY4APA98JrA64FHgtd5gGzw/Ss8Jow/7rQfxbwiLsfcvf1wFpgWvhZ6+7r3P0w8EjoW1DS4yoL1+/UuIqIlLxcH6n8E/AFIP20qlqgzd07w+stQF2YrgM2A4T5u0P/I+09ljlRe8FJNiTYvucQG3ceiLoUEZGcylmomNkHgR3uviRX2ziNWm4zsyYza2ppacn79qfr+SoiUiZyeaRyNfAhM9tA6tTUtcA3gBozqwh9xgLNYboZGAcQ5g8Hdma291jmRO3Hcff73b3R3RtHjRp15u/sNJ07aggjh1RpXEVESl7OQsXd73L3se5eT2qg/Xl3/zjwAvCR0G0O8FSYfjq8Jsx/3lODEE8DN4arwxqAScAiYDEwKVxNVhW28XSu3s+ZMDOmNSR0x2IRKXlRfE/li8CfmtlaUmMmD4T2B4Da0P6nwJ0A7v4G8CiwEvg5cLu7d4Vxl08D80ldXfZo6FuQptUnaG5rZ3OrxlVEpHRV9N3lzLn7i8CLYXodqSu3evY5CHz0BMt/BfhKL+3PAM9ksdScOfp9lVbGJQZFXI2ISG7oG/V5Mnn0UGoGVer5KiJS0hQqeRKLGVfWa1xFREqbQiWPkg0JNrUeYNvu9qhLERHJCYVKHun5KiJS6hQqeXThOcMYOrBCp8BEpGQpVPIofmRcRYP1IlKaFCp5lmxIsK5lPzv2Hoy6FBGRrFOo5Fn6+yqLdApMREqQQiXPLhkzjMFVcQ3Wi0hJUqjkWUU8xlSNq4hIiVKoRCDZkOB32/fRuv9w1KWIiGSVQiUC6eerLNLRioiUGIVKBC6tq2FgZYxXNK4iIiVGoRKBqooYUyeM0JcgRaTkKFQiMq2+lt++tYfdBzqiLkVEJGsUKhFJTkzgDos26GhFREqHQiUil4+roaoipueriEhJUahEZGBlnMvH1WhcRURKikIlQtMbEryxdTd7DmpcRURKg0IlQsmJtXQ7LNm4K+pSRESyQqESoSvGj6AybroPmIiUDIVKhKqr4rxjbI3uAyYiJUOhErFkQ4LlW3Zz4HBn1KWIiJwxhUrEkhNr6ex2jauISElQqERs6oQRxGMaVxGR0nDSUDGzgWY2qpf2UWY2MHdllY8hAyq4pG64xlVEpCT0daRyH/CuXtrfCXw9++WUp+kNCV7fvJuDHV1RlyIickb6CpWp7v54z0Z3fwK4JjcllZ/kxASHu7p5dZPGVUSkuPUVKoPOYFk5RY31CczQuIqIFL2+gmGHmU3r2WhmVwItuSmp/AwbWMlF5wzTuIqIFL2KPubfATxqZg8CS0JbI3ALcGMO6yo7yYZaHl64kUOdXQyoiEddjohIv5z0SMXdFwFJwIC54ceApLsvzHVx5SQ5McGhzm5e37w76lJERPqtryMV3H07cHceailr0+oTACxct5NpDYmIqxER6Z+ThoqZLQe8t1mAu/s7clJVGRoxuIoLzh7KwvWtfCbqYkRE+qmvI5UP5qUKAVL3AXu0aQsdXd1UxnVxnYgUn77GVDb2/AH2A5vCtGRRcmIt7R1dLG/WuIqIFKe+btMy3cxeNLPHzWyKma0AVgDbzWxmfkosH+mxFH1fRUSKVV/nWP4Z+FvgR8DzwP9097NJfZv+qydbMNw3bJGZvW5mb5jZX4f2BjNbaGZrzezHZlYV2geE12vD/PqMdd0V2leb2Q0Z7TND21ozu7Mf77+gjBwygPPOGqLvq4hI0eorVCrc/Rfu/h/AW+7+CoC7//YU1n0IuNbdLwMuB2aa2XTga8DX3f08YBdwa+h/K7ArtH899MPMLiL1nZiLgZnAt8wsbmZx4F+A9wEXATeFvkUt2ZCgacMuOru6oy5FROS09RUqmXu29h7zersq7OjMlH3hZWX4ceBa4LHQPg+YHaZnhdeE+deZmYX2R9z9kLuvB9YC08LPWndf5+6HgUdC36KWnFjLvkOdrNy2J+pSREROW1+hcpmZ7TGzvcA7wnT69aV9rTwcUbwG7AAWAG8Cbe6efszhFqAuTNcBmwHC/N1AbWZ7j2VO1N5bHbeZWZOZNbW0FPbdZaZrXEVEilhfV3/F3X2Yuw9194ownX5d2dfK3b3L3S8HxpI6srggS3WfFne/390b3b1x1KjjHg9TUM4aNpCGkYM1riIiRSkvX4Zw9zbgBeAqoMbM0t+PGQs0h+lmYBxAmD8c2JnZ3mOZE7UXvWRDgkXrW+nqPukZRhGRgpOzUAlPh6wJ09XA7wOrSIXLR0K3OcBTYfrp8Jow/3l399B+Y7g6rAGYBCwCFgOTwtVkVaQG85/O1fvJp+TEBHsOdvLbtzSuIiLFpc97f52Bc4B54SqtGPCou//UzFYCj5jZ3wBLgQdC/weAfzOztUAr4S7I7v6GmT0KrAQ6gdvdvQvAzD4NzAfiwPfd/Y0cvp+8STbUAqlxlYvHDI+4GhGRU2epg4Hy0djY6E1NTVGX0ad3fu15Lh4zjO9+ojHqUkSkzJnZEnc/pZ2RbjBVoJINtSxa30q3xlVEpIgoVApUcmKCXQc6WLNjX9+dRUQKhEKlQE1Pj6vo0mIRKSIKlQI1LlHNOcMHsnC9vgQpIsVDoVKgzIxkQ4KF61opt4spRKR4KVQKWHJiLW/vO8S6t/dHXYqIyClRqBSwpO4DJiJFRqFSwF7f3EbM4M+fWM7V9zzPk0tL4i40IlLCFCoF6smlzfz5EytIf02lua2dux5frmARkYKmUClQ985fTXtH1zFt7R1d3Dt/dUQViYj0TaFSoLa29Xwm2snbRUQKgUKlQI2pqT6tdhGRQqBQKVB33DCZ6sr4MW2VceOOGyZHVJGISN9yeet7OQOzp6SejHzv/NU0t7VTGTdqBw9g1uVjIq5MROTEFCoFbPaUuiPh8mjTZr7w2DJeXreTGeeOjLgyEZHe6fRXkfjQZWMYMaiSeS9tiLoUEZETUqgUiYGVcW6aNp4FK7ezZdeBqMsREemVQqWI3Dx9AmbGv72yMepSRER6pVApImNqqrn+otH8ePFmDvb4YqSISCFQqBSZOTPqaTvQwVOv6XYtIlJ4FCpFJtmQ4IKzh/LgSxv1nBURKTgKlSJjZsydUc+qbXtYpKdCikiBUagUoVmX1zG8upJ5L2+IuhQRkWMoVIpQdVWcG68cx/w3tusGkyJSUBQqRerm6RNwdx5eqMuLRaRwKFSK1LjEIN574Wh+tEiXF4tI4VCoFLG5M+pp3X+Y/3x9a9SliIgACpWidtW5tZw/egjzXt6gy4tFpCAoVIqYmTFnRj0rmvfw6qZdUZcjIqJQKXZ/OKWOYQMr+MFvNkRdioiIQqXYDaqq4GON4/j5irfYvudg1OWISJlTqJSAW66qp8udh3X3YhGJmEKlBIyvHcR1F5zFvy/axKFOXV4sItFRqJSIOTPqeXvfYX62bFvUpYhIGVOolIh3njeSc0cN1uOGRSRSCpUSkb68+PUtu1mqy4tFJCI5CxUzG2dmL5jZSjN7w8w+G9oTZrbAzNaE3yNCu5nZfWa21syWmdkVGeuaE/qvMbM5Ge1TzWx5WOY+M7NcvZ9i8OErxjJ0QIWOVkQkMrk8UukE/szdLwKmA7eb2UXAncBz7j4JeC68BngfMCn83AZ8G1IhBNwNJIFpwN3pIAp9/jhjuZk5fD8Fb8iACj7SOJafLd/Gjr26vFhE8i9noeLu29z91TC9F1gF1AGzgHmh2zxgdpieBTzkKa8ANWZ2DnADsMDdW919F7AAmBnmDXP3Vzx1j5KHMtZVtm65qp6OLuffF26KuhQRKUN5GVMxs3pgCrAQGO3u6UuU3gJGh+k6YHPGYltC28nat/TSXtYaRg7m3ZNH8fDCTRzu7I66HBEpMzkPFTMbAvwE+BN335M5Lxxh5PxOiGZ2m5k1mVlTS0tLrjcXubkz6mnZe4hnV+jyYhHJr5yGiplVkgqUh9398dC8PZy6IvzeEdqbgXEZi48NbSdrH9tL+3Hc/X53b3T3xlGjRp3ZmyoC10waRcPIwTyoAXsRybNcXv1lwAPAKnf/x4xZTwPpK7jmAE9ltN8SrgKbDuwOp8nmA9eb2YgwQH89MD/M22Nm08O2bslYV1mLxYxbrprA0k1tLNvSFnU5IlJGcnmkcjXwCeBaM3st/LwfuAf4fTNbA7w3vAZ4BlgHrAX+FfgUgLu3Al8GFoefL4U2Qp/vhWXeBJ7N4fspKh+ZOpbBVXEdrYhIXlm5PdypsbHRm5qaoi4jL+5+agU/WrSZl+66lpFDBkRdjogUKTNb4u6Np9JX36gvYbfMqOdwVzc/0uXFIpInCpUSdu6oIbxr0kh+uHAjHV26vFhEck+hUuLmzqhn+55DzH/jrahLEZEyoFApce+ZfBYTagfpfmAikhcKlRIXixmfmD6BxRt2saJ5d9TliEiJU6iUgY82jqO6Mq6jFRHJOYVKGRheXcmHr6jjqde30rr/cNTliEgJU6iUibkz6jnc2c0ji3V5sYjkjkKlTEwaPZSrz6vlhy9vpFOXF4tIjihUysicq+rZuvsgC1Zuj7oUESlRCpUyct2Foxk7olr3AxORnFGolJF4uHvxwvWtrNq2p+8FREROk0KlzHyscRwDK2O6vFhEckKhUmZqBlXxh1PqePK1ZtoO6PJiEckuhUoZmjOjnoMd3fx48eaoSxGREqNQKUMXnD2M6RMTPPTyRrq6y+t5OiKSWwqVMjV3Rj3Nbe38cpUuLxaR7FGolKn3XjiaMcMHasBeRLJKoVKmKuIxbr5qAi+9uZPfbd8bdTkiUiIUKmXsxivHM6BClxeLSPYoVMpYYnAVsy4fw+OvNrP7QEfU5YhICVColLk5M+pp7+jiP5bo8mIROXMKlTJ38ZjhXFk/QpcXi0hWKFSEuTMa2NR6gBdX74i6FBEpcgoV4fqLR3P2sIG6e7GInDGFilAZj3Hz9PH815q3WbtjX9TliEgRU6gIADdOG09VPMZDL2+IuhQRKWIKFQFg5JAB/MFlY/jJki3sOajLi0WkfxQqcsTcGfXsP9zFY01boi5FRIqUQkWOuHTscK4YX8NDL2+gW5cXi0g/KFTkGHNm1LNh5wF+taYl6lJEpAgpVOQY77vkHM4aOkD3AxORflGoyDGqKmJ8PDmBF1e3sK5FlxeLyOmpiLoAKTw3Jcdx3/Nr+INv/j8OHO5iTE01d9wwmdlT6qIuTaRoPbm0mXvnr2ZrW3te/6byvV2FihznpbU7wWH/4S4Amtvauevx5QAKFilqUe7Y73p8Oe0d+f2benJpM3f+ZBkHO7vztl2Fihzn3vmr6fJjr/5q7+jir55+g+GDKhk1ZABnDR1A7ZABxGMWUZVypqLawUa17dPZsbs7nd1OZ5fT0d1NZ5fT2dVNR3f43eV0hvaOrm46u8PvLqer+/i2L/905ZHtprV3dPF/n1zB6u17j1lnR2fGNrtDe5ifXu8x/buObqdn/8Nd3cf9d2jv6OLe+atz9t/b3Mvr0tHGxkZvamqKuoyC1nDnzziVfxUxg8TgVMCMCj/HTg880jZ4wKl9fim3HV1U2+25gwWorozz1Q9f2u9tu/sxO7iOHjvd9I7ul6u288/Prz1mh1cVj/Hx6eO5YvyIjB3jsTvJzJ10bzv6VPuJd/rLtrTR0XX8v+yYwbDqyiN9u7pTgZIvlXGjIhajIm5UxmNHXlfGjYp4jIpYRnvP+bEYlRUxKmNGRXp+LN0vxnd+9Wav2zRg/T0fOOUazWyJuzeeSt+cHamY2feBDwI73P2S0JYAfgzUAxuAj7n7LjMz4BvA+4EDwFx3fzUsMwf4i7Dav3H3eaF9KvAgUA08A3zWyy0hc2RMTTXNbe3HtY8eOoBv3TyVlr0Hadl7iJa9h9gRfrfsO8Tqt/by9r5Dvf5BDqqKHxM66cAZNWQAo4alfi/ZuIuvPrMqr4fqaf05PeHuuIMD3Uemw+8w3e2hH+DdR+d3h7Znl2/jKz879j1/8SfL2La7nWvOHxU+9aY+/aZ3rOkdbfoTcVd3egecOa+7l+WO9n9syZZePzl/8SfLeGzJll4/ER8bFMfvtM/k0QmHu7r5wW828IPfbOizb8zosfM8dqdccUx7anpgZazXQAHodvjQZWOOrOPI8r2sp7dtVcbTyx1dPrOtMm587Lsvs33PoeO2PWb4QF6667p+/3fry3++vrXXv+UxNdU522bOjlTM7BpgH/BQRqj8HdDq7veY2Z3ACHf/opm9H/gMqVBJAt9w92QIoSagkdTf7hJgagiiRcD/ARaSCpX73P3ZvurSkUrfzuRTbHe309bewY5egif1++CR6b0HO0+pnoqYMS4x6OjOOWPnTXid+n38fD8yP6N/z77u7D3YecKjs6qK2JEA6c5YrphUxo14zKgMO75dJ3nS5xXja3r9RNz7Tjv9Cbr3T9e9ffr+5A9fPeG2F3zumuOXT++sw3Ssn6dcr77n+V53sHU11fzmzmv7tc5TlYsjw3xutyCOVNz912ZW36N5FvDuMD0PeBH4Ymh/KBxpvGJmNWZ2Tui7wN1bAcxsATDTzF4Ehrn7K6H9IWA20GeoSN/S/9j6c0omFjMSg6tIDK7igrNP3vdgR9cxofPJHy7ptV9nt3NJ3XAMMCP8NgxSLwDDMuZlvA6dMucd399Oetv//3F1w5G+MUuv146sL5Yxben5GDHrWYuFdRzt95dPvXHC7X7n5qlhB57awcZjFsIhdqQ9vaOOxzL6xGLEMz4xp7eX6WQ72Mc/dfUJa8qGuhMcCdfVVDNp9NCcbfeOGyb3uoO944bJOdtm2pn8TRXbdvM9UD/a3beF6beA0WG6Dsh8nu2W0Hay9i29tEuWzJ5Sl/N/8AMr44xLDGJcYhBw8p3NN2+aktNaFqzcfsJt3/m+C3K23e/+at0Jtzvzkj5S+QxEuYONattR7dgztx/F1ZP53m5kV3+5u5tZXk4imNltwG0A48ePz8cmpR/KcUdXjjvYqLety+JzK9+hst3MznH3beH0Vvr5tc3AuIx+Y0NbM0dPl6XbXwztY3vp3yt3vx+4H1JjKmf2FiRXot7ZRLHtqN9zVDtY7dxLV04vKQ5jKj/NGKi/F9iZMVCfcPcvmNkHgE9zdKD+PnefFgbqlwBXhFW+SmqgvrWXgfpvuvszfdWkgXoRkdNTEAP1ZvYjUkcZI81sC3A3cA/wqJndCmwEPha6P0MqUNaSuqT4jwBCeHwZWBz6fSk9aA98iqOXFD+LBulFRCKnLz+KiMhJnc6Riu5SLCIiWaNQERGRrFGoiIhI1pTdmIqZtZC6SKCYjATejrqIPNN7Lg96z8VhgruPOpWOZRcqxcjMmk51kKxU6D2XB73n0qPTXyIikjUKFRERyRqFSnG4P+oCIqD3XB70nkuMxlRERCRrdKQiIiJZo1ApYGY2zsxeMLOVZvaGmX026pryxcziZrbUzH4adS35EB5M95iZ/dbMVpnZVVHXlEtm9rnwb3qFmf3IzAZGXVO2mdn3zWyHma3IaEuY2QIzWxN+j4iyxlxQqBS2TuDP3P0iYDpwu5ldFHFN+fJZYFXUReTRN4Cfu/sFwGWU8Hs3szpSdxhvDHcwjwM3RltVTjwIzOzRdifwnLtPAp4Lr0uKQqWAufs2d381TO8ltaMp+YdQmNlY4APA96KuJR/MbDhwDfAAgLsfdve2aKvKuQqg2swqgEHA1ojryTp3/zXQ2qN5FqlHqRN+z85rUXmgUCkS4dk0U0g9P6bU/RPwBaA76kLypAFoAX4QTvl9z8wGR11Urrh7M/D3wCZgG7Db3X8RbVV5c6JHqpcMhUoRMLMhwE+AP3H3PVHXk0tm9kFgh7svibqWPKog9SC6b7v7FGA/JXhaJC2MI8wiFaZjgMFmdnO0VeWfpy69LbnLbxUqBc7MKkkFysPu/njU9eTB1cCHzGwD8AhwrZn9MNqScm4LsMXd00ehj3H0aael6L3AendvcfcO4HFgRsQ15cv28Ch1ejxSvWQoVAqYmRmp8+yr3P0fo64nH9z9Lncf6+71pAZvn3f3kv4U6+5vAZvNbHJoug5YGWFJubYJmG5mg8K/8eso4QsTengamBOm5wBPRVhLTihUCtvVwCdIfVp/Lfy8P+qiJCc+AzxsZsuAy4G/jbienAlHZI8BrwLLSe2HSu5b5uGR6i8Dk81sS3iM+j3A75vZGlJHbPdEWWMu6Bv1IiKSNTpSERGRrFGoiIhI1ihUREQkaxQqIiKSNQoVERHJGoWKFC0zczP7h4zXnzezv8rSuh80s49kY119bOej4a7EL/Rorzez9nAZ+Uoz+46Z9fvv1czmmtk/h+lPmtktJ+lbb2b/vb/bkvKmUJFidgj4sJmNjLqQTOEmiafqVuCP3f09vcx7090vB94BXESPmw+e5naOcPfvuPtDJ+lSD5xWqPS3FinzoECkAAADyklEQVQ9ChUpZp2kvjT3uZ4zeh5pmNm+8PvdZvYrM3vKzNaZ2T1m9nEzW2Rmy83s3IzVvNfMmszsd+GeZOnnvNxrZovNbJmZ/a+M9f6XmT1NL9+GN7ObwvpXmNnXQttfAu8EHjCze0/0Jt29E3gJOK+37ZjZzaH+18zsu2YWD+1/FGpfROqLtOla/srMPh+mzzOzX5rZ62b2anj/9wDvCuv7nJkNNLMfhPqXmtl7wrJzzexpM3seeM7MzjGzX4flVpjZu/r4/yclSJ8upNj9C7DMzP7uNJa5DLiQ1G3J1wHfc/dplnoI2meAPwn96oFpwLnAC2Z2HnALqbvqXmlmA4DfmFn6DrtXAJe4+/rMjZnZGOBrwFRgF/ALM5vt7l8ys2uBz7t704mKNbNBpG5l8pc9t2NmFwL/Dbja3TvM7FvAx81sAfDXYZu7gReApb2s/mHgHnd/wlIPyoqRupnl5909HaR/Rur+h5ea2QWh/vMzanmHu7eGfvPd/Ssh2Aad6D1J6VKoSFFz9z1m9hCphz61n+Jii9O3HzezN4F0KCwHMk9DPeru3cAaM1sHXABcD7wj4yhoODAJOAws6hkowZXAi+7eErb5MKnnpzzZR53nmtlrpO5k+5S7P2tm7+6xnetIBcfi1G20qCZ1k8Jkj23+GDg/c+VmNhSoc/cnANz9YGjvWcc7gW+GPr81s40Z61rg7ulnhiwGvm+pm6A+6e6v9fH+pAQpVKQU/BOp+0j9IKOtk3B6NwxwV2XMO5Qx3Z3xuptj/yZ63sPIAQM+4+7zM2eEnf3+/pV/QukxlZ4yt2PAPHe/q0c9+Xr405Fa3P3XZnYNqQesPWhm/9jH2I2UII2pSNELn5QfJTXonbaB1Cd4gA8Blf1Y9UfNLBbGGSYCq4H5wP8On8Yxs/Ot7wdqLQJ+z8xGhtNCNwG/6kc9vXkO+IiZnRXqSZjZBFIPc/s9M6sNtX6054LhaaJb0gFkZgPCqba9wNCMrv8FfDz0OR8YT+q/xTHCdre7+7+SempnKd++X05ARypSKv4B+HTG638FnjKz14Gf07+jiE2kAmEY8El3P2hm3yM11vKqpc4TtdDHI2HdfZuZ3UlqXMOAn7l7Vm557u4rzewvSI1zxIAO4HZ3f8VSl1e/DLQBJzoV9Qngu2b2pbDsR4FlQFf4b/cg8C3g22a2nNQR4Fx3P9TLabJ3A3eYWQewj9T4k5QZ3aVYRESyRqe/REQkaxQqIiKSNQoVERHJGoWKiIhkjUJFRESyRqEiIiJZo1AREZGsUaiIiEjW/H/bOr+c/XkD2QAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "import math\n",
    "temp['BIC'] = 1/m *(temp.RSS + math.log(m)*temp.index * hat_sigma_squared)\n",
    "plt.scatter(temp.index, temp.BIC)\n",
    "plt.plot(temp.index, temp.BIC)\n",
    "plt.xlabel('Number of Predictors')\n",
    "plt.ylabel('BIC')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Shrinkage Methods(Regularization)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Two popular Regularization methods- Ridge and Lasso (sometimes elastic-net is also popular, but not as much as ridge and lasso)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Ridge Regression"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "metadata": {},
   "outputs": [],
   "source": [
    "#load Hitters Dataset\n",
    "df= pd.read_csv('Dataset/Hitters.csv').dropna()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Unnamed: 0</th>\n",
       "      <th>AtBat</th>\n",
       "      <th>Hits</th>\n",
       "      <th>HmRun</th>\n",
       "      <th>Runs</th>\n",
       "      <th>RBI</th>\n",
       "      <th>Walks</th>\n",
       "      <th>Years</th>\n",
       "      <th>CAtBat</th>\n",
       "      <th>CHits</th>\n",
       "      <th>...</th>\n",
       "      <th>CRuns</th>\n",
       "      <th>CRBI</th>\n",
       "      <th>CWalks</th>\n",
       "      <th>League</th>\n",
       "      <th>Division</th>\n",
       "      <th>PutOuts</th>\n",
       "      <th>Assists</th>\n",
       "      <th>Errors</th>\n",
       "      <th>Salary</th>\n",
       "      <th>NewLeague</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>-Alan Ashby</td>\n",
       "      <td>315</td>\n",
       "      <td>81</td>\n",
       "      <td>7</td>\n",
       "      <td>24</td>\n",
       "      <td>38</td>\n",
       "      <td>39</td>\n",
       "      <td>14</td>\n",
       "      <td>3449</td>\n",
       "      <td>835</td>\n",
       "      <td>...</td>\n",
       "      <td>321</td>\n",
       "      <td>414</td>\n",
       "      <td>375</td>\n",
       "      <td>N</td>\n",
       "      <td>W</td>\n",
       "      <td>632</td>\n",
       "      <td>43</td>\n",
       "      <td>10</td>\n",
       "      <td>475.0</td>\n",
       "      <td>N</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>-Alvin Davis</td>\n",
       "      <td>479</td>\n",
       "      <td>130</td>\n",
       "      <td>18</td>\n",
       "      <td>66</td>\n",
       "      <td>72</td>\n",
       "      <td>76</td>\n",
       "      <td>3</td>\n",
       "      <td>1624</td>\n",
       "      <td>457</td>\n",
       "      <td>...</td>\n",
       "      <td>224</td>\n",
       "      <td>266</td>\n",
       "      <td>263</td>\n",
       "      <td>A</td>\n",
       "      <td>W</td>\n",
       "      <td>880</td>\n",
       "      <td>82</td>\n",
       "      <td>14</td>\n",
       "      <td>480.0</td>\n",
       "      <td>A</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>-Andre Dawson</td>\n",
       "      <td>496</td>\n",
       "      <td>141</td>\n",
       "      <td>20</td>\n",
       "      <td>65</td>\n",
       "      <td>78</td>\n",
       "      <td>37</td>\n",
       "      <td>11</td>\n",
       "      <td>5628</td>\n",
       "      <td>1575</td>\n",
       "      <td>...</td>\n",
       "      <td>828</td>\n",
       "      <td>838</td>\n",
       "      <td>354</td>\n",
       "      <td>N</td>\n",
       "      <td>E</td>\n",
       "      <td>200</td>\n",
       "      <td>11</td>\n",
       "      <td>3</td>\n",
       "      <td>500.0</td>\n",
       "      <td>N</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>-Andres Galarraga</td>\n",
       "      <td>321</td>\n",
       "      <td>87</td>\n",
       "      <td>10</td>\n",
       "      <td>39</td>\n",
       "      <td>42</td>\n",
       "      <td>30</td>\n",
       "      <td>2</td>\n",
       "      <td>396</td>\n",
       "      <td>101</td>\n",
       "      <td>...</td>\n",
       "      <td>48</td>\n",
       "      <td>46</td>\n",
       "      <td>33</td>\n",
       "      <td>N</td>\n",
       "      <td>E</td>\n",
       "      <td>805</td>\n",
       "      <td>40</td>\n",
       "      <td>4</td>\n",
       "      <td>91.5</td>\n",
       "      <td>N</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>-Alfredo Griffin</td>\n",
       "      <td>594</td>\n",
       "      <td>169</td>\n",
       "      <td>4</td>\n",
       "      <td>74</td>\n",
       "      <td>51</td>\n",
       "      <td>35</td>\n",
       "      <td>11</td>\n",
       "      <td>4408</td>\n",
       "      <td>1133</td>\n",
       "      <td>...</td>\n",
       "      <td>501</td>\n",
       "      <td>336</td>\n",
       "      <td>194</td>\n",
       "      <td>A</td>\n",
       "      <td>W</td>\n",
       "      <td>282</td>\n",
       "      <td>421</td>\n",
       "      <td>25</td>\n",
       "      <td>750.0</td>\n",
       "      <td>A</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 21 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "          Unnamed: 0  AtBat  Hits  HmRun  Runs  RBI  Walks  Years  CAtBat  \\\n",
       "1        -Alan Ashby    315    81      7    24   38     39     14    3449   \n",
       "2       -Alvin Davis    479   130     18    66   72     76      3    1624   \n",
       "3      -Andre Dawson    496   141     20    65   78     37     11    5628   \n",
       "4  -Andres Galarraga    321    87     10    39   42     30      2     396   \n",
       "5   -Alfredo Griffin    594   169      4    74   51     35     11    4408   \n",
       "\n",
       "   CHits    ...      CRuns  CRBI  CWalks  League Division PutOuts  Assists  \\\n",
       "1    835    ...        321   414     375       N        W     632       43   \n",
       "2    457    ...        224   266     263       A        W     880       82   \n",
       "3   1575    ...        828   838     354       N        E     200       11   \n",
       "4    101    ...         48    46      33       N        E     805       40   \n",
       "5   1133    ...        501   336     194       A        W     282      421   \n",
       "\n",
       "   Errors  Salary  NewLeague  \n",
       "1      10   475.0          N  \n",
       "2      14   480.0          A  \n",
       "3       3   500.0          N  \n",
       "4       4    91.5          N  \n",
       "5      25   750.0          A  \n",
       "\n",
       "[5 rows x 21 columns]"
      ]
     },
     "execution_count": 60,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>AtBat</th>\n",
       "      <th>Hits</th>\n",
       "      <th>HmRun</th>\n",
       "      <th>Runs</th>\n",
       "      <th>RBI</th>\n",
       "      <th>Walks</th>\n",
       "      <th>Years</th>\n",
       "      <th>CAtBat</th>\n",
       "      <th>CHits</th>\n",
       "      <th>CHmRun</th>\n",
       "      <th>CRuns</th>\n",
       "      <th>CRBI</th>\n",
       "      <th>CWalks</th>\n",
       "      <th>PutOuts</th>\n",
       "      <th>Assists</th>\n",
       "      <th>Errors</th>\n",
       "      <th>Salary</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>count</th>\n",
       "      <td>263.000000</td>\n",
       "      <td>263.000000</td>\n",
       "      <td>263.000000</td>\n",
       "      <td>263.000000</td>\n",
       "      <td>263.000000</td>\n",
       "      <td>263.000000</td>\n",
       "      <td>263.000000</td>\n",
       "      <td>263.000000</td>\n",
       "      <td>263.000000</td>\n",
       "      <td>263.000000</td>\n",
       "      <td>263.000000</td>\n",
       "      <td>263.000000</td>\n",
       "      <td>263.000000</td>\n",
       "      <td>263.000000</td>\n",
       "      <td>263.000000</td>\n",
       "      <td>263.000000</td>\n",
       "      <td>263.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>mean</th>\n",
       "      <td>403.642586</td>\n",
       "      <td>107.828897</td>\n",
       "      <td>11.619772</td>\n",
       "      <td>54.745247</td>\n",
       "      <td>51.486692</td>\n",
       "      <td>41.114068</td>\n",
       "      <td>7.311787</td>\n",
       "      <td>2657.543726</td>\n",
       "      <td>722.186312</td>\n",
       "      <td>69.239544</td>\n",
       "      <td>361.220532</td>\n",
       "      <td>330.418251</td>\n",
       "      <td>260.266160</td>\n",
       "      <td>290.711027</td>\n",
       "      <td>118.760456</td>\n",
       "      <td>8.593156</td>\n",
       "      <td>535.925882</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>std</th>\n",
       "      <td>147.307209</td>\n",
       "      <td>45.125326</td>\n",
       "      <td>8.757108</td>\n",
       "      <td>25.539816</td>\n",
       "      <td>25.882714</td>\n",
       "      <td>21.718056</td>\n",
       "      <td>4.793616</td>\n",
       "      <td>2286.582929</td>\n",
       "      <td>648.199644</td>\n",
       "      <td>82.197581</td>\n",
       "      <td>331.198571</td>\n",
       "      <td>323.367668</td>\n",
       "      <td>264.055868</td>\n",
       "      <td>279.934575</td>\n",
       "      <td>145.080577</td>\n",
       "      <td>6.606574</td>\n",
       "      <td>451.118681</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>min</th>\n",
       "      <td>19.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>19.000000</td>\n",
       "      <td>4.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>2.000000</td>\n",
       "      <td>3.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>67.500000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25%</th>\n",
       "      <td>282.500000</td>\n",
       "      <td>71.500000</td>\n",
       "      <td>5.000000</td>\n",
       "      <td>33.500000</td>\n",
       "      <td>30.000000</td>\n",
       "      <td>23.000000</td>\n",
       "      <td>4.000000</td>\n",
       "      <td>842.500000</td>\n",
       "      <td>212.000000</td>\n",
       "      <td>15.000000</td>\n",
       "      <td>105.500000</td>\n",
       "      <td>95.000000</td>\n",
       "      <td>71.000000</td>\n",
       "      <td>113.500000</td>\n",
       "      <td>8.000000</td>\n",
       "      <td>3.000000</td>\n",
       "      <td>190.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>50%</th>\n",
       "      <td>413.000000</td>\n",
       "      <td>103.000000</td>\n",
       "      <td>9.000000</td>\n",
       "      <td>52.000000</td>\n",
       "      <td>47.000000</td>\n",
       "      <td>37.000000</td>\n",
       "      <td>6.000000</td>\n",
       "      <td>1931.000000</td>\n",
       "      <td>516.000000</td>\n",
       "      <td>40.000000</td>\n",
       "      <td>250.000000</td>\n",
       "      <td>230.000000</td>\n",
       "      <td>174.000000</td>\n",
       "      <td>224.000000</td>\n",
       "      <td>45.000000</td>\n",
       "      <td>7.000000</td>\n",
       "      <td>425.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>75%</th>\n",
       "      <td>526.000000</td>\n",
       "      <td>141.500000</td>\n",
       "      <td>18.000000</td>\n",
       "      <td>73.000000</td>\n",
       "      <td>71.000000</td>\n",
       "      <td>57.000000</td>\n",
       "      <td>10.000000</td>\n",
       "      <td>3890.500000</td>\n",
       "      <td>1054.000000</td>\n",
       "      <td>92.500000</td>\n",
       "      <td>497.500000</td>\n",
       "      <td>424.500000</td>\n",
       "      <td>328.500000</td>\n",
       "      <td>322.500000</td>\n",
       "      <td>192.000000</td>\n",
       "      <td>13.000000</td>\n",
       "      <td>750.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>max</th>\n",
       "      <td>687.000000</td>\n",
       "      <td>238.000000</td>\n",
       "      <td>40.000000</td>\n",
       "      <td>130.000000</td>\n",
       "      <td>121.000000</td>\n",
       "      <td>105.000000</td>\n",
       "      <td>24.000000</td>\n",
       "      <td>14053.000000</td>\n",
       "      <td>4256.000000</td>\n",
       "      <td>548.000000</td>\n",
       "      <td>2165.000000</td>\n",
       "      <td>1659.000000</td>\n",
       "      <td>1566.000000</td>\n",
       "      <td>1377.000000</td>\n",
       "      <td>492.000000</td>\n",
       "      <td>32.000000</td>\n",
       "      <td>2460.000000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "            AtBat        Hits       HmRun        Runs         RBI       Walks  \\\n",
       "count  263.000000  263.000000  263.000000  263.000000  263.000000  263.000000   \n",
       "mean   403.642586  107.828897   11.619772   54.745247   51.486692   41.114068   \n",
       "std    147.307209   45.125326    8.757108   25.539816   25.882714   21.718056   \n",
       "min     19.000000    1.000000    0.000000    0.000000    0.000000    0.000000   \n",
       "25%    282.500000   71.500000    5.000000   33.500000   30.000000   23.000000   \n",
       "50%    413.000000  103.000000    9.000000   52.000000   47.000000   37.000000   \n",
       "75%    526.000000  141.500000   18.000000   73.000000   71.000000   57.000000   \n",
       "max    687.000000  238.000000   40.000000  130.000000  121.000000  105.000000   \n",
       "\n",
       "            Years        CAtBat        CHits      CHmRun        CRuns  \\\n",
       "count  263.000000    263.000000   263.000000  263.000000   263.000000   \n",
       "mean     7.311787   2657.543726   722.186312   69.239544   361.220532   \n",
       "std      4.793616   2286.582929   648.199644   82.197581   331.198571   \n",
       "min      1.000000     19.000000     4.000000    0.000000     2.000000   \n",
       "25%      4.000000    842.500000   212.000000   15.000000   105.500000   \n",
       "50%      6.000000   1931.000000   516.000000   40.000000   250.000000   \n",
       "75%     10.000000   3890.500000  1054.000000   92.500000   497.500000   \n",
       "max     24.000000  14053.000000  4256.000000  548.000000  2165.000000   \n",
       "\n",
       "              CRBI       CWalks      PutOuts     Assists      Errors  \\\n",
       "count   263.000000   263.000000   263.000000  263.000000  263.000000   \n",
       "mean    330.418251   260.266160   290.711027  118.760456    8.593156   \n",
       "std     323.367668   264.055868   279.934575  145.080577    6.606574   \n",
       "min       3.000000     1.000000     0.000000    0.000000    0.000000   \n",
       "25%      95.000000    71.000000   113.500000    8.000000    3.000000   \n",
       "50%     230.000000   174.000000   224.000000   45.000000    7.000000   \n",
       "75%     424.500000   328.500000   322.500000  192.000000   13.000000   \n",
       "max    1659.000000  1566.000000  1377.000000  492.000000   32.000000   \n",
       "\n",
       "            Salary  \n",
       "count   263.000000  \n",
       "mean    535.925882  \n",
       "std     451.118681  \n",
       "min      67.500000  \n",
       "25%     190.000000  \n",
       "50%     425.000000  \n",
       "75%     750.000000  \n",
       "max    2460.000000  "
      ]
     },
     "execution_count": 61,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.describe()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "Int64Index: 263 entries, 1 to 321\n",
      "Data columns (total 21 columns):\n",
      "Unnamed: 0    263 non-null object\n",
      "AtBat         263 non-null int64\n",
      "Hits          263 non-null int64\n",
      "HmRun         263 non-null int64\n",
      "Runs          263 non-null int64\n",
      "RBI           263 non-null int64\n",
      "Walks         263 non-null int64\n",
      "Years         263 non-null int64\n",
      "CAtBat        263 non-null int64\n",
      "CHits         263 non-null int64\n",
      "CHmRun        263 non-null int64\n",
      "CRuns         263 non-null int64\n",
      "CRBI          263 non-null int64\n",
      "CWalks        263 non-null int64\n",
      "League        263 non-null object\n",
      "Division      263 non-null object\n",
      "PutOuts       263 non-null int64\n",
      "Assists       263 non-null int64\n",
      "Errors        263 non-null int64\n",
      "Salary        263 non-null float64\n",
      "NewLeague     263 non-null object\n",
      "dtypes: float64(1), int64(16), object(4)\n",
      "memory usage: 45.2+ KB\n"
     ]
    }
   ],
   "source": [
    "df.info()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0"
      ]
     },
     "execution_count": 63,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df['Salary'].isnull().sum()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 64,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "A    139\n",
       "N    124\n",
       "Name: League, dtype: int64"
      ]
     },
     "execution_count": 64,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.League.value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 65,
   "metadata": {},
   "outputs": [],
   "source": [
    "dummies = pd.get_dummies(df[['League', 'Division', 'NewLeague']])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 66,
   "metadata": {},
   "outputs": [],
   "source": [
    "y= df.Salary\n",
    "X_ = df.drop(['League', 'Division', 'NewLeague', 'Salary','Unnamed: 0'],axis =1).astype('float64')\n",
    "X= pd.concat([X_, dummies[['League_N', 'Division_E', 'NewLeague_N']]],axis =1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 67,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Index(['AtBat', 'Hits', 'HmRun', 'Runs', 'RBI', 'Walks', 'Years', 'CAtBat',\n",
       "       'CHits', 'CHmRun', 'CRuns', 'CRBI', 'CWalks', 'PutOuts', 'Assists',\n",
       "       'Errors', 'League_N', 'Division_E', 'NewLeague_N'],\n",
       "      dtype='object')"
      ]
     },
     "execution_count": 67,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X.columns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 68,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>AtBat</th>\n",
       "      <th>Hits</th>\n",
       "      <th>HmRun</th>\n",
       "      <th>Runs</th>\n",
       "      <th>RBI</th>\n",
       "      <th>Walks</th>\n",
       "      <th>Years</th>\n",
       "      <th>CAtBat</th>\n",
       "      <th>CHits</th>\n",
       "      <th>CHmRun</th>\n",
       "      <th>CRuns</th>\n",
       "      <th>CRBI</th>\n",
       "      <th>CWalks</th>\n",
       "      <th>PutOuts</th>\n",
       "      <th>Assists</th>\n",
       "      <th>Errors</th>\n",
       "      <th>League_N</th>\n",
       "      <th>Division_E</th>\n",
       "      <th>NewLeague_N</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>315.0</td>\n",
       "      <td>81.0</td>\n",
       "      <td>7.0</td>\n",
       "      <td>24.0</td>\n",
       "      <td>38.0</td>\n",
       "      <td>39.0</td>\n",
       "      <td>14.0</td>\n",
       "      <td>3449.0</td>\n",
       "      <td>835.0</td>\n",
       "      <td>69.0</td>\n",
       "      <td>321.0</td>\n",
       "      <td>414.0</td>\n",
       "      <td>375.0</td>\n",
       "      <td>632.0</td>\n",
       "      <td>43.0</td>\n",
       "      <td>10.0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>479.0</td>\n",
       "      <td>130.0</td>\n",
       "      <td>18.0</td>\n",
       "      <td>66.0</td>\n",
       "      <td>72.0</td>\n",
       "      <td>76.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>1624.0</td>\n",
       "      <td>457.0</td>\n",
       "      <td>63.0</td>\n",
       "      <td>224.0</td>\n",
       "      <td>266.0</td>\n",
       "      <td>263.0</td>\n",
       "      <td>880.0</td>\n",
       "      <td>82.0</td>\n",
       "      <td>14.0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>496.0</td>\n",
       "      <td>141.0</td>\n",
       "      <td>20.0</td>\n",
       "      <td>65.0</td>\n",
       "      <td>78.0</td>\n",
       "      <td>37.0</td>\n",
       "      <td>11.0</td>\n",
       "      <td>5628.0</td>\n",
       "      <td>1575.0</td>\n",
       "      <td>225.0</td>\n",
       "      <td>828.0</td>\n",
       "      <td>838.0</td>\n",
       "      <td>354.0</td>\n",
       "      <td>200.0</td>\n",
       "      <td>11.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>321.0</td>\n",
       "      <td>87.0</td>\n",
       "      <td>10.0</td>\n",
       "      <td>39.0</td>\n",
       "      <td>42.0</td>\n",
       "      <td>30.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>396.0</td>\n",
       "      <td>101.0</td>\n",
       "      <td>12.0</td>\n",
       "      <td>48.0</td>\n",
       "      <td>46.0</td>\n",
       "      <td>33.0</td>\n",
       "      <td>805.0</td>\n",
       "      <td>40.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>594.0</td>\n",
       "      <td>169.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>74.0</td>\n",
       "      <td>51.0</td>\n",
       "      <td>35.0</td>\n",
       "      <td>11.0</td>\n",
       "      <td>4408.0</td>\n",
       "      <td>1133.0</td>\n",
       "      <td>19.0</td>\n",
       "      <td>501.0</td>\n",
       "      <td>336.0</td>\n",
       "      <td>194.0</td>\n",
       "      <td>282.0</td>\n",
       "      <td>421.0</td>\n",
       "      <td>25.0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   AtBat   Hits  HmRun  Runs   RBI  Walks  Years  CAtBat   CHits  CHmRun  \\\n",
       "1  315.0   81.0    7.0  24.0  38.0   39.0   14.0  3449.0   835.0    69.0   \n",
       "2  479.0  130.0   18.0  66.0  72.0   76.0    3.0  1624.0   457.0    63.0   \n",
       "3  496.0  141.0   20.0  65.0  78.0   37.0   11.0  5628.0  1575.0   225.0   \n",
       "4  321.0   87.0   10.0  39.0  42.0   30.0    2.0   396.0   101.0    12.0   \n",
       "5  594.0  169.0    4.0  74.0  51.0   35.0   11.0  4408.0  1133.0    19.0   \n",
       "\n",
       "   CRuns   CRBI  CWalks  PutOuts  Assists  Errors  League_N  Division_E  \\\n",
       "1  321.0  414.0   375.0    632.0     43.0    10.0         1           0   \n",
       "2  224.0  266.0   263.0    880.0     82.0    14.0         0           0   \n",
       "3  828.0  838.0   354.0    200.0     11.0     3.0         1           1   \n",
       "4   48.0   46.0    33.0    805.0     40.0     4.0         1           1   \n",
       "5  501.0  336.0   194.0    282.0    421.0    25.0         0           0   \n",
       "\n",
       "   NewLeague_N  \n",
       "1            1  \n",
       "2            0  \n",
       "3            1  \n",
       "4            1  \n",
       "5            0  "
      ]
     },
     "execution_count": 68,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 69,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "19"
      ]
     },
     "execution_count": 69,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X.shape[1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 70,
   "metadata": {},
   "outputs": [],
   "source": [
    "#very very slow to run this\n",
    "\n",
    "#lm= LinearRegression()\n",
    "#efs = EFS(lm,min_features=1, max_features=19, scoring='neg_mean_squared_error',print_progress=True, cv=5)\n",
    "#efsfit= efs.fit(X,y)\n",
    "#print('Best negtive mean squared error: %.2f' % efsfit.best_score_)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Forward Subset Selection"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 71,
   "metadata": {},
   "outputs": [],
   "source": [
    "from mlxtend.feature_selection import SequentialFeatureSelector as SFS"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 90,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=1)]: Done   1 out of   1 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  19 out of  19 | elapsed:    0.2s finished\n",
      "\n",
      "[2019-04-16 14:24:22] Features: 1/7 -- score: -142142.865462403[Parallel(n_jobs=1)]: Done   1 out of   1 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  18 out of  18 | elapsed:    0.1s finished\n",
      "\n",
      "[2019-04-16 14:24:22] Features: 2/7 -- score: -124277.83941532117[Parallel(n_jobs=1)]: Done   1 out of   1 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  17 out of  17 | elapsed:    0.1s finished\n",
      "\n",
      "[2019-04-16 14:24:22] Features: 3/7 -- score: -120082.1567462722[Parallel(n_jobs=1)]: Done   1 out of   1 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  16 out of  16 | elapsed:    0.1s finished\n",
      "\n",
      "[2019-04-16 14:24:22] Features: 4/7 -- score: -117217.98393288904[Parallel(n_jobs=1)]: Done   1 out of   1 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  15 out of  15 | elapsed:    0.1s finished\n",
      "\n",
      "[2019-04-16 14:24:22] Features: 5/7 -- score: -114112.22125262395[Parallel(n_jobs=1)]: Done   1 out of   1 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  14 out of  14 | elapsed:    0.1s finished\n",
      "\n",
      "[2019-04-16 14:24:22] Features: 6/7 -- score: -110162.26143793529[Parallel(n_jobs=1)]: Done   1 out of   1 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  13 out of  13 | elapsed:    0.1s finished\n",
      "\n",
      "[2019-04-16 14:24:22] Features: 7/7 -- score: -110553.2314637053"
     ]
    },
    {
     "data": {
      "text/plain": [
       "SequentialFeatureSelector(clone_estimator=True, cv=5,\n",
       "             estimator=LinearRegression(copy_X=True, fit_intercept=True, n_jobs=1, normalize=False),\n",
       "             floating=False, forward=True, k_features=7, n_jobs=1,\n",
       "             pre_dispatch='2*n_jobs', scoring='neg_mean_squared_error',\n",
       "             verbose=2)"
      ]
     },
     "execution_count": 90,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "lm= LinearRegression()\n",
    "sfs1 = SFS(lm,\n",
    "           k_features=7,\n",
    "           forward=True,\n",
    "           floating=False,\n",
    "           verbose=2,\n",
    "           scoring='neg_mean_squared_error',\n",
    "           cv=5)\n",
    "\n",
    "sfs1.fit(X,y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 91,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{1: {'feature_idx': (11,),\n",
       "  'cv_scores': array([ -69552.92336717, -213009.92207213, -118946.76030707,\n",
       "         -207644.13810831, -101560.58345733]),\n",
       "  'avg_score': -142142.865462403,\n",
       "  'feature_names': ('CRBI',)},\n",
       " 2: {'feature_idx': (1, 11),\n",
       "  'cv_scores': array([ -53798.49669675, -163254.8070404 , -104610.93335577,\n",
       "         -201906.21884294,  -97818.74114074]),\n",
       "  'avg_score': -124277.83941532117,\n",
       "  'feature_names': ('Hits', 'CRBI')},\n",
       " 3: {'feature_idx': (1, 11, 17),\n",
       "  'cv_scores': array([ -62163.90514053, -153941.83393405,  -97468.14894897,\n",
       "         -191620.10821684,  -95216.78749097]),\n",
       "  'avg_score': -120082.1567462722,\n",
       "  'feature_names': ('Hits', 'CRBI', 'Division_E')},\n",
       " 4: {'feature_idx': (1, 11, 13, 17),\n",
       "  'cv_scores': array([ -65576.39003923, -141278.46897399,  -82554.98231355,\n",
       "         -188319.24800473, -108360.83033295]),\n",
       "  'avg_score': -117217.98393288904,\n",
       "  'feature_names': ('Hits', 'CRBI', 'PutOuts', 'Division_E')},\n",
       " 5: {'feature_idx': (0, 1, 11, 13, 17),\n",
       "  'cv_scores': array([ -64199.0854547 , -133171.50527159,  -83687.1147473 ,\n",
       "         -186210.0847334 , -103293.31605613]),\n",
       "  'avg_score': -114112.22125262395,\n",
       "  'feature_names': ('AtBat', 'Hits', 'CRBI', 'PutOuts', 'Division_E')},\n",
       " 6: {'feature_idx': (0, 1, 5, 11, 13, 17),\n",
       "  'cv_scores': array([ -60075.53412214, -132306.39833205,  -76593.1966602 ,\n",
       "         -185946.79180744,  -95889.38626785]),\n",
       "  'avg_score': -110162.26143793529,\n",
       "  'feature_names': ('AtBat',\n",
       "   'Hits',\n",
       "   'Walks',\n",
       "   'CRBI',\n",
       "   'PutOuts',\n",
       "   'Division_E')},\n",
       " 7: {'feature_idx': (0, 1, 3, 5, 11, 13, 17),\n",
       "  'cv_scores': array([ -60330.25929007, -132346.01741313,  -76649.84186021,\n",
       "         -187442.11344863,  -95997.92530648]),\n",
       "  'avg_score': -110553.2314637053,\n",
       "  'feature_names': ('AtBat',\n",
       "   'Hits',\n",
       "   'Runs',\n",
       "   'Walks',\n",
       "   'CRBI',\n",
       "   'PutOuts',\n",
       "   'Division_E')}}"
      ]
     },
     "execution_count": 91,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sfs1.subsets_"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Backward Subset Selection"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 95,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=1)]: Done   1 out of   1 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  19 out of  19 | elapsed:    0.2s finished\n",
      "\n",
      "[2019-04-16 14:26:50] Features: 18/7 -- score: -118980.27868752561[Parallel(n_jobs=1)]: Done   1 out of   1 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  18 out of  18 | elapsed:    0.1s finished\n",
      "\n",
      "[2019-04-16 14:26:50] Features: 17/7 -- score: -117604.40985668839[Parallel(n_jobs=1)]: Done   1 out of   1 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  17 out of  17 | elapsed:    0.1s finished\n",
      "\n",
      "[2019-04-16 14:26:50] Features: 16/7 -- score: -116518.59452589511[Parallel(n_jobs=1)]: Done   1 out of   1 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  16 out of  16 | elapsed:    0.1s finished\n",
      "\n",
      "[2019-04-16 14:26:50] Features: 15/7 -- score: -115875.63685350744[Parallel(n_jobs=1)]: Done   1 out of   1 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  15 out of  15 | elapsed:    0.1s finished\n",
      "\n",
      "[2019-04-16 14:26:50] Features: 14/7 -- score: -114090.43222176089[Parallel(n_jobs=1)]: Done   1 out of   1 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  14 out of  14 | elapsed:    0.1s finished\n",
      "\n",
      "[2019-04-16 14:26:50] Features: 13/7 -- score: -113437.09377154875[Parallel(n_jobs=1)]: Done   1 out of   1 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  13 out of  13 | elapsed:    0.1s finished\n",
      "\n",
      "[2019-04-16 14:26:50] Features: 12/7 -- score: -112034.22059086624[Parallel(n_jobs=1)]: Done   1 out of   1 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  12 out of  12 | elapsed:    0.1s finished\n",
      "\n",
      "[2019-04-16 14:26:50] Features: 11/7 -- score: -111504.22174979348[Parallel(n_jobs=1)]: Done   1 out of   1 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  11 out of  11 | elapsed:    0.1s finished\n",
      "\n",
      "[2019-04-16 14:26:50] Features: 10/7 -- score: -111215.33180039287[Parallel(n_jobs=1)]: Done   1 out of   1 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  10 out of  10 | elapsed:    0.1s finished\n",
      "\n",
      "[2019-04-16 14:26:51] Features: 9/7 -- score: -111271.8595214715[Parallel(n_jobs=1)]: Done   1 out of   1 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done   9 out of   9 | elapsed:    0.0s finished\n",
      "\n",
      "[2019-04-16 14:26:51] Features: 8/7 -- score: -111195.70503836502[Parallel(n_jobs=1)]: Done   1 out of   1 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done   8 out of   8 | elapsed:    0.0s finished\n",
      "\n",
      "[2019-04-16 14:26:51] Features: 7/7 -- score: -112446.76724753054"
     ]
    },
    {
     "data": {
      "text/plain": [
       "SequentialFeatureSelector(clone_estimator=True, cv=5,\n",
       "             estimator=LinearRegression(copy_X=True, fit_intercept=True, n_jobs=1, normalize=False),\n",
       "             floating=False, forward=False, k_features=7, n_jobs=1,\n",
       "             pre_dispatch='2*n_jobs', scoring='neg_mean_squared_error',\n",
       "             verbose=2)"
      ]
     },
     "execution_count": 95,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sfs2 = SFS(lm,\n",
    "           k_features=7,\n",
    "           forward=False,\n",
    "           floating=False,\n",
    "           verbose=2,\n",
    "           scoring='neg_mean_squared_error',\n",
    "           cv=5)\n",
    "\n",
    "sfs2.fit(X.values,y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 96,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Sequential Backward Selection (k=7):\n",
      "(0, 1, 5, 10, 11, 13, 17)\n",
      "CV Score:\n",
      "-112446.76724753054\n"
     ]
    }
   ],
   "source": [
    "print('\\nSequential Backward Selection (k=7):')\n",
    "print(sfs2.k_feature_idx_)\n",
    "print('CV Score:')\n",
    "print(sfs2.k_score_)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Ridge Regression"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 101,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.preprocessing import scale \n",
    "from sklearn.model_selection import train_test_split\n",
    "import matplotlib.pyplot as plt\n",
    "X_train, X_test, y_train, y_Test = train_test_split(X,y, test_size =0.5, random_state =1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 102,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAZAAAAEaCAYAAAA/lAFyAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4zLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvIxREBQAAIABJREFUeJzsnXecHlW5+L/PzLz9fbdvNptsekICSSghJPSOUi0oigIqIOK1X73We/Wn91pQr9fC1YsKKoKigkgTRFroBNIoaaSX3Wyvb39n5vz+mEnyJmyym81uNrs538/nvDNzzplznpk57zxz2nNEKYVGo9FoNAeKMdwCaDQajWZkohWIRqPRaAaEViAajUajGRBagWg0Go1mQGgFotFoNJoBoRWIRqPRaAaEViD9RERuEZGv7ydcicj0QynTQBGRiIg8KCJdInK37/dtEWkVkUYRmSgiSREx+0jnDBFZe2ikPrwQkRoReUZEekTkR4c476SITD3Eeb6lzPTjnEUi8tGhlm2wEZGzRWT7QZx/lYj8czBl8tP9mojcOtjpHgzWcAtwuCAim4EawAGSwD+ATymlkgBKqY8Pn3SDznvxrrVSKWWLyETgC8AkpVSzHyfeVyJKqWeBmYMhkH//P6qUenww0jsEfAxoBUrUEE6mEpFFwJ1KqV0vDqVUn89mCNijzOwdKCLfBKYrpa4+1IIdbiil/gD84WDSEJGz8Z57XVG63z1I0QYdXQPZk8v8P+fxwAnAV4dZnqFiEvBm0YtgItBWpDw0fTMJWDWUyuMwY+8yM2yIyGH74Xs4yzYkKKW0894Bm4Hzi45/APy96Ph3wLeLjr8I7AAagOsAhfcFBlAJPAh0A68A3waeKzp3FvAY0A6sBd63H7kqgN/6+XQA9xWF3QCs99N5ABjXVx7At4A8UMCrad0IZADXP/4dMNm/Hmt/MgBnA9uL8hwH/BVoATYBnykK+ybwF+D3QA+wEpjvh93h55/xZfgSEAbuBNqATv8+1uzjHn0F2OCnuwp4d1HYdOBpoAuvxvDn/dzru4FGP+4zwOx9xPudf//yvrzn91I+9r43m4F/A17z0/8zEC4Kfyewwi8zG4ALge/g1Yizfj7/68ctLmul/j1tAbYA/wEYfthHgOeA//af2ybgov1c/9HAIv9+rwTesY8yc/1e5124V/irvv8i4L+A5/1n80+gqui8k4EX/PxeBc7u4//5Zf/+5fBaT/ZX3iLA7f51r/bLVPHz2HUP9/5/9/Ls9le+PuJf34/xyuq3d953P/xL/j3Z6QrA7/ywa33ZeoCNwI2+f4w9/5NJ/1q/iVcr2Zn3O/zn1Onf66P7W94G7b052AmOVEeRAgHqgNeBn+6jgF0INAFz/If9R/b8U//Jd1HgGGBbUYGK+cfX+n+CE/BebMfsQ66/+w+/HAgAZ/n+5/rnzQNCwM3AM/3Jo5eCuPcfZjJ7KpB9ybDrPLza7FLgG0AQmOr/Kd5elGcWuBgwge8BL/V2//3jG/GUcNSPfyJec1Fv9+gK/w9mAO8HUkCtH3YX8O9+WBg4fT9l4Dog4d/PnwAr9hN3V3nYx/He93Qz8LIvZwXei+PjftgCvD/5Bb6c44FZftgivKa94ryLy9rvgft9uScDb+K/4PFeZAW8Dw0T+Be8jwDp5XoCeB8jX/Of37l4L7aZvZWZXs5/S7gv+wbgKLwX+iLgJj9sPN4L92L/mi/wj6v38/9cAUzw0+qrvN2E9+FQjvd/fo2BK5D9la+PADbwabz/WoQiBbLXNUzw7/9F/vElwDRAgLOANDCvNxn2vsf+PU359y2Ap6jWA8G+yttgOt2EtSf3iUgP3su3Gfh/+4j3PuC3Sqk3lFIpvAcLgN/x/B7g/yml0kqpVXhfQju5FNislPqtUspWSi3H+4q6Yu9MRKQWuAjvwXcopQpKqaf94KuA3yillimlcnjNbaeIyOQDyaMv+pChmJPw/vz/qZTKK6U2Ar8GriyK85xS6mGllINX6zhuP1kX8Gpy05VSjlJqqVKqu7eISqm7lVINSilXKfVnYB3eS3lnOpPwamdZpdRz+8pQKfUbpVSPfz+/CRwnIqX7kfFA+ZkvZzuecjze978e71k+5l9DvVJqTV+J+WXtSuCrvtybgR8B1xRF26KU+rV/z28HavH6MvbmZLx+r5v85/ck8BDwgYFd6i5+q5R6UymVwauB7rzmq4GH/fLgKqUeA5bgKZR98TOl1DY/rb7K2/uA7/pldjvws4FeQB/lC6BBKXWz/1/L9JaGiESA+/A+Sh/x0/27UmqD8ngar4Z2Rj/Fej9eC8ljSqkCXi0zApxaFGdf5W3Q0ApkT96llErgaf9ZQNU+4o3DUzI72VK0X433JVIcXrw/CVgoIp07HZ4yGNtLPhOAdqVUxz5k2JWv8jr72/C+7A4kj77YnwzFTALG7ZXn19jzZdVYtJ8GwvtpM74DeBT4k4g0iMgPRCTQW0QR+ZCIrCjKdw67n92X8L7wXhaRlSJy3T7SMEXkJhHZICLdeF9wsO8yMBD2vv6dneET8L7UD5QqvK/P4vK3Ba8MvCVPpVTa3+2tE34csE0p5e4nrYGwr2ueBFyxV3k5HU/B7Yu9/0f7K297/0eL9w+IPspXf9O+DVirlPp+UboXichLItLup3sx/S9ve///XV+OXp89e977QePI6vDpJ0qpp0Xkd3ha/V29RNmB96ffycSi/Ra8Km0dXnMCe8XdBjytlLqgH6JsAypEpEwp1blXWAPenwgAEYnhfbHXH2AeByPD3vE2KaVmDDCfPTqj/a+qbwHf8mtVD+P15dxWHE9EJuF9eZ4HvKiUckRkBZ7SQCnViNeEg4icDjwuIs8opdbvlf8H8fohzsdTHqV47efST/lTeM1tOzkQZb0NrymjN/bXSd/K7hrWKt9vIl4ZOFAagAkiYhQpkYnsLsN9caCDCbYBdyilbjiAc4rz6Ku87cD7D+68LxP2Ck/z1uf1lqG7fZWvXuR6CyLyFbwmpzOK/EJ4rQIfAu5XShVE5L6idPu6nw3A3KL0BO8aB/LsB4yugeybnwAXiEhvzSx/AT4iIseISJSipi6/qeBe4JsiEhWRWXiFZCcPAUeJyDUiEvDdSSJy9N6ZKKV2AI8AvxCRcj/umX7wXcC1InK8Xxi/Cyz2mzH6nUdf9CFDMS8DPSLyZX/OgCkic0TkpH5m1YTXjg2AiJwjInP9ZppuvBel28t5Mbw/W4t/3rV4X4g707lCRHYOhezw4/aWTgKvc7YN78VyoEMmVwAXi0iFiIwFPncA596G9yzPExFDRMb75Qb2ui/F+GXtL8B3RCThv+w+jzf44EBZjPdS/ZL/jM8GLsPry+sPTcBkEenvO+VO4DIRebtfVsLizb+o6/NMj77K21+Ar/pldjzwqb3OXwF80D/vQrw+iN7Yb/nqCxG5CPgMXsd7cfNWEK+vrQWw/XhvKwpvAir304T6F+ASv8wE8Ibh5/AGJRwytALZB0qpFrwOym/0EvYInoJ5Eq/j6sm9onwK7wu2Ea8p5i68h4tSqgevoFyJ9xXRCHwfrzD1xjV4L881eP0yn/PTeRz4Ot5XzA68L9grB5hHX/QqQzH+y+xSvHbWTXhfx7fi3Yf+8D3gP/xmgn/D+yK8B095rMbrEL2jl3xX4bX7v4j3p5uLNypmJycBi0UkiTdS7bN+e/ne/B6vSaAe76v1pX7KvZM78EYSbcZry/5zf09USr2MN+Dhx3id6U+zu3b5U+C9ItIhIr21438ar/azEW/E1R+B3xyg7Cil8ngK4yK8Z/cL4EP96Yvx2Tm5sE1ElvUjv214Nb6v4b1Et+GNbOzXO6kf5e0/8WoUm4DH8cpSriiJz+Jd787m3fv2kU9f5asv3o/XrL1avAmgSRG5xf+PfgZPEXTg1YAfKMp3Dd57Y6P/nxi3l1xr8fqRbvav/TK8aQj5A5DtoBGlDrTmqTlQROT7wFil1IeHWxaN5khERP4FuFIpta+ahmYA6BrIECAis0TkWPFYgDfK5m/DLZdGc6QgIrUicprfJDgTr4lH/wcHGd2JPjQk8Kqf4/CqvT/CG6uv0WgODUHgl8AUvGaqP+E1y2kGEd2EpdFoNJoBoZuwNBqNRjMgtALRaDQazYAY1X0gVVVVavLkycMthkaj0Ywoli5d2qqUqu4r3qhWIJMnT2bJkiXDLYZGo9GMKERkS9+xdBOWRqPRaAaIViAajUajGRBagWg0Go1mQGgFotFoNJoBoRWIRqPRaAaEViAajUajGRCjehivRqPRjGYKjksya5PM2aTzDum8TabgkCu4RIMmC6dWDmn+WoFoNBrNYYRSirZUni1tKRo6s+zoyrCjK0trMk9bMkd7Kk9nukBXpkCm4OwzneMmlHH/J08bUlm1AtFoNJphIltweL2+i1UN3axq6GZ1YzebWlL05Ow94sWCJtWJEJXxEBMqohxbF6AkHKA0EiAetoiFLGJBi2jQJBwwiQRNSiOBIZdfKxCNRqM5RDiuYtnWDhatbeblTe28uq2LvOOtsFwRC3J0bYJ3zxvP5MoYk6ui1JVHGVsaJhGy8JY9P7zQCkSj0WiGEKUUL25s477l9Tyxupm2VB7TEOaOL+Xa0yYzf3IFx9aVMiYROiyVxP7QCkSj0WiGgI5Unj+9so0/v7KVzW1pEiGLs2eN4YJjajh7ZjUl4aFvYhpqtALRaDSaQaQtmePW5zbx+xc2k8o7LJhSwWfPn8FFc2oJB8zhFm9Q0QpEo9FoBoGc7XDLoo388pkNZAoOlx47jk+dM52ZYxPDLdqQoRWIRqPRHCTPr2/l6/e9wcbWFJfMreVfL5jB9DGjV3HsRCsQjUajGSCZvMO3HlzJn17ZxqTKKL+/bgFnHtXnOkyjBq1ANBqNZgBsbEnyiT8sY01jDx8/axqfO3/GqOvj6AutQDQajeYA+ccbjfzb3a8SMIXfXnsS58wcM9wiDQtagWg0Gs0B8IfFW/iP+97g2Loy/u+qeYwriwy3SMOGViAajUbTD5RS/GLRBn746FrOmVnNL646kUjwyGqy2hutQDQajaYPlFJ8/x9rueXpDbzr+HH88IrjCJh6NQytQDQajaYPfvnMRm55egNXLZzIf71zDoYxskyODBXDrkJFxBSR5SLykH88RUQWi8h6EfmziAR9/5B/vN4Pnzyccms0miODu5ds46ZH1nDpsbVaeezFsCsQ4LPA6qLj7wM/VkpNBzqA633/64EO3//HfjyNRqMZMp5c08RX7n2d06dX8aP3HaeVx14MqwIRkTrgEuBW/1iAc4F7/Ci3A+/y99/pH+OHnycjzXSlRqMZMaxv7uHTf1zO0bUJbrnmRELWkd1h3hvDXQP5CfAlwPWPK4FOpdTO1VS2A+P9/fHANgA/vMuPvwci8jERWSIiS1paWoZSdo1GM0pJ5Ww+fucywgGTX39oPvGQ7i7ujWFTICJyKdCslFo6mOkqpX6llJqvlJpfXX3kmBTQaDSDg1KKr9z7Ohtbktz8gROoLT1y53n0xXCq1dOAd4jIxUAYKAF+CpSJiOXXMuqAej9+PTAB2C4iFlAKtB16sTUazWjm9hc28+CrDXzpwpmcOr1quMU5rBm2GohS6qtKqTql1GTgSuBJpdRVwFPAe/1oHwbu9/cf8I/xw59USqlDKLJGoxnlrGns5jsPr+b8o8fw8TOnDbc4hz3D3QfSG18GPi8i6/H6OG7z/W8DKn3/zwNfGSb5NBrNKKTguHzhL69SEg7w/fccq0dc9YPDomdIKbUIWOTvbwQW9BInC1xxSAXTaDRHDL94agMrG7q55eoTqYyHhlucEcHhWAPRaDSaQ8ob9V3c/OQ63nn8OC6cM3a4xRkxaAWi0WiOaAqOy7/d/SrlsSDfesfs4RZnRHFYNGFpNBrNcHH7C5tZ09jDLVefSFk0ONzijCh0DUSj0RyxNHdn+cnj6zh7ZjVvn10z3OKMOLQC0Wg0RyzffXg1edvlm5fNRltGOnC0AtFoNEckize2cd+KBm48ayqTq2LDLc6IRCsQjUZzxGE7Lt+4fyXjyyJ84uzpwy3OiEV3oms0miOOu5duZ21TD7dcPW/kLUurFLgOKMffukX7ClCen5gQe4u92UFFKxCNRnNEkc7b/M9jbzJ/Ujlvnz1Ecz6UglwPpFog3ea7dsh2Qa4bst2QT0I+5Tk7A4Wst7VznnPy4BR2b13bc8rpnwzj58MNTwzN9floBaLRaI4obn12Ey09OW65+sSD6zgvZKB1HbS+Ce0boXMLdG6Fru3Q0wSF1L7PDcYhGPNcIAaBiOciZWAGwQp5WzMIZgCMgL+1wDC9rRjevphF+4bvBGJjBn5t/UQrEI1Gc8TQ0pPjl09v4KI5YzlxUnn/T3QKUL8Mtr8CO1ZAw3Jo2wAU2XONj4WyiVB7PBxVC4mxEB8D0SqIlkOkAsKlECoBc3S8ekfHVWg0Gk0/+NkT68jZLl98+8y+I3dshtUPwYYnYetLu2sUJXUw7niYewVUHQXVM6FiqleDOMLQCkSj0RwRbGpN8ceXt3LVwolMrY73HinZDCv+ACv/Bjte9fyqZsLxH4QpZ8DEU7xahQbQCkSj0Rwh/PTxNwmaBp86t5dhu1tehJd/BasfBLcAdSfBBf8FR18GFVMOvbAjBK1ANBrNqGddUw/3v9rAx86cyphEeHfA1sXw1Ldh0zNe/8SCG+DEa6H6qOETdgShFYhGoxn1/PjxN4kGTG7cucpg+yZ45Muw7lGIVcPbvwcnfgSC0WGVc6ShFYhGoxnVrGzo4uHXG/nMudOpCBvwws3w5He8obDnfxMWfMwbTqs5YLQC0Wg0o5ofP7aOkrDF9XNMuO0CaFgGR10El/wISscPt3gjGq1ANBrNqOW17Z08vrqJL8wzKf39eYCC9/4GZl/uTbbTHBRagWg0mlHLTx9fR2nA4SMrPwpjp8L779SjqgYRbY1Xo9GMSl7f1sETa5r5qPoribkXw/X/1MpjkNEKRKPRjD4cm5/98V5KSPHh06bBe27VHeVDgFYgGo1mdOEUWHnnF3isYyzXT09RcuHXdX/HEKEViEajGT0oBQ9+lp+tLSNhOXzkqqu18hhCtALRaDSjh0XfY/WyZ3jUXcC1Z82iNBIYbolGNXoUlkajGR0svR2e/j7/m/gR8aTF9aeN0g5zpXYvRJVPQiHdy2JUOc80y/Tzh1QUrUA0Gs3IZ8NT8NC/8mbde3l4Qy2fPHsypdERVPtwXehp8EzIdzd4rqexaEXDVm81w2yXt5ph8Tok+2L8iVqBaDQazX7p2g5/vR6qZ/K/kRuJBNq5/vTDuPbRtd1bnKppJTSvhJa1nuJw8nvGC8Y9O12xKigZDzVzvMWowiUQSvgrGsb91QyjYIV9FwQzBKF9mKwfRLQC0Wg0Ixc7D3d/BOw868+7lQd/t5kbz5xGeSw43JLtpmMzrH8cNj4N25d4NQ0AxFuIaszRMPMiKJ/suZI6KKn1lMRhjlYgGo1m5PLY171lZq+4nZ+vKBC2TD56xmFQ+2haCa/f7a0v0rbe8yudAJNOhQkLoG4+jDlmxK9iqBWIRqMZmay8DxbfAid/gk01F3D/nYu4/vQpVMVDwyNPptNbzXDZHdCyGsSEqWfBSTfA9POgcvqoG1KsFYhGoxl5dO+ABz8L4+bBBf/JzX9dScA0uOHMqYdelrYN8NIvYMVd3rrpdQvg4v+G2e/2+i9GMVqBaDSakYVScP8nveGql/+aje057ltez3WnTdlztcGhpms7PP19WP4HMEyYe4W3tsi44w+dDMOMViAajWZk8cqtsOEJ7yu/ajo3/3kFQcvgxrOmHZr8c0lPcSy+xTs+6aNwxhcgUXNo8j+M0ApEo9GMHFrXwT+/DtPOg5M+yoaWJPevqOf606dQnTgEfR9r/g4Pfwm6t8PxV8HZX4WyCUOf72GKViAajWZk4Dpe05UVgnf+HES4+Yl1hCxz6Gsf6XZ48DPeqKoxs+G9t8HEk4c2zxGAViAajWZk8MqtsG0xvOsWKKllfXOSB15t4IYzpg7tyKuNi+BvH4dUq7eG+imfAnMEzXIfQobNmKKITBCRp0RklYisFJHP+v4VIvKYiKzzt+W+v4jIz0RkvYi8JiLzhkt2jUZziOnYAo9/yzPNcdyVAPz4sTcJB0w+NlQjr1wHHv8m/P5d3qS+G56A0/9VK48ihtMarw18QSl1DHAy8EkROQb4CvCEUmoG8IR/DHARMMN3HwP+79CLrNFoDjm+iXZE4NKfgAivbe/k76/v4KNnTKVyKGofmQ74w3vhuR/DvGvgY4ug9rjBz2eEM2xNWEqpHcAOf79HRFYD44F3Amf70W4HFgFf9v1/r5RSwEsiUiYitX46Go1mtLLij7DxKW/Uld9h/cNH11IRC3LDUMw6b14Nd33AG6Z72c/gxA8Pfh6jhMNiPRARmQycACwGaoqUQiOwc2zceGBb0Wnbfb+90/qYiCwRkSUtLS1DJrNGozkEJFvg0a/BxFNg/vUAPL++lWfXtfLJc6aTCA9yc9KGJ+HWC6CQgWsf1sqjD4ZdgYhIHPgr8DmlVHdxmF/b6Ifd4j3O+ZVSar5San51dfUgSqrRaA45//iKt97FZT8Fw0ApxQ/+sYbxZRGuWjhxcPNa8Uf4wxVQPglueNKzWaXZL8OqQEQkgKc8/qCUutf3bhKRWj+8Fmj2/euB4gHXdb6fRqMZjax7DN64x5ukVz0TgEfeaOTV7V187vwZhAPm4OSjFDz9Q7jvX2Dy6XDtI1D6lsYNTS8M5ygsAW4DViul/qco6AFgZ73xw8D9Rf4f8kdjnQx06f4PjWaUkk/BQ5+HqpneyCcgW3C46ZE1HFUT5/J5dYOTj+vCI1+Gp74Nx30APni3t96Gpl8M5zyQ04BrgNdFZIXv9zXgJuAvInI9sAV4nx/2MHAxsB5IA9ceWnE1Gs0h46nvQtdWuPYf3sRB4DfPb2Jre5o7r1+IaQyCVVunAPd9Al7/ize3423fHnXWcoea4RyF9Rywr6d1Xi/xFfDJIRVKo9EMP/VLPeu2J14Lk04BoLk7y8+fXM8Fx9Rw+oxBsHBbyHgLUb35DzjvG3D657XyGAB6JrpGozl8cArwwGcgXgMXfGuX9w8eXUvecfn3i48++Dyy3XDXlbDlBbj0xzD/uoNP8whFKxCNRnP48PxPoekNuPIuCJcC8Nr2Tu5Zup0bz5rK5KrYwaWfaoU7L/dWDHzPrTD3vYMg9JGLViAajebwoHUdPP0DOOZdMOtiAGzH5T/ue4OqeIhPnTP94NLv2g53vBs6t3oK6qi3DYLQRzZagWg0muHHdeD+T3lrhF/8w13ev3thM69t7+J/P3jCwU0abF4Nd1wO+SRcfS9MPm0QhNZoBaLRaIafl34B216Cd/8S4mMA2NqW5r//uZbzjx7DJXNrB572lhe8Po9A1JvjMXbOIAmtGfaZ6BqN5gineQ088V8w61I49v0AKKX49/texxThP985BxnoCKnX7/Gs6cbGwPX/1MpjkNEKRKPRDB+ODfd9HIIxb0SUryj+uqyeZ9e18uWLZjGuLHLg6bqup5T+ej2MPxGuexTKBtn0iUY3YWk0mmHkuR9Dw3K44nd7NF1984GVzJ9UztULJx14mtluzyzJmodg3ofg4h+BFRxcuYcJpZRnegVQqF2WAovNBvrBiIBpDe3aJVqBaDSa4WHrYlj0PZjzHpj9bgAKjsun/7QcEfjx+4/HONAZ5/XL4J7roHMLXHgTLPz4gCcIOrZNpqebbE832VSSXDpFLp0mn06Tz2YoZDMUcjnsfM7bFgo4hTx2Po9r29h2Ade2cW0bx3FQroPrOLiOi+s6KNfFdV2U66BchVIKpVxwva2nK1zYuT1AaqfP5IPf+dGArr2/aAWi0WgOPel270VfWuc1Xfn8z2Nv8uq2Tn7+wXlMqIj2Pz3XhZd+Do9/CxWvJvP+WyjUzsDpeAHXzSESwDQjmGaEYLCaQKCSVEcHnU076GzaQVdTEz1tLSTb20i2t5Hu6iSbSu4/TxECwRBWKIQVDGIFQ1iBAGYggGkFCITCmDELw7QwTRMxzV1bwzAQw0CMon0REMEwDBBBYA9/EF8Xyu4+IQHZadBDZI++onhFZf/v3wDRCkSj0RxalIL7PwnJJrj+0V0TBp9b18otT2/gAwsmcMmxfY+6UsollVpHatPfSTz9S6KtjbRWRVk5I4vd8DVo2M+5LuSTAXJdQbLtIbIdEciNJRSoo3LCRCbOPZ5oSSmRklIiiQShWJxwLE4oGiUYiRIMR7BCoYF37o8StALRaDSHlsW/hLUPw9u/53Vw4/V7fPquZUyvjvONS2fv81TXzdHe/gItrY/T3vQ449ZvYdL2DI5psOWEeWRmnsGkcB3ZTmjd2kTLpq20bN6O6+QQyyWcCFE2voR4lUWixKWkvI38xHqUagfqsax1lJQcR3nZAsrLZ5FIzMUw9GtyX/TrzojINGC7UionImcDx+ItL9s5lMJpNJpRxqZn4J//DkddBCf/CwDd2QLX3f4KroJffWg+keBb1/lIJtfSsONuGhvvw861U9esWLA1QyCbwZ5zGerc75Fft43NTy5mw7JFZHu8temqJ05m+nHvYtyMWYydPpPSMTVvqTW4rk0ms5nu7lfp7FpGV9cyNmz0+g5MM055+UIqKs6gsuJ0IpHJR3ytoxhRqu8F/3xz6/OByXhm1e8HZiulLh5S6Q6S+fPnqyVLlgy3GBqNBqBtA/z6XM9Q4kcfg3AptuNy3e1LeGF9K3dcv5BTpu1ut1dK0da2iC1bf0Vn58sYymJGegq16zZidjehJpxM49SrWLGqlXUvv0ghmyEcizNl3klMO3EBE2YfS7SkdECi5vNtdHQupqP9BdrbnyeT3QpAODyBysozqaw4k/Lyk7Gs+KDcmsMNEVmqlJrfZ7x+KpBlSql5IvJFIKuUullEliulThgMYYcKrUA0msOETCfcej6k2+CGJ6BiKkopvn7/G9z50lZuunwuVy7w5mko5dLU9BCbt/wfqdSbRI0xzEpOpmzNEiTZjFM9m3Wxc3nmle30tLYSisaYsfA0jj79LOqOnoNhDtJKhUWk05tL2i7RAAAgAElEQVRpa3+W9vZn6eh4EcdJI2JRUnI8FRWnUVF+KiUlczGM0KDnPRz0V4H0t3GvICIfwFsh8DLfb2gHGGs0mtGBnYO7Pwwdm+FD9+9SHt/++2rufGkrN541lSsXTEQpRWvrE2zc+D8kU2updsYxt3020fWvIIVV5GoXskLO54Xn63GdFUyccxxnXX09005ciBUc2nke0ehkotHJTKi7BtfN09m1lPb252lvf45Nm37Gpk0/xTBClJacQFnZSZSUHk9pyXEEAuVDKtdw018Fci3wceA7SqlNIjIFuGPoxNJoNKMCpwB3XwsbF8E7fwGTT0MpxU2PrOG25zbxkVMn85ULZ9HVtZx1675Dd+cy6rpKOKGlgmDjaygrTM+4c3hxa5w3nmwkGGnlhAsv49jzL6Zi3PCsW24YQSrKT6Gi/BSY9m8UCp10dr5MR+fLdHa8zKbNPwe8eRuRyEQS8dnE47OIJ44mFp1KODxh1HTM9/cqLlBKfWbnga9EskMkk0ajGQ04Ntx7A6z9O1z0QzjhKk95/GMNv3xmI9ecPIkvnV/CypWfo2vL/UxsNjihKYeZ3YSqmErLzOt4ckWG7Y9uJ1EZ4ayrr2PueW8nFD3INUEGmUCgjOrqt1Fd7ZmHt+0k3T2v0931Kt09r9GTXElzyyO74osEiEQmEYnUEQlPIBypIxSqIRSsIRSqJhiswjTjI6Kz/oD6QPby030gGo2mdxzbm+vx2p+8tcZP/TTZgsOX//oa969o4IMLxnHt3CdIv/Zz6urTVLTnQAyYeTHb4wt54qlVtG3fRnntOBa88wqOPuMcTGvkfrXbdo83ZyW1kXR6A+n0JjLZerLZbdh2z1viiwQIBiqwAiVYlu/MGKYVwzLjGGYE04hgmCFMI4xhBDGMEGIEMCSIGAECViklJXMHJO+g9IH4/R4fBKaIyANFQQmgfUCSaTSa0U2ux2u2Wv8YnPt1OPXTtKfy3HjHEl7Z3MFnFvbwzsyV1N7bTDTjomJVcMan2WjM4dmH/knb9kepGD+Biz/zRWaecjqGMfid4ocay0pQWjqP0tJ5bwkrFLrJ5ZvI55rJ5ZrIF9op5NvJF9qx7W5su5t8rpm0k8RxUth2CtfN9JlnScnxnDT/r0NxObvoS6W/AOwAqoBioyo9wGtDJZRGoxmhdO+AP77PWzL20p/A/GtZsa2Tz9y1HKtrM/fW3cqxb6zEchR27TGoi/6VdakxvHjvPbRufYmKcXVc8pkvclQ/FYdyHFShUORscGyUbaMcBxwH5bje1HPXRbnK21fKO1Y7DRLusko4pLdnXxhAmDBhJgG9GJA0feejlEK5Nq7K4roFlCr4WxtX2eDamDKwIcwHQr+asEYquglLozmEbHvZq3lkOuB9t+NMO59bFq3n6Sce5MbAQ5zDUhDITl1I8KSvsX5tO6seuJdUfT1liVKmHz2Hyspq3GQKt6cHN5XCTac9l8mgMhncbBaVzeIWCqhcDhxnuK/6sCV83LFM+fOfB3TuoA7jFZHLge8DY8Cz8QUopVTJgKTTaDSjB9fxzLI/9V0oHQ/XPsy6VAXPfe1fOWPHk7wv1UYubbGJqSi7ksK9zZDzZqEfvSuRRnh9LW0iGPE4RiKOGYthRGMYsShmeTlGOIxEwhihEBIKI6EgEgggQX9rBRDLQgIW+EYLxTTBMMH0DBbiGyYs3vcsEgq7bRIe/p3X/cGID/0kx/72Sv0AuEwptXoohdFoNCML1bqB/O8/Qfa1V8max5PKVJO+4yOYqRwn+3FaKcGoqqAQj9OeTZFMhDGn1DHxtDOoW3AyVkUlZnkZZmkpRizmvdw1I4L+KpAmrTw0Go2ybbIrV5J64TkyT95HZu1WnLwBlIPZSKh0KxVjCrSXx+D4CZSecj0b1rXx+nNPk81mGbPwBGaecQ7Vk6dRsG02FwreuhlNTTgNDd76GEpR3LQuvply8U2dG4aB6ZtEN30T6cXOsqz9Hu88V3Pw7LcPxG+6AjgLGAvcB+R2hiul7h1S6Q4S3Qei0Rw8haYmkoueJvnMM6QXL8ZNeutkBEsKhCYkcMttQokemuNVvGwdjV09FlNqaG/uIZPLocwADIF5kYNBRPZQJntv93bFyqtYoe3L7cxj53Zvv/3t7y3ngfgXU15ezplnntm/G/LW9AelD+Syov008LaiYwUc1gpEo9EMjNyGDXQ/+ig9jz9ObpXX+BCoLiM2KUuq2qJhzAS6jBjNVNKoqnHFVxAKaHKRQisBoLy6huD4CSTDUVrFpFEZ1LvQogTbtLANE8cwcA2DkqBFWSBAzLIosUzCpoEhYChv+dZ0wSFl23TlbZqyORzXwXRdDKWoC5rMigSZFwsxNxoihMJxHBzHwfZXBNy577rurmPHcXYdF2/3do7thRVsG9dR/kqC3qqBrlL+vipy3s3wVhVUu/b9hWj3GPm15ze8euue2tN/X+wdozxROWAF0l/2q0CUUtcOae4ajeawIbdpE91/f5jufzxCfv0GAALHzSV7+Wk0Gx1sNqtplioc/7VhKJuCBYlIAaO+kc6CQVvlWDKz59E2bjLrlUmHvXuUVGXAYlo0xOxIiCmREBMiQSaEg9SGAowJWgQPoFnJVYrmvM36dJbl3WmWd6d5rCvFn3pszGSOhaVx3lNTzjvGlJGwPOXmuopMT550V550d550d45MT4FsukA25bl8xqaQtilkbfJZh3zWxrX3//Iessaw3f36u1YoLA7bHUfe6g/UhId+jFN/Z6L/rBfvLmCJUur+QZdqkNBNWBrN/ik0NdP997/T/dBDZFetQolQOHkBLZMr2C55trrluJgYONSoFvIKGqwEPXELO2OzI1xHU+0kmsaMJ2N69lWDIsyKh5kbj3BMPMKsWJhZsQiVwaGdSW47Li9v6+S5Te2s3NZFviNHZdplUk4oyygKPQVvHsheGJYQiQUIxQKEohahiEUwYhEMWwTCJoGQ56ygiRUwMAMGVtDENAXT8o4NUzDMnVvBMAQxdm/FwG8G87SCGOIPApNdimLX/mEwCmywrfGGgVnA3f7xe4BNwHEico5S6nMDE1Oj0RxqnGSSnn8+RteDD5B+aTFKKXrmHUv9u89iczBOt8RBQbVqZbZaQ5MbY22ohhcSs+kuraC5agLJmPd1awFHx8OcWRLj+ESUYxMRZsYiBIyhewm6jktnc4a2+iTtO1J07EjT0ZiiqzmDY7vEgAV4L+l8wqIxrFhXLkQnxzmxrpSTx5eRKAsRSQSJJAIEQuZh8dIeifRXgRwLnKaUcgBE5P+AZ4HTgdeHSDaNRjNIuPk8qWefpevBh0g+9SROvkDL5Frq33Yi9aVjSRsxDBzq1A4q3Ea2mmU8UnI0ayrfSbK2EiwDUYpJlvC2ihLmlSU4IRFldjxC2By6EU123qF1e5KWrT20bOuhdVuS9oYUju1ZuxWBRFWEirFRJs6upGxMhNIxUUqrI8TKQhiGkLId/tzYzm3bW/lTJs34bIF/CY7hAxVxgoPYuW+7Nt35brpyXSTzSXoKPaQKKdKFNGk7TcbOkLWzZJ0seSdPzsmRd/IUnAK2sik4BQqqgO3aOK6Do5xdW1e5u7Y7nULt2gevr8XF74tBcUzFMdx83s2Ddn290V8FUg7E8ZqtAGJAhVLKEZHcvk/TaDTDhcqmaXv4HlofuA93+ZuovEvL+GqaTj+GzZUTyRgRTGUTlR5sy+T5imNYW30JblkQy4RJJpyXiDGvqpxjS6LMiUeImwYUCrj5PKqQR7WlKBTPBhfxJvWFQhjBIHIA63QoV9HemKJpUzdNm7tp3txNe30KcRWmQCQaoKo2yuSTayirilBaGSZRHsIQAVd5zVOOAttB1SfJbu8BpRAF71fwvkA5K7MZnmro5pUNm1htbOGMsgSnlsWImsbuXuhd290d3ik7RWe2k65cFz2FHnpyPSQLSU9B2ClShTQ5p3+vQlNMLMMiKCYRw8KUCKZhYoiBIQamGEjRVhCMPbZ4/n6tyffZPdLL7wiJpA+viYQrRGQRXnPdmcB3RSQGPD5Esmk0mn6S62qifvVzdKx5BnPZciLrW2C7S94J0Dx+DDtOP4EtlRMoGEEUio5wlNerp7Jx7AQqyTM+k+K4TJJ3Nb7GhJVNTGpuJNTdjZHswepJQjpNfSbjmQ85APNHEg5jlpdjlpURqBmLNW4qVnUdZlUdJMaQbHNIt2TId+ZwUwVMpQiIMNGAqYZgluz1impNew4ocOAWXccDV+/hkyEP5PtxbhyIE8Zr0a8+wJwPPcEJiSHPo9+2sESkFq9pEeAVpVTDkEk1SOhOdM2ow3Vp3bqSbaueIL11KZH2DdS2NxBuzJFsCJNqCpIKxaivG8fmqZNoL60CEfKmRXOilO5wHNcQSjJpSjNJEtl0v0YRWUoIYBIWi4gRIGoGSVghSoMRykIRIgELUxRSEKy8QnJg5EFyoAomyg2BRN7S16CUoqDAFoVELAIlYUJlIUKlIYywiQRNjKCBWAYSMMESb980vH1DwBDE9LeGb5bE77imuGNa/GPD04GN6R080bSOv7W6rHeqULgE0y8TTj6OZTdRG6tlUukkxifqGB8fz7j4OGrjtVRHqokHDq81SXrFEIzwwAYuDJY591lKqTUistMG8TZ/O1ZExiqllg1IOo1G0ydOPsvW1c/SsOZpcvUrqUxuZZqznUC2QLwtDM1h1I4QTblSWquq2DyxloaTa3GD3sstY+boCLbSY/WQlQIBO0SoM0qskCCWLyWRqyCem4ShTM+4HYD4cxbE9Z2DEhtlFHCNAlnDxjSEgBjYWBSURU5ZhNwYEeU3J/m4SpF2IaMUaQcyyiXrKnIIBBVBM000uZmSNU8TbViNoWys2bMJXnIJidMvJlBTM2j30lUu6zrW8UrjKyxrXsby5uW0ZloBCJthjq46me7Y+awOn0qh4nTOrYhxQ10NZ1Uk9rgmzZ70NRP9V0qpj4nIU70EK6XUuUMn2sGjayCakYByHVq2rWLDqkW0bVqB2bWJgNuIsjppdg2SnUGkLUC82WRcEwSllI6KcpqrKthRW00+4pntdsSmPdhJR7CT9mAn3YEMjgiOOBSMHK6RA3PPxhpLQkyIT+G46mNYOO4k5pefSHkugd2exe7IYXdkcTqyOB053M4cKmPvcb4TgKzl0uXk6MzlSBaEbCFIxrbIKkAgEg8QjgWwgibKVeSyNumuPE7B3ZWOCMSCBaI99UQbVhFP1VMzo4raK99BybnnIANYTGpbzzZebHiRl3a8xCuNr9CZ6wRgfHw8x485nhOqT+C4MccxvWw6lr/EbGve5rf1Lfy+oY2WvM20SIgPj6/k8poKqoZ4GPLhRH9rICPOnLuIXAj8FM86/q1KqZv2FVcrEM3hgnJdWnasY93al9i2bSUdHetxCi24RicFSZLMC7mkhdFtUtIp1LabVKVjWBKnJ5Ggq7SU9ooykokEyl8nw0QoU1HGuuVMtKupUaVY7H9UUaeZpiHYTafk6FEFwCGBQZUbZoxdRpmzV7u5KZhlIYySEHbYJGcISUfRmbJpas/S1pTZPSLKEEqqghjxAj2FZlpS23GCKWbOmsGCBQuYMmXKriYspRSZngI9bVm6WtJ0NKXp2JGmfUeKzsbUrm6WQL6H0nwjY2dUMPnSU6idPZbgPpplUoUUL+94mecbnuf5+ufZntwOQE20hoW1C1kwdgELxi6gNl7b5/PKuS4PNXdyW30ry7rTBER4W1UJ7xtbwVnliSEdeXY4MKgKRESiwOeBiX6NZAYwUyn10MGL2n9ExATeBC4AtgOvAB9QSq3qLb5WIJqhJJ/L0NqylY31b7J1x1pa2raSzLSRt7txnBRWPo1ZyGDk80jeoZA3IRcilA+RyAaJ2kFCdogAYXKhCNlImHQ0SioWIxcO75GXiTfhzQwJVtTASpiYCQsJGgQJEHQDBAsmwYJDMO8QzLsE8zZBWxF0FCFXsAREHJS4IA5KvMWUlBJsJXQrlxZVoMt1SNlCNhfDzI4lkhyPKhQpJoFEeZjy2hgV42JU1EapqktQXhvFCuyO19XVxZIlS1i6dCnpdJq6ujouvPBC6urq9ntf7bxDW0OKls2dbH9pHc0bu+iREm/JWxSVtRHGz6qidnop6ao2lvS8xHP1z7G8eTm2axOxIiwcu5BTxp3CKeNOYXLJ5IOa57E6meFPje3c3dhOe8Ehbhq8vaqUi6tLObM8sWum+2hisBXIn4GlwIeUUnN8hfKCUur4gxe1/4jIKcA3lVJv94+/CqCU+l5v8QeqQJY9+xhrXrjrYETVDCmq6Pet/tJr2P7TYteZoHAB6c0qkecve/oWHyt/eQnlO9cA11C4AsrwcxDlpe5vRRQGigCCZQhB0yBoeC5kCEHDwDS8fggkD0YOtdOZOVwzh2tlUWZ/xhIdOK4ScnYQwyijIjGNkpI6wpFxRMLjCUcmEAnXEQrVINL7V3mhUODVV19l0aJFJJNJjj32WM4//3xKSvpvaqNrxUo2/PYBdrzZTmfpNLrKpqDwZr53hptIV7dSPTXOiccfzcIZ8wha/R8+3F8KruLZjh4ebOnkkZYuOm0HS+Ck0hhnlSdYWBbn+ESUyCionQy2AlmilJovIsuVUif4fq8qpY4bBFn7jYi8F7hQKfVR//gaYKFS6lNFcT4GfAxg4sSJJ27ZsuWA87nrZ//FmDm/GxSZNZqBIo4FykIcC3EtcDynbAtsE3enK5g4tuds28J2LGwVBGKYRgLTTGCFygiGyglFKwhFEwQCAaywRShsEAgbBMJCymintWsz6R07iLTkqMwEwUqzPr6WjYk3sYIZyk2oDgYIkd1DVsMIEg5PJBqdRCQyiWhkMtHoZCKRSYTDtYiY5HI5nn32WV588UWCwSCXX345M2bM2O89aE43s7x5OUublvJK4yv0bFzHu15yOfN1g57SSTQvOAc14RTaGxxyaa9/Jl4eonZ6GeOmlzJ2WhkV42KeCZFBpOAqlnSneLKtmyfbu1mZ9O5HQIS5iQhz4hHmJiLMjkeYEQ2PuFrKYCuQF4DzgOeVUvNEZBpwl1JqQR+nDir9USDFDLQGsn7Vqzx7zy8PSlYN7GHZ7SBjey0Q0mtMMUAwigzLiTc01QvAEMt/gRgELBPDsDAsi4AVJBgMYQUDWMEwVsDCME1MMRHTm9S1yxS3wrNr5O8Xf20rpXYtaudJ58mgRHn7UjzRy2uGEd8suELAMBHDQClBLBNBUGKgUKidFmNtG8cu4BQK2Pm873IUcjny2QyFTIZcJk0+nSabSpJN9pBJ9vQ6ZyMYiZKorKKkqpqS6hpKqsdQOqaGsppaysbWEop6o7jcnE3Lq1toXryN0nqDVZEN/LT2D2wPNWEqRVXAYHZZDTNi5dSGLMqMPAGnAyffiOvuVjAiASKRCUQiEwiHJ+A65SxZuomWZpvjjjubs85+B4jBjtQONnZtZG37WtZ2rGVl60oaUt5sgYgVYd6YeSyo9foxpqXidPzy13Tdfz8SCFB25QeQy66iuUXRsK6ThvWdpLvy/vVa1EwpoWZKCWOnllIzqYRwPNCvMtlf2gs2S7pSLO5KsbQrxcpkhh5n9yCBmqDF1GiICeEgdb6rCQaoCVrUhAKUWxbWEJp/OVAGW4FcAPwHcAzwT+A04CNKqUUHKecBcaiasDSa0YDrOmSTSTLdXaQ6O0h1tJPsaKenrZWetla6W5vpbmkmm+zZ47xISSnlY8dRXjuO8trxlNeOo6RkDJ1rCqSWtrAytoRf1/yNHivJlOgUutxuWrItu84XFBMipcyIlTA2aFBhOpQaOcIqRVj1YO01bU8pSLpCjwM9rpB2BTHjxEM1jIlPYHzJNGpLphA0YxhmGNMII2JhGAHsxlY67/4rqaefQYIhSi++hJJ3vwszkSDVmaVlWzet23poq0/S1ZIGcQGXeHmAsrERymrClFYFKakOE4wKKBelHJRyvabMt+z7w5xxvbgof1t8rHCVQ6dt05Qr0JYv0Fawac8X6HZskrYD3qfDzjsAQMgQIoYQ9psuQ4ZnmDJgCAHxnCleX5ZlCCaCN/3F3wKG/xFlCERDY1kw7cMDKjuDrUDuBF4DMsBGYLFSqnVAkh0EImLhdaKfB9TjdaJ/UCm1srf4WoFoNH2TS6fpam6kq6mRjsYGOpt20LmjgY4d9SQ79pzrnaispmCWIrkQ2yo2s6JiE2NKxvPFc78BFQE2dW2iIdlAY7qRxlQj3bluz/RHvmeXzaaQuNQELcYEg4zLhinvEsKJHOW1FlHDwSKHY3dj290oZfcmsmYfuEVqqZGjuObchweUzmArkHOAM3w3DVgOPKOU+umApDsIRORi4Cd4A1N+o5T6zr7iagWi0Rwc+WyGDl+ZdDTU09HYQEfDdlrq63Gy6V3xFIpgOEb1uImU1NZQUlVNorKaWEUF8bIKYmXlREpKCITCb8lj1apV3HPPPYwZM4ZrrrmGWMxrQlNK4bp5HKcHx0njOFlcN4vj5lBuAaVsXFXwbVY5oBSFxka6H32UzPJliGkRO/kUEuedh1VZBb49KW9rImKQz7p0N2fpbM7S3ZqhuzVLd0uWQm7nCDVACeGYRSQRIhIPEY4FCIdMQhYElIFpg+W6UFCQdXCzNm5O4eYLqLxC5V1UwUG5O4d3eM2ge64jJexabKoI2aue4gB5A2xDsAVsw/Nz/K0r4IrgCJRUlHLpFy5jIAz6PBB/CO1JwDnAx4GMUmrWgKQ7RGgFotEMDUopNtc38++3P0W++U2q4y9CJsOMrrHEnADpXDeu67zlPCsQJJxIEIrGCEajhCJRAqEw6WyWLdu2EY5EOWb2bALBgPei3zlvZOeKfztXCXRsXNvBdWy/j6iA6zjYhQKubVNIJsk3NVHo7kKJQDQKkQiuIW8590Bse40kqiZO58M//MmAzh3U9UBE5Ak8C7wv4plxP0kp1TwgyTQazYhHRJhSV8NtX7yCT/xhKY+8OYcFpz7Kc6kXuKjnDD6x7XqkzsKYEyVfYpPu6iTT073L5dNpcpk02WQPPW2tOIUCcSdPpqmD11t2ELBMXLdopjriDTwwBDH8tcxNC8MyMU1vUIRhmpiBAKZlES4vJ1ZdjdgO9tat2Nu2Ic1tBCoriRx1FOEpUzDDYUzTRAxvHXSVdlHdBZyuAqq7gJu0EXe35VszHsJKhDDjQQgHcC0DxzQoKKGgoOAo7KxLIedg2wqnoCjkHZyCi2MrzxUUjqNwbccbNFF0hUU3tz9PoM8YgVj5gTzSAdHfufmvAScCc/BMuneKyItKqcyQSabRaA57IkGT/7v6RD5+JyxachlvP72OR/gLmfkuX1h3NfJoilhNlLFnHEv0zDGItf85EosXL+aRRx7h1FNP5W1ve9ugyekkU3T97W903Hkn+bsfwCitIHHhlYRmLMBJRyhs60H5plWMWIDAUXGCtTECtTECY2NYlREkMLjzO1xX4dreeuv/v707D4+qvP8+/v7Oln0hO2sWRCBAUJBQligCKqi4PfZnsWprbZFq61Ovn3X5abXVWmtba+1TBGlra/kptYpWLKjFspMgIIvIHjFCgABJIGSbzHY/f8yAAck2JDNJ+L6u61wzc86Zk+/NhPnkbPft8xp8XoPX49/TOvnaGH839/69r5OH9U4e/zL4fHzZ7bwvcBAssDgiquO7XmlTVyYiEgd8G3gAyDDGRHRQXe1CD2EpFRpOt5e7533Mit1HuWVSCYsPzmFU+iX8MuknmMLjuMtqsUTbiB6ZTkx+BvbU6Ca3tWjRItavX8+0adMYOXJku9RnPD5c+07gLD5O/ZZSPOVeEIv/Cqu6w9hSLERfnEXM2Fx/WJznHSi290n0H+A/gT4SKMF/GGuVMWbpOdbZoTRAlAodp9vLHS+vY/P+49x33Qn+vPOX5KbkMmfSHOz7PdR+VEb9tgrwGRyZ8UTlpRA9LAVr/Ol/h3q9Xl577TVKSkqYMWMG6UH0yms8Plz7q2nYW0XD51U0lJwAjw8E7L1iieifiL2nA1fxBqr/vYjawkLwerGmpBB76aXEjBtL9KhR2NPS2uufp0tp7wB5AH9ofGy60HV1GiBKhVZlrYsbZq2hzuXlwZtcPLPhUYamDGXOFXOIscfgrXZRu+Ew9VuO4C6r83+h9/Z/oUfmJODIjMcSaaOmpobZs2cTGxvL9773PWzN9MZrvD485fW4y2px7avGtb8a18Ea8Pi/2+wZMUT0TyCifyIR2QlYznJox3PsGLWrV1OzbDk1q1fjO3ECAEd2NlEjRxA1dBhRecOIGDAAsbfvTYidUbftjbctNECUCr09h6u56cVC+iRFc881tfyk8GHyUvOYPXk2MY0GYnIfqaN+aznO3cdw7a8Gn/+7yBrvwJYezT5rOe+WrGBUv2FcNmgMxmvA58PX4MVX7cZ7ogHP8QY85fX+oWwBbBYcvWNx9IsjIiuBiOx4LNFt+8I3Hg/OHTuoW7eO2nXrcG7egrcqMJq33U5EVhYRAwbg6J+Do18mjsx+2Pv0wdqjR7c59KUBggaIUuGyfNcRvvPX9Uwb3ourRx/hoVUPMTJ9JC9OfpEI61dPnfpcXlwlJ3AdrMFzpA73kTq8xxtY2bCVnZYDXOMaQU8TuKpIwBJrxxofgTXegT0tGlt6NPb0GOwZ0f7RCtuRMQZ3aSn1n3xCw85dNOzZQ8Pu3bgPnj4oq0REYMtIx56WjjUlGVtyCrbkJCwJCdgSE7HEJ2CNi8USG4slJgZLVBSW6OhOuUejAYIGiFLh9Pv/7OG3S3bz3NeHE9FjM4+seoTL+17Obyf89tQATi1paGjgpTkv4fV6uee7M7FHOhCrxT+EbZj5nE7cpaW49u3DXVqK+1AZ7rJDeI4cxVtejqeyEl91dcsbstuxOBxIZCQS4cBidyAOhz9Y7DbEZkdsNsRqBZsVsdrAakEs1i8fA5c4Ixaw+GWj4YIAABhtSURBVPuAc/TtR8rMu4NqmwYIGiBKhZPXZ5j+x7VsO1DFovsKKDz6Ds+se4br+l/HU+OewtJE9+9nKikp4a9//SuXXnopEyd26kFQv8LncuGrqsIbmHw1NXhravDV1OKrr8M4nfjq6jENTnzOBozTiXG7/ZPLhfF4Tk14PP4bKj0e8HoxPi94fRC4yRKvv98uDODzETl4MH3nzA6q7na9kVAppdrKahF+d8tFTH1hFff9fRNvzvwGVa4qXtz8IgkRCfz4kh+36pxBVlYWw4YNY82aNQwfPpzk5OQQVN8+LA4HltRUbKmp4S6lQ3T9kU+UUp1Wr8Qonv0/eXxSWsXvPtzNzLyZ3DroVuZtn8cr215p9XauvPJKrFYr7733Ht35qElXowGilOpQU4Zm8PWRfXhp5V62HTzBQ/kPcVXWVTz38XO8+9m7rdpGXFwcEyZMoLi4mF27dnVwxaq1NECUUh3usWtySYpx8OCbn+D1wS/G/4L8jHweX/M4aw6sadU2Ro8eTWpqKu+//z4eT5e5Ha1b0wBRSnW4hGg7T10/hO2HTjB35V4cVge/u/x39E/sz/3L72db+VmH9DmN1WplypQpHD9+HL04pnPQAFFKhcSUoT2ZOjSDF/6zh+IjNcQ54pg9eTY9Inpwz3/uYd+JfS1uIycnh6ysLFauXElDQ0MIqlbN0QBRSoXMz64fQpTdyqNvb8UYQ2p0KnOumIPP+Jj54UzK65sf6FREmDRpEnV1daxduzZEVaumaIAopUImLS6SB6cM5KPPK1m4xX8nd3ZCNrMmzaK8vpx7PryHGldNs9vo27cvAwcOpLCwkLq6umbXVR1LA0QpFVLfGNWPvD4JPL1oB9VONwB5qXk8d9lz7Dm2h/uW3UeDt/nDUxMnTqShoYHVq1eHomTVBA0QpVRIWS3CU9cP5WhNAy98uOfU/II+Bfx8/M/ZULaBH6/4MR5f01dapaenk5eXx7p166huTXchqkNogCilQm5430S+MaoffyksYVfZlwFwTc41PJz/MMv2L+OJwifwGV+T27jsssvwer0UFRWFomR1FhogSqmwePCqgcRF2nhi4aen3V1+6+Bbufeie1n42UKeWvtUk3eeJycnM2TIEDZs2KDnQsJEA0QpFRY9Yhz895UDWbu3kg+2lZ227O68u/nusO/y5u43eWbdM02GSEFBAS6Xi48++igUJaszaIAopcJm+qi+DMqI4+eLduB0e0/NFxHuu/g+7si9g/k75/PrDb8+a4ikp6czcOBAPvroI70vJAw0QJRSYWOzWnj82lxKj9Xz59Wfn7ZMRHjgkgdOdb749EdPn/WcyKWXXorT6WT9+vWhKlsFaIAopcJq7AUpXDUknVnLijl8wnnaMhHh4fyHuXPonby+63UeW/3YV67O6t27Nzk5ORQVFeF2u0NZ+nlPA0QpFXaPXp2Lx2t49r2dX1kmItw/4n5+ePEPeXfvuzyw4gGcntODpqCggNraWrZs2RKqkhUaIEqpTqBfcjTfLcjmrU0H2LTv2FeWiwgz8mbwcP7DLN23lLv+fRcV9RWnlmdlZdGrVy+Kiorw+Zq+9Fe1Lw0QpVSncM/lF5AWF8HP3t2Oz3f2q66+OfibPD/heXZX7uabi7/J3uN7AX/AjB07loqKCnbv3h3Kss9rGiBKqU4hNsLGQ1MGsXn/cf65+UCT603KnMRfpvwFp8fJbYtv4z/7/gPA4MGDSUhIoLCwMFQln/c0QJRSncaNF/dmeN9EfvneTmobmu7KZGjKUF675jX6xvflR8t+xG/W/waf+BgzZgz79u2jtLQ0hFWfvzRAlFKdhsUiPH5tLkeqG5i1rLjZdXvF9mLe1HncMvAWXtn+Cnd9cBepA1KJjIzUvZAQ0QBRSnUqIzN7cOPFvfnTqs8pKa9tdl2H1cFjX3uMZwueZc+xPUx/fzr1g+rZvmM7lZWVIar4/KUBopTqdB6ZOgiHzcJP393WZDcmjV2dczVvX/82I9JG8FbVW6zMWMk7a94JQaXnNw0QpVSnkxYfyY8mD2D5rqMs2X64Ve/JiMlg9uTZPDn2Saojq3m+4nmeXPMkx5xfvSxYtQ8NEKVUp/StsVlcmB7Lz97dTr3L2/Ib8F/Oe+OAG5k3YR7Z1dksKF7ANW9fw5wtczjhOtHBFZ9/NECUUp2S3WrhyeuHcuB4PbOXN39C/UyDMgdxY9yNXHf8OkamjWTW5llc9eZV/H7j7zlad7SDKj7/aIAopTqtr+Ukc8NFvZizYi97Drdt5MExY8ZgPWZlRtoM3pj2BmN7jeVPW//EFW9ewf3L7qfwYGGzA1aploUlQETk1yKyU0Q+EZG3RSSx0bJHRKRYRHaJyFWN5k8JzCsWkYfDUbdSKvQeuzaX6AgrDy34BG8Td6ifzYABA0hOTqaoqIiBPQby3ITn+NeN/+KO3DvYcHgDdy+5myveuIJn1z3L5iObNUyCIK25wqHdf6jIlcBSY4xHRJ4FMMY8JCK5wHwgH+gFfAhcGHjbbuAKoBRYD0w3xmxv7udccsklZsOGDR3UCqVUqLy9qZT7X9/CE9NyuXNcdqvft379ehYtWsSdd95JZmbmqfkur4ul+5by3ufvsfrAalw+F0mRSYzpNYZxvcaRn5FPekx6RzSlSxCRj40xl7S0ni0UxZzJGPPvRi/XAjcHnl8P/N0Y0wB8LiLF+MMEoNgYsxdARP4eWLfZAFFKdQ83XNSbdzYf5Ffv72Ly4HT6JkW36n3Dhw9n6dKlFBYWnhYgDquDKdlTmJI9hRpXDctLl7P6wGqKDhaxaO8iAHrF9OKitIsYljKMgUkDGZQ0iDhHXIe0r6sKS4Cc4TvA64HnvfEHykmlgXkA+8+YP/psGxORGcAMgH79+rVroUqp8BARnr5xGFf+dgWPvLWVv30nH4tFWnyfw+Fg1KhRrFy5kvLyclJSUr6yTqwjlmtzruXanGvxGR87Knew8fBGNh3ZxPqy9Sz+fPGpdTNiMsiOzyYrIYvM+Ex6xfSiV2wvesb2JM4eh0jLNXUnHRYgIvIhkHGWRY8aY94JrPMo4AFeba+fa4yZC8wF/yGs9tquUiq8eidG8T/XDObRtz/l5TWf892CnFa9Lz8/nzVr1lBUVMS0adOaXdciFoYkD2FI8hBuz70dgPL6cnZU7GBn5U4+q/qMkqoS3il+hzpP3WnvjbRGkhKVQkpUCj0ie5AUmURCRALxjnjiI+KJs8cRbY8mxh5DtC2aKFsUkbZIomxROKwOHBYHVos1uH+cMOmwADHGTG5uuYh8G7gWmGS+PBFzAOjbaLU+gXk0M18pdZ64Nb8fK3Yd5dn3dzI6O5lhfRJafE9sbCzDhw9ny5YtTJw4kZiYmDb9zJSoFAr6FFDQp+DUPGMMlc5KDtUe4kDNAcpqyzhad5Qj9UeoqK+gtKaUreVbOe48jsc03SnkmWxiw261Y7fYsVls2Cw27BY7VrFitVj9j2LFIpZTjycnEfE/xwIC/RP688joR9rU1rYKyyEsEZkCPAhcZoxpHOMLgddE5Lf4T6IPANYBAgwQkWz8wfEN4NbQVq2UCjcR4Vc353H1C6v4wfyN/OuH44mLtLf4vjFjxrBx40bWr1/PhAkT2qWO5KhkkqOSGZoytMn1jDHUe+qpdlVT7aqm1lNLnbuOOncd9d56nB4nTo+TBm8DLp+LBk8DHp8Ht8+N2+fG4/PgNV7cPjc+48Pr8+IxHnzGd9pkjMFrvHh9Xrx48RkfDd6Gc25nS8J1DuQPQASwJHDMcK0xZqYxZpuI/AP/yXEPcK8xxgsgIj8APgCswMvGmG3hKV0pFU6J0Q5emH4xt7xUxGP//JTf3XJRi+ceUlNTufDCC1m3bh3jxo3Dbm85dNqDiBBtjybaHt0tr+oKy30gxpgLjDF9jTEXBaaZjZY9bYzpb4wZaIx5r9H8xcaYCwPLng5H3UqpzmFUVhL3T76QdzYf5I+r9rbqPWPHjqWuro5NmzZ1cHXnD70TXSnVJd17+QVck9eTXyzeyXtbD7W4fmZmJn369GHNmjV4va3rW0s1TwNEKdUlWSzCc18fzsX9EvnR65vZtK/5XndFhIKCAqqqqti6dWuIquzeNECUUl1WpN3KH++4hLT4CL73tw0UH2m+v6wLL7yQ9PR0Vq1ahc+nXZecKw0QpVSXlhIbwV++nQ8It7y0lm0Hq5pc9+ReSEVFBTt27Ahdkd2UBohSqsu7IC2WN2aOIcJmYfrctc0ezsrNzSU5OZlVq1a1arRD1TQNEKVUt5CdEsM/Zo6hR4yD2/70UZMn1i0WC+PHj6esrIw9e/aEuMruRQNEKdVt9OkRzRt3j+GCtFi+/+pGfrpwGw2er15xlZeXR2JiIkuXLtVzIedAA0Qp1a2kxUfyxsyxfGdcNn8tLOG/5hSxq+z0k+tWq5XLL7+csrIytm/XTr2DpQGilOp2HDYLj0/L5aXbR/JFZR1TX1jJT/75KZW1rlPrDBs2jNTUVJYtW6b3hQRJA0Qp1W1dNSSD5Q9M4I4xWby2bh8Tfr2M33ywi0NV9VgsFiZNmkRFRQWbN28Od6ldkgaIUqpbS4x28NPrhvD+/y3gaznJzFpezPhnl3HPqx+zz5tIz169WbFiBW63O9yldjlhGdI2VHRIW6XUmfZX1vG/a7/g9Q37OV7npp+9monWnSQOGMmESwvI7RmPw3Z+/23d2iFtNUCUUucll8fHus8rWbK9jPItS0n0HuethmF4bZEMSIslJzWW/qkx9EuKpmdCFD0TIkmLjyDa0RkGcu1YGiBogCilWqeyspJZs2aR2Cub6p4j2X24ms+O1lB6rJ4zvyIj7RaSoh0kRDuIi7QRH2kjNsJGlMNKlN1GlMNCpM1KhN1ChM2Kw2bBbrVgtwo2iwWbVbBZBJvVglUEq8U/WcTfv5dFAs9FEAFBsFj8jyLgH8n35DJOdWXvf07gueCwWchIiAzq36O1AdL9o1QppVqQlJTE+PHjWbFiBd+aOI7sqwcD4HR7OXi8nrIqJwernBytbuBYnYvKWhfH69xUO90cPO6k1uWhzuWl3uWlzuXB1wn+Lr+obyL/vHdch/4MDRCllALGjx/Pli1bWLx4MTNnzsRqtRJpt5KT6j+c1RZur48Gjw+n24vHa0699vr8zz0+g9dn8BmDx+t/9PoMXmPAcOq1wT+qoc+Af9HJ5/6EOrl3ZDBfPg88JsU42ulfpmkaIEopBdjtdqZOncr8+fMpKipi/PjxwW/L6j9sFRvRvb9iz+9LDZRSqpGBAwcyaNAgli1bRllZWbjL6fQ0QJRSqpFp06YRFRXFggUL9N6QFmiAKKVUIzExMdxwww0cPXqUJUuWhLucTk0DRCmlznDBBRcwevRo1q1bp12+N0MDRCmlzmLy5MmkpaWxYMECjh49Gu5yOiUNEKWUOgu73c706dOxWq28+uqr1NbWhrukTkcDRCmlmtCjRw+mT59OTU0N8+fP15PqZ9AAUUqpZvTp04ebbrqJ0tJSFixYgMfjCXdJnYYGiFJKtSA3N5epU6eyc+dO5s+fj8vlavlN5wENEKWUaoXRo0dz3XXXsXfvXubNm0d9fX24Swo7DRCllGqlESNGcPPNN3PgwAFefvlljhw5Eu6SwkoDRCml2mDIkCHcdttt1NXVMXfuXDZu3Eh3HhajORogSinVRjk5OcycOZO+ffuycOFC3njjDaqqqsJdVshpgCilVBDi4uK4/fbbmThxIrt27eIPf/gDy5cvP69OsOuIhEopdY6OHTvGkiVL2L59O3FxceTn5zNy5Eiio6PDXVpQdEhbNECUUqFVUlLCypUr2bt3Lzabjby8PPLy8ujXrx8WS9c54KND2iqlVIhlZWWRlZXF4cOHWbt2LZ988gkbN24kNjaWwYMHk5OTQ2ZmZpfdMzmT7oEopVQHcblc7N69m23btlFcXHyqK5T09HR69+5NRkYGPXv2JDk5uVOFih7CQgNEKdV5eDweDhw4QElJCV988QWHDh067WbEqKgokpKSSExMJD4+nvj4eGJiYoiJiSE6OpqoqCgiIyNxOBwdfjisSxzCEpH/Bn4DpBpjykVEgBeAq4E64NvGmI2Bdb8FPBZ468+NMa+Eo2allAqGzWYjMzOTzMxMAIwxnDhxgrKyMioqKqioqKCyspKysjJ27drVZJ9bIoLD4Tg12e127HY7NpsNm82G1WrFZrORkpLChAkTOrZNHbr1ZohIX+BKYF+j2VOBAYFpNDAbGC0iScATwCWAAT4WkYXGmGOhrVoppdqHiJCQkEBCQsJXlhljcDqd1NbWnpqcTuepyeVy4XK5aGhowO124/F4cLvdOJ1OvF4vHo8Hr9fb4W0I5x7I88CDwDuN5l0P/M34j6utFZFEEekJTACWGGMqAURkCTAFmB/akpVSquOJCFFRUURFRZGSkhLucpoUluvKROR64IAxZssZi3oD+xu9Lg3Ma2r+2bY9Q0Q2iMgGHUVMKaU6ToftgYjIh0DGWRY9CvwP/sNX7c4YMxeYC/6T6B3xM5RSSnVggBhjJp9tvogMA7KBLf5z5vQBNopIPnAA6Nto9T6BeQfwH8ZqPH95uxetlFKq1UJ+CMsYs9UYk2aMyTLGZOE/HDXCGFMGLATuEL+vAVXGmEPAB8CVItJDRHrg33v5INS1K6WU+lJnuxN9Mf5LeIvxX8Z7J4AxplJEngLWB9Z78uQJdaWUUuER9gAJ7IWcfG6Ae5tY72Xg5RCVpZRSqgVdp3cvpZRSnYoGiFJKqaB0676wROQo8EWYy0gBysNcQ3vRtnRO3aUt3aUd0PXbkmmMSW1ppW4dIJ2BiGxoTadkXYG2pXPqLm3pLu2A7tWW5ughLKWUUkHRAFFKKRUUDZCONzfcBbQjbUvn1F3a0l3aAd2rLU3ScyBKKaWConsgSimlgqIBopRSKigaIEoppYKiARJiIpIrIv8QkdkicnO46zkXIlIgInNE5E8iUhjues6FiEwQkVWB9kwIdz3BEpHBgTa8KSLfD3c950JEckTkzyLyZrhraU/dqV0aIO1ARF4WkSMi8ukZ86eIyC4RKRaRhwOzpwL/zxjzfeCOkBfbgra0xRizyhgzE/gX8Eo46m1OGz8XA9QAkfiHGOg02viZ7Ah8Jv8FjAtHvc1pY1v2GmPuCk+lZ9fG36mz6oztCpoxRqdznIBLgRHAp43mWYHPgBzAAWwBcoE0YBbwa2BNuGs/l7Y0Wv4PIC7ctZ/j52IJLE8HXg137efymQDXAe8Bt4a79nb6/Xoz3HUH+Ts1DP8fV42ntM7YrmAn3QNpB8aYlcCZ45PkA8XG/9eGC/g7cL0x5ogx5l7gYTphXzltaQuAiPTDP/BXdWgrbVkbPxdfYPkxICKEZbaorZ+JMWahMWYq8M3QVtqytrals2nj79RWY8y1Z0xHQl50B9IA6Ti9gf2NXpcCvUUkS0TmAn/DvxfSFZy1LYHndwF/CXlFwWvqc7lJRF4C5gF/CEtlbdNUOyaIyO8DbVkcntLarKm2JIvIHOBiEXkkPKW1SnP/P76iC7WrRWEfUOp8Y4wpAWaEu472Yox5Itw1tAdjzFvAW+Gu41wZY5YDy8NcRrswxlQAM8NdR3vrTu3SPZCOcwDo2+h1n8C8rkjb0vl0l3ZA129LV68/aBogHWc9MEBEskXEAXwDWBjmmoKlbel8uks7oOu3pavXHzQNkHYgIvOBImCgiJSKyF3GGA/wA+ADYAfwD2PMtnDW2Rrals6nu7QDun5bunr97U07U1RKKRUU3QNRSikVFA0QpZRSQdEAUUopFRQNEKWUUkHRAFFKKRUUDRCllFJB0QBRqgOJSImIpJzrOkp1RhogSimlgqIBolQ7EZF/isjHIrJNRGacsSxLRHaKyKsisiMwYmB0o1V+KCIbRWSriAwKvCdfRIpEZJOIFIrIwJA2SKkWaIAo1X6+Y4wZCVwC3CciyWcsHwi8aIwZDJwA7mm0rNwYMwKYDTwQmLcTKDDGXAw8DvyiQ6tXqo00QJRqP/eJyBZgLf7eWQecsXy/MWZN4Pn/AuMbLTvZlfzHQFbgeQLwRmD41OeBIR1RtFLB0gBRqh2IyARgMjDGGDMc2IR/fPXGzux4rvHrhsCjly/H6XkKWGaMGQpMO8v2lAorDRCl2kcCcMwYUxc4h/G1s6zTT0TGBJ7fCqxuxTZPjivx7XapUql2pAGiVPt4H7CJyA7gl/gPY51pF3BvYJ0e+M93NOdXwDMisgkdPVR1Qtqdu1IhICJZwL8Ch6OU6hZ0D0QppVRQdA9EKaVUUHQPRCmlVFA0QJRSSgVFA0QppVRQNECUUkoFRQNEKaVUUDRAlFJKBeX/A7I2C4MqrATvAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "from sklearn.linear_model import Ridge\n",
    "alphas = 10**np.linspace(10,-2,100)*0.5\n",
    "ridge = Ridge()\n",
    "coefs = []\n",
    "\n",
    "for a in alphas:\n",
    "    ridge.set_params(alpha=a)\n",
    "    ridge.fit(scale(X), y)\n",
    "    coefs.append(ridge.coef_)\n",
    "\n",
    "ax = plt.gca()\n",
    "ax.plot(alphas, coefs)\n",
    "ax.set_xscale('log')\n",
    "ax.set_xlim(ax.get_xlim()[::-1])  # reverse axis\n",
    "plt.axis('tight')\n",
    "plt.xlabel('alpha')\n",
    "plt.ylabel('weights')\n",
    "plt.title('Ridge coefficients as a function of the regularization');"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}