Energy Efficiency Analysis .ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Loading required libraries\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "%matplotlib inline\n",
    "import matplotlib.pyplot as plt\n",
    "import seaborn as sns\n",
    "plt.style.use('seaborn') #set same style for all plots\n",
    "from sklearn.tree import DecisionTreeRegressor\n",
    "from sklearn.neighbors import KNeighborsRegressor\n",
    "from sklearn.ensemble import RandomForestRegressor\n",
    "from sklearn.metrics import r2_score\n",
    "from sklearn.model_selection import GridSearchCV\n",
    "from sklearn.multioutput import MultiOutputRegressor\n",
    "pd.options.mode.chained_assignment = None\n",
    "from sklearn.linear_model import LinearRegression\n",
    "from sklearn.preprocessing import MinMaxScaler\n",
    "from scipy.stats import randint as sp_randint\n",
    "from sklearn.svm import SVR\n",
    "from sklearn.ensemble import GradientBoostingRegressor,BaggingRegressor,AdaBoostRegressor,BaggingRegressor\n",
    "from sklearn.model_selection import train_test_split\n",
    "from keras.models import Sequential\n",
    "from keras.layers import Dense\n",
    "from keras.wrappers.scikit_learn import KerasClassifier\n",
    "import keras\n",
    "\n",
    "output = pd.DataFrame(index=None, columns=['model','train_r2_score','test_r2_score'])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 1. Data import"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>X1</th>\n",
       "      <th>X2</th>\n",
       "      <th>X3</th>\n",
       "      <th>X4</th>\n",
       "      <th>X5</th>\n",
       "      <th>X6</th>\n",
       "      <th>X7</th>\n",
       "      <th>X8</th>\n",
       "      <th>Y1</th>\n",
       "      <th>Y2</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0.98</td>\n",
       "      <td>514.5</td>\n",
       "      <td>294.0</td>\n",
       "      <td>110.25</td>\n",
       "      <td>7.0</td>\n",
       "      <td>2</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0</td>\n",
       "      <td>15.55</td>\n",
       "      <td>21.33</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0.98</td>\n",
       "      <td>514.5</td>\n",
       "      <td>294.0</td>\n",
       "      <td>110.25</td>\n",
       "      <td>7.0</td>\n",
       "      <td>3</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0</td>\n",
       "      <td>15.55</td>\n",
       "      <td>21.33</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0.98</td>\n",
       "      <td>514.5</td>\n",
       "      <td>294.0</td>\n",
       "      <td>110.25</td>\n",
       "      <td>7.0</td>\n",
       "      <td>4</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0</td>\n",
       "      <td>15.55</td>\n",
       "      <td>21.33</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0.98</td>\n",
       "      <td>514.5</td>\n",
       "      <td>294.0</td>\n",
       "      <td>110.25</td>\n",
       "      <td>7.0</td>\n",
       "      <td>5</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0</td>\n",
       "      <td>15.55</td>\n",
       "      <td>21.33</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0.90</td>\n",
       "      <td>563.5</td>\n",
       "      <td>318.5</td>\n",
       "      <td>122.50</td>\n",
       "      <td>7.0</td>\n",
       "      <td>2</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0</td>\n",
       "      <td>20.84</td>\n",
       "      <td>28.28</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "     X1     X2     X3      X4   X5  X6   X7  X8     Y1     Y2\n",
       "0  0.98  514.5  294.0  110.25  7.0   2  0.0   0  15.55  21.33\n",
       "1  0.98  514.5  294.0  110.25  7.0   3  0.0   0  15.55  21.33\n",
       "2  0.98  514.5  294.0  110.25  7.0   4  0.0   0  15.55  21.33\n",
       "3  0.98  514.5  294.0  110.25  7.0   5  0.0   0  15.55  21.33\n",
       "4  0.90  563.5  318.5  122.50  7.0   2  0.0   0  20.84  28.28"
      ]
     },
     "execution_count": 35,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data = pd.read_csv('ENB2012_data.csv')\n",
    "data.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "data.columns = ['Relative_Compactness', 'Surface_Area', 'Wall_Area', 'Roof_Area', 'Overall_Height',\n",
    "                'Orientation', 'Glazing_Area', 'Glazing_Area_Distribution', 'Heating_Load', 'Cooling_Load']"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Spliting the data in X and Y"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "X1    0\n",
      "X2    0\n",
      "X3    0\n",
      "X4    0\n",
      "X5    0\n",
      "X6    0\n",
      "X7    0\n",
      "X8    0\n",
      "dtype: int64\n"
     ]
    }
   ],
   "source": [
    "X = data[['Relative_Compactness', 'Surface_Area', 'Wall_Area', 'Roof_Area', 'Overall_Height',\n",
    "                'Orientation', 'Glazing_Area', 'Glazing_Area_Distribution']]\n",
    "Y = data[[ 'Heating_Load', 'Cooling_Load']]\n",
    "Y1=data[['Heating_Load']]\n",
    "Y2=data[['Cooling_Load']]\n",
    "print(X.isnull().sum())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "No null treatment or data preprocessing required."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Dividing each observation in either of the 3 categories (High Efficient, Average Efficient & Low efficient)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [],
   "source": [
    "Z = data[['Heating_Load','Cooling_Load']]\n",
    "\n",
    "# The total load is the sum of the individual heating and cooling load\n",
    "Z['Overall_load'] = Z['Heating_Load'] + Z['Cooling_Load']\n",
    "Z['class'] = 1\n",
    "\n",
    "Z.loc[Z.Overall_load < 42,['class']] = 0\n",
    "Z.loc[Z.Overall_load > 70 , ['class']] = 2\n",
    "\n",
    "#Efficiency classes\n",
    "yclass=Z['class']\n",
    "\n",
    "X_train_class, X_test_class, y_train_class, y_test_class = train_test_split(X, yclass, random_state = 5)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Using MinMax scaler for scaling"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.preprocessing import MinMaxScaler\n",
    "MinMax = MinMaxScaler(feature_range= (0,1))\n",
    "X_train_class = MinMax.fit_transform(X_train_class)\n",
    "X_test_class = MinMax.transform(X_test_class)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Now dividing the whole dataset into train and test"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Test-Train split of data for regression models\n",
    "X_train, X_test, y_train, y_test = train_test_split(X, Y, random_state = 5)\n",
    "\n",
    "X_train_div, X_test_div, y1_train, y1_test = train_test_split(X, Y1, random_state = 5)\n",
    "\n",
    "X_train_div, X_test_div, y2_train, y2_test = train_test_split(X, Y2, random_state = 5)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Scaling the data using MinMax Scaler"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [],
   "source": [
    "# MinMax Scaling done\n",
    "from sklearn.preprocessing import MinMaxScaler\n",
    "MinMax = MinMaxScaler(feature_range= (0,1))\n",
    "X_train = MinMax.fit_transform(X_train)\n",
    "X_test = MinMax.transform(X_test)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 2. Modeling"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Linear Regression\n",
    "#### As the output variables are 2 we will be using Multioutput regressor"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>model</th>\n",
       "      <th>train_r2_score</th>\n",
       "      <th>test_r2_score</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Linear Regressor</td>\n",
       "      <td>0.902539</td>\n",
       "      <td>0.899354</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "              model  train_r2_score  test_r2_score\n",
       "0  Linear Regressor        0.902539       0.899354"
      ]
     },
     "execution_count": 40,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "linear = LinearRegression()\n",
    "multiOutput_linear = MultiOutputRegressor(linear,n_jobs=-1)\n",
    "multiOutput_linear.fit(X_train,y_train)\n",
    "train_r2_score=r2_score(y_train,multiOutput_linear.predict(X_train))\n",
    "test_r2_score=r2_score(y_test,multiOutput_linear.predict(X_test))\n",
    "output = output.append(pd.Series({'model':'Linear Regressor', 'train_r2_score':train_r2_score,'test_r2_score':test_r2_score}),ignore_index=True )\n",
    "output"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### KNN regressor"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>model</th>\n",
       "      <th>train_r2_score</th>\n",
       "      <th>test_r2_score</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Linear Regressor</td>\n",
       "      <td>0.902539</td>\n",
       "      <td>0.899354</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>KNN Regressor</td>\n",
       "      <td>0.935276</td>\n",
       "      <td>0.904540</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "              model  train_r2_score  test_r2_score\n",
       "0  Linear Regressor        0.902539       0.899354\n",
       "1     KNN Regressor        0.935276       0.904540"
      ]
     },
     "execution_count": 41,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "param_grid = {'weights':['distance', 'uniform'], 'n_neighbors':range(3,75)}\n",
    "grid_search_knn = MultiOutputRegressor(GridSearchCV(KNeighborsRegressor(), param_grid, cv=5,return_train_score=True))\n",
    "grid_search_knn.fit(X_train, y_train)\n",
    "train_r2_score=r2_score(y_train,grid_search_knn.predict(X_train))\n",
    "test_r2_score=r2_score(y_test,grid_search_knn.predict(X_test))\n",
    "output = output.append(pd.Series({'model':'KNN Regressor', 'train_r2_score':train_r2_score,'test_r2_score':test_r2_score}),ignore_index=True )\n",
    "output"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Random Forest Regressor"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>model</th>\n",
       "      <th>train_r2_score</th>\n",
       "      <th>test_r2_score</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Linear Regressor</td>\n",
       "      <td>0.902539</td>\n",
       "      <td>0.899354</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>KNN Regressor</td>\n",
       "      <td>0.935276</td>\n",
       "      <td>0.904540</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Random Forest Regressor</td>\n",
       "      <td>0.997856</td>\n",
       "      <td>0.984413</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                     model  train_r2_score  test_r2_score\n",
       "0         Linear Regressor        0.902539       0.899354\n",
       "1            KNN Regressor        0.935276       0.904540\n",
       "2  Random Forest Regressor        0.997856       0.984413"
      ]
     },
     "execution_count": 42,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model = RandomForestRegressor( random_state=5, n_estimators= 700, n_jobs = -1)\n",
    "\n",
    "param_grid = {'max_features':['auto', 'log2'], 'max_depth':[5,10,15,20,30,50,60]}\n",
    "\n",
    "grid_search_rf = MultiOutputRegressor(GridSearchCV(model, param_grid, cv= 5,return_train_score=True))\n",
    "grid_search_rf.fit(X_train, y_train)\n",
    "train_r2_score=r2_score(y_train,grid_search_rf.predict(X_train))\n",
    "test_r2_score=r2_score(y_test,grid_search_rf.predict(X_test))\n",
    "output = output.append(pd.Series({'model':'Random Forest Regressor', 'train_r2_score':train_r2_score,'test_r2_score':test_r2_score}),ignore_index=True )\n",
    "output"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Linear SVM"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>model</th>\n",
       "      <th>train_r2_score</th>\n",
       "      <th>test_r2_score</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Linear Regressor</td>\n",
       "      <td>0.902539</td>\n",
       "      <td>0.899354</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>KNN Regressor</td>\n",
       "      <td>0.935276</td>\n",
       "      <td>0.904540</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Random Forest Regressor</td>\n",
       "      <td>0.997856</td>\n",
       "      <td>0.984413</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>SVM Linear</td>\n",
       "      <td>0.897939</td>\n",
       "      <td>0.896891</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                     model  train_r2_score  test_r2_score\n",
       "0         Linear Regressor        0.902539       0.899354\n",
       "1            KNN Regressor        0.935276       0.904540\n",
       "2  Random Forest Regressor        0.997856       0.984413\n",
       "3               SVM Linear        0.897939       0.896891"
      ]
     },
     "execution_count": 43,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "param_grid = {\"C\": [1e0, 1e1, 1e2, 1e3, 1e4],\"gamma\": np.logspace(-2, 1, 2, 3, 5)}\n",
    "grid_search_svm = MultiOutputRegressor(GridSearchCV(SVR(kernel='linear'), param_grid, cv=10,return_train_score=True))\n",
    "\n",
    "grid_search_svm.fit(X_train, y_train)\n",
    "train_r2_score=r2_score(y_train,grid_search_svm.predict(X_train))\n",
    "test_r2_score=r2_score(y_test,grid_search_svm.predict(X_test))\n",
    "output = output.append(pd.Series({'model':'SVM Linear', 'train_r2_score':train_r2_score,'test_r2_score':test_r2_score}),ignore_index=True )\n",
    "output"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Kernalized SVM "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>model</th>\n",
       "      <th>train_r2_score</th>\n",
       "      <th>test_r2_score</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Linear Regressor</td>\n",
       "      <td>0.902539</td>\n",
       "      <td>0.899354</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>KNN Regressor</td>\n",
       "      <td>0.935276</td>\n",
       "      <td>0.904540</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Random Forest Regressor</td>\n",
       "      <td>0.997856</td>\n",
       "      <td>0.984413</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>SVM Linear</td>\n",
       "      <td>0.897939</td>\n",
       "      <td>0.896891</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>SVM RBF</td>\n",
       "      <td>0.992589</td>\n",
       "      <td>0.986182</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                     model  train_r2_score  test_r2_score\n",
       "0         Linear Regressor        0.902539       0.899354\n",
       "1            KNN Regressor        0.935276       0.904540\n",
       "2  Random Forest Regressor        0.997856       0.984413\n",
       "3               SVM Linear        0.897939       0.896891\n",
       "4                  SVM RBF        0.992589       0.986182"
      ]
     },
     "execution_count": 44,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "param_grid = {\"C\": [1e0, 1e1, 1e2, 1e3, 1e4],\"gamma\": np.logspace(-2, 1, 2, 3, 5)}\n",
    "grid_search_svm_rbf = MultiOutputRegressor(GridSearchCV(SVR(kernel='rbf'), param_grid, cv=10,return_train_score=True))\n",
    "\n",
    "grid_search_svm_rbf.fit(X_train, y_train)\n",
    "train_r2_score=r2_score(y_train,grid_search_svm_rbf.predict(X_train))\n",
    "test_r2_score=r2_score(y_test,grid_search_svm_rbf.predict(X_test))\n",
    "output = output.append(pd.Series({'model':'SVM RBF', 'train_r2_score':train_r2_score,'test_r2_score':test_r2_score}),ignore_index=True )\n",
    "output"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 3. Ensembling"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Bagging:\n",
    "#### Bagging ensembler using Decision Tree Regressor as base model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>model</th>\n",
       "      <th>train_r2_score</th>\n",
       "      <th>test_r2_score</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Linear Regressor</td>\n",
       "      <td>0.902539</td>\n",
       "      <td>0.899354</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>KNN Regressor</td>\n",
       "      <td>0.935276</td>\n",
       "      <td>0.904540</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Random Forest Regressor</td>\n",
       "      <td>0.997856</td>\n",
       "      <td>0.984413</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>SVM Linear</td>\n",
       "      <td>0.897939</td>\n",
       "      <td>0.896891</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>SVM RBF</td>\n",
       "      <td>0.992589</td>\n",
       "      <td>0.986182</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>Multi Output DT Bagging</td>\n",
       "      <td>0.875499</td>\n",
       "      <td>0.880045</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                     model  train_r2_score  test_r2_score\n",
       "0         Linear Regressor        0.902539       0.899354\n",
       "1            KNN Regressor        0.935276       0.904540\n",
       "2  Random Forest Regressor        0.997856       0.984413\n",
       "3               SVM Linear        0.897939       0.896891\n",
       "4                  SVM RBF        0.992589       0.986182\n",
       "5  Multi Output DT Bagging        0.875499       0.880045"
      ]
     },
     "execution_count": 45,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn.ensemble import BaggingRegressor\n",
    "\n",
    "\n",
    "param_grid = {'max_samples':[5,10], 'max_features':[1,2,3,4,5,6,7]}\n",
    "bagging_DT = MultiOutputRegressor(GridSearchCV(BaggingRegressor(DecisionTreeRegressor(),n_estimators=750), \n",
    "                                                          param_grid,cv= 10,return_train_score=True))\n",
    "\n",
    "bagging_DT.fit(X_train, y_train)\n",
    "train_r2_score=r2_score(y_train,bagging_DT.predict(X_train))\n",
    "test_r2_score=r2_score(y_test,bagging_DT.predict(X_test))\n",
    "output = output.append(pd.Series({'model':'Multi Output DT Bagging', 'train_r2_score':train_r2_score,'test_r2_score':test_r2_score}),ignore_index=True )\n",
    "output\n",
    "\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Ada Boosting:\n",
    "#### Ada Boosting on Decision Tree Regressor"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>model</th>\n",
       "      <th>train_r2_score</th>\n",
       "      <th>test_r2_score</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Linear Regressor</td>\n",
       "      <td>0.902539</td>\n",
       "      <td>0.899354</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>KNN Regressor</td>\n",
       "      <td>0.935276</td>\n",
       "      <td>0.904540</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Random Forest Regressor</td>\n",
       "      <td>0.997856</td>\n",
       "      <td>0.984413</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>SVM Linear</td>\n",
       "      <td>0.897939</td>\n",
       "      <td>0.896891</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>SVM RBF</td>\n",
       "      <td>0.992589</td>\n",
       "      <td>0.986182</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>Multi Output DT Bagging</td>\n",
       "      <td>0.875499</td>\n",
       "      <td>0.880045</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>Adaboost_DecisionTree</td>\n",
       "      <td>0.999908</td>\n",
       "      <td>0.983499</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                     model  train_r2_score  test_r2_score\n",
       "0         Linear Regressor        0.902539       0.899354\n",
       "1            KNN Regressor        0.935276       0.904540\n",
       "2  Random Forest Regressor        0.997856       0.984413\n",
       "3               SVM Linear        0.897939       0.896891\n",
       "4                  SVM RBF        0.992589       0.986182\n",
       "5  Multi Output DT Bagging        0.875499       0.880045\n",
       "6    Adaboost_DecisionTree        0.999908       0.983499"
      ]
     },
     "execution_count": 46,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "param =  { \"n_estimators\": [100,500] }\n",
    "\n",
    "base_dt=DecisionTreeRegressor()\n",
    "ada_dt = AdaBoostRegressor(base_estimator=base_dt,learning_rate = 0.7,random_state=5)\n",
    "adaboost_dt = MultiOutputRegressor(GridSearchCV(ada_dt,param_grid=param))\n",
    "adaboost_dt.fit(X_train, y_train)\n",
    "train_r2_score=r2_score(y_train,adaboost_dt.predict(X_train))\n",
    "test_r2_score=r2_score(y_test,adaboost_dt.predict(X_test))\n",
    "output = output.append(pd.Series({'model':'Adaboost_DecisionTree', 'train_r2_score':train_r2_score,'test_r2_score':test_r2_score}),ignore_index=True )\n",
    "output"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Ada Boosting on KNN regressor"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>model</th>\n",
       "      <th>train_r2_score</th>\n",
       "      <th>test_r2_score</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Linear Regressor</td>\n",
       "      <td>0.902539</td>\n",
       "      <td>0.899354</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>KNN Regressor</td>\n",
       "      <td>0.935276</td>\n",
       "      <td>0.904540</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Random Forest Regressor</td>\n",
       "      <td>0.997856</td>\n",
       "      <td>0.984413</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>SVM Linear</td>\n",
       "      <td>0.897939</td>\n",
       "      <td>0.896891</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>SVM RBF</td>\n",
       "      <td>0.992589</td>\n",
       "      <td>0.986182</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>Multi Output DT Bagging</td>\n",
       "      <td>0.875499</td>\n",
       "      <td>0.880045</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>Adaboost_DecisionTree</td>\n",
       "      <td>0.999908</td>\n",
       "      <td>0.983499</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>KNN Adaboost</td>\n",
       "      <td>0.972943</td>\n",
       "      <td>0.929250</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                     model  train_r2_score  test_r2_score\n",
       "0         Linear Regressor        0.902539       0.899354\n",
       "1            KNN Regressor        0.935276       0.904540\n",
       "2  Random Forest Regressor        0.997856       0.984413\n",
       "3               SVM Linear        0.897939       0.896891\n",
       "4                  SVM RBF        0.992589       0.986182\n",
       "5  Multi Output DT Bagging        0.875499       0.880045\n",
       "6    Adaboost_DecisionTree        0.999908       0.983499\n",
       "7             KNN Adaboost        0.972943       0.929250"
      ]
     },
     "execution_count": 47,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "param =  { \"n_estimators\": [100,500,1000] }\n",
    "\n",
    "base_knn=KNeighborsRegressor()\n",
    "ada_knn = AdaBoostRegressor(base_estimator=base_knn,learning_rate = 0.7,random_state=10)\n",
    "adaboost_knn = MultiOutputRegressor(GridSearchCV(ada_knn,param_grid=param))\n",
    "adaboost_knn.fit(X_train, y_train)\n",
    "train_r2_score=r2_score(y_train,adaboost_knn.predict(X_train))\n",
    "test_r2_score=r2_score(y_test,adaboost_knn.predict(X_test))\n",
    "output = output.append(pd.Series({'model':'KNN Adaboost', 'train_r2_score':train_r2_score,'test_r2_score':test_r2_score}),ignore_index=True )\n",
    "output"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Ada Boosting on Linear SVM regressor"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>model</th>\n",
       "      <th>train_r2_score</th>\n",
       "      <th>test_r2_score</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Linear Regressor</td>\n",
       "      <td>0.902539</td>\n",
       "      <td>0.899354</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>KNN Regressor</td>\n",
       "      <td>0.935276</td>\n",
       "      <td>0.904540</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Random Forest Regressor</td>\n",
       "      <td>0.997856</td>\n",
       "      <td>0.984413</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>SVM Linear</td>\n",
       "      <td>0.897939</td>\n",
       "      <td>0.896891</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>SVM RBF</td>\n",
       "      <td>0.992589</td>\n",
       "      <td>0.986182</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>Multi Output DT Bagging</td>\n",
       "      <td>0.875499</td>\n",
       "      <td>0.880045</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>Adaboost_DecisionTree</td>\n",
       "      <td>0.999908</td>\n",
       "      <td>0.983499</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>KNN Adaboost</td>\n",
       "      <td>0.972943</td>\n",
       "      <td>0.929250</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>Adaboost_LinearSVM</td>\n",
       "      <td>0.896437</td>\n",
       "      <td>0.893522</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                     model  train_r2_score  test_r2_score\n",
       "0         Linear Regressor        0.902539       0.899354\n",
       "1            KNN Regressor        0.935276       0.904540\n",
       "2  Random Forest Regressor        0.997856       0.984413\n",
       "3               SVM Linear        0.897939       0.896891\n",
       "4                  SVM RBF        0.992589       0.986182\n",
       "5  Multi Output DT Bagging        0.875499       0.880045\n",
       "6    Adaboost_DecisionTree        0.999908       0.983499\n",
       "7             KNN Adaboost        0.972943       0.929250\n",
       "8       Adaboost_LinearSVM        0.896437       0.893522"
      ]
     },
     "execution_count": 48,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "param =  { \"n_estimators\": [100,500,1000] }\n",
    "\n",
    "\n",
    "base_svr=SVR(kernel='linear')\n",
    "ada_svr = AdaBoostRegressor(base_estimator=base_svr,learning_rate = 0.7,random_state=10)\n",
    "adaboost_svr = MultiOutputRegressor(GridSearchCV(ada_svr,param_grid=param,n_jobs=-1),n_jobs=-1)\n",
    "adaboost_svr.fit(X_train, y_train)\n",
    "train_r2_score=r2_score(y_train,adaboost_svr.predict(X_train))\n",
    "test_r2_score=r2_score(y_test,adaboost_svr.predict(X_test))\n",
    "output = output.append(pd.Series({'model':'Adaboost_LinearSVM', 'train_r2_score':train_r2_score,'test_r2_score':test_r2_score}),ignore_index=True )\n",
    "output"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Gradient Boosting regressor"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>model</th>\n",
       "      <th>train_r2_score</th>\n",
       "      <th>test_r2_score</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Linear Regressor</td>\n",
       "      <td>0.902539</td>\n",
       "      <td>0.899354</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>KNN Regressor</td>\n",
       "      <td>0.935276</td>\n",
       "      <td>0.904540</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Random Forest Regressor</td>\n",
       "      <td>0.997856</td>\n",
       "      <td>0.984413</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>SVM Linear</td>\n",
       "      <td>0.897939</td>\n",
       "      <td>0.896891</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>SVM RBF</td>\n",
       "      <td>0.992589</td>\n",
       "      <td>0.986182</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>Multi Output DT Bagging</td>\n",
       "      <td>0.875499</td>\n",
       "      <td>0.880045</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>Adaboost_DecisionTree</td>\n",
       "      <td>0.999908</td>\n",
       "      <td>0.983499</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>KNN Adaboost</td>\n",
       "      <td>0.972943</td>\n",
       "      <td>0.929250</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>Adaboost_LinearSVM</td>\n",
       "      <td>0.896437</td>\n",
       "      <td>0.893522</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>Gradient Boosting Regressor</td>\n",
       "      <td>0.999541</td>\n",
       "      <td>0.996787</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                         model  train_r2_score  test_r2_score\n",
       "0             Linear Regressor        0.902539       0.899354\n",
       "1                KNN Regressor        0.935276       0.904540\n",
       "2      Random Forest Regressor        0.997856       0.984413\n",
       "3                   SVM Linear        0.897939       0.896891\n",
       "4                      SVM RBF        0.992589       0.986182\n",
       "5      Multi Output DT Bagging        0.875499       0.880045\n",
       "6        Adaboost_DecisionTree        0.999908       0.983499\n",
       "7                 KNN Adaboost        0.972943       0.929250\n",
       "8           Adaboost_LinearSVM        0.896437       0.893522\n",
       "9  Gradient Boosting Regressor        0.999541       0.996787"
      ]
     },
     "execution_count": 49,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "param = {\"min_samples_split\": [3,4,5],\n",
    "         \"max_features\": [2,3,4,5,6],\n",
    "         'n_estimators': [100,500,1000]}\n",
    "\n",
    "gradient = GradientBoostingRegressor(learning_rate =1.1,random_state=10)\n",
    "gradient_mr = MultiOutputRegressor(GridSearchCV(gradient,param_grid=param,n_jobs=-1),n_jobs=-1)\n",
    "gradient_mr.fit(X_train, y_train)\n",
    "train_r2_score=r2_score(y_train,gradient_mr.predict(X_train))\n",
    "test_r2_score=r2_score(y_test,gradient_mr.predict(X_test))\n",
    "output = output.append(pd.Series({'model':'Gradient Boosting Regressor', 'train_r2_score':train_r2_score,'test_r2_score':test_r2_score}),ignore_index=True )\n",
    "output"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 4. Classification modeling\n",
    "### After running multioutput regressors we will now be running classification models"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.metrics import accuracy_score, f1_score\n",
    "from sklearn.linear_model import LogisticRegression\n",
    "from sklearn.svm import LinearSVC\n",
    "from sklearn.svm import SVC\n",
    "from sklearn.tree import DecisionTreeClassifier\n",
    "from sklearn.ensemble import GradientBoostingClassifier\n",
    "from sklearn.ensemble import RandomForestClassifier"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Best parameters for efficiency classification {'C': 10}\n",
      "The Train Accuracy score for Logistic Reression is 0.875\n",
      "The Test Accuracy score for Logistic Reression is 0.8802083333333334\n"
     ]
    }
   ],
   "source": [
    "#Logistic Regression\n",
    "\n",
    "param_grid = {'C':[0.01, 0.1, 0.5, 1, 2, 5, 10, 15, 20]}\n",
    "\n",
    "model = LogisticRegression()\n",
    "\n",
    "grid_search = GridSearchCV(model, param_grid, cv=10)\n",
    "grid_search.fit(X_train_class, y_train_class)\n",
    "\n",
    "print('Best parameters for efficiency classification {}'.format(grid_search.best_params_))\n",
    "\n",
    "print('The Train Accuracy score for Logistic Reression is',accuracy_score(y_train_class, grid_search.predict(X_train_class)))\n",
    "print('The Test Accuracy score for Logistic Reression is',accuracy_score(y_test_class, grid_search.predict(X_test_class)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Best parameters for efficiency classification {'C': 10}\n",
      "The Train Accuracy score for Linear SVC is 0.875\n",
      "The Test Accuracy score for Linear SVC is 0.8802083333333334\n"
     ]
    }
   ],
   "source": [
    "#Linear SVM\n",
    "\n",
    "param_grid = {'C':[0.001, 0.01, 0.1, 0.5, 1, 10]}\n",
    "\n",
    "model = LinearSVC()\n",
    "\n",
    "model = LogisticRegression()\n",
    "\n",
    "grid_search_Log_Class = GridSearchCV(model, param_grid, cv=5)\n",
    "grid_search_Log_Class.fit(X_train_class, y_train_class)\n",
    "\n",
    "print('Best parameters for efficiency classification {}'.format(grid_search_Log_Class.best_params_))\n",
    "\n",
    "print('The Train Accuracy score for Linear SVC is',accuracy_score(y_train_class, grid_search_Log_Class.predict(X_train_class)))\n",
    "print('The Test Accuracy score for Linear SVC is',accuracy_score(y_test_class, grid_search_Log_Class.predict(X_test_class)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Best parameters for efficiency classification {'C': 50, 'gamma': 0.5}\n",
      "The Train Accuracy score for SVM rbf is 0.9704861111111112\n",
      "The Test Accuracy score for SVM rbf is 0.984375\n"
     ]
    }
   ],
   "source": [
    "param_grid = {'C':[0.001, 0.01, 0.1, 0.5, 1, 10, 50, 100, 1000], 'gamma':[0.001, 0.01, 0.1, 0.5, 1, 10]}\n",
    "\n",
    "model = SVC(kernel='rbf')\n",
    "\n",
    "grid_search_SVC_Class = GridSearchCV(model, param_grid, cv=10)\n",
    "grid_search_SVC_Class.fit(X_train_class, y_train_class)\n",
    "\n",
    "print('Best parameters for efficiency classification {}'.format(grid_search_SVC_Class.best_params_))\n",
    "\n",
    "print('The Train Accuracy score for SVM rbf is',accuracy_score(y_train_class, grid_search_SVC_Class.predict(X_train_class)))\n",
    "print('The Test Accuracy score for SVM rbf is',accuracy_score(y_test_class, grid_search_SVC_Class.predict(X_test_class)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Best parameters for efficiency classification {'max_depth': 10, 'max_features': 'auto'}\n",
      "The Train Accuracy score for random forest classifier is 1.0\n",
      "The Test Accuracy score for random forest classifier is 0.96875\n"
     ]
    }
   ],
   "source": [
    "param_grid = {'max_features':['auto', 'log2'], 'max_depth':[5,10,15,20,50,60]}\n",
    "\n",
    "model = RandomForestClassifier(n_estimators= 700, n_jobs = -1, random_state = 5)\n",
    "\n",
    "grid_search_RF_Class = GridSearchCV(model, param_grid, cv=5)\n",
    "grid_search_RF_Class.fit(X_train_class, y_train_class)\n",
    "\n",
    "print('Best parameters for efficiency classification {}'.format(grid_search_RF_Class.best_params_))\n",
    "\n",
    "print('The Train Accuracy score for random forest classifier is',accuracy_score(y_train_class, grid_search_RF_Class.predict(X_train_class)))\n",
    "print('The Test Accuracy score for random forest classifier is',accuracy_score(y_test_class, grid_search_RF_Class.predict(X_test_class)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Best parameters for efficiency classification {'learning_rate': 0.01, 'max_depth': 5, 'max_features': 'log2'}\n",
      "The Train Accuracy score for Gradient Boosting Classifier is 1.0\n",
      "The Test Accuracy score for Gradient Boosting Classifier is 0.9791666666666666\n"
     ]
    }
   ],
   "source": [
    "param_grid = {'max_features':['auto', 'log2'], 'max_depth':[5,10,15,20,50,60], 'learning_rate':[0.01,0.1,0.7,1]}\n",
    "\n",
    "model =  GradientBoostingClassifier(random_state = 5, n_estimators = 500)\n",
    "\n",
    "\n",
    "grid_search_GB_Class = GridSearchCV(model, param_grid, cv=5)\n",
    "grid_search_GB_Class.fit(X_train_class, y_train_class)\n",
    "\n",
    "print('Best parameters for efficiency classification {}'.format(grid_search_GB_Class.best_params_))\n",
    "\n",
    "print('The Train Accuracy score for Gradient Boosting Classifier is',accuracy_score(y_train_class, grid_search_GB_Class.predict(X_train_class)))\n",
    "print('The Test Accuracy score for Gradient Boosting Classifier is',accuracy_score(y_test_class, grid_search_GB_Class.predict(X_test_class)))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 4. Neural Network Regression model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "metadata": {},
   "outputs": [],
   "source": [
    "from keras.models import Sequential\n",
    "from keras.layers import Dense\n",
    "\n",
    "# fix random seed for reproducibility\n",
    "np.random.seed(5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "metadata": {},
   "outputs": [],
   "source": [
    "from keras.wrappers.scikit_learn import KerasRegressor\n",
    "\n",
    "def create_model_regressor_heating():\n",
    "    #create model\n",
    "    model = Sequential()\n",
    "    model.add(Dense(8, input_dim=8, kernel_initializer='normal', activation='relu'))\n",
    "    model.add(Dense(4, kernel_initializer = 'normal', activation = 'relu')) # hidden layer\n",
    "    model.add(Dense(10, kernel_initializer = 'normal', activation = 'linear'))\n",
    "    model.add(Dense(1, kernel_initializer='normal'))\n",
    "    #compile model\n",
    "    model.compile(loss='mean_squared_error', optimizer='adam', metrics=['mean_squared_error'])\n",
    "    return model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "metadata": {},
   "outputs": [],
   "source": [
    "from keras.wrappers.scikit_learn import KerasRegressor\n",
    "\n",
    "def create_model_regressor_cooling():\n",
    "    #create model\n",
    "    model2 = Sequential()\n",
    "    model2.add(Dense(8, input_dim=8, kernel_initializer='normal', activation='relu'))\n",
    "    model2.add(Dense(4, kernel_initializer = 'normal', activation = 'relu')) # hidden layer\n",
    "    model2.add(Dense(10, kernel_initializer = 'normal', activation = 'linear'))\n",
    "    model2.add(Dense(1, kernel_initializer='normal'))\n",
    "    #compile model\n",
    "    model2.compile(loss='mean_squared_error', optimizer='adam', metrics=['mean_squared_error'])\n",
    "    #return model\n",
    "    return model2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Model training\n",
    "\n",
    "param_grid = {'epochs':[50, 100, 200] , 'batch_size':[20, 50, 100]}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Best parameters for cooling load {'batch_size': 20, 'epochs': 200}\n",
      "The Train R2 score is 0.8447339385412662\n",
      "The Test R2 score is 0.8184779982513859\n"
     ]
    }
   ],
   "source": [
    "model1 = KerasRegressor(build_fn = create_model_regressor_cooling , verbose = 0)\n",
    "\n",
    "grid_search_Keras_Reg = GridSearchCV(model1 , param_grid , cv =5)\n",
    "\n",
    "grid_search_Keras_Reg.fit(X_train_div, y1_train)\n",
    "\n",
    "print('Best parameters for cooling load {}'.format(grid_search_Keras_Reg.best_params_))\n",
    "\n",
    "print('The Train R2 score is',r2_score(y1_train, grid_search_Keras_Reg.predict(X_train_div)))\n",
    "print('The Test R2 score is',r2_score(y1_test, grid_search_Keras_Reg.predict(X_test_div)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Best parameters for heating load {'batch_size': 20, 'epochs': 200}\n",
      "The Train R2 score is 0.8120563702859663\n",
      "The Test R2 score is 0.7998327631743293\n"
     ]
    }
   ],
   "source": [
    "model2 = KerasRegressor(build_fn = create_model_regressor_heating , verbose = 0)\n",
    "grid_search_Keras2 = GridSearchCV(model2 , param_grid , cv =10)\n",
    "\n",
    "grid_search_Keras2.fit(X_train_div, y2_train)\n",
    "print('Best parameters for heating load {}'.format(grid_search_Keras2.best_params_))\n",
    "\n",
    "print('The Train R2 score is',r2_score(y2_train, grid_search_Keras2.predict(X_train_div)))\n",
    "print('The Test R2 score is',r2_score(y2_test, grid_search_Keras2.predict(X_test_div)))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 5. Neural Network Classification model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "metadata": {},
   "outputs": [],
   "source": [
    "from keras.utils import np_utils\n",
    "\n",
    "# convert integers to dummy variables (i.e. one hot encoded)\n",
    "y_train = np_utils.to_categorical(y_train_class)\n",
    "y_test = np_utils.to_categorical(y_test_class)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Creating model function for NN"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "metadata": {},
   "outputs": [],
   "source": [
    "from keras.wrappers.scikit_learn import KerasClassifier\n",
    "\n",
    "def model_classifier():\n",
    "    #create model\n",
    "    model = Sequential()\n",
    "    model.add(Dense(8, input_dim=8, activation='relu'))\n",
    "    model.add(Dense(4, activation='relu')) #hidden layer\n",
    "    model.add(Dense(3,activation='sigmoid')) #output layer\n",
    "    model.compile(loss='categorical_crossentropy', optimizer='adam', metrics=['accuracy'])\n",
    "    return model\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Grid Search for NN"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 64,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Best parameters for efficiency classification {'batch_size': 10, 'epochs': 200}\n"
     ]
    }
   ],
   "source": [
    "param_grid = {'epochs':[50, 200] , 'batch_size':[10, 50, 100]}\n",
    "\n",
    "model = KerasClassifier(build_fn = model_classifier , verbose = 0)\n",
    "\n",
    "grid_search_Keras_Class = GridSearchCV(model , param_grid , cv =10)\n",
    "\n",
    "grid_search_Keras_Class.fit(X_train_class, y_train)\n",
    "\n",
    "print('Best parameters for efficiency classification {}'.format(grid_search_Keras_Class.best_params_))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 65,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "The Train Accuracy score is 0.9635416666666666\n",
      "The Test Accuracy score is 0.9635416666666666\n"
     ]
    }
   ],
   "source": [
    "from sklearn.metrics import accuracy_score\n",
    "print('The Train Accuracy score is',accuracy_score(y_train_class, grid_search_Keras_Class.predict(X_train_class)))\n",
    "print('The Test Accuracy score is',accuracy_score(y_test_class, grid_search_Keras_Class.predict(X_test_class)))"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}