5. Algorithm Selection.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### V. Algorithm Selection\n",
    "\n",
    "In this part, we create a model, we try different algorithms and see which one delivers the best results. Then we chose the best algorithm and fine tune it. \n",
    "\n",
    "This notebook presents the following parts:\n",
    "\n",
    "    1) Model creation\n",
    "    2) Algorithm testing : Linear Regression (simple, lasso, ridge), Boosted decision tree regressor , Random forest regressor, Bayesian linear regressor\n",
    "    3) Chosing best algorithm\n",
    "    4) Save model\n",
    "\n",
    "\n",
    "     "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "# import libraries\n",
    "import pandas as pd\n",
    "from sklearn import preprocessing\n",
    "import sklearn.model_selection as ms\n",
    "from sklearn import linear_model\n",
    "import sklearn.metrics as sklm\n",
    "import numpy as np\n",
    "import numpy.random as nr\n",
    "import matplotlib.pyplot as plt\n",
    "import seaborn as sns\n",
    "import scipy.stats as ss\n",
    "import math\n",
    "from sklearn.model_selection import train_test_split \n",
    "from sklearn.linear_model import LinearRegression\n",
    "from sklearn import metrics\n",
    "\n",
    "\n",
    "%matplotlib inline\n",
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(1399, 18)"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#import data set\n",
    "df=pd.read_csv('dfprepared3.csv')\n",
    "df.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>capacity</th>\n",
       "      <th>failure_rate</th>\n",
       "      <th>margin</th>\n",
       "      <th>price</th>\n",
       "      <th>prod_cost</th>\n",
       "      <th>product_type:_auto-portee</th>\n",
       "      <th>product_type:_electrique</th>\n",
       "      <th>product_type:_essence</th>\n",
       "      <th>Quality:_Basic</th>\n",
       "      <th>Quality:_High</th>\n",
       "      <th>Quality:_Medium</th>\n",
       "      <th>Warranty_years:_1</th>\n",
       "      <th>Warranty_years:_2</th>\n",
       "      <th>Warranty_years:_3</th>\n",
       "      <th>Perc_Margin:_High</th>\n",
       "      <th>Perc_Margin:_Low</th>\n",
       "      <th>Perc_Margin:_Medium</th>\n",
       "      <th>attractiveness</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>-1.873473</td>\n",
       "      <td>-1.683579</td>\n",
       "      <td>2.342817</td>\n",
       "      <td>2.187839</td>\n",
       "      <td>0.911872</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0.650648</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>-1.380486</td>\n",
       "      <td>-1.746504</td>\n",
       "      <td>2.854882</td>\n",
       "      <td>2.395929</td>\n",
       "      <td>0.583018</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0.699792</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   capacity  failure_rate    margin     price  prod_cost  \\\n",
       "0 -1.873473     -1.683579  2.342817  2.187839   0.911872   \n",
       "1 -1.380486     -1.746504  2.854882  2.395929   0.583018   \n",
       "\n",
       "   product_type:_auto-portee  product_type:_electrique  product_type:_essence  \\\n",
       "0                          1                         0                      0   \n",
       "1                          1                         0                      0   \n",
       "\n",
       "   Quality:_Basic  Quality:_High  Quality:_Medium  Warranty_years:_1  \\\n",
       "0               1              0                0                  0   \n",
       "1               1              0                0                  0   \n",
       "\n",
       "   Warranty_years:_2  Warranty_years:_3  Perc_Margin:_High  Perc_Margin:_Low  \\\n",
       "0                  0                  1                  0                 0   \n",
       "1                  0                  1                  0                 0   \n",
       "\n",
       "   Perc_Margin:_Medium  attractiveness  \n",
       "0                    1        0.650648  \n",
       "1                    1        0.699792  "
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.head(2)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**1) Model Creation**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "((979, 17), (420, 17))"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# separate train and test sets\n",
    "X_train, X_test, y_train, y_test = train_test_split(\n",
    "    df.drop(labels=['attractiveness'], axis=1),\n",
    "    df['attractiveness'],\n",
    "    test_size=0.3,\n",
    "    random_state=0)\n",
    "\n",
    "X_train.shape, X_test.shape"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**2) Algoirthm testing**"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "2.1Algorithm: **Linear Regression**"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "2.1.1 Linear Regression Simple"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Train the Model and predict\n",
    "from sklearn.linear_model import LinearRegression\n",
    "from sklearn.metrics import mean_squared_log_error\n",
    "lm = LinearRegression()\n",
    "lm.fit(X_train,y_train)\n",
    "lm_predictions = lm.predict(X_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Simple Regression RMSLE is 0.07242206659110598\n"
     ]
    }
   ],
   "source": [
    "#print RMSLE\n",
    "print ('Simple Regression RMSLE is', np.sqrt(mean_squared_log_error(y_test, lm_predictions)))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "2.1.2 Lasso Linear Regression (l1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Train the Model and predict\n",
    "from sklearn.linear_model import Lasso\n",
    "from sklearn.metrics import mean_squared_log_error\n",
    "Lasso = Lasso()\n",
    "Lasso.fit(X_train,y_train)\n",
    "Lasso_predictions = Lasso.predict(X_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Lasso Regression RMSLE is 0.07920657308951531\n"
     ]
    }
   ],
   "source": [
    "#print RMSLE\n",
    "print ('Lasso Regression RMSLE is', np.sqrt(mean_squared_log_error(y_test, Lasso_predictions)))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "2.1.3 Ridge Regression (l2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Train the Model and predict\n",
    "from sklearn.linear_model import Ridge\n",
    "from sklearn.metrics import mean_squared_log_error\n",
    "Ridge = Ridge()\n",
    "Ridge.fit(X_train,y_train)\n",
    "Ridge_predictions = Ridge.predict(X_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Ridge Regression RMSLE is 0.07165632155144348\n"
     ]
    }
   ],
   "source": [
    "#print RMSLE\n",
    "print ('Ridge Regression RMSLE is', np.sqrt(mean_squared_log_error(y_test, Ridge_predictions)))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "2.2 Algorithm: **Boosted Decision Tree Regressor**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Train the Model and predict\n",
    "from sklearn.tree import DecisionTreeRegressor\n",
    "from sklearn.metrics import mean_squared_log_error\n",
    "Tree = DecisionTreeRegressor()\n",
    "Tree.fit(X_train,y_train)\n",
    "Tree_predictions = Tree.predict(X_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Boosted Decision Tree Regression RMSLE is 0.06004657556100968\n"
     ]
    }
   ],
   "source": [
    "#print RMSLE\n",
    "print ('Boosted Decision Tree Regression RMSLE is', np.sqrt(mean_squared_log_error(y_test, Tree_predictions)))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "2.3 Algorithm: **Random Forest Regressor**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/anaconda3/envs/featsel/lib/python3.6/site-packages/sklearn/ensemble/forest.py:245: FutureWarning: The default value of n_estimators will change from 10 in version 0.20 to 100 in 0.22.\n",
      "  \"10 in version 0.20 to 100 in 0.22.\", FutureWarning)\n"
     ]
    }
   ],
   "source": [
    "#Train the Model and predict\n",
    "from sklearn.ensemble import RandomForestRegressor\n",
    "from sklearn.metrics import mean_squared_log_error\n",
    "Forest = RandomForestRegressor()\n",
    "Forest.fit(X_train,y_train)\n",
    "Forest_predictions = Forest.predict(X_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Random Forest Regression RMSLE is 0.04796913613049845\n"
     ]
    }
   ],
   "source": [
    "#print RMSLE\n",
    "print ('Random Forest Regression RMSLE is', np.sqrt(mean_squared_log_error(y_test, Forest_predictions)))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "2.4 Algorithm: **Bayesian Linear Regressor**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Train the Model and predict\n",
    "from sklearn.linear_model import BayesianRidge\n",
    "from sklearn.metrics import mean_squared_log_error\n",
    "Bayesian = BayesianRidge()\n",
    "Bayesian.fit(X_train,y_train)\n",
    "Bayesian_predictions = Bayesian.predict(X_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Bayesian Ridge Regression RMSLE is 0.07204981804528336\n"
     ]
    }
   ],
   "source": [
    "#print RMSLE\n",
    "print ('Bayesian Ridge Regression RMSLE is', np.sqrt(mean_squared_log_error(y_test, Bayesian_predictions)))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**3. Compare and chose best model**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Simple Regression RMSLE is 0.07242206659110598\n",
      "Lasso Regression RMSLE is 0.07920657308951531\n",
      "Ridge Regression RMSLE is 0.07165632155144348\n",
      "Boosted Decision Tree Regression RMSLE is 0.06004657556100968\n",
      "Random Forest Regression RMSLE is 0.04796913613049845\n",
      "Bayesian Ridge Regression RMSLE is 0.07204981804528336\n"
     ]
    }
   ],
   "source": [
    "print ('Simple Regression RMSLE is', np.sqrt(mean_squared_log_error(y_test, lm_predictions)))\n",
    "print ('Lasso Regression RMSLE is', np.sqrt(mean_squared_log_error(y_test, Lasso_predictions)))\n",
    "print ('Ridge Regression RMSLE is', np.sqrt(mean_squared_log_error(y_test, Ridge_predictions)))\n",
    "print ('Boosted Decision Tree Regression RMSLE is', np.sqrt(mean_squared_log_error(y_test, Tree_predictions)))\n",
    "print ('Random Forest Regression RMSLE is', np.sqrt(mean_squared_log_error(y_test, Forest_predictions)))\n",
    "print ('Bayesian Ridge Regression RMSLE is', np.sqrt(mean_squared_log_error(y_test, Bayesian_predictions)))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Clearly, Random Forest Regressor is the best model. \n",
    "\n",
    "**4. Save the model**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "df.to_csv(\"dftobeimproved.csv\", index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}