L1 and L2 Regularization.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<h2 style='color:blue' align='center'>L1 and L2 Regularization</h2>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "# import libraries\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "import pandas as pd\n",
    "import seaborn as sns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Suppress Warnings for clean notebook\n",
    "import warnings\n",
    "warnings.filterwarnings('ignore')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**We are going to use Melbourne House Price Dataset where we'll predict House Predictions based on various features.**\n",
    "#### The Dataset Link is\n",
    "https://www.kaggle.com/anthonypino/melbourne-housing-market"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "# read dataset\n",
    "dataset = pd.read_csv('./Melbourne_housing_FULL.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Suburb</th>\n",
       "      <th>Address</th>\n",
       "      <th>Rooms</th>\n",
       "      <th>Type</th>\n",
       "      <th>Price</th>\n",
       "      <th>Method</th>\n",
       "      <th>SellerG</th>\n",
       "      <th>Date</th>\n",
       "      <th>Distance</th>\n",
       "      <th>Postcode</th>\n",
       "      <th>...</th>\n",
       "      <th>Bathroom</th>\n",
       "      <th>Car</th>\n",
       "      <th>Landsize</th>\n",
       "      <th>BuildingArea</th>\n",
       "      <th>YearBuilt</th>\n",
       "      <th>CouncilArea</th>\n",
       "      <th>Lattitude</th>\n",
       "      <th>Longtitude</th>\n",
       "      <th>Regionname</th>\n",
       "      <th>Propertycount</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Abbotsford</td>\n",
       "      <td>68 Studley St</td>\n",
       "      <td>2</td>\n",
       "      <td>h</td>\n",
       "      <td>NaN</td>\n",
       "      <td>SS</td>\n",
       "      <td>Jellis</td>\n",
       "      <td>3/09/2016</td>\n",
       "      <td>2.5</td>\n",
       "      <td>3067.0</td>\n",
       "      <td>...</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>126.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Yarra City Council</td>\n",
       "      <td>-37.8014</td>\n",
       "      <td>144.9958</td>\n",
       "      <td>Northern Metropolitan</td>\n",
       "      <td>4019.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Abbotsford</td>\n",
       "      <td>85 Turner St</td>\n",
       "      <td>2</td>\n",
       "      <td>h</td>\n",
       "      <td>1480000.0</td>\n",
       "      <td>S</td>\n",
       "      <td>Biggin</td>\n",
       "      <td>3/12/2016</td>\n",
       "      <td>2.5</td>\n",
       "      <td>3067.0</td>\n",
       "      <td>...</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>202.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Yarra City Council</td>\n",
       "      <td>-37.7996</td>\n",
       "      <td>144.9984</td>\n",
       "      <td>Northern Metropolitan</td>\n",
       "      <td>4019.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Abbotsford</td>\n",
       "      <td>25 Bloomburg St</td>\n",
       "      <td>2</td>\n",
       "      <td>h</td>\n",
       "      <td>1035000.0</td>\n",
       "      <td>S</td>\n",
       "      <td>Biggin</td>\n",
       "      <td>4/02/2016</td>\n",
       "      <td>2.5</td>\n",
       "      <td>3067.0</td>\n",
       "      <td>...</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>156.0</td>\n",
       "      <td>79.0</td>\n",
       "      <td>1900.0</td>\n",
       "      <td>Yarra City Council</td>\n",
       "      <td>-37.8079</td>\n",
       "      <td>144.9934</td>\n",
       "      <td>Northern Metropolitan</td>\n",
       "      <td>4019.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Abbotsford</td>\n",
       "      <td>18/659 Victoria St</td>\n",
       "      <td>3</td>\n",
       "      <td>u</td>\n",
       "      <td>NaN</td>\n",
       "      <td>VB</td>\n",
       "      <td>Rounds</td>\n",
       "      <td>4/02/2016</td>\n",
       "      <td>2.5</td>\n",
       "      <td>3067.0</td>\n",
       "      <td>...</td>\n",
       "      <td>2.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Yarra City Council</td>\n",
       "      <td>-37.8114</td>\n",
       "      <td>145.0116</td>\n",
       "      <td>Northern Metropolitan</td>\n",
       "      <td>4019.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Abbotsford</td>\n",
       "      <td>5 Charles St</td>\n",
       "      <td>3</td>\n",
       "      <td>h</td>\n",
       "      <td>1465000.0</td>\n",
       "      <td>SP</td>\n",
       "      <td>Biggin</td>\n",
       "      <td>4/03/2017</td>\n",
       "      <td>2.5</td>\n",
       "      <td>3067.0</td>\n",
       "      <td>...</td>\n",
       "      <td>2.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>134.0</td>\n",
       "      <td>150.0</td>\n",
       "      <td>1900.0</td>\n",
       "      <td>Yarra City Council</td>\n",
       "      <td>-37.8093</td>\n",
       "      <td>144.9944</td>\n",
       "      <td>Northern Metropolitan</td>\n",
       "      <td>4019.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 21 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "       Suburb             Address  Rooms Type      Price Method SellerG  \\\n",
       "0  Abbotsford       68 Studley St      2    h        NaN     SS  Jellis   \n",
       "1  Abbotsford        85 Turner St      2    h  1480000.0      S  Biggin   \n",
       "2  Abbotsford     25 Bloomburg St      2    h  1035000.0      S  Biggin   \n",
       "3  Abbotsford  18/659 Victoria St      3    u        NaN     VB  Rounds   \n",
       "4  Abbotsford        5 Charles St      3    h  1465000.0     SP  Biggin   \n",
       "\n",
       "        Date  Distance  Postcode  ...  Bathroom  Car  Landsize  BuildingArea  \\\n",
       "0  3/09/2016       2.5    3067.0  ...       1.0  1.0     126.0           NaN   \n",
       "1  3/12/2016       2.5    3067.0  ...       1.0  1.0     202.0           NaN   \n",
       "2  4/02/2016       2.5    3067.0  ...       1.0  0.0     156.0          79.0   \n",
       "3  4/02/2016       2.5    3067.0  ...       2.0  1.0       0.0           NaN   \n",
       "4  4/03/2017       2.5    3067.0  ...       2.0  0.0     134.0         150.0   \n",
       "\n",
       "   YearBuilt         CouncilArea Lattitude  Longtitude             Regionname  \\\n",
       "0        NaN  Yarra City Council  -37.8014    144.9958  Northern Metropolitan   \n",
       "1        NaN  Yarra City Council  -37.7996    144.9984  Northern Metropolitan   \n",
       "2     1900.0  Yarra City Council  -37.8079    144.9934  Northern Metropolitan   \n",
       "3        NaN  Yarra City Council  -37.8114    145.0116  Northern Metropolitan   \n",
       "4     1900.0  Yarra City Council  -37.8093    144.9944  Northern Metropolitan   \n",
       "\n",
       "  Propertycount  \n",
       "0        4019.0  \n",
       "1        4019.0  \n",
       "2        4019.0  \n",
       "3        4019.0  \n",
       "4        4019.0  \n",
       "\n",
       "[5 rows x 21 columns]"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dataset.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Suburb             351\n",
       "Address          34009\n",
       "Rooms               12\n",
       "Type                 3\n",
       "Price             2871\n",
       "Method               9\n",
       "SellerG            388\n",
       "Date                78\n",
       "Distance           215\n",
       "Postcode           211\n",
       "Bedroom2            15\n",
       "Bathroom            11\n",
       "Car                 15\n",
       "Landsize          1684\n",
       "BuildingArea       740\n",
       "YearBuilt          160\n",
       "CouncilArea         33\n",
       "Lattitude        13402\n",
       "Longtitude       14524\n",
       "Regionname           8\n",
       "Propertycount      342\n",
       "dtype: int64"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dataset.nunique()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "# let's use limited columns which makes more sense for serving our purpose\n",
    "cols_to_use = ['Suburb', 'Rooms', 'Type', 'Method', 'SellerG', 'Regionname', 'Propertycount', \n",
    "               'Distance', 'CouncilArea', 'Bedroom2', 'Bathroom', 'Car', 'Landsize', 'BuildingArea', 'Price']\n",
    "dataset = dataset[cols_to_use]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Suburb</th>\n",
       "      <th>Rooms</th>\n",
       "      <th>Type</th>\n",
       "      <th>Method</th>\n",
       "      <th>SellerG</th>\n",
       "      <th>Regionname</th>\n",
       "      <th>Propertycount</th>\n",
       "      <th>Distance</th>\n",
       "      <th>CouncilArea</th>\n",
       "      <th>Bedroom2</th>\n",
       "      <th>Bathroom</th>\n",
       "      <th>Car</th>\n",
       "      <th>Landsize</th>\n",
       "      <th>BuildingArea</th>\n",
       "      <th>Price</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Abbotsford</td>\n",
       "      <td>2</td>\n",
       "      <td>h</td>\n",
       "      <td>SS</td>\n",
       "      <td>Jellis</td>\n",
       "      <td>Northern Metropolitan</td>\n",
       "      <td>4019.0</td>\n",
       "      <td>2.5</td>\n",
       "      <td>Yarra City Council</td>\n",
       "      <td>2.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>126.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Abbotsford</td>\n",
       "      <td>2</td>\n",
       "      <td>h</td>\n",
       "      <td>S</td>\n",
       "      <td>Biggin</td>\n",
       "      <td>Northern Metropolitan</td>\n",
       "      <td>4019.0</td>\n",
       "      <td>2.5</td>\n",
       "      <td>Yarra City Council</td>\n",
       "      <td>2.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>202.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1480000.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Abbotsford</td>\n",
       "      <td>2</td>\n",
       "      <td>h</td>\n",
       "      <td>S</td>\n",
       "      <td>Biggin</td>\n",
       "      <td>Northern Metropolitan</td>\n",
       "      <td>4019.0</td>\n",
       "      <td>2.5</td>\n",
       "      <td>Yarra City Council</td>\n",
       "      <td>2.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>156.0</td>\n",
       "      <td>79.0</td>\n",
       "      <td>1035000.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Abbotsford</td>\n",
       "      <td>3</td>\n",
       "      <td>u</td>\n",
       "      <td>VB</td>\n",
       "      <td>Rounds</td>\n",
       "      <td>Northern Metropolitan</td>\n",
       "      <td>4019.0</td>\n",
       "      <td>2.5</td>\n",
       "      <td>Yarra City Council</td>\n",
       "      <td>3.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Abbotsford</td>\n",
       "      <td>3</td>\n",
       "      <td>h</td>\n",
       "      <td>SP</td>\n",
       "      <td>Biggin</td>\n",
       "      <td>Northern Metropolitan</td>\n",
       "      <td>4019.0</td>\n",
       "      <td>2.5</td>\n",
       "      <td>Yarra City Council</td>\n",
       "      <td>3.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>134.0</td>\n",
       "      <td>150.0</td>\n",
       "      <td>1465000.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "       Suburb  Rooms Type Method SellerG             Regionname  \\\n",
       "0  Abbotsford      2    h     SS  Jellis  Northern Metropolitan   \n",
       "1  Abbotsford      2    h      S  Biggin  Northern Metropolitan   \n",
       "2  Abbotsford      2    h      S  Biggin  Northern Metropolitan   \n",
       "3  Abbotsford      3    u     VB  Rounds  Northern Metropolitan   \n",
       "4  Abbotsford      3    h     SP  Biggin  Northern Metropolitan   \n",
       "\n",
       "   Propertycount  Distance         CouncilArea  Bedroom2  Bathroom  Car  \\\n",
       "0         4019.0       2.5  Yarra City Council       2.0       1.0  1.0   \n",
       "1         4019.0       2.5  Yarra City Council       2.0       1.0  1.0   \n",
       "2         4019.0       2.5  Yarra City Council       2.0       1.0  0.0   \n",
       "3         4019.0       2.5  Yarra City Council       3.0       2.0  1.0   \n",
       "4         4019.0       2.5  Yarra City Council       3.0       2.0  0.0   \n",
       "\n",
       "   Landsize  BuildingArea      Price  \n",
       "0     126.0           NaN        NaN  \n",
       "1     202.0           NaN  1480000.0  \n",
       "2     156.0          79.0  1035000.0  \n",
       "3       0.0           NaN        NaN  \n",
       "4     134.0         150.0  1465000.0  "
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dataset.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(34857, 15)"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dataset.shape"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Checking for Nan values"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Suburb               0\n",
       "Rooms                0\n",
       "Type                 0\n",
       "Method               0\n",
       "SellerG              0\n",
       "Regionname           3\n",
       "Propertycount        3\n",
       "Distance             1\n",
       "CouncilArea          3\n",
       "Bedroom2          8217\n",
       "Bathroom          8226\n",
       "Car               8728\n",
       "Landsize         11810\n",
       "BuildingArea     21115\n",
       "Price             7610\n",
       "dtype: int64"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dataset.isna().sum()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Handling Missing values"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Some feature's missing values can be treated as zero (another class for NA values or absence of that feature)\n",
    "# like 0 for Propertycount, Bedroom2 will refer to other class of NA values\n",
    "# like 0 for Car feature will mean that there's no car parking feature with house\n",
    "cols_to_fill_zero = ['Propertycount', 'Distance', 'Bedroom2', 'Bathroom', 'Car']\n",
    "dataset[cols_to_fill_zero] = dataset[cols_to_fill_zero].fillna(0)\n",
    "\n",
    "# other continuous features can be imputed with mean for faster results since our focus is on Reducing overfitting\n",
    "# using Lasso and Ridge Regression\n",
    "dataset['Landsize'] = dataset['Landsize'].fillna(dataset.Landsize.mean())\n",
    "dataset['BuildingArea'] = dataset['BuildingArea'].fillna(dataset.BuildingArea.mean())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**Drop NA values of Price, since it's our predictive variable we won't impute it**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "dataset.dropna(inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(27244, 15)"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dataset.shape"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Let's one hot encode the categorical features"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "dataset = pd.get_dummies(dataset, drop_first=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Rooms</th>\n",
       "      <th>Propertycount</th>\n",
       "      <th>Distance</th>\n",
       "      <th>Bedroom2</th>\n",
       "      <th>Bathroom</th>\n",
       "      <th>Car</th>\n",
       "      <th>Landsize</th>\n",
       "      <th>BuildingArea</th>\n",
       "      <th>Price</th>\n",
       "      <th>Suburb_Aberfeldie</th>\n",
       "      <th>...</th>\n",
       "      <th>CouncilArea_Moorabool Shire Council</th>\n",
       "      <th>CouncilArea_Moreland City Council</th>\n",
       "      <th>CouncilArea_Nillumbik Shire Council</th>\n",
       "      <th>CouncilArea_Port Phillip City Council</th>\n",
       "      <th>CouncilArea_Stonnington City Council</th>\n",
       "      <th>CouncilArea_Whitehorse City Council</th>\n",
       "      <th>CouncilArea_Whittlesea City Council</th>\n",
       "      <th>CouncilArea_Wyndham City Council</th>\n",
       "      <th>CouncilArea_Yarra City Council</th>\n",
       "      <th>CouncilArea_Yarra Ranges Shire Council</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2</td>\n",
       "      <td>4019.0</td>\n",
       "      <td>2.5</td>\n",
       "      <td>2.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>202.0</td>\n",
       "      <td>160.2564</td>\n",
       "      <td>1480000.0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2</td>\n",
       "      <td>4019.0</td>\n",
       "      <td>2.5</td>\n",
       "      <td>2.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>156.0</td>\n",
       "      <td>79.0000</td>\n",
       "      <td>1035000.0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>3</td>\n",
       "      <td>4019.0</td>\n",
       "      <td>2.5</td>\n",
       "      <td>3.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>134.0</td>\n",
       "      <td>150.0000</td>\n",
       "      <td>1465000.0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>3</td>\n",
       "      <td>4019.0</td>\n",
       "      <td>2.5</td>\n",
       "      <td>3.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>94.0</td>\n",
       "      <td>160.2564</td>\n",
       "      <td>850000.0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>4</td>\n",
       "      <td>4019.0</td>\n",
       "      <td>2.5</td>\n",
       "      <td>3.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>120.0</td>\n",
       "      <td>142.0000</td>\n",
       "      <td>1600000.0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 745 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "   Rooms  Propertycount  Distance  Bedroom2  Bathroom  Car  Landsize  \\\n",
       "1      2         4019.0       2.5       2.0       1.0  1.0     202.0   \n",
       "2      2         4019.0       2.5       2.0       1.0  0.0     156.0   \n",
       "4      3         4019.0       2.5       3.0       2.0  0.0     134.0   \n",
       "5      3         4019.0       2.5       3.0       2.0  1.0      94.0   \n",
       "6      4         4019.0       2.5       3.0       1.0  2.0     120.0   \n",
       "\n",
       "   BuildingArea      Price  Suburb_Aberfeldie  ...  \\\n",
       "1      160.2564  1480000.0                  0  ...   \n",
       "2       79.0000  1035000.0                  0  ...   \n",
       "4      150.0000  1465000.0                  0  ...   \n",
       "5      160.2564   850000.0                  0  ...   \n",
       "6      142.0000  1600000.0                  0  ...   \n",
       "\n",
       "   CouncilArea_Moorabool Shire Council  CouncilArea_Moreland City Council  \\\n",
       "1                                    0                                  0   \n",
       "2                                    0                                  0   \n",
       "4                                    0                                  0   \n",
       "5                                    0                                  0   \n",
       "6                                    0                                  0   \n",
       "\n",
       "   CouncilArea_Nillumbik Shire Council  CouncilArea_Port Phillip City Council  \\\n",
       "1                                    0                                      0   \n",
       "2                                    0                                      0   \n",
       "4                                    0                                      0   \n",
       "5                                    0                                      0   \n",
       "6                                    0                                      0   \n",
       "\n",
       "   CouncilArea_Stonnington City Council  CouncilArea_Whitehorse City Council  \\\n",
       "1                                     0                                    0   \n",
       "2                                     0                                    0   \n",
       "4                                     0                                    0   \n",
       "5                                     0                                    0   \n",
       "6                                     0                                    0   \n",
       "\n",
       "   CouncilArea_Whittlesea City Council  CouncilArea_Wyndham City Council  \\\n",
       "1                                    0                                 0   \n",
       "2                                    0                                 0   \n",
       "4                                    0                                 0   \n",
       "5                                    0                                 0   \n",
       "6                                    0                                 0   \n",
       "\n",
       "   CouncilArea_Yarra City Council  CouncilArea_Yarra Ranges Shire Council  \n",
       "1                               1                                       0  \n",
       "2                               1                                       0  \n",
       "4                               1                                       0  \n",
       "5                               1                                       0  \n",
       "6                               1                                       0  \n",
       "\n",
       "[5 rows x 745 columns]"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dataset.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Let's bifurcate our dataset into train and test dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "X = dataset.drop('Price', axis=1)\n",
    "y = dataset['Price']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.model_selection import train_test_split\n",
    "train_X, test_X, train_y, test_y = train_test_split(X, y, test_size=0.3, random_state=2)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Let's train our Linear Regression Model on training dataset and check the accuracy on test set"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.linear_model import LinearRegression\n",
    "reg = LinearRegression().fit(train_X, train_y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.13853683161500907"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "reg.score(test_X, test_y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.6827792395792723"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "reg.score(train_X, train_y)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**Here training score is 68% but test score is 13.85% which is very low**"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<h4 style='color:purple'>Normal Regression is clearly overfitting the data, let's try other models</h4>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Using Lasso (L1 Regularized) Regression Model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Lasso(alpha=50, max_iter=100, tol=0.1)"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn import linear_model\n",
    "lasso_reg = linear_model.Lasso(alpha=50, max_iter=100, tol=0.1)\n",
    "lasso_reg.fit(train_X, train_y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.6636111369404488"
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "lasso_reg.score(test_X, test_y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.6766985624766824"
      ]
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "lasso_reg.score(train_X, train_y)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Using Ridge (L2 Regularized) Regression Model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Ridge(alpha=50, max_iter=100, tol=0.1)"
      ]
     },
     "execution_count": 48,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn.linear_model import Ridge\n",
    "ridge_reg= Ridge(alpha=50, max_iter=100, tol=0.1)\n",
    "ridge_reg.fit(train_X, train_y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.6670848945194959"
      ]
     },
     "execution_count": 49,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "ridge_reg.score(test_X, test_y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.6622376739684328"
      ]
     },
     "execution_count": 50,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "ridge_reg.score(train_X, train_y)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**We see that Lasso and Ridge Regularizations prove to be beneficial when our Simple Linear Regression Model overfits. These results may not be that contrast but significant in most cases.Also that L1 & L2 Regularizations are used in Neural Networks too**"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}