Kaggle_Solutions/House-Prices-Advanced-Regression-Techniques.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "import seaborn as sns\n",
    "\n",
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "pd.set_option(\"display.max_columns\",None)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Lifecycle\n",
    "\n",
    "* Data Analytics\n",
    "* Feature Engineering\n",
    "* Feature Selection\n",
    "* Model Building\n",
    "* Model Deployment\n",
    "\n",
    "Dataset:- https://www.kaggle.com/c/house-prices-advanced-regression-techniques/data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Id</th>\n",
       "      <th>MSSubClass</th>\n",
       "      <th>MSZoning</th>\n",
       "      <th>LotFrontage</th>\n",
       "      <th>LotArea</th>\n",
       "      <th>Street</th>\n",
       "      <th>Alley</th>\n",
       "      <th>LotShape</th>\n",
       "      <th>LandContour</th>\n",
       "      <th>Utilities</th>\n",
       "      <th>LotConfig</th>\n",
       "      <th>LandSlope</th>\n",
       "      <th>Neighborhood</th>\n",
       "      <th>Condition1</th>\n",
       "      <th>Condition2</th>\n",
       "      <th>BldgType</th>\n",
       "      <th>HouseStyle</th>\n",
       "      <th>OverallQual</th>\n",
       "      <th>OverallCond</th>\n",
       "      <th>YearBuilt</th>\n",
       "      <th>YearRemodAdd</th>\n",
       "      <th>RoofStyle</th>\n",
       "      <th>RoofMatl</th>\n",
       "      <th>Exterior1st</th>\n",
       "      <th>Exterior2nd</th>\n",
       "      <th>MasVnrType</th>\n",
       "      <th>MasVnrArea</th>\n",
       "      <th>ExterQual</th>\n",
       "      <th>ExterCond</th>\n",
       "      <th>Foundation</th>\n",
       "      <th>BsmtQual</th>\n",
       "      <th>BsmtCond</th>\n",
       "      <th>BsmtExposure</th>\n",
       "      <th>BsmtFinType1</th>\n",
       "      <th>BsmtFinSF1</th>\n",
       "      <th>BsmtFinType2</th>\n",
       "      <th>BsmtFinSF2</th>\n",
       "      <th>BsmtUnfSF</th>\n",
       "      <th>TotalBsmtSF</th>\n",
       "      <th>Heating</th>\n",
       "      <th>HeatingQC</th>\n",
       "      <th>CentralAir</th>\n",
       "      <th>Electrical</th>\n",
       "      <th>1stFlrSF</th>\n",
       "      <th>2ndFlrSF</th>\n",
       "      <th>LowQualFinSF</th>\n",
       "      <th>GrLivArea</th>\n",
       "      <th>BsmtFullBath</th>\n",
       "      <th>BsmtHalfBath</th>\n",
       "      <th>FullBath</th>\n",
       "      <th>HalfBath</th>\n",
       "      <th>BedroomAbvGr</th>\n",
       "      <th>KitchenAbvGr</th>\n",
       "      <th>KitchenQual</th>\n",
       "      <th>TotRmsAbvGrd</th>\n",
       "      <th>Functional</th>\n",
       "      <th>Fireplaces</th>\n",
       "      <th>FireplaceQu</th>\n",
       "      <th>GarageType</th>\n",
       "      <th>GarageYrBlt</th>\n",
       "      <th>GarageFinish</th>\n",
       "      <th>GarageCars</th>\n",
       "      <th>GarageArea</th>\n",
       "      <th>GarageQual</th>\n",
       "      <th>GarageCond</th>\n",
       "      <th>PavedDrive</th>\n",
       "      <th>WoodDeckSF</th>\n",
       "      <th>OpenPorchSF</th>\n",
       "      <th>EnclosedPorch</th>\n",
       "      <th>3SsnPorch</th>\n",
       "      <th>ScreenPorch</th>\n",
       "      <th>PoolArea</th>\n",
       "      <th>PoolQC</th>\n",
       "      <th>Fence</th>\n",
       "      <th>MiscFeature</th>\n",
       "      <th>MiscVal</th>\n",
       "      <th>MoSold</th>\n",
       "      <th>YrSold</th>\n",
       "      <th>SaleType</th>\n",
       "      <th>SaleCondition</th>\n",
       "      <th>SalePrice</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>60</td>\n",
       "      <td>RL</td>\n",
       "      <td>65.0</td>\n",
       "      <td>8450</td>\n",
       "      <td>Pave</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Reg</td>\n",
       "      <td>Lvl</td>\n",
       "      <td>AllPub</td>\n",
       "      <td>Inside</td>\n",
       "      <td>Gtl</td>\n",
       "      <td>CollgCr</td>\n",
       "      <td>Norm</td>\n",
       "      <td>Norm</td>\n",
       "      <td>1Fam</td>\n",
       "      <td>2Story</td>\n",
       "      <td>7</td>\n",
       "      <td>5</td>\n",
       "      <td>2003</td>\n",
       "      <td>2003</td>\n",
       "      <td>Gable</td>\n",
       "      <td>CompShg</td>\n",
       "      <td>VinylSd</td>\n",
       "      <td>VinylSd</td>\n",
       "      <td>BrkFace</td>\n",
       "      <td>196.0</td>\n",
       "      <td>Gd</td>\n",
       "      <td>TA</td>\n",
       "      <td>PConc</td>\n",
       "      <td>Gd</td>\n",
       "      <td>TA</td>\n",
       "      <td>No</td>\n",
       "      <td>GLQ</td>\n",
       "      <td>706</td>\n",
       "      <td>Unf</td>\n",
       "      <td>0</td>\n",
       "      <td>150</td>\n",
       "      <td>856</td>\n",
       "      <td>GasA</td>\n",
       "      <td>Ex</td>\n",
       "      <td>Y</td>\n",
       "      <td>SBrkr</td>\n",
       "      <td>856</td>\n",
       "      <td>854</td>\n",
       "      <td>0</td>\n",
       "      <td>1710</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>1</td>\n",
       "      <td>Gd</td>\n",
       "      <td>8</td>\n",
       "      <td>Typ</td>\n",
       "      <td>0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Attchd</td>\n",
       "      <td>2003.0</td>\n",
       "      <td>RFn</td>\n",
       "      <td>2</td>\n",
       "      <td>548</td>\n",
       "      <td>TA</td>\n",
       "      <td>TA</td>\n",
       "      <td>Y</td>\n",
       "      <td>0</td>\n",
       "      <td>61</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>2008</td>\n",
       "      <td>WD</td>\n",
       "      <td>Normal</td>\n",
       "      <td>208500</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2</td>\n",
       "      <td>20</td>\n",
       "      <td>RL</td>\n",
       "      <td>80.0</td>\n",
       "      <td>9600</td>\n",
       "      <td>Pave</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Reg</td>\n",
       "      <td>Lvl</td>\n",
       "      <td>AllPub</td>\n",
       "      <td>FR2</td>\n",
       "      <td>Gtl</td>\n",
       "      <td>Veenker</td>\n",
       "      <td>Feedr</td>\n",
       "      <td>Norm</td>\n",
       "      <td>1Fam</td>\n",
       "      <td>1Story</td>\n",
       "      <td>6</td>\n",
       "      <td>8</td>\n",
       "      <td>1976</td>\n",
       "      <td>1976</td>\n",
       "      <td>Gable</td>\n",
       "      <td>CompShg</td>\n",
       "      <td>MetalSd</td>\n",
       "      <td>MetalSd</td>\n",
       "      <td>None</td>\n",
       "      <td>0.0</td>\n",
       "      <td>TA</td>\n",
       "      <td>TA</td>\n",
       "      <td>CBlock</td>\n",
       "      <td>Gd</td>\n",
       "      <td>TA</td>\n",
       "      <td>Gd</td>\n",
       "      <td>ALQ</td>\n",
       "      <td>978</td>\n",
       "      <td>Unf</td>\n",
       "      <td>0</td>\n",
       "      <td>284</td>\n",
       "      <td>1262</td>\n",
       "      <td>GasA</td>\n",
       "      <td>Ex</td>\n",
       "      <td>Y</td>\n",
       "      <td>SBrkr</td>\n",
       "      <td>1262</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1262</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>3</td>\n",
       "      <td>1</td>\n",
       "      <td>TA</td>\n",
       "      <td>6</td>\n",
       "      <td>Typ</td>\n",
       "      <td>1</td>\n",
       "      <td>TA</td>\n",
       "      <td>Attchd</td>\n",
       "      <td>1976.0</td>\n",
       "      <td>RFn</td>\n",
       "      <td>2</td>\n",
       "      <td>460</td>\n",
       "      <td>TA</td>\n",
       "      <td>TA</td>\n",
       "      <td>Y</td>\n",
       "      <td>298</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0</td>\n",
       "      <td>5</td>\n",
       "      <td>2007</td>\n",
       "      <td>WD</td>\n",
       "      <td>Normal</td>\n",
       "      <td>181500</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>3</td>\n",
       "      <td>60</td>\n",
       "      <td>RL</td>\n",
       "      <td>68.0</td>\n",
       "      <td>11250</td>\n",
       "      <td>Pave</td>\n",
       "      <td>NaN</td>\n",
       "      <td>IR1</td>\n",
       "      <td>Lvl</td>\n",
       "      <td>AllPub</td>\n",
       "      <td>Inside</td>\n",
       "      <td>Gtl</td>\n",
       "      <td>CollgCr</td>\n",
       "      <td>Norm</td>\n",
       "      <td>Norm</td>\n",
       "      <td>1Fam</td>\n",
       "      <td>2Story</td>\n",
       "      <td>7</td>\n",
       "      <td>5</td>\n",
       "      <td>2001</td>\n",
       "      <td>2002</td>\n",
       "      <td>Gable</td>\n",
       "      <td>CompShg</td>\n",
       "      <td>VinylSd</td>\n",
       "      <td>VinylSd</td>\n",
       "      <td>BrkFace</td>\n",
       "      <td>162.0</td>\n",
       "      <td>Gd</td>\n",
       "      <td>TA</td>\n",
       "      <td>PConc</td>\n",
       "      <td>Gd</td>\n",
       "      <td>TA</td>\n",
       "      <td>Mn</td>\n",
       "      <td>GLQ</td>\n",
       "      <td>486</td>\n",
       "      <td>Unf</td>\n",
       "      <td>0</td>\n",
       "      <td>434</td>\n",
       "      <td>920</td>\n",
       "      <td>GasA</td>\n",
       "      <td>Ex</td>\n",
       "      <td>Y</td>\n",
       "      <td>SBrkr</td>\n",
       "      <td>920</td>\n",
       "      <td>866</td>\n",
       "      <td>0</td>\n",
       "      <td>1786</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>1</td>\n",
       "      <td>Gd</td>\n",
       "      <td>6</td>\n",
       "      <td>Typ</td>\n",
       "      <td>1</td>\n",
       "      <td>TA</td>\n",
       "      <td>Attchd</td>\n",
       "      <td>2001.0</td>\n",
       "      <td>RFn</td>\n",
       "      <td>2</td>\n",
       "      <td>608</td>\n",
       "      <td>TA</td>\n",
       "      <td>TA</td>\n",
       "      <td>Y</td>\n",
       "      <td>0</td>\n",
       "      <td>42</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0</td>\n",
       "      <td>9</td>\n",
       "      <td>2008</td>\n",
       "      <td>WD</td>\n",
       "      <td>Normal</td>\n",
       "      <td>223500</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>4</td>\n",
       "      <td>70</td>\n",
       "      <td>RL</td>\n",
       "      <td>60.0</td>\n",
       "      <td>9550</td>\n",
       "      <td>Pave</td>\n",
       "      <td>NaN</td>\n",
       "      <td>IR1</td>\n",
       "      <td>Lvl</td>\n",
       "      <td>AllPub</td>\n",
       "      <td>Corner</td>\n",
       "      <td>Gtl</td>\n",
       "      <td>Crawfor</td>\n",
       "      <td>Norm</td>\n",
       "      <td>Norm</td>\n",
       "      <td>1Fam</td>\n",
       "      <td>2Story</td>\n",
       "      <td>7</td>\n",
       "      <td>5</td>\n",
       "      <td>1915</td>\n",
       "      <td>1970</td>\n",
       "      <td>Gable</td>\n",
       "      <td>CompShg</td>\n",
       "      <td>Wd Sdng</td>\n",
       "      <td>Wd Shng</td>\n",
       "      <td>None</td>\n",
       "      <td>0.0</td>\n",
       "      <td>TA</td>\n",
       "      <td>TA</td>\n",
       "      <td>BrkTil</td>\n",
       "      <td>TA</td>\n",
       "      <td>Gd</td>\n",
       "      <td>No</td>\n",
       "      <td>ALQ</td>\n",
       "      <td>216</td>\n",
       "      <td>Unf</td>\n",
       "      <td>0</td>\n",
       "      <td>540</td>\n",
       "      <td>756</td>\n",
       "      <td>GasA</td>\n",
       "      <td>Gd</td>\n",
       "      <td>Y</td>\n",
       "      <td>SBrkr</td>\n",
       "      <td>961</td>\n",
       "      <td>756</td>\n",
       "      <td>0</td>\n",
       "      <td>1717</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>3</td>\n",
       "      <td>1</td>\n",
       "      <td>Gd</td>\n",
       "      <td>7</td>\n",
       "      <td>Typ</td>\n",
       "      <td>1</td>\n",
       "      <td>Gd</td>\n",
       "      <td>Detchd</td>\n",
       "      <td>1998.0</td>\n",
       "      <td>Unf</td>\n",
       "      <td>3</td>\n",
       "      <td>642</td>\n",
       "      <td>TA</td>\n",
       "      <td>TA</td>\n",
       "      <td>Y</td>\n",
       "      <td>0</td>\n",
       "      <td>35</td>\n",
       "      <td>272</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>2006</td>\n",
       "      <td>WD</td>\n",
       "      <td>Abnorml</td>\n",
       "      <td>140000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>5</td>\n",
       "      <td>60</td>\n",
       "      <td>RL</td>\n",
       "      <td>84.0</td>\n",
       "      <td>14260</td>\n",
       "      <td>Pave</td>\n",
       "      <td>NaN</td>\n",
       "      <td>IR1</td>\n",
       "      <td>Lvl</td>\n",
       "      <td>AllPub</td>\n",
       "      <td>FR2</td>\n",
       "      <td>Gtl</td>\n",
       "      <td>NoRidge</td>\n",
       "      <td>Norm</td>\n",
       "      <td>Norm</td>\n",
       "      <td>1Fam</td>\n",
       "      <td>2Story</td>\n",
       "      <td>8</td>\n",
       "      <td>5</td>\n",
       "      <td>2000</td>\n",
       "      <td>2000</td>\n",
       "      <td>Gable</td>\n",
       "      <td>CompShg</td>\n",
       "      <td>VinylSd</td>\n",
       "      <td>VinylSd</td>\n",
       "      <td>BrkFace</td>\n",
       "      <td>350.0</td>\n",
       "      <td>Gd</td>\n",
       "      <td>TA</td>\n",
       "      <td>PConc</td>\n",
       "      <td>Gd</td>\n",
       "      <td>TA</td>\n",
       "      <td>Av</td>\n",
       "      <td>GLQ</td>\n",
       "      <td>655</td>\n",
       "      <td>Unf</td>\n",
       "      <td>0</td>\n",
       "      <td>490</td>\n",
       "      <td>1145</td>\n",
       "      <td>GasA</td>\n",
       "      <td>Ex</td>\n",
       "      <td>Y</td>\n",
       "      <td>SBrkr</td>\n",
       "      <td>1145</td>\n",
       "      <td>1053</td>\n",
       "      <td>0</td>\n",
       "      <td>2198</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>4</td>\n",
       "      <td>1</td>\n",
       "      <td>Gd</td>\n",
       "      <td>9</td>\n",
       "      <td>Typ</td>\n",
       "      <td>1</td>\n",
       "      <td>TA</td>\n",
       "      <td>Attchd</td>\n",
       "      <td>2000.0</td>\n",
       "      <td>RFn</td>\n",
       "      <td>3</td>\n",
       "      <td>836</td>\n",
       "      <td>TA</td>\n",
       "      <td>TA</td>\n",
       "      <td>Y</td>\n",
       "      <td>192</td>\n",
       "      <td>84</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0</td>\n",
       "      <td>12</td>\n",
       "      <td>2008</td>\n",
       "      <td>WD</td>\n",
       "      <td>Normal</td>\n",
       "      <td>250000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>6</td>\n",
       "      <td>50</td>\n",
       "      <td>RL</td>\n",
       "      <td>85.0</td>\n",
       "      <td>14115</td>\n",
       "      <td>Pave</td>\n",
       "      <td>NaN</td>\n",
       "      <td>IR1</td>\n",
       "      <td>Lvl</td>\n",
       "      <td>AllPub</td>\n",
       "      <td>Inside</td>\n",
       "      <td>Gtl</td>\n",
       "      <td>Mitchel</td>\n",
       "      <td>Norm</td>\n",
       "      <td>Norm</td>\n",
       "      <td>1Fam</td>\n",
       "      <td>1.5Fin</td>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "      <td>1993</td>\n",
       "      <td>1995</td>\n",
       "      <td>Gable</td>\n",
       "      <td>CompShg</td>\n",
       "      <td>VinylSd</td>\n",
       "      <td>VinylSd</td>\n",
       "      <td>None</td>\n",
       "      <td>0.0</td>\n",
       "      <td>TA</td>\n",
       "      <td>TA</td>\n",
       "      <td>Wood</td>\n",
       "      <td>Gd</td>\n",
       "      <td>TA</td>\n",
       "      <td>No</td>\n",
       "      <td>GLQ</td>\n",
       "      <td>732</td>\n",
       "      <td>Unf</td>\n",
       "      <td>0</td>\n",
       "      <td>64</td>\n",
       "      <td>796</td>\n",
       "      <td>GasA</td>\n",
       "      <td>Ex</td>\n",
       "      <td>Y</td>\n",
       "      <td>SBrkr</td>\n",
       "      <td>796</td>\n",
       "      <td>566</td>\n",
       "      <td>0</td>\n",
       "      <td>1362</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>TA</td>\n",
       "      <td>5</td>\n",
       "      <td>Typ</td>\n",
       "      <td>0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Attchd</td>\n",
       "      <td>1993.0</td>\n",
       "      <td>Unf</td>\n",
       "      <td>2</td>\n",
       "      <td>480</td>\n",
       "      <td>TA</td>\n",
       "      <td>TA</td>\n",
       "      <td>Y</td>\n",
       "      <td>40</td>\n",
       "      <td>30</td>\n",
       "      <td>0</td>\n",
       "      <td>320</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>MnPrv</td>\n",
       "      <td>Shed</td>\n",
       "      <td>700</td>\n",
       "      <td>10</td>\n",
       "      <td>2009</td>\n",
       "      <td>WD</td>\n",
       "      <td>Normal</td>\n",
       "      <td>143000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>7</td>\n",
       "      <td>20</td>\n",
       "      <td>RL</td>\n",
       "      <td>75.0</td>\n",
       "      <td>10084</td>\n",
       "      <td>Pave</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Reg</td>\n",
       "      <td>Lvl</td>\n",
       "      <td>AllPub</td>\n",
       "      <td>Inside</td>\n",
       "      <td>Gtl</td>\n",
       "      <td>Somerst</td>\n",
       "      <td>Norm</td>\n",
       "      <td>Norm</td>\n",
       "      <td>1Fam</td>\n",
       "      <td>1Story</td>\n",
       "      <td>8</td>\n",
       "      <td>5</td>\n",
       "      <td>2004</td>\n",
       "      <td>2005</td>\n",
       "      <td>Gable</td>\n",
       "      <td>CompShg</td>\n",
       "      <td>VinylSd</td>\n",
       "      <td>VinylSd</td>\n",
       "      <td>Stone</td>\n",
       "      <td>186.0</td>\n",
       "      <td>Gd</td>\n",
       "      <td>TA</td>\n",
       "      <td>PConc</td>\n",
       "      <td>Ex</td>\n",
       "      <td>TA</td>\n",
       "      <td>Av</td>\n",
       "      <td>GLQ</td>\n",
       "      <td>1369</td>\n",
       "      <td>Unf</td>\n",
       "      <td>0</td>\n",
       "      <td>317</td>\n",
       "      <td>1686</td>\n",
       "      <td>GasA</td>\n",
       "      <td>Ex</td>\n",
       "      <td>Y</td>\n",
       "      <td>SBrkr</td>\n",
       "      <td>1694</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1694</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>3</td>\n",
       "      <td>1</td>\n",
       "      <td>Gd</td>\n",
       "      <td>7</td>\n",
       "      <td>Typ</td>\n",
       "      <td>1</td>\n",
       "      <td>Gd</td>\n",
       "      <td>Attchd</td>\n",
       "      <td>2004.0</td>\n",
       "      <td>RFn</td>\n",
       "      <td>2</td>\n",
       "      <td>636</td>\n",
       "      <td>TA</td>\n",
       "      <td>TA</td>\n",
       "      <td>Y</td>\n",
       "      <td>255</td>\n",
       "      <td>57</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0</td>\n",
       "      <td>8</td>\n",
       "      <td>2007</td>\n",
       "      <td>WD</td>\n",
       "      <td>Normal</td>\n",
       "      <td>307000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>8</td>\n",
       "      <td>60</td>\n",
       "      <td>RL</td>\n",
       "      <td>NaN</td>\n",
       "      <td>10382</td>\n",
       "      <td>Pave</td>\n",
       "      <td>NaN</td>\n",
       "      <td>IR1</td>\n",
       "      <td>Lvl</td>\n",
       "      <td>AllPub</td>\n",
       "      <td>Corner</td>\n",
       "      <td>Gtl</td>\n",
       "      <td>NWAmes</td>\n",
       "      <td>PosN</td>\n",
       "      <td>Norm</td>\n",
       "      <td>1Fam</td>\n",
       "      <td>2Story</td>\n",
       "      <td>7</td>\n",
       "      <td>6</td>\n",
       "      <td>1973</td>\n",
       "      <td>1973</td>\n",
       "      <td>Gable</td>\n",
       "      <td>CompShg</td>\n",
       "      <td>HdBoard</td>\n",
       "      <td>HdBoard</td>\n",
       "      <td>Stone</td>\n",
       "      <td>240.0</td>\n",
       "      <td>TA</td>\n",
       "      <td>TA</td>\n",
       "      <td>CBlock</td>\n",
       "      <td>Gd</td>\n",
       "      <td>TA</td>\n",
       "      <td>Mn</td>\n",
       "      <td>ALQ</td>\n",
       "      <td>859</td>\n",
       "      <td>BLQ</td>\n",
       "      <td>32</td>\n",
       "      <td>216</td>\n",
       "      <td>1107</td>\n",
       "      <td>GasA</td>\n",
       "      <td>Ex</td>\n",
       "      <td>Y</td>\n",
       "      <td>SBrkr</td>\n",
       "      <td>1107</td>\n",
       "      <td>983</td>\n",
       "      <td>0</td>\n",
       "      <td>2090</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>1</td>\n",
       "      <td>TA</td>\n",
       "      <td>7</td>\n",
       "      <td>Typ</td>\n",
       "      <td>2</td>\n",
       "      <td>TA</td>\n",
       "      <td>Attchd</td>\n",
       "      <td>1973.0</td>\n",
       "      <td>RFn</td>\n",
       "      <td>2</td>\n",
       "      <td>484</td>\n",
       "      <td>TA</td>\n",
       "      <td>TA</td>\n",
       "      <td>Y</td>\n",
       "      <td>235</td>\n",
       "      <td>204</td>\n",
       "      <td>228</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Shed</td>\n",
       "      <td>350</td>\n",
       "      <td>11</td>\n",
       "      <td>2009</td>\n",
       "      <td>WD</td>\n",
       "      <td>Normal</td>\n",
       "      <td>200000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   Id  MSSubClass MSZoning  LotFrontage  LotArea Street Alley LotShape  \\\n",
       "0   1          60       RL         65.0     8450   Pave   NaN      Reg   \n",
       "1   2          20       RL         80.0     9600   Pave   NaN      Reg   \n",
       "2   3          60       RL         68.0    11250   Pave   NaN      IR1   \n",
       "3   4          70       RL         60.0     9550   Pave   NaN      IR1   \n",
       "4   5          60       RL         84.0    14260   Pave   NaN      IR1   \n",
       "5   6          50       RL         85.0    14115   Pave   NaN      IR1   \n",
       "6   7          20       RL         75.0    10084   Pave   NaN      Reg   \n",
       "7   8          60       RL          NaN    10382   Pave   NaN      IR1   \n",
       "\n",
       "  LandContour Utilities LotConfig LandSlope Neighborhood Condition1  \\\n",
       "0         Lvl    AllPub    Inside       Gtl      CollgCr       Norm   \n",
       "1         Lvl    AllPub       FR2       Gtl      Veenker      Feedr   \n",
       "2         Lvl    AllPub    Inside       Gtl      CollgCr       Norm   \n",
       "3         Lvl    AllPub    Corner       Gtl      Crawfor       Norm   \n",
       "4         Lvl    AllPub       FR2       Gtl      NoRidge       Norm   \n",
       "5         Lvl    AllPub    Inside       Gtl      Mitchel       Norm   \n",
       "6         Lvl    AllPub    Inside       Gtl      Somerst       Norm   \n",
       "7         Lvl    AllPub    Corner       Gtl       NWAmes       PosN   \n",
       "\n",
       "  Condition2 BldgType HouseStyle  OverallQual  OverallCond  YearBuilt  \\\n",
       "0       Norm     1Fam     2Story            7            5       2003   \n",
       "1       Norm     1Fam     1Story            6            8       1976   \n",
       "2       Norm     1Fam     2Story            7            5       2001   \n",
       "3       Norm     1Fam     2Story            7            5       1915   \n",
       "4       Norm     1Fam     2Story            8            5       2000   \n",
       "5       Norm     1Fam     1.5Fin            5            5       1993   \n",
       "6       Norm     1Fam     1Story            8            5       2004   \n",
       "7       Norm     1Fam     2Story            7            6       1973   \n",
       "\n",
       "   YearRemodAdd RoofStyle RoofMatl Exterior1st Exterior2nd MasVnrType  \\\n",
       "0          2003     Gable  CompShg     VinylSd     VinylSd    BrkFace   \n",
       "1          1976     Gable  CompShg     MetalSd     MetalSd       None   \n",
       "2          2002     Gable  CompShg     VinylSd     VinylSd    BrkFace   \n",
       "3          1970     Gable  CompShg     Wd Sdng     Wd Shng       None   \n",
       "4          2000     Gable  CompShg     VinylSd     VinylSd    BrkFace   \n",
       "5          1995     Gable  CompShg     VinylSd     VinylSd       None   \n",
       "6          2005     Gable  CompShg     VinylSd     VinylSd      Stone   \n",
       "7          1973     Gable  CompShg     HdBoard     HdBoard      Stone   \n",
       "\n",
       "   MasVnrArea ExterQual ExterCond Foundation BsmtQual BsmtCond BsmtExposure  \\\n",
       "0       196.0        Gd        TA      PConc       Gd       TA           No   \n",
       "1         0.0        TA        TA     CBlock       Gd       TA           Gd   \n",
       "2       162.0        Gd        TA      PConc       Gd       TA           Mn   \n",
       "3         0.0        TA        TA     BrkTil       TA       Gd           No   \n",
       "4       350.0        Gd        TA      PConc       Gd       TA           Av   \n",
       "5         0.0        TA        TA       Wood       Gd       TA           No   \n",
       "6       186.0        Gd        TA      PConc       Ex       TA           Av   \n",
       "7       240.0        TA        TA     CBlock       Gd       TA           Mn   \n",
       "\n",
       "  BsmtFinType1  BsmtFinSF1 BsmtFinType2  BsmtFinSF2  BsmtUnfSF  TotalBsmtSF  \\\n",
       "0          GLQ         706          Unf           0        150          856   \n",
       "1          ALQ         978          Unf           0        284         1262   \n",
       "2          GLQ         486          Unf           0        434          920   \n",
       "3          ALQ         216          Unf           0        540          756   \n",
       "4          GLQ         655          Unf           0        490         1145   \n",
       "5          GLQ         732          Unf           0         64          796   \n",
       "6          GLQ        1369          Unf           0        317         1686   \n",
       "7          ALQ         859          BLQ          32        216         1107   \n",
       "\n",
       "  Heating HeatingQC CentralAir Electrical  1stFlrSF  2ndFlrSF  LowQualFinSF  \\\n",
       "0    GasA        Ex          Y      SBrkr       856       854             0   \n",
       "1    GasA        Ex          Y      SBrkr      1262         0             0   \n",
       "2    GasA        Ex          Y      SBrkr       920       866             0   \n",
       "3    GasA        Gd          Y      SBrkr       961       756             0   \n",
       "4    GasA        Ex          Y      SBrkr      1145      1053             0   \n",
       "5    GasA        Ex          Y      SBrkr       796       566             0   \n",
       "6    GasA        Ex          Y      SBrkr      1694         0             0   \n",
       "7    GasA        Ex          Y      SBrkr      1107       983             0   \n",
       "\n",
       "   GrLivArea  BsmtFullBath  BsmtHalfBath  FullBath  HalfBath  BedroomAbvGr  \\\n",
       "0       1710             1             0         2         1             3   \n",
       "1       1262             0             1         2         0             3   \n",
       "2       1786             1             0         2         1             3   \n",
       "3       1717             1             0         1         0             3   \n",
       "4       2198             1             0         2         1             4   \n",
       "5       1362             1             0         1         1             1   \n",
       "6       1694             1             0         2         0             3   \n",
       "7       2090             1             0         2         1             3   \n",
       "\n",
       "   KitchenAbvGr KitchenQual  TotRmsAbvGrd Functional  Fireplaces FireplaceQu  \\\n",
       "0             1          Gd             8        Typ           0         NaN   \n",
       "1             1          TA             6        Typ           1          TA   \n",
       "2             1          Gd             6        Typ           1          TA   \n",
       "3             1          Gd             7        Typ           1          Gd   \n",
       "4             1          Gd             9        Typ           1          TA   \n",
       "5             1          TA             5        Typ           0         NaN   \n",
       "6             1          Gd             7        Typ           1          Gd   \n",
       "7             1          TA             7        Typ           2          TA   \n",
       "\n",
       "  GarageType  GarageYrBlt GarageFinish  GarageCars  GarageArea GarageQual  \\\n",
       "0     Attchd       2003.0          RFn           2         548         TA   \n",
       "1     Attchd       1976.0          RFn           2         460         TA   \n",
       "2     Attchd       2001.0          RFn           2         608         TA   \n",
       "3     Detchd       1998.0          Unf           3         642         TA   \n",
       "4     Attchd       2000.0          RFn           3         836         TA   \n",
       "5     Attchd       1993.0          Unf           2         480         TA   \n",
       "6     Attchd       2004.0          RFn           2         636         TA   \n",
       "7     Attchd       1973.0          RFn           2         484         TA   \n",
       "\n",
       "  GarageCond PavedDrive  WoodDeckSF  OpenPorchSF  EnclosedPorch  3SsnPorch  \\\n",
       "0         TA          Y           0           61              0          0   \n",
       "1         TA          Y         298            0              0          0   \n",
       "2         TA          Y           0           42              0          0   \n",
       "3         TA          Y           0           35            272          0   \n",
       "4         TA          Y         192           84              0          0   \n",
       "5         TA          Y          40           30              0        320   \n",
       "6         TA          Y         255           57              0          0   \n",
       "7         TA          Y         235          204            228          0   \n",
       "\n",
       "   ScreenPorch  PoolArea PoolQC  Fence MiscFeature  MiscVal  MoSold  YrSold  \\\n",
       "0            0         0    NaN    NaN         NaN        0       2    2008   \n",
       "1            0         0    NaN    NaN         NaN        0       5    2007   \n",
       "2            0         0    NaN    NaN         NaN        0       9    2008   \n",
       "3            0         0    NaN    NaN         NaN        0       2    2006   \n",
       "4            0         0    NaN    NaN         NaN        0      12    2008   \n",
       "5            0         0    NaN  MnPrv        Shed      700      10    2009   \n",
       "6            0         0    NaN    NaN         NaN        0       8    2007   \n",
       "7            0         0    NaN    NaN        Shed      350      11    2009   \n",
       "\n",
       "  SaleType SaleCondition  SalePrice  \n",
       "0       WD        Normal     208500  \n",
       "1       WD        Normal     181500  \n",
       "2       WD        Normal     223500  \n",
       "3       WD       Abnorml     140000  \n",
       "4       WD        Normal     250000  \n",
       "5       WD        Normal     143000  \n",
       "6       WD        Normal     307000  \n",
       "7       WD        Normal     200000  "
      ]
     },
     "execution_count": 47,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df = pd.read_csv(\"./datasets/house-prices-advanced-regression-techniques/train.csv\")\n",
    "df.head(8)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(1460, 81)"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.shape"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## In Data Analysis we will analyze to find out the below stuff\n",
    "\n",
    "* Missing Values\n",
    "* All The Numerical Variables\n",
    "* Distribution of the Numerical Variables (if it is skewed then we have to do transformation)\n",
    "* Categorical Variables\n",
    "* Cardinality of Categorical Variables\n",
    "* Outliers\n",
    "* Relationship between independent and dependent feature (SalePrice)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Missing value"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "LotFrontage  :  17.73972602739726\n",
      "Alley  :  93.76712328767123\n",
      "MasVnrType  :  0.547945205479452\n",
      "MasVnrArea  :  0.547945205479452\n",
      "BsmtQual  :  2.5342465753424657\n",
      "BsmtCond  :  2.5342465753424657\n",
      "BsmtExposure  :  2.6027397260273974\n",
      "BsmtFinType1  :  2.5342465753424657\n",
      "BsmtFinType2  :  2.6027397260273974\n",
      "FireplaceQu  :  47.26027397260274\n",
      "GarageType  :  5.5479452054794525\n",
      "GarageYrBlt  :  5.5479452054794525\n",
      "GarageFinish  :  5.5479452054794525\n",
      "GarageQual  :  5.5479452054794525\n",
      "GarageCond  :  5.5479452054794525\n",
      "PoolQC  :  99.52054794520548\n",
      "Fence  :  80.75342465753424\n",
      "MiscFeature  :  96.30136986301369\n",
      "\n",
      "Total Features with null values:  18\n"
     ]
    }
   ],
   "source": [
    "features_with_na = [feature for feature in df.columns if df[feature].isnull().sum() >1]\n",
    "\n",
    "for feature in features_with_na:\n",
    "    print(feature,\" : \", df[feature].isnull().mean() *100)\n",
    "\n",
    "print(\"\\nTotal Features with null values: \",len(features_with_na))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['LotFrontage',\n",
       " 'MasVnrType',\n",
       " 'MasVnrArea',\n",
       " 'BsmtQual',\n",
       " 'BsmtCond',\n",
       " 'BsmtExposure',\n",
       " 'BsmtFinType1',\n",
       " 'BsmtFinType2',\n",
       " 'FireplaceQu',\n",
       " 'GarageType',\n",
       " 'GarageYrBlt',\n",
       " 'GarageFinish',\n",
       " 'GarageQual',\n",
       " 'GarageCond']"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Dropping columns having high % of null values\n",
    "df1 = df.drop(columns = [\"Alley\",\"PoolQC\",\"Fence\",\"MiscFeature\"])\n",
    "features_with_na2 = list1 = [ele for ele in features_with_na if ele not in [\"Alley\",\"PoolQC\",\"Fence\",\"MiscFeature\"]] \n",
    "features_with_na2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "RangeIndex: 1460 entries, 0 to 1459\n",
      "Data columns (total 14 columns):\n",
      " #   Column        Non-Null Count  Dtype  \n",
      "---  ------        --------------  -----  \n",
      " 0   LotFrontage   1460 non-null   float64\n",
      " 1   MasVnrType    1452 non-null   object \n",
      " 2   MasVnrArea    1460 non-null   float64\n",
      " 3   BsmtQual      1423 non-null   object \n",
      " 4   BsmtCond      1423 non-null   object \n",
      " 5   BsmtExposure  1422 non-null   object \n",
      " 6   BsmtFinType1  1423 non-null   object \n",
      " 7   BsmtFinType2  1422 non-null   object \n",
      " 8   FireplaceQu   770 non-null    object \n",
      " 9   GarageType    1379 non-null   object \n",
      " 10  GarageYrBlt   1460 non-null   float64\n",
      " 11  GarageFinish  1379 non-null   object \n",
      " 12  GarageQual    1379 non-null   object \n",
      " 13  GarageCond    1379 non-null   object \n",
      "dtypes: float64(3), object(11)\n",
      "memory usage: 159.8+ KB\n"
     ]
    }
   ],
   "source": [
    "df1[features_with_na2].info()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>LotFrontage</th>\n",
       "      <th>MasVnrArea</th>\n",
       "      <th>GarageYrBlt</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>count</th>\n",
       "      <td>1201.000000</td>\n",
       "      <td>1452.000000</td>\n",
       "      <td>1379.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>mean</th>\n",
       "      <td>70.049958</td>\n",
       "      <td>103.685262</td>\n",
       "      <td>1978.506164</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>std</th>\n",
       "      <td>24.284752</td>\n",
       "      <td>181.066207</td>\n",
       "      <td>24.689725</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>min</th>\n",
       "      <td>21.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>1900.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25%</th>\n",
       "      <td>59.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>1961.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>50%</th>\n",
       "      <td>69.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>1980.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>75%</th>\n",
       "      <td>80.000000</td>\n",
       "      <td>166.000000</td>\n",
       "      <td>2002.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>max</th>\n",
       "      <td>313.000000</td>\n",
       "      <td>1600.000000</td>\n",
       "      <td>2010.000000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "       LotFrontage   MasVnrArea  GarageYrBlt\n",
       "count  1201.000000  1452.000000  1379.000000\n",
       "mean     70.049958   103.685262  1978.506164\n",
       "std      24.284752   181.066207    24.689725\n",
       "min      21.000000     0.000000  1900.000000\n",
       "25%      59.000000     0.000000  1961.000000\n",
       "50%      69.000000     0.000000  1980.000000\n",
       "75%      80.000000   166.000000  2002.000000\n",
       "max     313.000000  1600.000000  2010.000000"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Can observe mean and median are almost same, and max value are too high\n",
    "df1[features_with_na2].describe()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.0      861\n",
       "72.0       8\n",
       "180.0      8\n",
       "108.0      8\n",
       "120.0      7\n",
       "        ... \n",
       "651.0      1\n",
       "337.0      1\n",
       "415.0      1\n",
       "293.0      1\n",
       "621.0      1\n",
       "Name: MasVnrArea, Length: 327, dtype: int64"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df1[\"MasVnrArea\"].value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>LotFrontage</th>\n",
       "      <th>MasVnrArea</th>\n",
       "      <th>GarageYrBlt</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>count</th>\n",
       "      <td>1460.000000</td>\n",
       "      <td>1460.000000</td>\n",
       "      <td>1460.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>mean</th>\n",
       "      <td>70.049958</td>\n",
       "      <td>103.117123</td>\n",
       "      <td>1978.506164</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>std</th>\n",
       "      <td>22.024023</td>\n",
       "      <td>180.731373</td>\n",
       "      <td>23.994583</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>min</th>\n",
       "      <td>21.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>1900.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25%</th>\n",
       "      <td>60.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>1962.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>50%</th>\n",
       "      <td>70.049958</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>1978.506164</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>75%</th>\n",
       "      <td>79.000000</td>\n",
       "      <td>164.250000</td>\n",
       "      <td>2001.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>max</th>\n",
       "      <td>313.000000</td>\n",
       "      <td>1600.000000</td>\n",
       "      <td>2010.000000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "       LotFrontage   MasVnrArea  GarageYrBlt\n",
       "count  1460.000000  1460.000000  1460.000000\n",
       "mean     70.049958   103.117123  1978.506164\n",
       "std      22.024023   180.731373    23.994583\n",
       "min      21.000000     0.000000  1900.000000\n",
       "25%      60.000000     0.000000  1962.000000\n",
       "50%      70.049958     0.000000  1978.506164\n",
       "75%      79.000000   164.250000  2001.000000\n",
       "max     313.000000  1600.000000  2010.000000"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# For normal distribution replacing with mean values\n",
    "df1[\"LotFrontage\"] = df1[\"LotFrontage\"].fillna(df1[\"LotFrontage\"].mean())\n",
    "df1[\"GarageYrBlt\"] = df1[\"GarageYrBlt\"].fillna(df1[\"GarageYrBlt\"].mean())\n",
    "df1[\"MasVnrArea\"] = df1[\"MasVnrArea\"].fillna(0)\n",
    "df1[features_with_na2].describe()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "MasVnrType  :\n",
      "\n",
      "None       864\n",
      "BrkFace    445\n",
      "Stone      128\n",
      "BrkCmn      15\n",
      "Name: MasVnrType, dtype: int64\n",
      "\n",
      "\n",
      "\n",
      "BsmtQual  :\n",
      "\n",
      "TA    649\n",
      "Gd    618\n",
      "Ex    121\n",
      "Fa     35\n",
      "Name: BsmtQual, dtype: int64\n",
      "\n",
      "\n",
      "\n",
      "BsmtCond  :\n",
      "\n",
      "TA    1311\n",
      "Gd      65\n",
      "Fa      45\n",
      "Po       2\n",
      "Name: BsmtCond, dtype: int64\n",
      "\n",
      "\n",
      "\n",
      "BsmtExposure  :\n",
      "\n",
      "No    953\n",
      "Av    221\n",
      "Gd    134\n",
      "Mn    114\n",
      "Name: BsmtExposure, dtype: int64\n",
      "\n",
      "\n",
      "\n",
      "BsmtFinType1  :\n",
      "\n",
      "Unf    430\n",
      "GLQ    418\n",
      "ALQ    220\n",
      "BLQ    148\n",
      "Rec    133\n",
      "LwQ     74\n",
      "Name: BsmtFinType1, dtype: int64\n",
      "\n",
      "\n",
      "\n",
      "BsmtFinType2  :\n",
      "\n",
      "Unf    1256\n",
      "Rec      54\n",
      "LwQ      46\n",
      "BLQ      33\n",
      "ALQ      19\n",
      "GLQ      14\n",
      "Name: BsmtFinType2, dtype: int64\n",
      "\n",
      "\n",
      "\n",
      "FireplaceQu  :\n",
      "\n",
      "Gd    380\n",
      "TA    313\n",
      "Fa     33\n",
      "Ex     24\n",
      "Po     20\n",
      "Name: FireplaceQu, dtype: int64\n",
      "\n",
      "\n",
      "\n",
      "GarageType  :\n",
      "\n",
      "Attchd     870\n",
      "Detchd     387\n",
      "BuiltIn     88\n",
      "Basment     19\n",
      "CarPort      9\n",
      "2Types       6\n",
      "Name: GarageType, dtype: int64\n",
      "\n",
      "\n",
      "\n",
      "GarageFinish  :\n",
      "\n",
      "Unf    605\n",
      "RFn    422\n",
      "Fin    352\n",
      "Name: GarageFinish, dtype: int64\n",
      "\n",
      "\n",
      "\n",
      "GarageQual  :\n",
      "\n",
      "TA    1311\n",
      "Fa      48\n",
      "Gd      14\n",
      "Po       3\n",
      "Ex       3\n",
      "Name: GarageQual, dtype: int64\n",
      "\n",
      "\n",
      "\n",
      "GarageCond  :\n",
      "\n",
      "TA    1326\n",
      "Fa      35\n",
      "Gd       9\n",
      "Po       7\n",
      "Ex       2\n",
      "Name: GarageCond, dtype: int64\n",
      "\n",
      "\n",
      "\n"
     ]
    }
   ],
   "source": [
    "features_with_na3 = [ele for ele in features_with_na2 if ele not in [\"LotFrontage\",\"GarageYrBlt\",\"MasVnrArea\"]] \n",
    "\n",
    "for col in features_with_na3:\n",
    "    print(col,\" :\\n\")\n",
    "    print(df[col].value_counts())\n",
    "    print(\"\\n\\n\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>MasVnrType</th>\n",
       "      <th>BsmtQual</th>\n",
       "      <th>BsmtCond</th>\n",
       "      <th>BsmtExposure</th>\n",
       "      <th>BsmtFinType1</th>\n",
       "      <th>BsmtFinType2</th>\n",
       "      <th>FireplaceQu</th>\n",
       "      <th>GarageType</th>\n",
       "      <th>GarageFinish</th>\n",
       "      <th>GarageQual</th>\n",
       "      <th>GarageCond</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>None</td>\n",
       "      <td>TA</td>\n",
       "      <td>TA</td>\n",
       "      <td>No</td>\n",
       "      <td>Unf</td>\n",
       "      <td>Unf</td>\n",
       "      <td>Gd</td>\n",
       "      <td>Attchd</td>\n",
       "      <td>Unf</td>\n",
       "      <td>TA</td>\n",
       "      <td>TA</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "  MasVnrType BsmtQual BsmtCond BsmtExposure BsmtFinType1 BsmtFinType2  \\\n",
       "0       None       TA       TA           No          Unf          Unf   \n",
       "\n",
       "  FireplaceQu GarageType GarageFinish GarageQual GarageCond  \n",
       "0          Gd     Attchd          Unf         TA         TA  "
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df1[features_with_na3].mode()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "RangeIndex: 1460 entries, 0 to 1459\n",
      "Data columns (total 11 columns):\n",
      " #   Column        Non-Null Count  Dtype \n",
      "---  ------        --------------  ----- \n",
      " 0   MasVnrType    1452 non-null   object\n",
      " 1   BsmtQual      1460 non-null   object\n",
      " 2   BsmtCond      1460 non-null   object\n",
      " 3   BsmtExposure  1460 non-null   object\n",
      " 4   BsmtFinType1  1460 non-null   object\n",
      " 5   BsmtFinType2  1460 non-null   object\n",
      " 6   FireplaceQu   770 non-null    object\n",
      " 7   GarageType    1460 non-null   object\n",
      " 8   GarageFinish  1460 non-null   object\n",
      " 9   GarageQual    1460 non-null   object\n",
      " 10  GarageCond    1460 non-null   object\n",
      "dtypes: object(11)\n",
      "memory usage: 125.6+ KB\n"
     ]
    }
   ],
   "source": [
    "# Fill null values of categorical column with most probable value\n",
    "features_with_na4 = [\"MasVnrType\",\"FireplaceQu\"]\n",
    "df2 = df1.copy()\n",
    "for col in features_with_na3:\n",
    "    if col not in features_with_na4:\n",
    "        df2[col] = df1[col].fillna(df1[col].mode()[0])\n",
    "        \n",
    "df2[features_with_na3].info()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>MasVnrType</th>\n",
       "      <th>BsmtQual</th>\n",
       "      <th>BsmtCond</th>\n",
       "      <th>BsmtExposure</th>\n",
       "      <th>BsmtFinType1</th>\n",
       "      <th>BsmtFinType2</th>\n",
       "      <th>FireplaceQu</th>\n",
       "      <th>GarageType</th>\n",
       "      <th>GarageFinish</th>\n",
       "      <th>GarageQual</th>\n",
       "      <th>GarageCond</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>None</td>\n",
       "      <td>TA</td>\n",
       "      <td>TA</td>\n",
       "      <td>No</td>\n",
       "      <td>Unf</td>\n",
       "      <td>Unf</td>\n",
       "      <td>Gd</td>\n",
       "      <td>Attchd</td>\n",
       "      <td>Unf</td>\n",
       "      <td>TA</td>\n",
       "      <td>TA</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "  MasVnrType BsmtQual BsmtCond BsmtExposure BsmtFinType1 BsmtFinType2  \\\n",
       "0       None       TA       TA           No          Unf          Unf   \n",
       "\n",
       "  FireplaceQu GarageType GarageFinish GarageQual GarageCond  \n",
       "0          Gd     Attchd          Unf         TA         TA  "
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df2[features_with_na3].mode()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Since it has 2 many null values\n",
    "df3 = df2.drop(columns = features_with_na4)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1"
      ]
     },
     "execution_count": 39,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df3.isnull().sum().sum()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "RangeIndex: 1460 entries, 0 to 1459\n",
      "Data columns (total 75 columns):\n",
      " #   Column         Non-Null Count  Dtype  \n",
      "---  ------         --------------  -----  \n",
      " 0   Id             1460 non-null   int64  \n",
      " 1   MSSubClass     1460 non-null   int64  \n",
      " 2   MSZoning       1460 non-null   object \n",
      " 3   LotFrontage    1460 non-null   float64\n",
      " 4   LotArea        1460 non-null   int64  \n",
      " 5   Street         1460 non-null   object \n",
      " 6   LotShape       1460 non-null   object \n",
      " 7   LandContour    1460 non-null   object \n",
      " 8   Utilities      1460 non-null   object \n",
      " 9   LotConfig      1460 non-null   object \n",
      " 10  LandSlope      1460 non-null   object \n",
      " 11  Neighborhood   1460 non-null   object \n",
      " 12  Condition1     1460 non-null   object \n",
      " 13  Condition2     1460 non-null   object \n",
      " 14  BldgType       1460 non-null   object \n",
      " 15  HouseStyle     1460 non-null   object \n",
      " 16  OverallQual    1460 non-null   int64  \n",
      " 17  OverallCond    1460 non-null   int64  \n",
      " 18  YearBuilt      1460 non-null   int64  \n",
      " 19  YearRemodAdd   1460 non-null   int64  \n",
      " 20  RoofStyle      1460 non-null   object \n",
      " 21  RoofMatl       1460 non-null   object \n",
      " 22  Exterior1st    1460 non-null   object \n",
      " 23  Exterior2nd    1460 non-null   object \n",
      " 24  MasVnrArea     1460 non-null   float64\n",
      " 25  ExterQual      1460 non-null   object \n",
      " 26  ExterCond      1460 non-null   object \n",
      " 27  Foundation     1460 non-null   object \n",
      " 28  BsmtQual       1460 non-null   object \n",
      " 29  BsmtCond       1460 non-null   object \n",
      " 30  BsmtExposure   1460 non-null   object \n",
      " 31  BsmtFinType1   1460 non-null   object \n",
      " 32  BsmtFinSF1     1460 non-null   int64  \n",
      " 33  BsmtFinType2   1460 non-null   object \n",
      " 34  BsmtFinSF2     1460 non-null   int64  \n",
      " 35  BsmtUnfSF      1460 non-null   int64  \n",
      " 36  TotalBsmtSF    1460 non-null   int64  \n",
      " 37  Heating        1460 non-null   object \n",
      " 38  HeatingQC      1460 non-null   object \n",
      " 39  CentralAir     1460 non-null   object \n",
      " 40  Electrical     1459 non-null   object \n",
      " 41  1stFlrSF       1460 non-null   int64  \n",
      " 42  2ndFlrSF       1460 non-null   int64  \n",
      " 43  LowQualFinSF   1460 non-null   int64  \n",
      " 44  GrLivArea      1460 non-null   int64  \n",
      " 45  BsmtFullBath   1460 non-null   int64  \n",
      " 46  BsmtHalfBath   1460 non-null   int64  \n",
      " 47  FullBath       1460 non-null   int64  \n",
      " 48  HalfBath       1460 non-null   int64  \n",
      " 49  BedroomAbvGr   1460 non-null   int64  \n",
      " 50  KitchenAbvGr   1460 non-null   int64  \n",
      " 51  KitchenQual    1460 non-null   object \n",
      " 52  TotRmsAbvGrd   1460 non-null   int64  \n",
      " 53  Functional     1460 non-null   object \n",
      " 54  Fireplaces     1460 non-null   int64  \n",
      " 55  GarageType     1460 non-null   object \n",
      " 56  GarageYrBlt    1460 non-null   float64\n",
      " 57  GarageFinish   1460 non-null   object \n",
      " 58  GarageCars     1460 non-null   int64  \n",
      " 59  GarageArea     1460 non-null   int64  \n",
      " 60  GarageQual     1460 non-null   object \n",
      " 61  GarageCond     1460 non-null   object \n",
      " 62  PavedDrive     1460 non-null   object \n",
      " 63  WoodDeckSF     1460 non-null   int64  \n",
      " 64  OpenPorchSF    1460 non-null   int64  \n",
      " 65  EnclosedPorch  1460 non-null   int64  \n",
      " 66  3SsnPorch      1460 non-null   int64  \n",
      " 67  ScreenPorch    1460 non-null   int64  \n",
      " 68  PoolArea       1460 non-null   int64  \n",
      " 69  MiscVal        1460 non-null   int64  \n",
      " 70  MoSold         1460 non-null   int64  \n",
      " 71  YrSold         1460 non-null   int64  \n",
      " 72  SaleType       1460 non-null   object \n",
      " 73  SaleCondition  1460 non-null   object \n",
      " 74  SalePrice      1460 non-null   int64  \n",
      "dtypes: float64(3), int64(35), object(37)\n",
      "memory usage: 855.6+ KB\n"
     ]
    }
   ],
   "source": [
    "df3.info()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0"
      ]
     },
     "execution_count": 43,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df3[\"Electrical\"].fillna(df3[\"Electrical\"].mode()[0], inplace=True)\n",
    "df3.isnull().sum().sum()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "37"
      ]
     },
     "execution_count": 46,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cat_col = df3.select_dtypes(include=\"O\")\n",
    "len(cat_col.columns)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "tensor",
   "language": "python",
   "name": "tensor"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.12"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}