pre-processing.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Populating the interactive namespace from numpy and matplotlib\n"
     ]
    }
   ],
   "source": [
    "%pylab inline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.ensemble import RandomForestRegressor\n",
    "import pandas"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(1460, 81)"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df = pandas.read_csv('dataset/train.csv')\n",
    "df.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Id                  0\n",
       "MSSubClass          0\n",
       "MSZoning            0\n",
       "LotFrontage       259\n",
       "LotArea             0\n",
       "Street              0\n",
       "Alley            1369\n",
       "LotShape            0\n",
       "LandContour         0\n",
       "Utilities           0\n",
       "LotConfig           0\n",
       "LandSlope           0\n",
       "Neighborhood        0\n",
       "Condition1          0\n",
       "Condition2          0\n",
       "BldgType            0\n",
       "HouseStyle          0\n",
       "OverallQual         0\n",
       "OverallCond         0\n",
       "YearBuilt           0\n",
       "YearRemodAdd        0\n",
       "RoofStyle           0\n",
       "RoofMatl            0\n",
       "Exterior1st         0\n",
       "Exterior2nd         0\n",
       "MasVnrType          8\n",
       "MasVnrArea          8\n",
       "ExterQual           0\n",
       "ExterCond           0\n",
       "Foundation          0\n",
       "                 ... \n",
       "BedroomAbvGr        0\n",
       "KitchenAbvGr        0\n",
       "KitchenQual         0\n",
       "TotRmsAbvGrd        0\n",
       "Functional          0\n",
       "Fireplaces          0\n",
       "FireplaceQu       690\n",
       "GarageType         81\n",
       "GarageYrBlt        81\n",
       "GarageFinish       81\n",
       "GarageCars          0\n",
       "GarageArea          0\n",
       "GarageQual         81\n",
       "GarageCond         81\n",
       "PavedDrive          0\n",
       "WoodDeckSF          0\n",
       "OpenPorchSF         0\n",
       "EnclosedPorch       0\n",
       "3SsnPorch           0\n",
       "ScreenPorch         0\n",
       "PoolArea            0\n",
       "PoolQC           1453\n",
       "Fence            1179\n",
       "MiscFeature      1406\n",
       "MiscVal             0\n",
       "MoSold              0\n",
       "YrSold              0\n",
       "SaleType            0\n",
       "SaleCondition       0\n",
       "SalePrice           0\n",
       "Length: 81, dtype: int64"
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sum(pandas.isnull(df['MasVnrArea']))\n",
    "sum(pandas.isnull(df[df.columns]),axis=0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "LotFrontage     259\n",
      "Alley          1369\n",
      "FireplaceQu     690\n",
      "PoolQC         1453\n",
      "Fence          1179\n",
      "MiscFeature    1406\n",
      "dtype: int64\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "Id                  730.5\n",
       "MSSubClass           50.0\n",
       "LotArea            9478.5\n",
       "OverallQual           6.0\n",
       "OverallCond           5.0\n",
       "YearBuilt          1973.0\n",
       "YearRemodAdd       1994.0\n",
       "MasVnrArea            0.0\n",
       "BsmtFinSF1          383.5\n",
       "BsmtFinSF2            0.0\n",
       "BsmtUnfSF           477.5\n",
       "TotalBsmtSF         991.5\n",
       "1stFlrSF           1087.0\n",
       "2ndFlrSF              0.0\n",
       "LowQualFinSF          0.0\n",
       "GrLivArea          1464.0\n",
       "BsmtFullBath          0.0\n",
       "BsmtHalfBath          0.0\n",
       "FullBath              2.0\n",
       "HalfBath              0.0\n",
       "BedroomAbvGr          3.0\n",
       "KitchenAbvGr          1.0\n",
       "TotRmsAbvGrd          6.0\n",
       "Fireplaces            1.0\n",
       "GarageYrBlt        1980.0\n",
       "GarageCars            2.0\n",
       "GarageArea          480.0\n",
       "WoodDeckSF            0.0\n",
       "OpenPorchSF          25.0\n",
       "EnclosedPorch         0.0\n",
       "3SsnPorch             0.0\n",
       "ScreenPorch           0.0\n",
       "PoolArea              0.0\n",
       "MiscVal               0.0\n",
       "MoSold                6.0\n",
       "YrSold             2008.0\n",
       "SalePrice        163000.0\n",
       "dtype: float64"
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# counting the number of nulls and removing those columns which have more than or \n",
    "# equal to 100 nulls.\n",
    "count_null = sum(pandas.isnull(df[df.columns]),axis=0)\n",
    "#printing the columns neglected\n",
    "print(count_null[count_null>=100])\n",
    "processed_df = df[count_null[count_null < 100].index]\n",
    "processed_df.columns\n",
    "processed_df.median(axis=0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.0"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}