Data_mining_for_property24.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from bs4 import BeautifulSoup\n",
    "import requests\n",
    "import time\n",
    "import pandas as pd\n",
    "from urllib import urlopen\n",
    "import re"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Change the url if you change switch the page number of the website\n",
    "url = 'https://www.property24.com.ph/property-for-sale-in-san-lorenzo-s22639?Page=2'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "html = urlopen(url)\n",
    "# Get the html of the website"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "housing_info = BeautifulSoup(html,'lxml')\n",
    "# Get the html of the website\n",
    "# housing_info"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# find al all the listing price\n",
    "pricing = housing_info.find_all(\"div\",class_ = \"sc_listingTilePrice\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "10500000\n",
      "325000\n",
      "120000000\n",
      "150000000\n",
      "50000000\n",
      "150000000\n",
      "120000000\n",
      "150000000\n",
      "150000000\n",
      "120000000\n",
      "150000000\n",
      "120000000\n",
      "120000000\n",
      "135000000\n",
      "8500000\n",
      "150000000\n",
      "120000000\n",
      "150000000\n",
      "120000000\n",
      "258500000\n",
      "150000000\n"
     ]
    }
   ],
   "source": [
    "# import the regular expresstion\n",
    "# Get all the pricing for the house\n",
    "# the counting is to check everything is well-aligned\n",
    "\n",
    "housing_pricing = []\n",
    "k = 0\n",
    "for i in pricing:\n",
    "    k = k + 1 # counting to match with other type\n",
    "    price = i.get_text()\n",
    "    # print(price)\n",
    "    # get \n",
    "    price = \" \".join(re.split(\"\\s+\", price, flags=re.UNICODE)) # delete the extra space in the unicode string\n",
    "    b = price.split(\" \") # split the array by space\n",
    "    b = str(b[2]).split(\",\") # split the array by ','\n",
    "    b = int(''.join(b)) # Get the numerical value for the pricing of the housing \n",
    "    print(b)\n",
    "    \n",
    "    housing_pricing.append(b) # append the numerical value of the pricing to the array 'housing_pricing' "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Get the title of the renting\n",
    "detail = housing_info.find_all(\"div\",class_ = \"sc_listingTileArea\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "housing_info_detail = []\n",
    "k = 0 # counting on the listing \n",
    "for i in detail:\n",
    "    k = k + 1\n",
    "    b = i.get_text()\n",
    "    b = b.encode('utf-8') # change unicode to string\n",
    "    b = \" \".join(re.split(\"\\s+\", b, flags=re.UNICODE)) # kick out the space in the string\n",
    "    housing_info_detail.append(b)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[' Condominium for sale in San Lorenzo ',\n",
       " ' 2 bedroom Condominium for sale in San Lorenzo ',\n",
       " ' Lot for sale in San Lorenzo ',\n",
       " ' 5 bedroom House and Lot for sale in San Lorenzo ',\n",
       " ' 3 bedroom Condominium for sale in San Lorenzo ',\n",
       " ' House and Lot for sale in San Lorenzo ',\n",
       " ' Lot for sale in San Lorenzo ',\n",
       " ' House and Lot for sale in San Lorenzo ',\n",
       " ' House and Lot for sale in San Lorenzo ',\n",
       " ' Lot for sale in San Lorenzo ',\n",
       " ' 5 bedroom House and Lot for sale in San Lorenzo ',\n",
       " ' Lot for sale in San Lorenzo ',\n",
       " ' Lot for sale in San Lorenzo ',\n",
       " ' 4 bedroom House and Lot for sale in San Lorenzo ',\n",
       " ' 3 bedroom Condominium for sale in San Lorenzo ',\n",
       " ' 5 bedroom House and Lot for sale in San Lorenzo ',\n",
       " ' Lot for sale in San Lorenzo ',\n",
       " ' 5 bedroom House and Lot for sale in San Lorenzo ',\n",
       " ' Lot for sale in San Lorenzo ',\n",
       " ' 4 bedroom House and Lot for sale in San Lorenzo ',\n",
       " ' 5 bedroom House and Lot for sale in San Lorenzo ']"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "housing_info_detail# the title for each village"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Get the title of the renting\n",
    "room_info = housing_info.find_all(\"div\",class_ = \"sc_listingTileTeaser\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "k = 0\n",
    "bedroom_number = [] # array of the number of bedroom\n",
    "housing_area = [] # array of housing area\n",
    "\n",
    "for i in room_info:\n",
    "    k = k + 1\n",
    "    sentence = i.get_text()\n",
    "    sentence = sentence.encode('utf-8')\n",
    "    sentence = \" \".join(re.split(\"\\s+\", sentence, flags=re.UNICODE))\n",
    "    alist = sentence.split(\" \")\n",
    "    #print(alist)\n",
    "    if 'bedrooms,' in alist:\n",
    "        index1 = alist.index('bedrooms,')-1 # get the index of bedroom\n",
    "        # print( int(alist[index1]) )\n",
    "        bedroom_number.append(int( alist[index1] ))\n",
    "    else:\n",
    "        bedroom_number.append(0)\n",
    "        \n",
    "    if 'sqm' in alist:\n",
    "        index2 = alist.index('sqm')-1 # get the index of housing area\n",
    "        # print( int(alist[index1]) )\n",
    "        housing_area.append(int( alist[index2] ))\n",
    "    else:\n",
    "        housing_area.append(0)\n",
    "    \n",
    "\n",
    "# get the number of bedroom from the title of the listing\n",
    "for i in range(0, len(housing_info_detail)):\n",
    "    a = housing_info_detail[i]\n",
    "    # print a\n",
    "    \n",
    "    b = a.split(\" \")\n",
    "    if 'bedroom' in b:\n",
    "        index = b.index('bedroom') - 1; # the previous number before bedroom\n",
    "        if bedroom_number[i] == 0:\n",
    "            bedroom_number[i] = int(b[index])\n",
    "            # print(b[index])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "bedroom number:\n",
      "[0, 2, 0, 5, 3, 0, 0, 0, 0, 0, 5, 0, 0, 4, 3, 5, 0, 5, 0, 4, 5]\n",
      "21\n",
      "housing_area\n",
      "[0, 0, 543, 558, 0, 558, 543, 558, 558, 543, 558, 543, 543, 0, 0, 558, 543, 558, 543, 0, 558]\n",
      "21\n"
     ]
    }
   ],
   "source": [
    "print('bedroom number:')\n",
    "print(bedroom_number)\n",
    "print(len(bedroom_number))\n",
    "\n",
    "print('housing_area')\n",
    "print(housing_area)\n",
    "print(len(housing_area))\n",
    "# 0 means the missing data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# get the link of different address for users to check\n",
    "\n",
    "# the link is always in the class with the name sc_panel\n",
    "# find all \"a\" class with the name title\n",
    "\n",
    "housing_url = []\n",
    "k = 0 # counting number to check whether the length is assistant with the other length\n",
    "for link in housing_info.findAll(\"a\",{\"class\":\"title\"}):\n",
    "    k = k + 1\n",
    "    url = link.get('href') # Get the url link for the housing, well the type is also string which is good\n",
    "    url = \"https://www.property24.com.ph\"+url # add the front of the url link\n",
    "    housing_url.append(url)\n",
    "    # print(url)\n",
    "    \n",
    "# check the length of the array is assistant with the "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Store data into csv file\n",
    "final_data = {'House Information':housing_info_detail,\n",
    "             'House Pricing':housing_pricing,\n",
    "             'Bedroom Number':bedroom_number,\n",
    "             'House Area':housing_area,\n",
    "             'House Website':housing_url}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>House Information</th>\n",
       "      <th>House Pricing</th>\n",
       "      <th>Bedroom Number</th>\n",
       "      <th>House Area</th>\n",
       "      <th>House Website</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Condominium for sale in San Lorenzo</td>\n",
       "      <td>10500000</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>https://www.property24.com.ph/condominium-for-...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2 bedroom Condominium for sale in San Lorenzo</td>\n",
       "      <td>325000</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>https://www.property24.com.ph/2-bedroom-condom...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Lot for sale in San Lorenzo</td>\n",
       "      <td>120000000</td>\n",
       "      <td>0</td>\n",
       "      <td>543</td>\n",
       "      <td>https://www.property24.com.ph/lot-for-sale-in-...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>5 bedroom House and Lot for sale in San Lorenzo</td>\n",
       "      <td>150000000</td>\n",
       "      <td>5</td>\n",
       "      <td>558</td>\n",
       "      <td>https://www.property24.com.ph/5-bedroom-house-...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>3 bedroom Condominium for sale in San Lorenzo</td>\n",
       "      <td>50000000</td>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>https://www.property24.com.ph/3-bedroom-condom...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>House and Lot for sale in San Lorenzo</td>\n",
       "      <td>150000000</td>\n",
       "      <td>0</td>\n",
       "      <td>558</td>\n",
       "      <td>https://www.property24.com.ph/house-and-lot-fo...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>Lot for sale in San Lorenzo</td>\n",
       "      <td>120000000</td>\n",
       "      <td>0</td>\n",
       "      <td>543</td>\n",
       "      <td>https://www.property24.com.ph/lot-for-sale-in-...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>House and Lot for sale in San Lorenzo</td>\n",
       "      <td>150000000</td>\n",
       "      <td>0</td>\n",
       "      <td>558</td>\n",
       "      <td>https://www.property24.com.ph/house-and-lot-fo...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>House and Lot for sale in San Lorenzo</td>\n",
       "      <td>150000000</td>\n",
       "      <td>0</td>\n",
       "      <td>558</td>\n",
       "      <td>https://www.property24.com.ph/house-and-lot-fo...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>Lot for sale in San Lorenzo</td>\n",
       "      <td>120000000</td>\n",
       "      <td>0</td>\n",
       "      <td>543</td>\n",
       "      <td>https://www.property24.com.ph/lot-for-sale-in-...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>5 bedroom House and Lot for sale in San Lorenzo</td>\n",
       "      <td>150000000</td>\n",
       "      <td>5</td>\n",
       "      <td>558</td>\n",
       "      <td>https://www.property24.com.ph/5-bedroom-house-...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>Lot for sale in San Lorenzo</td>\n",
       "      <td>120000000</td>\n",
       "      <td>0</td>\n",
       "      <td>543</td>\n",
       "      <td>https://www.property24.com.ph/lot-for-sale-in-...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>Lot for sale in San Lorenzo</td>\n",
       "      <td>120000000</td>\n",
       "      <td>0</td>\n",
       "      <td>543</td>\n",
       "      <td>https://www.property24.com.ph/lot-for-sale-in-...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>4 bedroom House and Lot for sale in San Lorenzo</td>\n",
       "      <td>135000000</td>\n",
       "      <td>4</td>\n",
       "      <td>0</td>\n",
       "      <td>https://www.property24.com.ph/4-bedroom-house-...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>3 bedroom Condominium for sale in San Lorenzo</td>\n",
       "      <td>8500000</td>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>https://www.property24.com.ph/3-bedroom-condom...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>5 bedroom House and Lot for sale in San Lorenzo</td>\n",
       "      <td>150000000</td>\n",
       "      <td>5</td>\n",
       "      <td>558</td>\n",
       "      <td>https://www.property24.com.ph/5-bedroom-house-...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16</th>\n",
       "      <td>Lot for sale in San Lorenzo</td>\n",
       "      <td>120000000</td>\n",
       "      <td>0</td>\n",
       "      <td>543</td>\n",
       "      <td>https://www.property24.com.ph/lot-for-sale-in-...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17</th>\n",
       "      <td>5 bedroom House and Lot for sale in San Lorenzo</td>\n",
       "      <td>150000000</td>\n",
       "      <td>5</td>\n",
       "      <td>558</td>\n",
       "      <td>https://www.property24.com.ph/5-bedroom-house-...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18</th>\n",
       "      <td>Lot for sale in San Lorenzo</td>\n",
       "      <td>120000000</td>\n",
       "      <td>0</td>\n",
       "      <td>543</td>\n",
       "      <td>https://www.property24.com.ph/lot-for-sale-in-...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19</th>\n",
       "      <td>4 bedroom House and Lot for sale in San Lorenzo</td>\n",
       "      <td>258500000</td>\n",
       "      <td>4</td>\n",
       "      <td>0</td>\n",
       "      <td>https://www.property24.com.ph/4-bedroom-house-...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20</th>\n",
       "      <td>5 bedroom House and Lot for sale in San Lorenzo</td>\n",
       "      <td>150000000</td>\n",
       "      <td>5</td>\n",
       "      <td>558</td>\n",
       "      <td>https://www.property24.com.ph/5-bedroom-house-...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                    House Information  House Pricing  \\\n",
       "0                Condominium for sale in San Lorenzo        10500000   \n",
       "1      2 bedroom Condominium for sale in San Lorenzo          325000   \n",
       "2                        Lot for sale in San Lorenzo       120000000   \n",
       "3    5 bedroom House and Lot for sale in San Lorenzo       150000000   \n",
       "4      3 bedroom Condominium for sale in San Lorenzo        50000000   \n",
       "5              House and Lot for sale in San Lorenzo       150000000   \n",
       "6                        Lot for sale in San Lorenzo       120000000   \n",
       "7              House and Lot for sale in San Lorenzo       150000000   \n",
       "8              House and Lot for sale in San Lorenzo       150000000   \n",
       "9                        Lot for sale in San Lorenzo       120000000   \n",
       "10   5 bedroom House and Lot for sale in San Lorenzo       150000000   \n",
       "11                       Lot for sale in San Lorenzo       120000000   \n",
       "12                       Lot for sale in San Lorenzo       120000000   \n",
       "13   4 bedroom House and Lot for sale in San Lorenzo       135000000   \n",
       "14     3 bedroom Condominium for sale in San Lorenzo         8500000   \n",
       "15   5 bedroom House and Lot for sale in San Lorenzo       150000000   \n",
       "16                       Lot for sale in San Lorenzo       120000000   \n",
       "17   5 bedroom House and Lot for sale in San Lorenzo       150000000   \n",
       "18                       Lot for sale in San Lorenzo       120000000   \n",
       "19   4 bedroom House and Lot for sale in San Lorenzo       258500000   \n",
       "20   5 bedroom House and Lot for sale in San Lorenzo       150000000   \n",
       "\n",
       "    Bedroom Number  House Area  \\\n",
       "0                0           0   \n",
       "1                2           0   \n",
       "2                0         543   \n",
       "3                5         558   \n",
       "4                3           0   \n",
       "5                0         558   \n",
       "6                0         543   \n",
       "7                0         558   \n",
       "8                0         558   \n",
       "9                0         543   \n",
       "10               5         558   \n",
       "11               0         543   \n",
       "12               0         543   \n",
       "13               4           0   \n",
       "14               3           0   \n",
       "15               5         558   \n",
       "16               0         543   \n",
       "17               5         558   \n",
       "18               0         543   \n",
       "19               4           0   \n",
       "20               5         558   \n",
       "\n",
       "                                        House Website  \n",
       "0   https://www.property24.com.ph/condominium-for-...  \n",
       "1   https://www.property24.com.ph/2-bedroom-condom...  \n",
       "2   https://www.property24.com.ph/lot-for-sale-in-...  \n",
       "3   https://www.property24.com.ph/5-bedroom-house-...  \n",
       "4   https://www.property24.com.ph/3-bedroom-condom...  \n",
       "5   https://www.property24.com.ph/house-and-lot-fo...  \n",
       "6   https://www.property24.com.ph/lot-for-sale-in-...  \n",
       "7   https://www.property24.com.ph/house-and-lot-fo...  \n",
       "8   https://www.property24.com.ph/house-and-lot-fo...  \n",
       "9   https://www.property24.com.ph/lot-for-sale-in-...  \n",
       "10  https://www.property24.com.ph/5-bedroom-house-...  \n",
       "11  https://www.property24.com.ph/lot-for-sale-in-...  \n",
       "12  https://www.property24.com.ph/lot-for-sale-in-...  \n",
       "13  https://www.property24.com.ph/4-bedroom-house-...  \n",
       "14  https://www.property24.com.ph/3-bedroom-condom...  \n",
       "15  https://www.property24.com.ph/5-bedroom-house-...  \n",
       "16  https://www.property24.com.ph/lot-for-sale-in-...  \n",
       "17  https://www.property24.com.ph/5-bedroom-house-...  \n",
       "18  https://www.property24.com.ph/lot-for-sale-in-...  \n",
       "19  https://www.property24.com.ph/4-bedroom-house-...  \n",
       "20  https://www.property24.com.ph/5-bedroom-house-...  "
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_housing = pd.DataFrame(final_data, columns = ['House Information','House Pricing','Bedroom Number',\n",
    "                                                 'House Area','House Website'])\n",
    "df_housing"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "df_housing.to_csv('housing_data_mining.csv', encoding='utf-8', index=False)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.13"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}