week-3-assignment-part-1.ipynb

{
  "cells": [
    {
      "metadata": {
        "trusted": true
      },
      "cell_type": "code",
      "source": "import requests\nimport pandas as pd",
      "execution_count": 4,
      "outputs": []
    },
    {
      "metadata": {
        "trusted": true
      },
      "cell_type": "code",
      "source": "#downloading page\nurl  = \"https://en.wikipedia.org/wiki/List_of_postal_codes_of_Canada:_M\"\npage = requests.get(url)\nif page.status_code == 200:\n    print('Page download successful')\nelse:\n    print('Page download error. Error code: {}'.format(page.status_code))",
      "execution_count": 2,
      "outputs": [
        {
          "output_type": "stream",
          "text": "Page download successful\n",
          "name": "stdout"
        }
      ]
    },
    {
      "metadata": {
        "trusted": true
      },
      "cell_type": "code",
      "source": "#Please note, I am using the pandas function \"read_html\" we can easily process the HTML string instead of beautifulsoup\n#We set header = 0 to use the first row as column names\n#Since we will discard the \"Not Assigned\" columns, we set them to NaN so we can later use the dropna method.\ndf_html = pd.read_html(url, header=0, na_values = ['Not assigned'])[0]\ndf_html.head()",
      "execution_count": 7,
      "outputs": [
        {
          "output_type": "execute_result",
          "execution_count": 7,
          "data": {
            "text/html": "<div>\n<style scoped>\n    .dataframe tbody tr th:only-of-type {\n        vertical-align: middle;\n    }\n\n    .dataframe tbody tr th {\n        vertical-align: top;\n    }\n\n    .dataframe thead th {\n        text-align: right;\n    }\n</style>\n<table border=\"1\" class=\"dataframe\">\n  <thead>\n    <tr style=\"text-align: right;\">\n      <th></th>\n      <th>Postcode</th>\n      <th>Borough</th>\n      <th>Neighbourhood</th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <th>0</th>\n      <td>M1A</td>\n      <td>NaN</td>\n      <td>NaN</td>\n    </tr>\n    <tr>\n      <th>1</th>\n      <td>M2A</td>\n      <td>NaN</td>\n      <td>NaN</td>\n    </tr>\n    <tr>\n      <th>2</th>\n      <td>M3A</td>\n      <td>North York</td>\n      <td>Parkwoods</td>\n    </tr>\n    <tr>\n      <th>3</th>\n      <td>M4A</td>\n      <td>North York</td>\n      <td>Victoria Village</td>\n    </tr>\n    <tr>\n      <th>4</th>\n      <td>M5A</td>\n      <td>Downtown Toronto</td>\n      <td>Harbourfront</td>\n    </tr>\n  </tbody>\n</table>\n</div>",
            "text/plain": "  Postcode           Borough     Neighbourhood\n0      M1A               NaN               NaN\n1      M2A               NaN               NaN\n2      M3A        North York         Parkwoods\n3      M4A        North York  Victoria Village\n4      M5A  Downtown Toronto      Harbourfront"
          },
          "metadata": {}
        }
      ]
    },
    {
      "metadata": {
        "trusted": true
      },
      "cell_type": "code",
      "source": "#Drop the the rows on which the Borough is empty\ndf_html.dropna(subset=['Borough'], inplace=True)",
      "execution_count": 8,
      "outputs": []
    },
    {
      "metadata": {
        "trusted": true
      },
      "cell_type": "code",
      "source": "#Check Neighborhood is empty but Borough exists\nn_empty_neighborhood = df_html[df_html['Neighbourhood'].isna()].shape[0]\nprint('Number of rows on which Neighborhood column is empty: {}'.format(n_empty_neighborhood))",
      "execution_count": 9,
      "outputs": [
        {
          "output_type": "stream",
          "text": "Number of rows on which Neighborhood column is empty: 1\n",
          "name": "stdout"
        }
      ]
    },
    {
      "metadata": {
        "trusted": true
      },
      "cell_type": "code",
      "source": "#Show which neighborhood is emtpy but Borough exists\ndf_html[df_html['Neighbourhood'].isna()]",
      "execution_count": 10,
      "outputs": [
        {
          "output_type": "execute_result",
          "execution_count": 10,
          "data": {
            "text/html": "<div>\n<style scoped>\n    .dataframe tbody tr th:only-of-type {\n        vertical-align: middle;\n    }\n\n    .dataframe tbody tr th {\n        vertical-align: top;\n    }\n\n    .dataframe thead th {\n        text-align: right;\n    }\n</style>\n<table border=\"1\" class=\"dataframe\">\n  <thead>\n    <tr style=\"text-align: right;\">\n      <th></th>\n      <th>Postcode</th>\n      <th>Borough</th>\n      <th>Neighbourhood</th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <th>8</th>\n      <td>M7A</td>\n      <td>Queen's Park</td>\n      <td>NaN</td>\n    </tr>\n  </tbody>\n</table>\n</div>",
            "text/plain": "  Postcode       Borough Neighbourhood\n8      M7A  Queen's Park           NaN"
          },
          "metadata": {}
        }
      ]
    },
    {
      "metadata": {
        "trusted": true
      },
      "cell_type": "code",
      "source": "#Replace empty Neighborhood with Borough name and check again\ndf_html['Neighbourhood'].fillna(df_html['Borough'], inplace=True)\nn_empty_neighborhood = df_html[df_html['Neighbourhood'].isna()].shape[0]\nprint('Number of rows on which Neighborhood column is empty: {}'.format(n_empty_neighborhood))",
      "execution_count": 11,
      "outputs": [
        {
          "output_type": "stream",
          "text": "Number of rows on which Neighborhood column is empty: 0\n",
          "name": "stdout"
        }
      ]
    },
    {
      "metadata": {
        "trusted": true
      },
      "cell_type": "code",
      "source": "#Confirm that Queen's Park Neighborhood is not empty now:\ndf_html[df_html['Borough']==\"Queen's Park\"]",
      "execution_count": 12,
      "outputs": [
        {
          "output_type": "execute_result",
          "execution_count": 12,
          "data": {
            "text/html": "<div>\n<style scoped>\n    .dataframe tbody tr th:only-of-type {\n        vertical-align: middle;\n    }\n\n    .dataframe tbody tr th {\n        vertical-align: top;\n    }\n\n    .dataframe thead th {\n        text-align: right;\n    }\n</style>\n<table border=\"1\" class=\"dataframe\">\n  <thead>\n    <tr style=\"text-align: right;\">\n      <th></th>\n      <th>Postcode</th>\n      <th>Borough</th>\n      <th>Neighbourhood</th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <th>8</th>\n      <td>M7A</td>\n      <td>Queen's Park</td>\n      <td>Queen's Park</td>\n    </tr>\n  </tbody>\n</table>\n</div>",
            "text/plain": "  Postcode       Borough Neighbourhood\n8      M7A  Queen's Park  Queen's Park"
          },
          "metadata": {}
        }
      ]
    },
    {
      "metadata": {
        "trusted": true
      },
      "cell_type": "code",
      "source": "#Group by Postcode / Borough\ndf_postcodes = df_html.groupby(['Postcode','Borough']).Neighbourhood.agg([('Neighbourhood', ', '.join)])\ndf_postcodes.reset_index(inplace=True)\ndf_postcodes.head(5)",
      "execution_count": 13,
      "outputs": [
        {
          "output_type": "execute_result",
          "execution_count": 13,
          "data": {
            "text/html": "<div>\n<style scoped>\n    .dataframe tbody tr th:only-of-type {\n        vertical-align: middle;\n    }\n\n    .dataframe tbody tr th {\n        vertical-align: top;\n    }\n\n    .dataframe thead th {\n        text-align: right;\n    }\n</style>\n<table border=\"1\" class=\"dataframe\">\n  <thead>\n    <tr style=\"text-align: right;\">\n      <th></th>\n      <th>Postcode</th>\n      <th>Borough</th>\n      <th>Neighbourhood</th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <th>0</th>\n      <td>M1B</td>\n      <td>Scarborough</td>\n      <td>Rouge, Malvern</td>\n    </tr>\n    <tr>\n      <th>1</th>\n      <td>M1C</td>\n      <td>Scarborough</td>\n      <td>Highland Creek, Rouge Hill, Port Union</td>\n    </tr>\n    <tr>\n      <th>2</th>\n      <td>M1E</td>\n      <td>Scarborough</td>\n      <td>Guildwood, Morningside, West Hill</td>\n    </tr>\n    <tr>\n      <th>3</th>\n      <td>M1G</td>\n      <td>Scarborough</td>\n      <td>Woburn</td>\n    </tr>\n    <tr>\n      <th>4</th>\n      <td>M1H</td>\n      <td>Scarborough</td>\n      <td>Cedarbrae</td>\n    </tr>\n  </tbody>\n</table>\n</div>",
            "text/plain": "  Postcode      Borough                           Neighbourhood\n0      M1B  Scarborough                          Rouge, Malvern\n1      M1C  Scarborough  Highland Creek, Rouge Hill, Port Union\n2      M1E  Scarborough       Guildwood, Morningside, West Hill\n3      M1G  Scarborough                                  Woburn\n4      M1H  Scarborough                               Cedarbrae"
          },
          "metadata": {}
        }
      ]
    },
    {
      "metadata": {
        "trusted": true
      },
      "cell_type": "code",
      "source": "#Lets check for Scarborough, so we can compare with the example dataframe shown in the assignment\ndf_postcodes[df_postcodes['Borough']=='Scarborough']",
      "execution_count": 14,
      "outputs": [
        {
          "output_type": "execute_result",
          "execution_count": 14,
          "data": {
            "text/html": "<div>\n<style scoped>\n    .dataframe tbody tr th:only-of-type {\n        vertical-align: middle;\n    }\n\n    .dataframe tbody tr th {\n        vertical-align: top;\n    }\n\n    .dataframe thead th {\n        text-align: right;\n    }\n</style>\n<table border=\"1\" class=\"dataframe\">\n  <thead>\n    <tr style=\"text-align: right;\">\n      <th></th>\n      <th>Postcode</th>\n      <th>Borough</th>\n      <th>Neighbourhood</th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <th>0</th>\n      <td>M1B</td>\n      <td>Scarborough</td>\n      <td>Rouge, Malvern</td>\n    </tr>\n    <tr>\n      <th>1</th>\n      <td>M1C</td>\n      <td>Scarborough</td>\n      <td>Highland Creek, Rouge Hill, Port Union</td>\n    </tr>\n    <tr>\n      <th>2</th>\n      <td>M1E</td>\n      <td>Scarborough</td>\n      <td>Guildwood, Morningside, West Hill</td>\n    </tr>\n    <tr>\n      <th>3</th>\n      <td>M1G</td>\n      <td>Scarborough</td>\n      <td>Woburn</td>\n    </tr>\n    <tr>\n      <th>4</th>\n      <td>M1H</td>\n      <td>Scarborough</td>\n      <td>Cedarbrae</td>\n    </tr>\n    <tr>\n      <th>5</th>\n      <td>M1J</td>\n      <td>Scarborough</td>\n      <td>Scarborough Village</td>\n    </tr>\n    <tr>\n      <th>6</th>\n      <td>M1K</td>\n      <td>Scarborough</td>\n      <td>East Birchmount Park, Ionview, Kennedy Park</td>\n    </tr>\n    <tr>\n      <th>7</th>\n      <td>M1L</td>\n      <td>Scarborough</td>\n      <td>Clairlea, Golden Mile, Oakridge</td>\n    </tr>\n    <tr>\n      <th>8</th>\n      <td>M1M</td>\n      <td>Scarborough</td>\n      <td>Cliffcrest, Cliffside, Scarborough Village West</td>\n    </tr>\n    <tr>\n      <th>9</th>\n      <td>M1N</td>\n      <td>Scarborough</td>\n      <td>Birch Cliff, Cliffside West</td>\n    </tr>\n    <tr>\n      <th>10</th>\n      <td>M1P</td>\n      <td>Scarborough</td>\n      <td>Dorset Park, Scarborough Town Centre, Wexford ...</td>\n    </tr>\n    <tr>\n      <th>11</th>\n      <td>M1R</td>\n      <td>Scarborough</td>\n      <td>Maryvale, Wexford</td>\n    </tr>\n    <tr>\n      <th>12</th>\n      <td>M1S</td>\n      <td>Scarborough</td>\n      <td>Agincourt</td>\n    </tr>\n    <tr>\n      <th>13</th>\n      <td>M1T</td>\n      <td>Scarborough</td>\n      <td>Clarks Corners, Sullivan, Tam O'Shanter</td>\n    </tr>\n    <tr>\n      <th>14</th>\n      <td>M1V</td>\n      <td>Scarborough</td>\n      <td>Agincourt North, L'Amoreaux East, Milliken, St...</td>\n    </tr>\n    <tr>\n      <th>15</th>\n      <td>M1W</td>\n      <td>Scarborough</td>\n      <td>L'Amoreaux West, Steeles West</td>\n    </tr>\n    <tr>\n      <th>16</th>\n      <td>M1X</td>\n      <td>Scarborough</td>\n      <td>Upper Rouge</td>\n    </tr>\n  </tbody>\n</table>\n</div>",
            "text/plain": "   Postcode      Borough                                      Neighbourhood\n0       M1B  Scarborough                                     Rouge, Malvern\n1       M1C  Scarborough             Highland Creek, Rouge Hill, Port Union\n2       M1E  Scarborough                  Guildwood, Morningside, West Hill\n3       M1G  Scarborough                                             Woburn\n4       M1H  Scarborough                                          Cedarbrae\n5       M1J  Scarborough                                Scarborough Village\n6       M1K  Scarborough        East Birchmount Park, Ionview, Kennedy Park\n7       M1L  Scarborough                    Clairlea, Golden Mile, Oakridge\n8       M1M  Scarborough    Cliffcrest, Cliffside, Scarborough Village West\n9       M1N  Scarborough                        Birch Cliff, Cliffside West\n10      M1P  Scarborough  Dorset Park, Scarborough Town Centre, Wexford ...\n11      M1R  Scarborough                                  Maryvale, Wexford\n12      M1S  Scarborough                                          Agincourt\n13      M1T  Scarborough            Clarks Corners, Sullivan, Tam O'Shanter\n14      M1V  Scarborough  Agincourt North, L'Amoreaux East, Milliken, St...\n15      M1W  Scarborough                      L'Amoreaux West, Steeles West\n16      M1X  Scarborough                                        Upper Rouge"
          },
          "metadata": {}
        }
      ]
    },
    {
      "metadata": {
        "trusted": true
      },
      "cell_type": "code",
      "source": "#Shape of the dataset\nprint('The shape of the dataset is:',df_postcodes.shape)",
      "execution_count": 15,
      "outputs": [
        {
          "output_type": "stream",
          "text": "The shape of the dataset is: (103, 3)\n",
          "name": "stdout"
        }
      ]
    },
    {
      "metadata": {
        "trusted": true
      },
      "cell_type": "code",
      "source": "",
      "execution_count": null,
      "outputs": []
    }
  ],
  "metadata": {
    "kernelspec": {
      "name": "python36",
      "display_name": "Python 3.6",
      "language": "python"
    },
    "language_info": {
      "mimetype": "text/x-python",
      "nbconvert_exporter": "python",
      "name": "python",
      "pygments_lexer": "ipython3",
      "version": "3.6.6",
      "file_extension": ".py",
      "codemirror_mode": {
        "version": 3,
        "name": "ipython"
      }
    }
  },
  "nbformat": 4,
  "nbformat_minor": 2
}