Process_DC_Cab_Pickup.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import cv2\n",
    "%matplotlib inline\n",
    "import glob\n",
    "import csv\n",
    "import pandas as pd\n",
    "import re\n",
    "import datetime"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def measure_geo_distance(pt1_longitude,pt1_latitude,pt2_longtide,pt2_latitude):\n",
    "    \"\"\"\n",
    "    This function compute the geo_distance between two points using Euclidean distance\n",
    "    \"\"\"\n",
    "    distance = ((pt1_longitude-pt2_longtide)**2+(pt1_latitude-pt2_latitude)**2)**0.5\n",
    "    return distance\n",
    "\n",
    "def filter_by_origin(data_pickup,data_regions):\n",
    "    \"\"\"\n",
    "    This function filter out the trips that are not originated from any of our selected region, and add a region tag\n",
    "    data_pickup\n",
    "    data_region\n",
    "    \"\"\"\n",
    "    # Create a copy\n",
    "    data_pickup_filtered = data_pickup.copy()\n",
    "\n",
    "    # Initialize the list to record which region the origin is located\n",
    "    in_which_region_list = []\n",
    "\n",
    "    # Loop through each row\n",
    "    for i in range(0, len(data_pickup)):\n",
    "        pickup_latitude = data_pickup['Pickup_Block_Latitude'][i]\n",
    "        pickup_longitude = data_pickup['Pickup_Block_Longitude'][i]\n",
    "        in_which_region = -1 # Initialize with -1\n",
    "        for j in range(0,len(data_regions)):\n",
    "            distance = measure_geo_distance(pickup_longitude,pickup_latitude,data_regions['Longitude'][j],data_regions['Latitude'][j])\n",
    "            if distance <= data_regions['Radius'][j]:\n",
    "                in_which_region = j\n",
    "        in_which_region_list.append(in_which_region)\n",
    "    data_pickup_filtered['Region'] = in_which_region_list\n",
    "    # Keep only those have real region indice\n",
    "    data_pickup_filtered = data_pickup_filtered[data_pickup_filtered.Region != -1] \n",
    "    # Reset the indice\n",
    "    data_pickup_filtered = data_pickup_filtered.dropna(how='any').reset_index(drop=True)\n",
    "    return data_pickup_filtered\n",
    "\n",
    "def filter_by_dest(data_pickup,data_regions,max_radius_multiple = 1.25):\n",
    "    \"\"\"\n",
    "    This function filter out the trips that the destination is out of their corresponding regions by 1.25 of the radius\n",
    "    data_pickup: pickup dataset\n",
    "    data_regions: dataset that record our regions\n",
    "    \"\"\"\n",
    "    # Create a copy\n",
    "    data_pickup_dest_filter = data_pickup.copy()\n",
    "    # Initialize the list to record whether the destination is within the same region\n",
    "    in_region_list =  [] \n",
    "\n",
    "    for i in range(0,len(data_pickup_dest_filter)):\n",
    "        in_region = 0 # Initialize with negative\n",
    "        region_id = data_pickup_dest_filter['Region'][i]\n",
    "        distance = measure_geo_distance(data_pickup_dest_filter['Pickup_Block_Longitude'][i],data_pickup_dest_filter['Pickup_Block_Latitude'][i],data_regions['Longitude'][region_id],data_regions['Latitude'][region_id])\n",
    "        if distance <= data_regions['Radius'][region_id]*max_radius_multiple:\n",
    "            in_region = 1\n",
    "        in_region_list.append(in_region)\n",
    "    # Add a column - for the convinience of removing rows\n",
    "    data_pickup_dest_filter['Dest_in_region'] = in_region_list\n",
    "    # Remove trips that are out of the region\n",
    "    data_pickup = data_pickup[data_pickup_dest_filter.Dest_in_region == 1] \n",
    "    # Reset the indice\n",
    "    data_pickup = data_pickup.dropna(how='any').reset_index(drop=True)\n",
    "\n",
    "    return data_pickup\n",
    "\n",
    "def date_to_weekday(data_pickup):\n",
    "    \"\"\"\n",
    "    This function conver the date in the dataset to day of week\n",
    "    data_pickup: the dataset of the pickup\n",
    "    \"\"\"\n",
    "    # Initialize the list of weekday\n",
    "    weekday_list = []\n",
    "    # Initialize the recorders that can be used to speed up the function\n",
    "    month_string_prev = '0'\n",
    "    day_prev= '-1'\n",
    "    weekday_prev = '-1'\n",
    "    for i in range(0,len(data_pickup)):\n",
    "        month_string = data_pickup['Month_of_Pickupdatetime_Tr'][i]\n",
    "        day = int(data_pickup['Day_of_Pickupdatetime_Tr'][i])\n",
    "\n",
    "        # We only compute the weekday again if either month or day has been changed\n",
    "        if  month_string != month_string_prev or day != day_prev:\n",
    "            year = 2016\n",
    "            if month_string == 'January':\n",
    "                month = 1\n",
    "            elif month_string == 'February':\n",
    "                month = 2\n",
    "            elif month_string == 'March':\n",
    "                month = 3\n",
    "            else:\n",
    "                print(\"Month name is wrong!\")\n",
    "                #return\n",
    "            date_input = datetime.datetime(year,month,day)\n",
    "            # Conver to weekday\n",
    "            # Note: the weekday starts from 0 and end in 6\n",
    "            weekday = date_input.weekday() +1\n",
    "        else:\n",
    "            # Simply use the previous weekday, since the date has not been changed\n",
    "            weekday = weekday_prev\n",
    "\n",
    "        # One cycle comes to the end: update the recorder\n",
    "        month_string_prev = month_string\n",
    "        day_prev = day\n",
    "        weekday_prev = weekday\n",
    "\n",
    "        # Add weekday to the lsit\n",
    "        weekday_list.append(weekday)\n",
    "    data_pickup['Weekday'] = weekday_list\n",
    "    return data_pickup\n",
    "\n",
    "def compute_avg_traffic(data_pickup):\n",
    "    \"\"\"\n",
    "    This function aggregates the damand across the Hour, Day, Month, Weekday, and Region; then it compute the average by weekday,hour and region\n",
    "    \"\"\"\n",
    "    df = data_pickup.copy()\n",
    "    # Remove unuseful geodata columns\n",
    "    df=df.drop(['Dropoff_Block_Latitude','Dropoff_Block_Longitude','Pickup_Block_Latitude','Pickup_Block_Longitude'],1)\n",
    "\n",
    "    # Group by to find sum\n",
    "    df = df.groupby([\"Day_of_Pickupdatetime_Tr\",\"Hour_of_Pickupdatetime_Tr\",\"Month_of_Pickupdatetime_Tr\",\"Weekday\",\"Region\"], as_index=False).sum()\n",
    "\n",
    "    # Drop the Day, Hour, Month\n",
    "    df=df.drop(['Day_of_Pickupdatetime_Tr','Month_of_Pickupdatetime_Tr',],1)\n",
    "\n",
    "    # Compute the average by Weekday and Region\n",
    "    df = df.groupby(['Weekday','Hour_of_Pickupdatetime_Tr','Region'], as_index=False).mean()\n",
    "    df.columns = ['Weekday', 'Hour','Region','Avg_Traffic']\n",
    "    return df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def process_pipeline(data_pickup, data_region):\n",
    "    \"\"\"\n",
    "    This function set up an automated pipeline of the processes on the pickup dataset to extract the useful information\n",
    "    \"\"\"\n",
    "    # Create a new copy\n",
    "    df = data_pickup.copy()\n",
    "\n",
    "    # Filter the trips by origins - whether the origins fall in one of the region\n",
    "    df = filter_by_origin(df,data_regions)\n",
    "\n",
    "    # Filter the trips by destination -  whether this is an intra-zone trip\n",
    "    df = filter_by_dest(df,data_regions)\n",
    "\n",
    "    # Convert date to weekday\n",
    "    df = date_to_weekday(df)\n",
    "\n",
    "    # Compute the average by weekday, hour, and region\n",
    "    df = compute_avg_traffic(df)\n",
    "    \n",
    "    return df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Read the Dataset\n",
    "Data_Jan_Pickup=pd.read_csv('Data/DC_Cab_Pickup/Jan_DC_Data.csv',sep = '\\t')\n",
    "Data_Feb_Pickup=pd.read_csv('Data/DC_Cab_Pickup/Feb_DC_Data.csv',sep = '\\t')\n",
    "Data_Mar_Pickup=pd.read_csv('Data/DC_Cab_Pickup/Mar_DC_Data.csv',sep = '\\t')\n",
    "\n",
    "# Merge three datasets together\n",
    "frames = [Data_Jan_Pickup, Data_Feb_Pickup, Data_Mar_Pickup]\n",
    "data_pickup = pd.concat(frames)\n",
    "data_pickup = data_pickup.dropna(how='any').reset_index(drop=True)\n",
    "\n",
    "# Read the Region Data Set\n",
    "data_regions=pd.read_csv('Data/DC_Cab_Pickup/CoordinateWrenches.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Process the dataset with the pipeline\n",
    "df = process_pipeline(data_pickup,data_regions)\n",
    "\n",
    "# Save Dataframe\n",
    "df.to_csv(\"Data/Processed_Data/Pickup_Demands.csv\")"
   ]
  }
 ],
 "metadata": {
  "anaconda-cloud": {},
  "kernelspec": {
   "display_name": "Python [conda root]",
   "language": "python",
   "name": "conda-root-py"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}