notebooks/05 Fetching the Nodes Locations on the Metr-LA Dataset.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<center>\n",
    "    <h1>Verbal Explanation of Spatial Temporal GNNs for Traffic Forecasting</h1>\n",
    "    <h2>Fetching the Nodes Locations on the Metr-LA Dataset</h2>\n",
    "</center>\n",
    "\n",
    "---\n",
    "\n",
    "In this notebook the street and kilometrage of each node in the dataset are fetched.\n",
    "\n",
    "Note: the extraction of the geolocation of the nodes is not deterministic, refer to [link](../data/metr-la/structured/node_locations.pkl) for the nodes streets and kilometrages extracted for the experiment."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import sys\n",
    "import os\n",
    "\n",
    "# Set the main path in the root folder of the project.\n",
    "sys.path.append(os.path.join('..'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Settings for autoreloading.\n",
    "%load_ext autoreload\n",
    "%autoreload 2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "from src.utils.seed import set_random_seed\n",
    "\n",
    "# Set the random seed for deterministic operations.\n",
    "SEED = 42\n",
    "set_random_seed(SEED)"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 1 Loading the Data\n",
    "In this section the data is loaded"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "\n",
    "BASE_DATA_DIR = os.path.join('..', 'data', 'metr-la')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from src.data.data_extraction import get_adjacency_matrix\n",
    "\n",
    "\n",
    "# Get the adjacency matrix\n",
    "adj_matrix_structure = get_adjacency_matrix(\n",
    "    os.path.join(BASE_DATA_DIR, 'raw', 'adj_mx_metr_la.pkl'))\n",
    "\n",
    "# Get the header of the adjacency matrix, the node indices and the\n",
    "# matrix itself.\n",
    "header, node_ids_dict, adj_matrix = adj_matrix_structure"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "from src.data.data_extraction import get_locations_dataframe\n",
    "\n",
    "# Get the dataframe containing the latitude and longitude of each sensor.\n",
    "locations_df = get_locations_dataframe(\n",
    "    os.path.join(BASE_DATA_DIR, 'raw', 'graph_sensor_locations_metr_la.csv'),\n",
    "    has_header=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Get the node positions dictionary.\n",
    "node_pos_dict = { i: id for id, i in node_ids_dict.items() }"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 2 Fetch the Nodes Locations and Kilometrage\n",
    "In this section the node locations and kilometrages is fetched."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 2.1 Fetch the Nodes Roads\n",
    "The roads of the nodes are fetched by geopy according to their latitude and longitude."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "from geopy.geocoders import Nominatim\n",
    "from geopy.extra.rate_limiter import RateLimiter\n",
    "\n",
    "geocoder = Nominatim(user_agent='metr-la')\n",
    "\n",
    "geocode = RateLimiter(\n",
    "    geocoder.geocode,\n",
    "    min_delay_seconds=.05,\n",
    "    return_value_on_exception=None)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "\n",
    "def get_road(row: pd.Series) -> str:\n",
    "    data = geocoder.reverse((row['latitude'], row['longitude'])).raw\n",
    "    road = data['address']['road']\n",
    "    return road"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "locations_df['road'] = locations_df.apply(get_road, axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'Arroyo Seco Parkway',\n",
       " 'East 4th Street',\n",
       " 'Foothill Freeway',\n",
       " 'Glendale Freeway',\n",
       " 'Golden State Freeway',\n",
       " 'Hollywood Freeway',\n",
       " 'San Diego Freeway',\n",
       " 'Santa Ana Freeway',\n",
       " 'Sherman Way',\n",
       " 'US 101',\n",
       " 'Ventura Freeway'}"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "set(road for road in locations_df['road'].values)"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Fixing wrong assignment of coordinates"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "US 101: coordinates (34.06491, -118.25126)\n",
      "US 101: coordinates (34.06461, -118.25102)\n",
      "East 4th Street: coordinates (34.04301, -118.21724)\n",
      "Sherman Way: coordinates (34.20112, -118.47361)\n",
      "US 101: coordinates (34.1527, -118.3754)\n",
      "US 101: coordinates (34.15133, -118.37456)\n"
     ]
    }
   ],
   "source": [
    "for row in locations_df.itertuples():\n",
    "    if row.road in ['East 4th Street', 'Sherman Way', 'US 101']:\n",
    "        print(f'{row.road}:',\n",
    "              'coordinates', \n",
    "              f'({row.latitude}, {row.longitude})')"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "By searching these coordinates, it can be observed that they actually refer to specific highways, hence we manually change this wrongly assigned reference"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "\n",
    "\n",
    "def change_road_name(row: pd.Series) -> str:\n",
    "    if row['road'] == 'East 4th Street':\n",
    "        return 'Golden State Freeway'\n",
    "\n",
    "    elif row['road'] == 'Sherman Way':\n",
    "        return 'San Diego Freeway'\n",
    "\n",
    "    elif row['road'] == 'US 101':\n",
    "        return 'Hollywood Freeway'\n",
    "\n",
    "    return row['road']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "locations_df['road'] = locations_df.apply(change_road_name, axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'Arroyo Seco Parkway',\n",
       " 'Foothill Freeway',\n",
       " 'Glendale Freeway',\n",
       " 'Golden State Freeway',\n",
       " 'Hollywood Freeway',\n",
       " 'San Diego Freeway',\n",
       " 'Santa Ana Freeway',\n",
       " 'Ventura Freeway'}"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "set(road for road in locations_df['road'].values)"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 2.2 Get the Road Beginnings\n",
    "\n",
    "In order to get the kilometrages of the nodes, the road beginnings are computed from [ArcGIS official website](https://www.arcgis.com/home/webmap/viewer.html?featurecollection=https%3A%2F%2Fgeo.dot.gov%2Fserver%2Frest%2Fservices%2FHosted%2FCalifornia_2018_PR%2FFeatureServer%3Ff%3Djson%26option%3Dfootprints&supportsProjection=true&supportsJSONP=true)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "road_beginnings = {\n",
    "    'Arroyo Seco Parkway': (34.06261, -118.24863),\n",
    "    'Foothill Freeway': (34.317596, -118.481173),\n",
    "    'Glendale Freeway': (34.207526, -118.215157),\n",
    "    'Golden State Freeway': (34.060422, -118.213057),\n",
    "    'Hollywood Freeway': (34.151159, -118.373795),\n",
    "    'San Diego Freeway': (34.294368, -118.469921),\n",
    "    'Santa Ana Freeway': (34.06312,\t-118.247073),\n",
    "    'Ventura Freeway': (34.147306, -118.160817)}"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 2.3 Get Nodes Kilometrages and Save Results\n",
    "Finally, the kilometrage of the nodes is computed by their distance from the coordinates of the beginnings of the roads.\n",
    "\n",
    "For each node a dictionary is computed containing a tuple with the name of the road it is part of and its kilometrage on the road."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "from geopy.distance import distance\n",
    "\n",
    "\n",
    "def get_kilometrage(\n",
    "    row: pd.Series,\n",
    "    ) -> float:\n",
    "    road = row['road']\n",
    "    road_beginning_coordinates = road_beginnings[road]\n",
    "    coordinates = (row['latitude'], row['longitude'])\n",
    "    return distance(road_beginning_coordinates, coordinates).km"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [],
   "source": [
    "locations_df['kilometrage'] = locations_df.apply(get_kilometrage, axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>sensor_id</th>\n",
       "      <th>latitude</th>\n",
       "      <th>longitude</th>\n",
       "      <th>road</th>\n",
       "      <th>kilometrage</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>index</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>773869</td>\n",
       "      <td>34.15497</td>\n",
       "      <td>-118.31829</td>\n",
       "      <td>Ventura Freeway</td>\n",
       "      <td>14.547154</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>767541</td>\n",
       "      <td>34.11621</td>\n",
       "      <td>-118.23799</td>\n",
       "      <td>Glendale Freeway</td>\n",
       "      <td>10.345752</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>767542</td>\n",
       "      <td>34.11641</td>\n",
       "      <td>-118.23819</td>\n",
       "      <td>Glendale Freeway</td>\n",
       "      <td>10.327808</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>717447</td>\n",
       "      <td>34.07248</td>\n",
       "      <td>-118.26772</td>\n",
       "      <td>Hollywood Freeway</td>\n",
       "      <td>13.112986</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>717446</td>\n",
       "      <td>34.07142</td>\n",
       "      <td>-118.26572</td>\n",
       "      <td>Hollywood Freeway</td>\n",
       "      <td>13.329056</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>717445</td>\n",
       "      <td>34.06913</td>\n",
       "      <td>-118.25932</td>\n",
       "      <td>Hollywood Freeway</td>\n",
       "      <td>13.940930</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>773062</td>\n",
       "      <td>34.05368</td>\n",
       "      <td>-118.23369</td>\n",
       "      <td>Santa Ana Freeway</td>\n",
       "      <td>1.619571</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>767620</td>\n",
       "      <td>34.13486</td>\n",
       "      <td>-118.22932</td>\n",
       "      <td>Glendale Freeway</td>\n",
       "      <td>8.165598</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>737529</td>\n",
       "      <td>34.20264</td>\n",
       "      <td>-118.47352</td>\n",
       "      <td>San Diego Freeway</td>\n",
       "      <td>10.180500</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>717816</td>\n",
       "      <td>34.15562</td>\n",
       "      <td>-118.46860</td>\n",
       "      <td>San Diego Freeway</td>\n",
       "      <td>15.391305</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "      sensor_id  latitude  longitude               road  kilometrage\n",
       "index                                                               \n",
       "0        773869  34.15497 -118.31829    Ventura Freeway    14.547154\n",
       "1        767541  34.11621 -118.23799   Glendale Freeway    10.345752\n",
       "2        767542  34.11641 -118.23819   Glendale Freeway    10.327808\n",
       "3        717447  34.07248 -118.26772  Hollywood Freeway    13.112986\n",
       "4        717446  34.07142 -118.26572  Hollywood Freeway    13.329056\n",
       "5        717445  34.06913 -118.25932  Hollywood Freeway    13.940930\n",
       "6        773062  34.05368 -118.23369  Santa Ana Freeway     1.619571\n",
       "7        767620  34.13486 -118.22932   Glendale Freeway     8.165598\n",
       "8        737529  34.20264 -118.47352  San Diego Freeway    10.180500\n",
       "9        717816  34.15562 -118.46860  San Diego Freeway    15.391305"
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "locations_df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [],
   "source": [
    "road_locations = {}\n",
    "\n",
    "for row in locations_df.itertuples():\n",
    "    node_id = row.sensor_id\n",
    "    road = row.road\n",
    "    kilometrage = row.kilometrage\n",
    "    road_locations[node_id] = (road, kilometrage)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "import pickle\n",
    "\n",
    "os.makedirs(os.path.join(BASE_DATA_DIR, 'structured'), exist_ok=True)\n",
    "\n",
    "with open(\n",
    "      os.path.join(BASE_DATA_DIR, 'structured', 'node_locations.pkl'), \n",
    "      'wb') as f:\n",
    "    pickle.dump(road_locations, f)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.0"
  },
  "orig_nbformat": 4
 },
 "nbformat": 4,
 "nbformat_minor": 2
}