multi-view_graph_construction.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2022-03-14 04:02:23.941280: I tensorflow/stream_executor/platform/default/dso_loader.cc:53] Successfully opened dynamic library libcudart.so.11.0\n"
     ]
    }
   ],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "import os\n",
    "import warnings\n",
    "import multiprocessing\n",
    "import json\n",
    "\n",
    "from shapely.geometry import Point, Polygon, MultiPolygon\n",
    "from datetime import datetime, date\n",
    "from scipy.spatial import distance\n",
    "from sklearn.preprocessing import MinMaxScaler\n",
    "from spektral.utils import normalized_laplacian\n",
    "\n",
    "warnings.filterwarnings('ignore')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "cities = ['Seoul', 'Busan', 'Daegu', 'Daejeon', 'Gwangju']\n",
    "years = ['2016', '2018']\n",
    "n_steps, length = 6, 12"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open('geojson/selected-city-gps.json') as f:\n",
    "    city_geo = json.load(f)\n",
    "    \n",
    "with open('geojson/selected-district-gps.json') as f:\n",
    "    district_geo = json.load(f)\n",
    "    \n",
    "with open('geojson/selected-subdistrict-gps.json') as f:\n",
    "    subdistrict_geo = json.load(f)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_polygons(g, level):\n",
    "    polygons = []\n",
    "    np_g = np.array(g['geometry']['coordinates_gps'])\n",
    "    \n",
    "    for pl in g['geometry']['coordinates_gps']:\n",
    "        if len(np_g.shape) == 3:\n",
    "            polygons.append(Polygon(pl))\n",
    "        else:\n",
    "            for p in pl:\n",
    "                polygons.append(Polygon(p))\n",
    "                \n",
    "    \n",
    "    return polygons"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "for selected_city in cities:\n",
    "    district_names = []\n",
    "    subdistrict_names = []\n",
    "    city_polygons = []\n",
    "    district_polygons = []    \n",
    "    subdistrict_polygons = []\n",
    "    new_city_geo = []\n",
    "    new_district_geo = []\n",
    "    new_subdistrict_geo = []\n",
    "\n",
    "    for city in city_geo:\n",
    "        if city['properties']['CTP_ENG_NM'] == selected_city:\n",
    "            new_city_geo.append(city)\n",
    "            for district in district_geo:\n",
    "                if district['properties']['SIG_CD'].startswith(city['properties']['CTPRVN_CD']):\n",
    "                    new_district_geo.append(district)\n",
    "                    for subdistrict in subdistrict_geo:\n",
    "                        if subdistrict['properties']['EMD_CD'].startswith(district['properties']['SIG_CD']):\n",
    "                            new_subdistrict_geo.append(subdistrict)\n",
    "\n",
    "    for city in new_city_geo:\n",
    "        polygons = get_polygons(city, 'city')\n",
    "        city_polygons.append({'polygon': MultiPolygon(polygons), 'code': city['properties']['CTPRVN_CD'], 'name': city['properties']['CTP_KOR_NM'], 'eng_name': city['properties']['CTP_ENG_NM']})\n",
    "    for district in new_district_geo:\n",
    "        polygons = get_polygons(district, 'district')\n",
    "        district_polygons.append({'polygon': MultiPolygon(polygons), 'code': district['properties']['SIG_CD'], 'name': district['properties']['SIG_KOR_NM'], 'eng_name': district['properties']['SIG_ENG_NM']})\n",
    "    for subdistrict in new_subdistrict_geo:\n",
    "        polygons = get_polygons(subdistrict, 'subdistrict')\n",
    "        subdistrict_polygons.append({'polygon': MultiPolygon(polygons), 'code': subdistrict['properties']['EMD_CD'], 'name': subdistrict['properties']['EMD_KOR_NM'], 'eng_name': subdistrict['properties']['EMD_ENG_NM']})\n",
    "\n",
    "    n_districts = len(district_polygons)\n",
    "    A = np.zeros([n_districts, n_districts], dtype=int)\n",
    "\n",
    "    for i in range(n_districts):\n",
    "        r = district_polygons[i]\n",
    "        district_names.append(r['name'])\n",
    "        for j in range(n_districts):\n",
    "            c = district_polygons[j]\n",
    "            # check polygons' adjacency\n",
    "            if r['polygon'].intersects(c['polygon']):\n",
    "                A[i][j] = 1\n",
    "            if selected_city == 'Busan' and r['name'] == '영도구' and c['name'] in ['서구', '중구', '남구']:\n",
    "                A[i][j] = 1\n",
    "            if selected_city == 'Busan' and c['name'] == '영도구' and r['name'] in ['서구', '중구', '남구']:\n",
    "                A[i][j] = 1\n",
    "\n",
    "    pd.DataFrame(A, columns=district_names, index=district_names).to_csv(f'./datasets/graph_data/{selected_city}-adjacency-district.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "for selected_city in cities:\n",
    "    A = pd.read_csv(f'./datasets/graph_data/{selected_city}-adjacency-district.csv', engine='c', index_col=0)\n",
    "    D = np.zeros(A.shape, dtype=int)\n",
    "    for i in range(A.shape[0]):\n",
    "        D[i][i] = np.sum(np.array(A), axis=1)[i]\n",
    "        pd.DataFrame(D, columns=A.columns.to_list(), index=A.columns.to_list()).to_csv(f'./datasets/graph_data/{selected_city}-degree-district.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "for selected_city in cities:\n",
    "    A = pd.read_csv(f'./datasets/graph_data/{selected_city}-adjacency-district.csv', engine='c', index_col=0)\n",
    "    D = pd.read_csv(f'./datasets/graph_data/{selected_city}-degree-district.csv', engine='c', index_col=0)\n",
    "    columns = A.columns.to_list()\n",
    "    A, D = np.array(A), np.array(D)\n",
    "    L = np.dot(np.dot(np.linalg.inv(np.sqrt(D)), A), np.linalg.inv(np.sqrt(D)))\n",
    "    pd.DataFrame(L, index=columns, columns=columns).to_csv(f'./datasets/graph_data/{selected_city}-normalized-district.csv')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Multi-view Graph Generation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "def normalized_matrix(city, year, data, rel_type, metric):\n",
    "    A_s = pd.read_csv(f'./datasets/graph_data/{city}-adjacency-district.csv', engine='c', index_col=0)\n",
    "    n_districts = len(A_s.columns)\n",
    "    \n",
    "    A = np.zeros([n_districts, n_districts])\n",
    "    for i in range(n_districts):\n",
    "        for j in range(n_districts):\n",
    "            v, u = data[:,i], data[:,j]\n",
    "            A[i][j] = 1 - metric(v, u)\n",
    "            \n",
    "    m_avg = np.average(A)\n",
    "    for idx, scale in enumerate([0.25, 0.5, 1., 2., 4.]):\n",
    "        eps = m_avg * scale\n",
    "        for i in range(n_districts):\n",
    "            for j in range(n_districts):\n",
    "                A[i][j] = 1 if A[i][j] >= eps else 0\n",
    "\n",
    "        D = np.zeros(A.shape, dtype=int)\n",
    "        for i in range(A.shape[0]):\n",
    "            D[i][i] = np.sum(np.array(A), axis=1)[i]\n",
    "\n",
    "        columns = A_s.columns.to_list()\n",
    "        L = np.dot(np.dot(np.linalg.inv(np.sqrt(D)), A), np.linalg.inv(np.sqrt(D)))\n",
    "        pd.DataFrame(L, index=columns, columns=columns).to_csv(f'./datasets/graph_data/{city}-{year}-{rel_type}-normalized-district-{metric.__name__}-{idx}.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      " 2016 - Seoul\n",
      "\n",
      " 2018 - Seoul\n"
     ]
    }
   ],
   "source": [
    "for year in years:\n",
    "    for city in ['Seoul']:#cities:\n",
    "        print('\\n', year, '-', city)\n",
    "        for metric in [distance.jaccard]:#[distance.cosine, distance.jaccard, distance.euclidean, distance.cityblock, distance.correlation]:\n",
    "            # static spatial distance\n",
    "            A = pd.read_csv(f'./datasets/graph_data/{city}-adjacency-district.csv', engine='c', index_col=0)\n",
    "            road_data = pd.read_csv(f'./datasets/roads/{city}-{year}-district-road-count.csv').drop(columns=['attribute'])[A.columns].to_numpy()\n",
    "            demo_data = pd.read_csv(f'./datasets/demographic/{city}-{year}-district.csv').drop(columns=['index'])[A.columns].to_numpy()\n",
    "            poi_data = pd.read_csv(f'./datasets/poi/{city}-{year}-district.csv').drop(columns=['location'])[A.columns].to_numpy()\n",
    "\n",
    "            normalized_matrix(city, year, MinMaxScaler().fit_transform(poi_data), 'poi', metric)\n",
    "            normalized_matrix(city, year, MinMaxScaler().fit_transform(demo_data), 'demo', metric)\n",
    "            normalized_matrix(city, year, MinMaxScaler().fit_transform(road_data), 'road', metric)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      " 2016 - Seoul\n",
      "\n",
      " 2018 - Seoul\n"
     ]
    }
   ],
   "source": [
    "for year in years:\n",
    "    for city in ['Seoul']:#cities:\n",
    "        print('\\n', year, '-', city)\n",
    "        for metric in [distance.jaccard]:#[distance.cosine, distance.jaccard, distance.euclidean, distance.cityblock, distance.correlation]:\n",
    "            for length in [12]:#[6, 12, 18]:\n",
    "                for idx, scale in enumerate([0.25, 0.5, 1., 2., 4.]):\n",
    "                    # dynamic spatial distance\n",
    "                    A = pd.read_csv(f'./datasets/graph_data/{city}-adjacency-district.csv', engine='c', index_col=0)\n",
    "\n",
    "                    risk_data = pd.read_csv(f'./datasets/risk_scores/{city}-{year}-district-hour-risk.csv')[A.columns].to_numpy()\n",
    "                    volume_data = pd.read_csv(f'./datasets/traffic_volume/{city}-{year}.csv').drop(columns=['date', 'hour'])[A.columns].to_numpy()\n",
    "                    speed_data = pd.read_csv(f'./datasets/traffic_speed/{city}-{year}.csv').drop(columns=['date', 'hour'])[A.columns].to_numpy()\n",
    "                    dtg_data = pd.read_csv(f'./datasets/dangerous_cases/{city}-{year}-date-hour-district-new.csv')[['district', 'OS', 'RA', 'QS', 'RD', 'SS', 'SLC', 'SO', 'ST', 'SUT']]\n",
    "\n",
    "                    volume_data = MinMaxScaler().fit_transform(volume_data)\n",
    "                    speed_data = MinMaxScaler().fit_transform(speed_data)\n",
    "\n",
    "                    districts = list(A.columns)\n",
    "                    n_districts = len(districts)\n",
    "                    traffic_data = np.zeros([risk_data.shape[0], len(districts), 2 + 9])\n",
    "\n",
    "                    for t in range(720):\n",
    "                        for i, di in enumerate(districts):\n",
    "                            dtg = [0]*9 if dtg_data[dtg_data['district'] == di].to_numpy()[t, 1:].sum() == 0 else dtg_data[dtg_data['district'] == di].to_numpy()[t, 1:] / dtg_data[dtg_data['district'] == di].to_numpy()[t, 1:].sum()\n",
    "                            traffic_data[t][i] = np.concatenate([[volume_data[t][i], speed_data[t][i]], dtg])\n",
    "\n",
    "                    A_traffic = np.zeros([720, n_districts, n_districts])\n",
    "\n",
    "                    for t in range(720):\n",
    "                        for i, di in enumerate(districts):\n",
    "                            for j, dj in enumerate(districts):\n",
    "                                vt, ut = traffic_data[t][i], traffic_data[t][j]\n",
    "                                A_traffic[t][i][j] = 1 - metric(vt, ut)\n",
    "\n",
    "                        m_avg = np.average(A_traffic[t])\n",
    "                        eps = m_avg * scale\n",
    "                        for i, di in enumerate(districts):\n",
    "                            for j, dj in enumerate(districts):\n",
    "                                A_traffic[t][i][j] = 1 if A_traffic[t][i][j] >= eps else 0\n",
    "\n",
    "\n",
    "                    # train-test split\n",
    "                    A_traffic_train, A_traffic_test = [], []\n",
    "                    for i in range(length, 721-n_steps):\n",
    "                        if i <= (24*24): # before date 25th\n",
    "                            A_traffic_train.append(A_traffic[i-length:i, :n_districts])\n",
    "                        else:\n",
    "                            A_traffic_test.append(A_traffic[i-length:i, :n_districts])\n",
    "\n",
    "                    A_traffic_train, A_traffic_test = np.array(A_traffic_train), np.array(A_traffic_test)\n",
    "\n",
    "                    D_traffic_train = np.zeros(A_traffic_train.shape, dtype=int)\n",
    "                    for t in range(A_traffic_train.shape[0]):\n",
    "                        for l in range(length):\n",
    "                            for i in range(n_districts):\n",
    "                                D_traffic_train[t][l][i][i] = np.sum(np.array(A_traffic_train[t][l]), axis=1)[i]\n",
    "\n",
    "                    L = np.zeros(A_traffic_train.shape, dtype=float)\n",
    "                    for t in range(A_traffic_train.shape[0]):\n",
    "                        for l in range(length):\n",
    "                            A, D = np.array(A_traffic_train), np.array(D_traffic_train)\n",
    "        #                     D[t][l] = D[t][l] + 0.0000001 * np.random.rand(D[t][l].shape[0], n_districts)\n",
    "                            INV_D = np.linalg.inv(np.sqrt(D[t][l] + 0.0000001 * np.random.rand(D[t][l].shape[0], n_districts)))\n",
    "                            L[t][l] = np.dot(np.dot(INV_D, A[t][l]), INV_D)\n",
    "\n",
    "                    with open(f'datasets/graph_data/{city}-{year}-traffic-district-normalized-train-{metric.__name__}-{length}-{idx}.npy', 'wb') as f:\n",
    "                        np.save(f, L)\n",
    "\n",
    "                    D_traffic_test = np.zeros(A_traffic_test.shape, dtype=int)\n",
    "                    for t in range(A_traffic_test.shape[0]):\n",
    "                        for l in range(length):\n",
    "                            for i in range(n_districts):\n",
    "                                D_traffic_test[t][l][i][i] = np.sum(np.array(A_traffic_test[t][l]), axis=1)[i]\n",
    "\n",
    "                    L = np.zeros(A_traffic_test.shape, dtype=float)\n",
    "                    for t in range(A_traffic_test.shape[0]):\n",
    "                        for l in range(length):\n",
    "                            A, D = np.array(A_traffic_test), np.array(D_traffic_test)\n",
    "        #                     D[t][l] = D[t][l] + 0.0000001 * np.random.rand(D[t][l].shape[0], n_districts)\n",
    "                            INV_D = np.linalg.inv(np.sqrt(D[t][l] + 0.0000001 * np.random.rand(D[t][l].shape[0], n_districts)))\n",
    "                            L[t][l] = np.dot(np.dot(INV_D, A[t][l]), INV_D)\n",
    "\n",
    "                    with open(f'datasets/graph_data/{city}-{year}-traffic-district-normalized-test-{metric.__name__}-{length}-{idx}.npy', 'wb') as f:\n",
    "                        np.save(f, L)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "mg-tar",
   "language": "python",
   "name": "mg-tar"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}