GPS-trace-optimization-via-Valhalla.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "import json\n",
    "import requests\n",
    "\n",
    "from keplergl import KeplerGl"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>gps_time</th>\n",
       "      <th>longitude</th>\n",
       "      <th>latitude</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>2020-07-31T19:05:46</td>\n",
       "      <td>67.153923</td>\n",
       "      <td>54.591755</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2020-07-31T19:04:34</td>\n",
       "      <td>67.154312</td>\n",
       "      <td>54.591812</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2020-07-31T19:04:27</td>\n",
       "      <td>67.154930</td>\n",
       "      <td>54.591877</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>2020-07-31T19:04:07</td>\n",
       "      <td>67.156960</td>\n",
       "      <td>54.591896</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>2020-07-31T19:03:48</td>\n",
       "      <td>67.158783</td>\n",
       "      <td>54.591942</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "              gps_time  longitude   latitude\n",
       "0  2020-07-31T19:05:46  67.153923  54.591755\n",
       "1  2020-07-31T19:04:34  67.154312  54.591812\n",
       "2  2020-07-31T19:04:27  67.154930  54.591877\n",
       "3  2020-07-31T19:04:07  67.156960  54.591896\n",
       "4  2020-07-31T19:03:48  67.158783  54.591942"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Getting the data to Pandas dataframe\n",
    "\n",
    "filename = 'gps-trace.json'\n",
    "\n",
    "df = pd.read_json(filename)\n",
    "df = df['points']\n",
    "df = pd.json_normalize(df)\n",
    "df = df[['gps_time', 'longitude', 'latitude']]\n",
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Adding some noise to the trace to make it more difficult\n",
    "\n",
    "mu, sigma = 0, 0.0002 \n",
    "noise = np.random.normal(mu, sigma, [len(df.index),])\n",
    "df['latitude'] = df['latitude'] + noise\n",
    "df['longitude'] = df['longitude'] + noise"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "User Guide: https://docs.kepler.gl/docs/keplergl-jupyter\n",
      "Map saved to Non-optimized.html!\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "bd67ecea6fbc4b16b71cf9ab042991fc",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "KeplerGl(data={'Non-optimized.html': {'index': [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, …"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# Plotting the data on the map and saving it to the html file\n",
    "\n",
    "title = 'Non-optimized.html'\n",
    "\n",
    "map = KeplerGl(height=600, width=600)\n",
    "# add data to keplergl map\n",
    "map.add_data(data=df, name=title)\n",
    "map.save_to_html(file_name=title)\n",
    "map"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "df['gps_time'] = pd.to_datetime(df['gps_time'])\n",
    "df = df.sort_values(by=['gps_time'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>lon</th>\n",
       "      <th>lat</th>\n",
       "      <th>time</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>81</th>\n",
       "      <td>67.147783</td>\n",
       "      <td>54.669725</td>\n",
       "      <td>2020-07-31 18:52:49</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>80</th>\n",
       "      <td>67.147004</td>\n",
       "      <td>54.669144</td>\n",
       "      <td>2020-07-31 18:52:53</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>79</th>\n",
       "      <td>67.147285</td>\n",
       "      <td>54.669464</td>\n",
       "      <td>2020-07-31 18:52:55</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>78</th>\n",
       "      <td>67.147856</td>\n",
       "      <td>54.670005</td>\n",
       "      <td>2020-07-31 18:52:57</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>77</th>\n",
       "      <td>67.147519</td>\n",
       "      <td>54.669461</td>\n",
       "      <td>2020-07-31 18:53:02</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "          lon        lat                time\n",
       "81  67.147783  54.669725 2020-07-31 18:52:49\n",
       "80  67.147004  54.669144 2020-07-31 18:52:53\n",
       "79  67.147285  54.669464 2020-07-31 18:52:55\n",
       "78  67.147856  54.670005 2020-07-31 18:52:57\n",
       "77  67.147519  54.669461 2020-07-31 18:53:02"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Optimizing data to get it to Valhalla's Meili\n",
    "\n",
    "df_trip_for_meili = df[['longitude', 'latitude', 'gps_time']].copy()\n",
    "df_trip_for_meili.columns = ['lon', 'lat', 'time']\n",
    "df_trip_for_meili.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Preparing the request to Valhalla's Meili\n",
    "\n",
    "meili_coordinates = df_trip_for_meili.to_json(orient='records')\n",
    "meili_head = '{\"shape\":'\n",
    "meili_tail = \"\"\",\"search_radius\": 150, \"shape_match\":\"map_snap\", \"costing\":\"auto\", \"format\":\"osrm\"}\"\"\"\n",
    "meili_request_body = meili_head + meili_coordinates + meili_tail"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Sending a request to Valhalla's Meili\n",
    "\n",
    "url = \"http://localhost:8002/trace_route\"\n",
    "headers = {'Content-type': 'application/json'}\n",
    "data = str(meili_request_body)\n",
    "\n",
    "r = requests.post(url, data=data, headers=headers)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Parsing the response from Valhalla's Meili\n",
    "\n",
    "if r.status_code == 200:\n",
    "    \n",
    "        response_text = json.loads(r.text)\n",
    "\n",
    "        resp = str(response_text['tracepoints'])\n",
    "\n",
    "        resp = resp.replace(\"'waypoint_index': None\", \"'waypoint_index': '#'\")\n",
    "        resp = resp.replace(\"None\", \"{'matchings_index': '#', 'name': '', 'waypoint_index': '#', 'alternatives_count': 0, 'distance': 0, 'location': [0.0, 0.0]}\")\n",
    "\n",
    "        resp = resp.replace(\"'\", '\"')\n",
    "\n",
    "        resp = json.dumps(resp)\n",
    "        resp = json.loads(resp)\n",
    "        \n",
    "        df_response = pd.read_json(resp)\n",
    "        df_response = df_response[['name', 'distance', 'location']]\n",
    "                \n",
    "        df_trip_optimized = pd.merge(df_trip_for_meili, df_response, left_index=True, right_index=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>lon</th>\n",
       "      <th>lat</th>\n",
       "      <th>time</th>\n",
       "      <th>name</th>\n",
       "      <th>distance</th>\n",
       "      <th>location</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>81</th>\n",
       "      <td>67.147783</td>\n",
       "      <td>54.669725</td>\n",
       "      <td>2020-07-31 18:52:49</td>\n",
       "      <td></td>\n",
       "      <td>17.835</td>\n",
       "      <td>[67.153885, 54.591877]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>80</th>\n",
       "      <td>67.147004</td>\n",
       "      <td>54.669144</td>\n",
       "      <td>2020-07-31 18:52:53</td>\n",
       "      <td></td>\n",
       "      <td>7.219</td>\n",
       "      <td>[67.154449, 54.591881]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>79</th>\n",
       "      <td>67.147285</td>\n",
       "      <td>54.669464</td>\n",
       "      <td>2020-07-31 18:52:55</td>\n",
       "      <td></td>\n",
       "      <td>35.674</td>\n",
       "      <td>[67.15461, 54.591881]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>78</th>\n",
       "      <td>67.147856</td>\n",
       "      <td>54.670005</td>\n",
       "      <td>2020-07-31 18:52:57</td>\n",
       "      <td></td>\n",
       "      <td>17.842</td>\n",
       "      <td>[67.15715, 54.591915]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>77</th>\n",
       "      <td>67.147519</td>\n",
       "      <td>54.669461</td>\n",
       "      <td>2020-07-31 18:53:02</td>\n",
       "      <td></td>\n",
       "      <td>7.660</td>\n",
       "      <td>[67.158714, 54.591949]</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "          lon        lat                time name  distance  \\\n",
       "81  67.147783  54.669725 2020-07-31 18:52:49         17.835   \n",
       "80  67.147004  54.669144 2020-07-31 18:52:53          7.219   \n",
       "79  67.147285  54.669464 2020-07-31 18:52:55         35.674   \n",
       "78  67.147856  54.670005 2020-07-31 18:52:57         17.842   \n",
       "77  67.147519  54.669461 2020-07-31 18:53:02          7.660   \n",
       "\n",
       "                  location  \n",
       "81  [67.153885, 54.591877]  \n",
       "80  [67.154449, 54.591881]  \n",
       "79   [67.15461, 54.591881]  \n",
       "78   [67.15715, 54.591915]  \n",
       "77  [67.158714, 54.591949]  "
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_trip_optimized.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Parsing the location column of Meili's response\n",
    "\n",
    "for i, row in df_trip_optimized.iterrows():\n",
    "        df_trip_optimized.at[i, 'longitude'] = df_trip_optimized.at[i,'location'][0]\n",
    "        df_trip_optimized.at[i, 'latitude'] = df_trip_optimized.at[i,'location'][1]\n",
    "        \n",
    "        #saving all the geopoints that couldn't be snapped to road\n",
    "        if df_trip_optimized.at[i, 'longitude'] == 0.0:\n",
    "            df_trip_optimized.at[i, 'longitude'] = df_trip_optimized.at[i,'lon']\n",
    "            df_trip_optimized.at[i, 'latitude'] = df_trip_optimized.at[i,'lat']\n",
    "            \n",
    "df_trip_optimized = df_trip_optimized.drop(['location', 'lon', 'lat'], 1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>time</th>\n",
       "      <th>name</th>\n",
       "      <th>distance</th>\n",
       "      <th>longitude</th>\n",
       "      <th>latitude</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>81</th>\n",
       "      <td>2020-07-31 18:52:49</td>\n",
       "      <td></td>\n",
       "      <td>17.835</td>\n",
       "      <td>67.153885</td>\n",
       "      <td>54.591877</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>80</th>\n",
       "      <td>2020-07-31 18:52:53</td>\n",
       "      <td></td>\n",
       "      <td>7.219</td>\n",
       "      <td>67.154449</td>\n",
       "      <td>54.591881</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>79</th>\n",
       "      <td>2020-07-31 18:52:55</td>\n",
       "      <td></td>\n",
       "      <td>35.674</td>\n",
       "      <td>67.154610</td>\n",
       "      <td>54.591881</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>78</th>\n",
       "      <td>2020-07-31 18:52:57</td>\n",
       "      <td></td>\n",
       "      <td>17.842</td>\n",
       "      <td>67.157150</td>\n",
       "      <td>54.591915</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>77</th>\n",
       "      <td>2020-07-31 18:53:02</td>\n",
       "      <td></td>\n",
       "      <td>7.660</td>\n",
       "      <td>67.158714</td>\n",
       "      <td>54.591949</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                  time name  distance  longitude   latitude\n",
       "81 2020-07-31 18:52:49         17.835  67.153885  54.591877\n",
       "80 2020-07-31 18:52:53          7.219  67.154449  54.591881\n",
       "79 2020-07-31 18:52:55         35.674  67.154610  54.591881\n",
       "78 2020-07-31 18:52:57         17.842  67.157150  54.591915\n",
       "77 2020-07-31 18:53:02          7.660  67.158714  54.591949"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_trip_optimized.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_trip_optimized['time'] = df_trip_optimized['time'].astype(str)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "User Guide: https://docs.kepler.gl/docs/keplergl-jupyter\n",
      "Map saved to Optimized.html!\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "cf455f6573b74e85b5059e9086fe0b29",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "KeplerGl(data={'Optimized.html': {'index': [81, 80, 79, 78, 77, 76, 75, 74, 73, 72, 71, 70, 69, 68, 67, 66, 65…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# Plotting the data on the map and saving it to the html file\n",
    "\n",
    "title = 'Optimized.html'\n",
    "\n",
    "map = KeplerGl(height=600, width=600)\n",
    "# add data to keplergl map\n",
    "map.add_data(data=df_trip_optimized, name=title)\n",
    "map.save_to_html(file_name=title)\n",
    "map"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.1"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}