scripts/standard-deviation-toponym-resolution-ambiguity.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "ce1797a9",
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import os"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "2f592cee",
   "metadata": {},
   "outputs": [],
   "source": [
    "dir_geoparsed_results = os.path.dirname(os.getcwd())"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "4191c7b5",
   "metadata": {},
   "source": [
    "## Calculate Standard Deviation of MdnED (Produced by Edinburgh Geoparser) for Top 10 Highly Ambiguous Toponym in WikToR"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "b0dc0f45",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>name</th>\n",
       "      <th>wikipedia_name</th>\n",
       "      <th>lat</th>\n",
       "      <th>lon</th>\n",
       "      <th>page</th>\n",
       "      <th>wiktor_key</th>\n",
       "      <th>toponym</th>\n",
       "      <th>old_coord</th>\n",
       "      <th>new_coord</th>\n",
       "      <th>mlg_key</th>\n",
       "      <th>geocoded_coordinates_list</th>\n",
       "      <th>median_error_distance</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Santa Maria</td>\n",
       "      <td>Santa Maria, California</td>\n",
       "      <td>34.9514</td>\n",
       "      <td>-120.4333</td>\n",
       "      <td>http://en.wikipedia.org/wiki/Santa_Maria%2C_Ca...</td>\n",
       "      <td>Santa Maria, California 34.9514,-120.4333</td>\n",
       "      <td>Santa Maria, California</td>\n",
       "      <td>34.9514 , -120.4333</td>\n",
       "      <td>34.951388888889 , -120.43333333333</td>\n",
       "      <td>Santa Maria, California 34.9514,-120.4333</td>\n",
       "      <td>[(34.95303, -120.43572), (34.95303, -120.43572...</td>\n",
       "      <td>0.283927</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Santa Maria</td>\n",
       "      <td>Santa Maria, Bulacan</td>\n",
       "      <td>14.8208</td>\n",
       "      <td>120.9636</td>\n",
       "      <td>http://en.wikipedia.org/wiki/Santa_Maria%2C_Bu...</td>\n",
       "      <td>Santa Maria, Bulacan 14.8208,120.9636</td>\n",
       "      <td>Santa Maria, Bulacan</td>\n",
       "      <td>14.8208 , 120.9636</td>\n",
       "      <td>14.818333333333 , 120.95633333333</td>\n",
       "      <td>Santa Maria, Bulacan 14.8208,120.9636</td>\n",
       "      <td>[(14.83333, 121.0), (14.83333, 121.0), (14.833...</td>\n",
       "      <td>4.981280</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Santa Maria</td>\n",
       "      <td>Santa Maria, Ilocos Sur</td>\n",
       "      <td>17.3585</td>\n",
       "      <td>120.4981</td>\n",
       "      <td>http://en.wikipedia.org/wiki/Santa_Maria%2C_Il...</td>\n",
       "      <td>Santa Maria, Ilocos Sur 17.3585,120.4981</td>\n",
       "      <td>Santa Maria, Ilocos Sur</td>\n",
       "      <td>17.3585 , 120.4981</td>\n",
       "      <td>17.366666666667 , 120.48333333333</td>\n",
       "      <td>Santa Maria, Ilocos Sur 17.3585,120.4981</td>\n",
       "      <td>[(16.9518, 120.4445), (16.9518, 120.4445), (16...</td>\n",
       "      <td>46.315270</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Santa Maria</td>\n",
       "      <td>Santa Maria, Romblon</td>\n",
       "      <td>12.4417</td>\n",
       "      <td>122.1042</td>\n",
       "      <td>http://en.wikipedia.org/wiki/Santa_Maria%2C_Ro...</td>\n",
       "      <td>Santa Maria, Romblon 12.4417,122.1042</td>\n",
       "      <td>Santa Maria, Romblon</td>\n",
       "      <td>12.4417 , 122.1042</td>\n",
       "      <td>12.441667 , 122.104167</td>\n",
       "      <td>Santa Maria, Romblon 12.4417,122.1042</td>\n",
       "      <td>[]</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Santa Cruz</td>\n",
       "      <td>Santa Cruz, Chile</td>\n",
       "      <td>-34.6372</td>\n",
       "      <td>-71.3631</td>\n",
       "      <td>http://en.wikipedia.org/wiki/Santa_Cruz%2C_Chile</td>\n",
       "      <td>Santa Cruz, Chile -34.6372,-71.3631</td>\n",
       "      <td>Santa Cruz, Chile</td>\n",
       "      <td>-34.6372 , -71.3631</td>\n",
       "      <td>-34.633333333333 , -71.366666666667</td>\n",
       "      <td>Santa Cruz, Chile -34.6372,-71.3631</td>\n",
       "      <td>[]</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "          name           wikipedia_name      lat       lon  \\\n",
       "0  Santa Maria  Santa Maria, California  34.9514 -120.4333   \n",
       "1  Santa Maria     Santa Maria, Bulacan  14.8208  120.9636   \n",
       "2  Santa Maria  Santa Maria, Ilocos Sur  17.3585  120.4981   \n",
       "3  Santa Maria     Santa Maria, Romblon  12.4417  122.1042   \n",
       "4   Santa Cruz        Santa Cruz, Chile -34.6372  -71.3631   \n",
       "\n",
       "                                                page  \\\n",
       "0  http://en.wikipedia.org/wiki/Santa_Maria%2C_Ca...   \n",
       "1  http://en.wikipedia.org/wiki/Santa_Maria%2C_Bu...   \n",
       "2  http://en.wikipedia.org/wiki/Santa_Maria%2C_Il...   \n",
       "3  http://en.wikipedia.org/wiki/Santa_Maria%2C_Ro...   \n",
       "4   http://en.wikipedia.org/wiki/Santa_Cruz%2C_Chile   \n",
       "\n",
       "                                  wiktor_key                   toponym  \\\n",
       "0  Santa Maria, California 34.9514,-120.4333  Santa Maria, California    \n",
       "1      Santa Maria, Bulacan 14.8208,120.9636     Santa Maria, Bulacan    \n",
       "2   Santa Maria, Ilocos Sur 17.3585,120.4981  Santa Maria, Ilocos Sur    \n",
       "3      Santa Maria, Romblon 12.4417,122.1042     Santa Maria, Romblon    \n",
       "4        Santa Cruz, Chile -34.6372,-71.3631        Santa Cruz, Chile    \n",
       "\n",
       "               old_coord                             new_coord  \\\n",
       "0   34.9514 , -120.4333     34.951388888889 , -120.43333333333   \n",
       "1    14.8208 , 120.9636      14.818333333333 , 120.95633333333   \n",
       "2    17.3585 , 120.4981      17.366666666667 , 120.48333333333   \n",
       "3    12.4417 , 122.1042                 12.441667 , 122.104167   \n",
       "4   -34.6372 , -71.3631    -34.633333333333 , -71.366666666667   \n",
       "\n",
       "                                     mlg_key  \\\n",
       "0  Santa Maria, California 34.9514,-120.4333   \n",
       "1      Santa Maria, Bulacan 14.8208,120.9636   \n",
       "2   Santa Maria, Ilocos Sur 17.3585,120.4981   \n",
       "3      Santa Maria, Romblon 12.4417,122.1042   \n",
       "4        Santa Cruz, Chile -34.6372,-71.3631   \n",
       "\n",
       "                           geocoded_coordinates_list  median_error_distance  \n",
       "0  [(34.95303, -120.43572), (34.95303, -120.43572...               0.283927  \n",
       "1  [(14.83333, 121.0), (14.83333, 121.0), (14.833...               4.981280  \n",
       "2  [(16.9518, 120.4445), (16.9518, 120.4445), (16...              46.315270  \n",
       "3                                                 []                    NaN  \n",
       "4                                                 []                    NaN  "
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_wiktor_result = pd.read_csv(dir_geoparsed_results+'/geoparsed-results/wiktor_geocoded_results_Edinburgh_Geoparser.csv')\n",
    "del df_wiktor_result['Unnamed: 0']\n",
    "df_wiktor_result.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "b5d24831",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "name\n",
       "Washington County    25\n",
       "Clinton              23\n",
       "Springfield          22\n",
       "Greenville           22\n",
       "Kingston             21\n",
       "Georgetown           21\n",
       "Hamilton             20\n",
       "Jefferson County     20\n",
       "Newport              20\n",
       "Franklin County      20\n",
       "Richmond             18\n",
       "Lexington            18\n",
       "Clifton              18\n",
       "Milton               18\n",
       "Burlington           18\n",
       "Franklin             18\n",
       "Jackson County       17\n",
       "Milford              17\n",
       "Weston               16\n",
       "Windsor              16\n",
       "dtype: int64"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_wiktor_result.groupby(['name']).size().sort_values(ascending=False).head(20)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "56cd1047",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Washington County      median_error_distance\n",
      "std             807.704385\n",
      "Clinton      median_error_distance\n",
      "std                    NaN\n",
      "Springfield      median_error_distance\n",
      "std                    NaN\n",
      "Greenville      median_error_distance\n",
      "std             407.517261\n",
      "Kingston      median_error_distance\n",
      "std             680.481725\n",
      "Georgetown      median_error_distance\n",
      "std                    NaN\n",
      "Hamilton      median_error_distance\n",
      "std            4105.574962\n",
      "Jefferson County      median_error_distance\n",
      "std                    NaN\n",
      "Newport      median_error_distance\n",
      "std            5705.194849\n",
      "Franklin County      median_error_distance\n",
      "std                    NaN\n"
     ]
    }
   ],
   "source": [
    "for name in df_wiktor_result.groupby(['name']).size().sort_values(ascending=False).head(10).index:\n",
    "    print(name, df_wiktor_result[['name','median_error_distance']].groupby(['name']).get_group(name).agg({'median_error_distance':['std']}))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "6c933e07",
   "metadata": {},
   "source": [
    "## Calculate Standard Deviation of MdnED (Produced by CamCoder) for Top 10 Highly Ambiguous Toponym in WikToR"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "d752d6f0",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>name</th>\n",
       "      <th>wikipedia_name</th>\n",
       "      <th>lat</th>\n",
       "      <th>lon</th>\n",
       "      <th>page</th>\n",
       "      <th>wiktor_key</th>\n",
       "      <th>toponym</th>\n",
       "      <th>old_coord</th>\n",
       "      <th>new_coord</th>\n",
       "      <th>mlg_key</th>\n",
       "      <th>geocoded_coordinates_list</th>\n",
       "      <th>median_error_distance</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Santa Maria</td>\n",
       "      <td>Santa Maria, California</td>\n",
       "      <td>34.9514</td>\n",
       "      <td>-120.4333</td>\n",
       "      <td>http://en.wikipedia.org/wiki/Santa_Maria%2C_Ca...</td>\n",
       "      <td>Santa Maria, California 34.9514,-120.4333</td>\n",
       "      <td>Santa Maria, California</td>\n",
       "      <td>34.9514 , -120.4333</td>\n",
       "      <td>34.951388888889 , -120.43333333333</td>\n",
       "      <td>Santa Maria, California 34.9514,-120.4333</td>\n",
       "      <td>[]</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Santa Maria</td>\n",
       "      <td>Santa Maria, Bulacan</td>\n",
       "      <td>14.8208</td>\n",
       "      <td>120.9636</td>\n",
       "      <td>http://en.wikipedia.org/wiki/Santa_Maria%2C_Bu...</td>\n",
       "      <td>Santa Maria, Bulacan 14.8208,120.9636</td>\n",
       "      <td>Santa Maria, Bulacan</td>\n",
       "      <td>14.8208 , 120.9636</td>\n",
       "      <td>14.818333333333 , 120.95633333333</td>\n",
       "      <td>Santa Maria, Bulacan 14.8208,120.9636</td>\n",
       "      <td>[(16.15687, 119.92664)]</td>\n",
       "      <td>185.275483</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Santa Maria</td>\n",
       "      <td>Santa Maria, Ilocos Sur</td>\n",
       "      <td>17.3585</td>\n",
       "      <td>120.4981</td>\n",
       "      <td>http://en.wikipedia.org/wiki/Santa_Maria%2C_Il...</td>\n",
       "      <td>Santa Maria, Ilocos Sur 17.3585,120.4981</td>\n",
       "      <td>Santa Maria, Ilocos Sur</td>\n",
       "      <td>17.3585 , 120.4981</td>\n",
       "      <td>17.366666666667 , 120.48333333333</td>\n",
       "      <td>Santa Maria, Ilocos Sur 17.3585,120.4981</td>\n",
       "      <td>[(17.3906, 120.4587), (17.3906, 120.4587), (17...</td>\n",
       "      <td>3.730377</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Santa Maria</td>\n",
       "      <td>Santa Maria, Romblon</td>\n",
       "      <td>12.4417</td>\n",
       "      <td>122.1042</td>\n",
       "      <td>http://en.wikipedia.org/wiki/Santa_Maria%2C_Ro...</td>\n",
       "      <td>Santa Maria, Romblon 12.4417,122.1042</td>\n",
       "      <td>Santa Maria, Romblon</td>\n",
       "      <td>12.4417 , 122.1042</td>\n",
       "      <td>12.441667 , 122.104167</td>\n",
       "      <td>Santa Maria, Romblon 12.4417,122.1042</td>\n",
       "      <td>[]</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Santa Cruz</td>\n",
       "      <td>Santa Cruz, Chile</td>\n",
       "      <td>-34.6372</td>\n",
       "      <td>-71.3631</td>\n",
       "      <td>http://en.wikipedia.org/wiki/Santa_Cruz%2C_Chile</td>\n",
       "      <td>Santa Cruz, Chile -34.6372,-71.3631</td>\n",
       "      <td>Santa Cruz, Chile</td>\n",
       "      <td>-34.6372 , -71.3631</td>\n",
       "      <td>-34.633333333333 , -71.366666666667</td>\n",
       "      <td>Santa Cruz, Chile -34.6372,-71.3631</td>\n",
       "      <td>[(-34.63881, -71.36576), (-34.63881, -71.36576...</td>\n",
       "      <td>0.614602</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "          name           wikipedia_name      lat       lon  \\\n",
       "0  Santa Maria  Santa Maria, California  34.9514 -120.4333   \n",
       "1  Santa Maria     Santa Maria, Bulacan  14.8208  120.9636   \n",
       "2  Santa Maria  Santa Maria, Ilocos Sur  17.3585  120.4981   \n",
       "3  Santa Maria     Santa Maria, Romblon  12.4417  122.1042   \n",
       "4   Santa Cruz        Santa Cruz, Chile -34.6372  -71.3631   \n",
       "\n",
       "                                                page  \\\n",
       "0  http://en.wikipedia.org/wiki/Santa_Maria%2C_Ca...   \n",
       "1  http://en.wikipedia.org/wiki/Santa_Maria%2C_Bu...   \n",
       "2  http://en.wikipedia.org/wiki/Santa_Maria%2C_Il...   \n",
       "3  http://en.wikipedia.org/wiki/Santa_Maria%2C_Ro...   \n",
       "4   http://en.wikipedia.org/wiki/Santa_Cruz%2C_Chile   \n",
       "\n",
       "                                  wiktor_key                   toponym  \\\n",
       "0  Santa Maria, California 34.9514,-120.4333  Santa Maria, California    \n",
       "1      Santa Maria, Bulacan 14.8208,120.9636     Santa Maria, Bulacan    \n",
       "2   Santa Maria, Ilocos Sur 17.3585,120.4981  Santa Maria, Ilocos Sur    \n",
       "3      Santa Maria, Romblon 12.4417,122.1042     Santa Maria, Romblon    \n",
       "4        Santa Cruz, Chile -34.6372,-71.3631        Santa Cruz, Chile    \n",
       "\n",
       "               old_coord                             new_coord  \\\n",
       "0   34.9514 , -120.4333     34.951388888889 , -120.43333333333   \n",
       "1    14.8208 , 120.9636      14.818333333333 , 120.95633333333   \n",
       "2    17.3585 , 120.4981      17.366666666667 , 120.48333333333   \n",
       "3    12.4417 , 122.1042                 12.441667 , 122.104167   \n",
       "4   -34.6372 , -71.3631    -34.633333333333 , -71.366666666667   \n",
       "\n",
       "                                     mlg_key  \\\n",
       "0  Santa Maria, California 34.9514,-120.4333   \n",
       "1      Santa Maria, Bulacan 14.8208,120.9636   \n",
       "2   Santa Maria, Ilocos Sur 17.3585,120.4981   \n",
       "3      Santa Maria, Romblon 12.4417,122.1042   \n",
       "4        Santa Cruz, Chile -34.6372,-71.3631   \n",
       "\n",
       "                           geocoded_coordinates_list  median_error_distance  \n",
       "0                                                 []                    NaN  \n",
       "1                            [(16.15687, 119.92664)]             185.275483  \n",
       "2  [(17.3906, 120.4587), (17.3906, 120.4587), (17...               3.730377  \n",
       "3                                                 []                    NaN  \n",
       "4  [(-34.63881, -71.36576), (-34.63881, -71.36576...               0.614602  "
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_wiktor_result = pd.read_csv(dir_geoparsed_results+'/geoparsed-results/wiktor_geocoded_results_CamCoder.csv')\n",
    "del df_wiktor_result['Unnamed: 0']\n",
    "df_wiktor_result.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "8183360a",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "name\n",
       "Washington County    25\n",
       "Clinton              23\n",
       "Springfield          22\n",
       "Greenville           22\n",
       "Kingston             21\n",
       "Georgetown           21\n",
       "Hamilton             20\n",
       "Jefferson County     20\n",
       "Newport              20\n",
       "Franklin County      20\n",
       "Richmond             18\n",
       "Lexington            18\n",
       "Clifton              18\n",
       "Milton               18\n",
       "Burlington           18\n",
       "Franklin             18\n",
       "Jackson County       17\n",
       "Milford              17\n",
       "Weston               16\n",
       "Windsor              16\n",
       "dtype: int64"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_wiktor_result.groupby(['name']).size().sort_values(ascending=False).head(20)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "2d1c121b",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Washington County      median_error_distance\n",
      "std             692.177184\n",
      "Clinton      median_error_distance\n",
      "std              539.43373\n",
      "Springfield      median_error_distance\n",
      "std            2770.634287\n",
      "Greenville      median_error_distance\n",
      "std             514.681684\n",
      "Kingston      median_error_distance\n",
      "std             226.447057\n",
      "Georgetown      median_error_distance\n",
      "std            2742.785136\n",
      "Hamilton      median_error_distance\n",
      "std             622.237286\n",
      "Jefferson County      median_error_distance\n",
      "std             384.380587\n",
      "Newport      median_error_distance\n",
      "std             145.050226\n",
      "Franklin County      median_error_distance\n",
      "std             414.985655\n"
     ]
    }
   ],
   "source": [
    "for name in df_wiktor_result.groupby(['name']).size().sort_values(ascending=False).head(10).index:\n",
    "    print(name, df_wiktor_result[['name','median_error_distance']].groupby(['name']).get_group(name).agg({'median_error_distance':['std']}))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "ee728f03",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.13"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}