d_matrix_building.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Code for constructing the distance matrix as described in eq. (3.1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 346
    },
    "id": "AUv90Ejml1SS",
    "outputId": "c5db59f5-a8f3-49ec-ea07-f4e286892bc6"
   },
   "outputs": [],
   "source": [
    "## Modules\n",
    "\n",
    "import geopandas as gpd\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "\n",
    "import json\n",
    "import urllib.request\n",
    "import geopandas as gpd\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "import geopy.distance"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "id": "LAUFIeGGl4aI"
   },
   "outputs": [],
   "source": [
    "## Utils\n",
    "over_18_pop_by_zip = pd.read_csv(r\"C:\\Users\\luosk\\Dropbox\\Research Stuff\\coverage project\\Full US Data\\over_18_pop_data.csv\", index_col = 0)\n",
    "cars_by_zip = pd.read_csv(r\"C:\\Users\\luosk\\Dropbox\\Research Stuff\\coverage project\\Full US Data\\full_us_car_data.csv\", index_col = 0)\n",
    "\n",
    "#get population for zip code\n",
    "def get_pop(zipcode):\n",
    "    foo = 'ZCTA5 '+zipcode\n",
    "    return int(over_18_pop_by_zip.loc[foo])\n",
    "\n",
    "#get car ownership data for zip code\n",
    "def get_cars(zipcode):\n",
    "    foo = 'ZCTA5 '+zipcode\n",
    "    return int(cars_by_zip.loc[foo])    \n",
    "\n",
    "#get zip code of polling site\n",
    "def get_zip(site, polls):\n",
    "    return polls.loc[site]['address.zip']\n",
    "\n",
    "#get fraction of cars/population\n",
    "def get_car_prop(site, polls):\n",
    "    zipcode = get_zip(site, polls)\n",
    "    pop = get_pop(zipcode)\n",
    "    cars = get_cars(zipcode)\n",
    "    if pop == 0:\n",
    "        return 0\n",
    "    else:\n",
    "        return min(cars/pop, 1)\n",
    "\n",
    "#construction of \\tilde{d} \n",
    "def calc_dtilde_matrix(polls, t_car, t_pub, t_walk):\n",
    "    N = polls.shape[0]\n",
    "    dtilde_matrix = np.zeros((N,N))\n",
    "    for i, a in polls.iterrows():\n",
    "        for j, b in polls.iterrows():\n",
    "            if i != j:\n",
    "                walk = t_walk[i,j] + t_walk[j,i]\n",
    "                pub = t_pub[i,j] + t_pub[j,i]\n",
    "                car = t_car[i,j] + t_car[j,i]\n",
    "                driver_min = min(walk, pub, car)\n",
    "                non_driver_min = min(walk, pub)\n",
    "                car_prop = get_car_prop(i, polls)\n",
    "                dtilde_matrix[i,j] = car_prop*driver_min + (1-car_prop)*non_driver_min\n",
    "    return dtilde_matrix\n",
    "\n",
    "#construction of d (as in eq. (3.1))\n",
    "def calc_d_matrix(dtilde_matrix, polls):\n",
    "    N = polls.shape[0]\n",
    "    d_matrix = np.zeros((N,N))\n",
    "    for i, a in polls.iterrows():\n",
    "        for j, b in polls.iterrows():\n",
    "            if i != j:\n",
    "                zipi = get_zip(i, polls)\n",
    "                zipj = get_zip(j, polls)\n",
    "                popi = get_pop(zipi)\n",
    "                popj = get_pop(zipj)\n",
    "                if popi == 0 and popj == 0:\n",
    "                    d_matrix[i,j] = 0\n",
    "                elif popi == 0:\n",
    "                    d_matrix[i,j] = dtilde_matrix[j,i]\n",
    "                elif popj == 0:\n",
    "                    d_matrix[i,j] = dtilde_matrix[i,j]\n",
    "                else:\n",
    "                    d_matrix[i,j] = (1/(popi+popj))*(popi*dtilde_matrix[i,j] + popj*dtilde_matrix[j,i])\n",
    "    return d_matrix"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Note: Some of the zip codes had to be manually adjusted, as described in the paper. "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "Wc6LqlS0moe0"
   },
   "source": [
    "SLC"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "id": "Ux-dp2DRmN_2"
   },
   "outputs": [],
   "source": [
    "slc_sites = gpd.read_file('Salt Lake City\\slc_polls.geojson')\n",
    "slc_sites.at[10,'address.zip'] = '84115'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "id": "NL9gxx61mt-w"
   },
   "outputs": [],
   "source": [
    "slc_tcar = np.load(\"Salt Lake City\\slc_tcar_matrix_completed.npy\")\n",
    "slc_tpub = np.load(\"Salt Lake City\\slc_tpub_matrix_completed.npy\")\n",
    "slc_twalk = np.load(\"walk_matrix_complete\\slc_walk.npy\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "id": "8l-H5ZNPm5bv"
   },
   "outputs": [],
   "source": [
    "slc_dtilde = calc_dtilde_matrix(slc_sites, slc_tcar, slc_tpub, slc_twalk)\n",
    "slc_d_matrix = calc_d_matrix(slc_dtilde, slc_sites)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "np.save(\"slc_d_matrix_updated.npy\", slc_d_matrix) "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Atlanta"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "atl_sites = gpd.read_file('Atlanta/atl_polls.geojson')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "atl_tcar = np.load(\"Atlanta/atl_tcar_matrix_completed.npy\")\n",
    "atl_tpub = np.load(\"Atlanta/atl_tpub_matrix_completed.npy\")\n",
    "atl_twalk = np.load(\"walk_matrix_complete/atl_walk.npy\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "atl_sites['address.zip'] = atl_sites['address.zip'].astype(float)\n",
    "atl_sites['address.zip'] = atl_sites['address.zip'].astype(int)\n",
    "atl_sites['address.zip'] = atl_sites['address.zip'].astype(str)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "atl_sites.at[16, 'address.zip'] = '30312'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "atl_dtilde = calc_dtilde_matrix(atl_sites, atl_tcar, atl_tpub, atl_twalk)\n",
    "atl_d_matrix = calc_d_matrix(atl_dtilde, atl_sites)\n",
    "np.save(\"atl_d_matrix_updated.npy\", atl_d_matrix) "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "LAC"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "lac_sites = gpd.read_file('Los Angeles/fixed_files/lac_NoCatalina_polls.geojson')\n",
    "for i,a in lac_sites.iterrows():\n",
    "    if a['address.zip'] == '90095':\n",
    "        lac_sites.at[i, 'address.zip'] = '90024'\n",
    "    elif a['address.zip'] == '90506':\n",
    "        lac_sites.at[i, 'address.zip'] = '90249'\n",
    "    elif a['address.zip'] == '91371':\n",
    "        lac_sites.at[i, 'address.zip'] = '91367'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "lac_tcar = np.load(\"Los Angeles/fixed_files/lac_tcar_matrix_completed.npy\")\n",
    "lac_tpub = np.load(\"Los Angeles/fixed_files/lac_tpub_matrix_completed.npy\")\n",
    "lac_twalk = np.load(\"walk_matrix_complete\\lac_walk.npy\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "lac_dtilde = calc_dtilde_matrix(lac_sites, lac_tcar, lac_tpub, lac_twalk)\n",
    "lac_d_matrix = calc_d_matrix(lac_dtilde, lac_sites)\n",
    "np.save(\"lac_d_matrix_updated.npy\", lac_d_matrix) "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Jacksonville"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "jax_sites = gpd.read_file('Jacksonville/jax_polls.geojson')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "jax_tcar = np.load(\"Jacksonville/jax_tcar_matrix_completed.npy\")\n",
    "jax_tpub = np.load(\"Jacksonville/jax_tpub_matrix_completed.npy\")\n",
    "jax_twalk = np.load(\"walk_matrix_complete\\jax_walk.npy\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "jax_dtilde = calc_dtilde_matrix(jax_sites, jax_tcar, jax_tpub, jax_twalk)\n",
    "jax_d_matrix = calc_d_matrix(jax_dtilde, jax_sites)\n",
    "np.save(\"jax_d_matrix_updated.npy\", jax_d_matrix) "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Chicago"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [],
   "source": [
    "chc_sites = gpd.read_file('Chicago/chc_polls.geojson')\n",
    "chc_sites.at[397, 'address.zip'] = '60656'\n",
    "chc_sites.at[453, 'address.zip'] = '60827'\n",
    "\n",
    "chc_tcar = np.load(\"Chicago/chc_tcar_matrix_completed.npy\")\n",
    "chc_tpub = np.load(\"Chicago/chc_tpub_matrix_completed.npy\")\n",
    "chc_twalk = np.load(\"walk_matrix_complete\\chc_walk.npy\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [],
   "source": [
    "chc_dtilde = calc_dtilde_matrix(chc_sites, chc_tcar, chc_tpub, chc_twalk)\n",
    "chc_d_matrix = calc_d_matrix(chc_dtilde, chc_sites)\n",
    "np.save(\"chc_d_matrix_updated.npy\", chc_d_matrix) "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "NYC"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [],
   "source": [
    "manbronx_sites = gpd.read_file('NYC/manbronx_polls.geojson')\n",
    "for i,a in manbronx_sites.iterrows():\n",
    "    if a['address.zip'] == '10103':\n",
    "        manbronx_sites.at[i, 'address.zip'] = '10019'\n",
    "\n",
    "manbronx_tcar = np.load(\"NYC/manbronx_tcar_matrix_completed.npy\")\n",
    "manbronx_tpub = np.load(\"NYC/manbronx_tpub_matrix_completed.npy\")\n",
    "manbronx_twalk = np.load(\"walk_matrix_complete\\manbronx_walk.npy\")\n",
    "\n",
    "manbronx_dtilde = calc_dtilde_matrix(manbronx_sites, manbronx_tcar, manbronx_tpub, manbronx_twalk)\n",
    "manbronx_d_matrix = calc_d_matrix(manbronx_dtilde, manbronx_sites)\n",
    "np.save(\"manbronx_d_matrix_updated.npy\", manbronx_d_matrix) "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "queensbrook_sites = gpd.read_file('NYC/queensbrook_polls.geojson')\n",
    "\n",
    "queensbrook_tcar = np.load(\"NYC/queensbrook_tcar_matrix_completed.npy\")\n",
    "queensbrook_tpub = np.load(\"NYC/queensbrook_tpub_matrix_completed.npy\")\n",
    "queensbrook_twalk = np.load(\"walk_matrix_complete\\queensbrook_walk.npy\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [],
   "source": [
    "queensbrook_dtilde = calc_dtilde_matrix(queensbrook_sites, queensbrook_tcar, queensbrook_tpub, queensbrook_twalk,)\n",
    "queensbrook_d_matrix = calc_d_matrix(queensbrook_dtilde, queensbrook_sites)\n",
    "np.save(\"queensbrook_d_matrix_updated.npy\", queensbrook_d_matrix) "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [],
   "source": [
    "stat_sites = gpd.read_file('NYC/stat_polls.geojson')\n",
    "\n",
    "stat_tcar = np.load(\"NYC/stat_tcar_matrix_completed.npy\")\n",
    "stat_tpub = np.load(\"NYC/stat_tpub_matrix_completed.npy\")\n",
    "stat_twalk = np.load(\"walk_matrix_complete\\stat_walk.npy\")\n",
    "\n",
    "stat_dtilde = calc_dtilde_matrix(stat_sites, stat_tcar, stat_tpub, stat_twalk)\n",
    "stat_d_matrix = calc_d_matrix(stat_dtilde, stat_sites)\n",
    "np.save(\"stat_d_matrix_updated.npy\", stat_d_matrix) "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "np.save(\"slc_twalk_matrix_completed.npy\", slc_twalk)\n",
    "np.save(\"atl_twalk_matrix_completed.npy\", atl_twalk)\n",
    "np.save(\"lac_twalk_matrix_completed.npy\", lac_twalk)\n",
    "np.save(\"chc_twalk_matrix_completed.npy\", chc_twalk)\n",
    "np.save(\"jax_twalk_matrix_completed.npy\", jax_twalk)\n",
    "np.save(\"queensbrook_twalk_matrix_completed.npy\", queensbrook_twalk)\n",
    "np.save(\"stat_twalk_matrix_completed.npy\", stat_twalk)\n",
    "np.save(\"manbronx_twalk_matrix_completed.npy\", manbronx_twalk)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "colab": {
   "collapsed_sections": [],
   "name": "d_matrix_building.ipynb",
   "provenance": []
  },
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.13"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}