pandasgo_joins_to_find_within_data.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import geojson\n",
    "import pandas as pd\n",
    "import geopandas\n",
    "import matplotlib.pyplot as plt\n",
    "import numpy as np \n",
    "from pandas import DataFrame\n",
    "import geopandas as gpd\n",
    "from shapely.geometry import Point, Polygon\n",
    "%matplotlib inline\n",
    "pd.set_option('display.max_colwidth', -1)\n",
    "from IPython.display import clear_output\n",
    "clear_output()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "def venue2geojson(df, filename):\n",
    "    features = []\n",
    "    insert_features = lambda X: features.append(\n",
    "            geojson.Feature(geometry=geojson.Point((X[\"longitude\"],\n",
    "                                                    X[\"latitude\"])),\n",
    "                            properties=dict(venueId=X[\"venueId\"],\n",
    "                                            venueCategory=X[\"venueCategory\"],\n",
    "                                            venueCategoryId=X[\"venueCategoryId\"],\n",
    "                                            timezoneOffset=X[\"timezoneOffset\"],\n",
    "                                            utcTimestamp=X[\"utcTimestamp\"],\n",
    "                                            userId=X[\"userId\"])))\n",
    "    df.apply(insert_features, axis=1)\n",
    "    with open(filename, 'w', encoding='utf8') as fp:\n",
    "        geojson.dump(geojson.FeatureCollection(features), fp, sort_keys=True, ensure_ascii=False)\n",
    "\n",
    "        \n",
    "def screen2geojson(df, filename):\n",
    "    features = []\n",
    "    insert_features = lambda X: features.append(\n",
    "            geojson.Feature(geometry=geojson.Point((X[\"longitude\"],\n",
    "                                                    X[\"latitude\"])),\n",
    "                            properties=dict(id=X[\"id\"],\n",
    "                                            venue=X[\"venue\"],\n",
    "                                            City=X[\"City\"])))\n",
    "    df.apply(insert_features, axis=1)\n",
    "    with open(filename, 'w', encoding='utf8') as fp:\n",
    "        geojson.dump(geojson.FeatureCollection(features), fp, sort_keys=True, ensure_ascii=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "user_visit_data_point = pd.read_csv('/folder_name/input_data.csv', engine='python')\n",
    "user_visit_data_point = user_visit_data_point.head(200)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "user_visit_data_point_geo = geopandas.GeoDataFrame(user_visit_data_point, geometry=gpd.points_from_xy(user_visit_data_point.longitude,\n",
    "                                                                                user_visit_data_point.latitude))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [],
   "source": [
    "user_visit_data_point_geo.crs = {'init' :'epsg:4326'}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "#user_visit_data_point_geo.drop(['latitude', 'longitude'], axis=1, inplace=True)\n",
    "clear_output()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [],
   "source": [
    "places_data = pd.read_csv('/folder_name/input_data_.csv', engine='python')\n",
    "clear_output()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [],
   "source": [
    "nyc_cities = [\"New York\"]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [],
   "source": [
    "places_data_nyc= places_data[places_data['City'].isin(nyc_cities)]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [],
   "source": [
    "places_data_nyc_geo = geopandas.GeoDataFrame(places_data_nyc, geometry=gpd.points_from_xy(places_data_nyc.longitude,\n",
    "                                                                                places_data_nyc.latitude))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {},
   "outputs": [],
   "source": [
    "places_data_nyc_geo.drop(['latitude', 'longitude'], axis=1, inplace=True)\n",
    "clear_output()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [],
   "source": [
    "filename = \"input_file.geojson\"\n",
    "screen2geojson(places_data_nyc, filename)\n",
    "places_data_nyc_geo = gpd.read_file(filename)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [],
   "source": [
    "def create_polygon_by_geo(gdf):\n",
    "    gdf.crs = {'init': 'epsg:4326'}\n",
    "    gdf = gdf.to_crs({'init': 'epsg:3857'})\n",
    "    gdf['geometry'] = gdf['geometry'].apply(lambda x: x.buffer(50).envelope)\n",
    "    gdf = gdf.to_crs({'init': 'epsg:4326'})\n",
    "    return gdf"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [],
   "source": [
    "places_data_present_geo = create_polygon_by_geo(places_data_nyc_geo)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [],
   "source": [
    "places_data_present_geo.to_crs({'init': 'epsg:4326'})\n",
    "clear_output()    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [],
   "source": [
    "venue_data_geo_venue = gpd.sjoin(places_data_present_geo, user_visit_data_point_geo,  how=\"inner\", op=\"within\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {},
   "outputs": [],
   "source": [
    "filename = \"/folder_name/output_data.geojson\"\n",
    "places_data_present_geo.to_file(filename, driver=\"GeoJSON\")\n",
    "clear_output()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {},
   "outputs": [],
   "source": [
    "filename = \"/folder_name/output_data.geojson\"\n",
    "user_visit_data_point_geo.to_file(filename, driver=\"GeoJSON\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "output_type": "stream",
     "name": "stdout",
     "text": "User Guide: https://github.com/keplergl/kepler.gl/blob/master/docs/keplergl-jupyter/user-guide.md\n"
    }
   ],
   "source": [
    "from keplergl import KeplerGl \n",
    "map_2 = KeplerGl(height=500)\n",
    "\n",
    "map_2.add_data(data=places_data_present_geo, name=\"screens\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "output_type": "display_data",
     "data": {
      "text/plain": "KeplerGl(data={'screens': {'index': [321, 396, 522, 765, 806, 1256, 1470, 2046, 2352, 2501, 2637, 2992, 3171, …",
      "application/vnd.jupyter.widget-view+json": {
       "version_major": 2,
       "version_minor": 0,
       "model_id": "cdda7feaaaa04557b6519f2b872f36eb"
      }
     },
     "metadata": {}
    }
   ],
   "source": [
    "map_2.add_data(data=user_visit_data_point_geo, name=\"venues\")\n",
    "map_2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.7-final"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}