From 047234a22bc8fa29ab5780314289b8d87ba14bb7 Mon Sep 17 00:00:00 2001 From: Winston <43570913+gumdropsteve@users.noreply.github.com> Date: Sun, 12 Jan 2020 22:28:26 -0800 Subject: [PATCH] NYC Taxi Dashboard update (#7) (#10) * [TEST] updated to process all Q1 data + rm elif cases & ChristmasNYC * wildcard data path; map if/elifs; sql input errors --- taxi_dashboard.ipynb | 360 ++++++++++++++++++------------------------- 1 file changed, 151 insertions(+), 209 deletions(-) diff --git a/taxi_dashboard.ipynb b/taxi_dashboard.ipynb index 5e98e02..1b95b08 100644 --- a/taxi_dashboard.ipynb +++ b/taxi_dashboard.ipynb @@ -1258,7 +1258,20 @@ "text/html": [ "\n", "\n", - "\n", "\n", "\n", @@ -2300,7 +2300,7 @@ { "data": { "text/plain": [ - "'/home/winston/turbo-telegram/data/nyc_taxi_jan15.csv'" + "'/home/winston/turbo-telegram/data/nyc_taxi_*'" ] }, "execution_count": 1, @@ -2337,7 +2337,7 @@ "# tag path to this notebook\n", "cwd = os.getcwd() \n", "# add taxi wildcard to complete file path of data\n", - "path = cwd + '/' + 'data/nyc_taxi_jan15.csv'\n", + "path = cwd + '/' + 'data/nyc_taxi_*'\n", "# how's it look?\n", "path" ] @@ -2392,106 +2392,97 @@ " mta_tax\n", " tip_amount\n", " tolls_amount\n", - " improvement_surcharge\n", " total_amount\n", " \n", " \n", " \n", " \n", - " 697498\n", - " 2\n", - " 2015-01-31 20:46:12\n", - " 2015-01-31 20:57:49\n", + " 1911983\n", " 1\n", - " 2.31\n", - " -8.237828e+06\n", - " 4.972035e+06\n", + " 2015-02-02 13:12:45\n", + " 2015-02-02 13:20:29\n", + " 1\n", + " 0.80\n", + " -8.233733e+06\n", + " 4.978858e+06\n", " 1\n", " N\n", - " -8.239360e+06\n", - " 4.969136e+06\n", + " -8.232789e+06\n", + " 4.980123e+06\n", " 2\n", - " 10.0\n", - " 0.5\n", + " 6.5\n", + " 0.0\n", " 0.5\n", " 0.0\n", " 0.0\n", - " 0.3\n", - " 11.3\n", + " 7.3\n", " \n", " \n", - " 697499\n", - " 1\n", - " 2015-01-31 21:04:48\n", - " 2015-01-31 21:20:53\n", - " 1\n", - " 3.10\n", - " -8.233061e+06\n", - " 4.978176e+06\n", + " 1911984\n", + " 2\n", + " 2015-02-12 21:39:55\n", + " 2015-02-12 21:47:53\n", + " 2\n", + " 1.04\n", + " -8.235069e+06\n", + " 4.976922e+06\n", " 1\n", " N\n", - " -8.236266e+06\n", - " 4.972510e+06\n", + " -8.233533e+06\n", + " 4.976631e+06\n", " 1\n", - " 13.0\n", + " 7.0\n", " 0.5\n", " 0.5\n", - " 1.0\n", + " 1.5\n", " 0.0\n", - " 0.3\n", - " 15.3\n", + " 9.8\n", " \n", " \n", - " 697500\n", - " 1\n", - " 2015-01-31 20:52:51\n", - " 2015-01-31 21:20:06\n", + " 1911985\n", + " 2\n", + " 2015-02-25 16:55:21\n", + " 2015-02-25 17:06:37\n", " 1\n", - " 6.90\n", - " -8.234850e+06\n", - " 4.981938e+06\n", + " 1.84\n", + " -8.234084e+06\n", + " 4.977811e+06\n", " 1\n", " N\n", - " -8.237785e+06\n", - " 4.971117e+06\n", - " 1\n", - " 25.5\n", - " 0.5\n", + " -8.231831e+06\n", + " 4.979868e+06\n", + " 2\n", + " 9.5\n", + " 1.0\n", " 0.5\n", - " 1.2\n", " 0.0\n", - " 0.3\n", - " 28.0\n", + " 0.0\n", + " 11.3\n", " \n", " \n", "\n", "" ], "text/plain": [ - " VendorID tpep_pickup_datetime tpep_dropoff_datetime passenger_count \\\n", - "697498 2 2015-01-31 20:46:12 2015-01-31 20:57:49 1 \n", - "697499 1 2015-01-31 21:04:48 2015-01-31 21:20:53 1 \n", - "697500 1 2015-01-31 20:52:51 2015-01-31 21:20:06 1 \n", - "\n", - " trip_distance pickup_x pickup_y RateCodeID \\\n", - "697498 2.31 -8.237828e+06 4.972035e+06 1 \n", - "697499 3.10 -8.233061e+06 4.978176e+06 1 \n", - "697500 6.90 -8.234850e+06 4.981938e+06 1 \n", - "\n", - " store_and_fwd_flag dropoff_x dropoff_y payment_type \\\n", - "697498 N -8.239360e+06 4.969136e+06 2 \n", - "697499 N -8.236266e+06 4.972510e+06 1 \n", - "697500 N -8.237785e+06 4.971117e+06 1 \n", - "\n", - " fare_amount extra mta_tax tip_amount tolls_amount \\\n", - "697498 10.0 0.5 0.5 0.0 0.0 \n", - "697499 13.0 0.5 0.5 1.0 0.0 \n", - "697500 25.5 0.5 0.5 1.2 0.0 \n", - "\n", - " improvement_surcharge total_amount \n", - "697498 0.3 11.3 \n", - "697499 0.3 15.3 \n", - "697500 0.3 28.0 " + " VendorID tpep_pickup_datetime tpep_dropoff_datetime passenger_count \\\n", + "1911983 1 2015-02-02 13:12:45 2015-02-02 13:20:29 1 \n", + "1911984 2 2015-02-12 21:39:55 2015-02-12 21:47:53 2 \n", + "1911985 2 2015-02-25 16:55:21 2015-02-25 17:06:37 1 \n", + "\n", + " trip_distance pickup_x pickup_y RateCodeID \\\n", + "1911983 0.80 -8.233733e+06 4.978858e+06 1 \n", + "1911984 1.04 -8.235069e+06 4.976922e+06 1 \n", + "1911985 1.84 -8.234084e+06 4.977811e+06 1 \n", + "\n", + " store_and_fwd_flag dropoff_x dropoff_y payment_type \\\n", + "1911983 N -8.232789e+06 4.980123e+06 2 \n", + "1911984 N -8.233533e+06 4.976631e+06 1 \n", + "1911985 N -8.231831e+06 4.979868e+06 2 \n", + "\n", + " fare_amount extra mta_tax tip_amount tolls_amount total_amount \n", + "1911983 6.5 0.0 0.5 0.0 0.0 7.3 \n", + "1911984 7.0 0.5 0.5 1.5 0.0 9.8 \n", + "1911985 9.5 1.0 0.5 0.0 0.0 11.3 " ] }, "execution_count": 2, @@ -2505,7 +2496,7 @@ " 'int', 'str', 'float64', 'float64', 'int', 'float64', 'float64', \n", " 'float64', 'float64', 'float64', 'float64', 'float64']\n", "bc.create_table('big_taxi', path, dtype=col_types, header=0)\n", - "# what's it look like?\n", + "# what's it look like? \n", "bc.sql('select * from big_taxi').tail(3)" ] }, @@ -2524,7 +2515,6 @@ "source": [ "# build outputs (each will have it's own tab)\n", "output = widgets.Output()\n", - "Christmas_NYC = widgets.Output()\n", "plot_output = widgets.Output()\n", "hv_output = widgets.Output()\n", "hv_output2 = widgets.Output()\n", @@ -2591,44 +2581,19 @@ "default_location = dropdown_location.value\n", "\n", "def common_filtering(day, month, year, n_riders, tol_cost, location):\n", - " # sql: day, month, year, n_riders, tol_cost\n", - " # visual dataframe: month, year, n_riders, tol_cost\n", - " # plot: month, year, n_riders, tol_cost, location\n", - " \n", - " # check if input boxes are blank or not\n", - " if n_riders != '':\n", - " # and strip extra blank space from start & end\n", - " n_riders = n_riders.strip()\n", - " # check again that it's not blank\n", - " if n_riders != '':\n", - " # make sure there's a direction\n", - " if n_riders[0] not in '<>=':\n", - " # give user time to sort it\n", - " time.sleep(4) \n", - " # call it\n", - " return 'number of riders input must start with one of the following:\\n >\\n <\\n =\\n >=\\n <=\\nor be blank\\n\\nfor example: > 2 or = 3'\n", - " if tol_cost != '':\n", - " # e.g. ' ' = '' or '> 5 ' = '> 5'\n", - " tol_cost = tol_cost.strip()\n", - " # check again that it's not blank\n", - " if tol_cost != '':\n", - " # make sure there's a direction\n", - " if tol_cost[0] not in '<>=':\n", - " # give user some time to figure it out\n", - " time.sleep(4)\n", - " # we need a direction\n", - " return 'fare amount input must start with one of the following:\\n >\\n <\\n =\\n >=\\n <=\\nor be blank\\n\\nfor example: > 2 or = 3'\n", - " \n", + " \"\"\"sql: day, month, year, n_riders, tol_cost\n", + " visual dataframe: month, year, n_riders, tol_cost\n", + " plot: month, year, n_riders, tol_cost, location\"\"\"\n", + " \n", " # clear outputs (replace instead of stacking, basically)\n", " output.clear_output()\n", - " Christmas_NYC.clear_output()\n", " plot_output.clear_output()\n", " hv_output.clear_output()\n", " hv_output2.clear_output()\n", - " \n", + "\n", " # start building our SQL query \n", " query = 'SELECT * FROM big_taxi'\n", - " \n", + "\n", " # has user focused specific days?\n", " if day != default_day:\n", " # go through days selected\n", @@ -2699,102 +2664,39 @@ " else:\n", " # add WHERE & total fare to query \n", " query += f' WHERE fare_amount {tol_cost}'\n", - " \n", + "\n", " # run query & deliver cuDF DataFrame of results\n", " query = query.replace('big_taxi', f'({time_table})')\n", " gdf = bc.sql(query).to_pandas()\n", - " \n", + "\n", " with output:\n", " print(f'{query}\\n') # print(gdf.info())\n", " # trim down columns to display \n", " focused = ['passenger_count', 'trip_distance', 'VendorID', 'RateCodeID', 'payment_type', \n", " 'store_and_fwd_flag', 'fare_amount', 'extra', 'mta_tax', 'tip_amount', \n", - " 'tolls_amount', 'improvement_surcharge', 'total_amount', 'tpep_pickup_datetime']\n", + " 'tolls_amount', 'total_amount', 'tpep_pickup_datetime']\n", " left_out = 'tpep_dropoff_datetime, pickup_x, pickup_y, dropoff_x, dropoff_y'\n", " print(f'DATAFRAME SAMPLE\\nexcluding columns: {left_out}')\n", " display(gdf[focused].sample(21))\n", - " \n", - " with Christmas_NYC:\n", - " map_tiles = EsriImagery().opts(alpha=0.5, width=900, height=480, bgcolor='red')\n", - " # check viability of graph\n", - " if len(gdf) >= 5000:\n", - " if dropdown_location.value == 'Pickup Location':\n", - " points = hv.Points(gdf, ['pickup_x', 'pickup_y'])\n", - " else:\n", - " points = hv.Points(gdf, ['dropoff_x', 'dropoff_y'])\n", - " taxi_trips = datashade(points, x_sampling=1, y_sampling=1, cmap=['green', 'white'], width=900, height=480)\n", - " display(map_tiles * taxi_trips)\n", - " # no dataframe edge case\n", - " elif len(gdf) == 0:\n", - " print(f'NO DATAFRAME TO DISPLAY\\nlen(gdf) == {len(gdf)}')\n", - " display(gdf)\n", - " # hyper-focused edge case \n", - " else: # in danger of unintrepretable map\n", - " # what's going on?\n", - " print(f'DISPLAY WARNING: len(gdf) < 5000\\n len(gdf) == {len(gdf)}')\n", - " extras = 5000 // len(gdf) # how short are we?\n", - " print(f'Graph will display DataFrame copied {1 + extras} times to saturate points')\n", - " print('Actual DataFrame WILL NOT change, this DOES NOT apply to other graphs')\n", - " # make temp dataframe (copied/saturated dataframe)\n", - " temp_df = pd.concat([gdf] + [gdf.copy() for i in range(1 + extras)], axis=0)\n", - " # and then we call the kdeplot method of seaborn by passing the number of visits:\n", - " if dropdown_location.value == 'Pickup Location':\n", - " points = hv.Points(temp_df, ['pickup_x', 'pickup_y'])\n", - " else:\n", - " points = hv.Points(temp_df, ['dropoff_x', 'dropoff_y'])\n", - " taxi_trips = datashade(points, x_sampling=1, y_sampling=1, cmap=fire, width=900, height=480)\n", - " # forget temp\n", - " temp_df = ''\n", - " display(map_tiles * taxi_trips)\n", - " # erase temp dataframe\n", - " temp_df = '' \n", - " # display NYC coord map \n", - " display(tf.set_background(tf.shade(agg, cmap=fire), \"black\")) \n", + " # nyc map \n", " with plot_output:\n", + " # determine pickup location \n", + " if dropdown_location.value == 'Pickup Location':\n", + " points = hv.Points(gdf, ['pickup_x', 'pickup_y'])\n", + " else:\n", + " points = hv.Points(gdf, ['dropoff_x', 'dropoff_y'])\n", + " # graph it\n", + " taxi_trips = datashade(points, x_sampling=1, y_sampling=1, cmap=fire, width=900, height=480)\n", + " # overlay NYC map & display the graph\n", " map_tiles = EsriImagery().opts(alpha=0.5, width=900, height=480, bgcolor='black')\n", - " # check viability of graph\n", - " if len(gdf) >= 5000:\n", - " # determine pickup location \n", - " if dropdown_location.value == 'Pickup Location':\n", - " points = hv.Points(gdf, ['pickup_x', 'pickup_y'])\n", - " else:\n", - " points = hv.Points(gdf, ['dropoff_x', 'dropoff_y'])\n", - " # graph it\n", - " taxi_trips = datashade(points, x_sampling=1, y_sampling=1, cmap=fire, width=900, height=480)\n", - " display(map_tiles * taxi_trips)\n", - " # no dataframe edge case\n", - " elif len(gdf) == 0:\n", - " print(f'NO DATAFRAME TO DISPLAY\\nlen(gdf) == {len(gdf)}')\n", - " display(gdf)\n", - " # hyper-focused edge case \n", - " else: # in danger of unintrepretable map\n", - " # what's going on?\n", - " print(f'DISPLAY WARNING: len(gdf) < 5000\\n len(gdf) == {len(gdf)}')\n", - " extras = 5000 // len(gdf) # how short are we?\n", - " print(f'Graph will display DataFrame copied {1 + extras} times to saturate points')\n", - " print('Actual DataFrame WILL NOT change, this DOES NOT apply to other graphs')\n", - " # make temp dataframe (copied/saturated dataframe)\n", - " temp_df = pd.concat([gdf] + [gdf.copy() for i in range(1 + extras)], axis=0)\n", - " # and then we call the kdeplot method of seaborn by passing the number of visits:\n", - " if dropdown_location.value == 'Pickup Location':\n", - " points = hv.Points(temp_df, ['pickup_x', 'pickup_y'])\n", - " else:\n", - " points = hv.Points(temp_df, ['dropoff_x', 'dropoff_y'])\n", - " # graph it\n", - " taxi_trips = datashade(points, x_sampling=1, y_sampling=1, cmap=fire, width=900, height=480)\n", - " # forget temp\n", - " temp_df = ''\n", - " display(map_tiles * taxi_trips)\n", - " # erase temp dataframe\n", - " temp_df = '' \n", - " # display NYC coord map \n", - " display(tf.set_background(tf.shade(agg, cmap=fire), \"black\")) \n", + " display(map_tiles * taxi_trips)\n", " # tip amount vs number of riders\n", " with hv_output:\n", " scatter = hv.Scatter(gdf, 'passenger_count', 'tip_amount')\n", - " ranged = scatter.redim.range(tip_amount=(0, 30), passenger_count=(-0.5, 6.5))\n", + " ranged = scatter.redim.range(tip_amount=(0, 60), passenger_count=(-0.5, 6.5))\n", " shaded = hd.spread(hd.datashade(ranged, x_sampling=0.15))\n", " labelled = shaded.redim.label(passenger_count=\"Passengers\", tip_amount=\"Tip ($)\")\n", + " # let's see\n", " display(labelled)\n", " # trip distance vs tip amount\n", " with hv_output2:\n", @@ -2802,27 +2704,66 @@ " ranged = scatter.redim.range(trip_distance=(0, 20), tip_amount=(0, 20))\n", " shaded = hd.spread(hd.datashade(ranged))\n", " labelled = shaded.redim.label(trip_distance=\"Trip Distance\", tip_amount=\"Tip ($)\")\n", + " # let's see\n", " display(labelled)\n", " \n", - "# The existing event handlers need to be adjusted \n", + " \n", "def dropdown_day_eventhandler(change):\n", " common_filtering(change.new, dropdown_month.value, dropdown_year.value, dropdown_riders.value, \n", " dropdown_fare.value, dropdown_location.value)\n", + " \n", "def dropdown_month_eventhandler(change):\n", " common_filtering(dropdown_day.value, change.new, dropdown_year.value, dropdown_riders.value, \n", " dropdown_fare.value, dropdown_location.value)\n", + " \n", "def dropdown_year_eventhandler(change):\n", " common_filtering(dropdown_day.value, dropdown_month.value, change.new, dropdown_riders.value, \n", " dropdown_fare.value, dropdown_location.value)\n", + " \n", "def dropdown_riders_eventhandler(change):\n", - " common_filtering(dropdown_day.value, dropdown_month.value, dropdown_year.value, change.new.replace('\"','').replace(\"'\",''), \n", - " dropdown_fare.value, dropdown_location.value) \n", + " # adjust change to remove errors \n", + " change = change.new.strip().replace('\"','').replace(\"'\",'')\n", + " # check change to make sure it works\n", + " valid = True\n", + " # is input blank \n", + " if change != '':\n", + " # make sure there's a direction\n", + " if change[0] not in '<>=':\n", + " # query not ready yet\n", + " valid = False\n", + " # make sure there's a number\n", + " if change[-1] not in '1234567890':\n", + " # query not ready yet\n", + " valid = False\n", + " # is the query ready?\n", + " if valid == True:\n", + " common_filtering(dropdown_day.value, dropdown_month.value, dropdown_year.value, \n", + " change, dropdown_fare.value, dropdown_location.value) \n", + " \n", "def dropdown_fare_eventhandler(change):\n", - " common_filtering(dropdown_day.value, dropdown_month.value, dropdown_year.value, dropdown_riders.value, \n", - " change.new.replace('\"','').replace(\"'\",''), dropdown_location.value) \n", + " # adjust change to remove errors \n", + " change = change.new.strip().replace('\"','').replace(\"'\",'')\n", + " # check change to make sure it works\n", + " valid = True\n", + " # is input blank \n", + " if change != '':\n", + " # make sure there's a direction\n", + " if change[0] not in '<>=':\n", + " # query not ready yet\n", + " valid = False\n", + " # make sure there's a number\n", + " if change[-1] not in '1234567890':\n", + " # query not ready yet\n", + " valid = False\n", + " # is the query ready?\n", + " if valid == True:\n", + " common_filtering(dropdown_day.value, dropdown_month.value, dropdown_year.value, \n", + " dropdown_riders.value, change, dropdown_location.value) \n", + " \n", "def dropdown_location_eventhandler(change):\n", " common_filtering(dropdown_day.value, dropdown_month.value, dropdown_year.value, dropdown_riders.value, \n", " dropdown_fare.value, change.new) \n", + " \n", "\n", "# We bind the handlers to the dropdowns, and that’s it!\n", "dropdown_day.observe(dropdown_day_eventhandler, names='value')\n", @@ -2832,6 +2773,7 @@ "dropdown_fare.observe(dropdown_fare_eventhandler, names='value')\n", "dropdown_location.observe(dropdown_location_eventhandler, names='value')\n", "\n", + "\n", "def input_dashboard():\n", " # The HBox will add widgets to it one at a time from left-to-right:\n", " input_widgets = widgets.HBox([dropdown_day, dropdown_month, dropdown_year])\n", @@ -2842,12 +2784,11 @@ " \n", "def display_dashboard():\n", " # dashboard outputs\n", - " tab = widgets.Tab([output, Christmas_NYC, plot_output, hv_output, hv_output2])\n", + " tab = widgets.Tab([output, plot_output, hv_output, hv_output2])\n", " tab.set_title(0, 'Dataset Exploration')\n", - " tab.set_title(1, 'Christmas Map')\n", - " tab.set_title(2, 'Coordinate Map')\n", - " tab.set_title(3, 'Tips vs # Riders')\n", - " tab.set_title(4, 'Tips vs Distance')\n", + " tab.set_title(1, 'Coordinate Map')\n", + " tab.set_title(2, 'Tips vs # Riders')\n", + " tab.set_title(3, 'Tips vs Distance')\n", " # display outputs \n", " display(tab)" ] @@ -2867,7 +2808,7 @@ { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "8203f2502706466ba1023f6167b80fcf", + "model_id": "542e1862ca0144549585a3ff55d35e62", "version_major": 2, "version_minor": 0 }, @@ -2881,7 +2822,7 @@ { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "283914e92b394ef989eec7e905e05748", + "model_id": "c55575a743dc4142be59f35c368249c1", "version_major": 2, "version_minor": 0 }, @@ -2894,6 +2835,7 @@ } ], "source": [ + "# to fix: running query when incomplete\n", "input_dashboard()" ] }, @@ -2912,12 +2854,12 @@ { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "c202eeed1770411ea5e93e39691f064e", + "model_id": "5539e063239841b49dc3245e86b84b1b", "version_major": 2, "version_minor": 0 }, "text/plain": [ - "Tab(children=(Output(), Output(), Output(), Output(), Output()), _titles={'0': 'Dataset Exploration', '1': 'Ch…" + "Tab(children=(Output(), Output(), Output(), Output()), _titles={'0': 'Dataset Exploration', '1': 'Coordinate M…" ] }, "metadata": {}, @@ -2945,7 +2887,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.7.3" + "version": "3.7.6" } }, "nbformat": 4,