From 58d7697ecc6bc8d2ac315b7fa604500b43dbeff5 Mon Sep 17 00:00:00 2001 From: louisg1337 Date: Thu, 8 Aug 2024 15:31:58 -0400 Subject: [PATCH 01/39] Added in support for emission common to public dashboard, and integrated the base mode colors into the plots --- viz_scripts/docker/Dockerfile.dev | 1 + viz_scripts/scaffolding.py | 15 +++++++++++++-- 2 files changed, 14 insertions(+), 2 deletions(-) diff --git a/viz_scripts/docker/Dockerfile.dev b/viz_scripts/docker/Dockerfile.dev index 1f306da..30a12ce 100644 --- a/viz_scripts/docker/Dockerfile.dev +++ b/viz_scripts/docker/Dockerfile.dev @@ -9,6 +9,7 @@ WORKDIR /usr/src/app RUN /bin/bash -c "source setup/activate.sh && conda env update --name emission --file setup/environment36.notebook.additions.yml" RUN /bin/bash -c "source setup/activate.sh && conda env update --name emission --file /environment36.dashboard.additions.yml" +RUN /bin/bash -c "source setup/activate.sh && pip3 install 'git+https://github.com/jgreenlee/e-mission-common.git'" ADD docker/start_notebook.sh /usr/src/app/.docker/start_notebook.sh RUN chmod u+x /usr/src/app/.docker/start_notebook.sh diff --git a/viz_scripts/scaffolding.py b/viz_scripts/scaffolding.py index e2abc57..b029057 100644 --- a/viz_scripts/scaffolding.py +++ b/viz_scripts/scaffolding.py @@ -4,10 +4,13 @@ import sys from collections import defaultdict from collections import OrderedDict +import difflib import emission.storage.timeseries.abstract_timeseries as esta import emission.storage.timeseries.tcquery as esttc import emission.core.wrapper.localdate as ecwl +import emcommon.diary.base_modes as base_modes + # Module for pretty-printing outputs (e.g. head) to help users # understand what is going on @@ -193,6 +196,14 @@ def translate_labels(labels): return defaultdict(lambda: 'Other', translation_mapping) dic_mapping = translate_labels(dynamic_labels[label_type]) return dic_mapping + +def find_closest_key(input_key, dictionary): + input_key = input_key.split(",")[0].split("/")[0].replace("share","").upper() + keys = list(dictionary.keys()) + closest_matches = difflib.get_close_matches(input_key, keys, n=1) + if closest_matches: + return closest_matches[0] + return "OTHER" # Function: Maps "MODE", "PURPOSE", and "REPLACED_MODE" to colors. # Input: dynamic_labels, dic_re, and dic_pur @@ -208,9 +219,9 @@ def mapping_color_labels(dynamic_labels, dic_re, dic_pur): combined_mode_values = (list(OrderedDict.fromkeys(dic_re.values())) + ['Other']) purpose_values = list(OrderedDict.fromkeys(dic_pur.values())) - colors_mode = dict(zip(combined_mode_values, plt.cm.tab20.colors[:len(combined_mode_values)])) + colors_mode = {x: base_modes.BASE_MODES[find_closest_key(x, base_modes.BASE_MODES)]["color"] for x in combined_mode_values} colors_purpose = dict(zip(purpose_values, plt.cm.tab20.colors[:len(purpose_values)])) - colors_sensed = dict(zip(sensed_values, plt.cm.tab20.colors[:len(sensed_values)])) + colors_sensed = dict(zip(sensed_values, [base_modes.BASE_MODES[x.upper()]['color'] for x in sensed_values])) return colors_mode, colors_purpose, colors_sensed From ee7cc2e4d4011965b8c1d2b17b05537f8436db44 Mon Sep 17 00:00:00 2001 From: louisg1337 Date: Thu, 8 Aug 2024 18:14:29 -0400 Subject: [PATCH 02/39] Added in edge case to deal with bike to BICYCLE --- viz_scripts/scaffolding.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/viz_scripts/scaffolding.py b/viz_scripts/scaffolding.py index b029057..fe72896 100644 --- a/viz_scripts/scaffolding.py +++ b/viz_scripts/scaffolding.py @@ -198,6 +198,9 @@ def translate_labels(labels): return dic_mapping def find_closest_key(input_key, dictionary): + # Edge case + if " bike" in input_key.lower(): + return "BICYCLING" input_key = input_key.split(",")[0].split("/")[0].replace("share","").upper() keys = list(dictionary.keys()) closest_matches = difflib.get_close_matches(input_key, keys, n=1) From 25259d2bf2193f89d2ad04f08385c58a4855f342 Mon Sep 17 00:00:00 2001 From: louisg1337 Date: Tue, 20 Aug 2024 12:06:41 -0400 Subject: [PATCH 03/39] Removed emcommon dependency install and adjusted code to work for dynamic labels --- viz_scripts/docker/Dockerfile.dev | 1 - viz_scripts/scaffolding.py | 12 ++++++++++-- 2 files changed, 10 insertions(+), 3 deletions(-) diff --git a/viz_scripts/docker/Dockerfile.dev b/viz_scripts/docker/Dockerfile.dev index 30a12ce..1f306da 100644 --- a/viz_scripts/docker/Dockerfile.dev +++ b/viz_scripts/docker/Dockerfile.dev @@ -9,7 +9,6 @@ WORKDIR /usr/src/app RUN /bin/bash -c "source setup/activate.sh && conda env update --name emission --file setup/environment36.notebook.additions.yml" RUN /bin/bash -c "source setup/activate.sh && conda env update --name emission --file /environment36.dashboard.additions.yml" -RUN /bin/bash -c "source setup/activate.sh && pip3 install 'git+https://github.com/jgreenlee/e-mission-common.git'" ADD docker/start_notebook.sh /usr/src/app/.docker/start_notebook.sh RUN chmod u+x /usr/src/app/.docker/start_notebook.sh diff --git a/viz_scripts/scaffolding.py b/viz_scripts/scaffolding.py index fe72896..eb6819e 100644 --- a/viz_scripts/scaffolding.py +++ b/viz_scripts/scaffolding.py @@ -213,16 +213,24 @@ def find_closest_key(input_key, dictionary): # Output: Dictionary mapping between color with mode/purpose/sensed def mapping_color_labels(dynamic_labels, dic_re, dic_pur): sensed_values = ["WALKING", "BICYCLING", "IN_VEHICLE", "AIR_OR_HSR", "UNKNOWN", "OTHER", "Other"] + colors_mode = {} if len(dynamic_labels) > 0: mode_values = list(mapping_labels(dynamic_labels, "MODE").values()) if "MODE" in dynamic_labels else [] replaced_mode_values = list(mapping_labels(dynamic_labels, "REPLACED_MODE").values()) if "REPLACED_MODE" in dynamic_labels else [] purpose_values = list(mapping_labels(dynamic_labels, "PURPOSE").values()) + ['Other'] if "PURPOSE" in dynamic_labels else [] combined_mode_values = mode_values + replaced_mode_values + ['Other'] + # Translation to baseMode mapping so we can map directly to corresponding baseMode + translations_to_basemodes = { + dynamic_labels["translations"]["en"][key]: mode["baseMode"] + for key in dynamic_labels["translations"]["en"].keys() + for mode in dynamic_labels["MODE"] + if mode["value"] == key + } + colors_mode = {mode: base_modes.BASE_MODES[translations_to_basemodes.get(mode, "UNKNOWN")]["color"] for mode in combined_mode_values} else: combined_mode_values = (list(OrderedDict.fromkeys(dic_re.values())) + ['Other']) purpose_values = list(OrderedDict.fromkeys(dic_pur.values())) - - colors_mode = {x: base_modes.BASE_MODES[find_closest_key(x, base_modes.BASE_MODES)]["color"] for x in combined_mode_values} + colors_mode = {x: base_modes.BASE_MODES[find_closest_key(x, base_modes.BASE_MODES)]["color"] for x in combined_mode_values} colors_purpose = dict(zip(purpose_values, plt.cm.tab20.colors[:len(purpose_values)])) colors_sensed = dict(zip(sensed_values, [base_modes.BASE_MODES[x.upper()]['color'] for x in sensed_values])) From 7b456a98f151af9a72003edd8b1b16b7754eb5a8 Mon Sep 17 00:00:00 2001 From: louisg1337 Date: Fri, 23 Aug 2024 13:48:21 -0400 Subject: [PATCH 04/39] Added in support for lighter colors if multiple of the same color appear --- viz_scripts/scaffolding.py | 29 +++++++++++++++++++++++++++-- 1 file changed, 27 insertions(+), 2 deletions(-) diff --git a/viz_scripts/scaffolding.py b/viz_scripts/scaffolding.py index eb6819e..39835cf 100644 --- a/viz_scripts/scaffolding.py +++ b/viz_scripts/scaffolding.py @@ -5,6 +5,7 @@ from collections import defaultdict from collections import OrderedDict import difflib +import colorsys import emission.storage.timeseries.abstract_timeseries as esta import emission.storage.timeseries.tcquery as esttc @@ -208,12 +209,36 @@ def find_closest_key(input_key, dictionary): return closest_matches[0] return "OTHER" +def lighten_color(hex_color, lightness): + # Cap lightness at 2 to avoid too light colors + lightness = min(2, lightness) + # Convert to RGB + r, g, b = int(hex_color[1:3], 16), int(hex_color[3:5], 16), int(hex_color[5:7], 16) + # Convert RGB to HLS + h, l, s = colorsys.rgb_to_hls(r/255, g/255, b/255) + # Modify lightness + l *= lightness + # Convert back to RGB + r, g, b = colorsys.hls_to_rgb(h, l, s) + # Convert to hex + return "#{:02x}{:02x}{:02x}".format(int(r*255), int(g*255), int(b*255)) + +def get_color(base_mode, base_mode_count): + # Keep track of how many base modes we have seen so we can lighten the color if necessary + base_mode_count[base_mode] += 1 + if base_mode_count[base_mode] <= 1: + return base_modes.BASE_MODES[base_mode]["color"] + else: + return lighten_color(base_modes.BASE_MODES[base_mode]["color"], (1 + base_mode_count[base_mode] * 0.2)) + + # Function: Maps "MODE", "PURPOSE", and "REPLACED_MODE" to colors. # Input: dynamic_labels, dic_re, and dic_pur # Output: Dictionary mapping between color with mode/purpose/sensed def mapping_color_labels(dynamic_labels, dic_re, dic_pur): sensed_values = ["WALKING", "BICYCLING", "IN_VEHICLE", "AIR_OR_HSR", "UNKNOWN", "OTHER", "Other"] colors_mode = {} + base_mode_count = defaultdict(int) if len(dynamic_labels) > 0: mode_values = list(mapping_labels(dynamic_labels, "MODE").values()) if "MODE" in dynamic_labels else [] replaced_mode_values = list(mapping_labels(dynamic_labels, "REPLACED_MODE").values()) if "REPLACED_MODE" in dynamic_labels else [] @@ -226,11 +251,11 @@ def mapping_color_labels(dynamic_labels, dic_re, dic_pur): for mode in dynamic_labels["MODE"] if mode["value"] == key } - colors_mode = {mode: base_modes.BASE_MODES[translations_to_basemodes.get(mode, "UNKNOWN")]["color"] for mode in combined_mode_values} + colors_mode = {mode: get_color(translations_to_basemodes.get(mode, "UNKNOWN"), base_mode_count) for mode in combined_mode_values} else: combined_mode_values = (list(OrderedDict.fromkeys(dic_re.values())) + ['Other']) purpose_values = list(OrderedDict.fromkeys(dic_pur.values())) - colors_mode = {x: base_modes.BASE_MODES[find_closest_key(x, base_modes.BASE_MODES)]["color"] for x in combined_mode_values} + colors_mode = {mode: (get_color(find_closest_key(mode, base_modes.BASE_MODES), base_mode_count)) for mode in combined_mode_values} colors_purpose = dict(zip(purpose_values, plt.cm.tab20.colors[:len(purpose_values)])) colors_sensed = dict(zip(sensed_values, [base_modes.BASE_MODES[x.upper()]['color'] for x in sensed_values])) From 7137a012ae20e90e88c36685dff8df87968f24a1 Mon Sep 17 00:00:00 2001 From: louisg1337 Date: Fri, 23 Aug 2024 17:14:05 -0400 Subject: [PATCH 05/39] Use exponential decay coefficient to account for infinite duplicate colors, and added in emcommon as a dependency --- viz_scripts/docker/environment36.dashboard.additions.yml | 1 + viz_scripts/scaffolding.py | 3 ++- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/viz_scripts/docker/environment36.dashboard.additions.yml b/viz_scripts/docker/environment36.dashboard.additions.yml index 59d26eb..df84f5e 100644 --- a/viz_scripts/docker/environment36.dashboard.additions.yml +++ b/viz_scripts/docker/environment36.dashboard.additions.yml @@ -7,3 +7,4 @@ dependencies: - pip: - nbparameterise==0.6 - devcron==0.4 + - git+https://github.com/JGreenlee/e-mission-common@master diff --git a/viz_scripts/scaffolding.py b/viz_scripts/scaffolding.py index 39835cf..9efe6cb 100644 --- a/viz_scripts/scaffolding.py +++ b/viz_scripts/scaffolding.py @@ -6,6 +6,7 @@ from collections import OrderedDict import difflib import colorsys +import math import emission.storage.timeseries.abstract_timeseries as esta import emission.storage.timeseries.tcquery as esttc @@ -229,7 +230,7 @@ def get_color(base_mode, base_mode_count): if base_mode_count[base_mode] <= 1: return base_modes.BASE_MODES[base_mode]["color"] else: - return lighten_color(base_modes.BASE_MODES[base_mode]["color"], (1 + base_mode_count[base_mode] * 0.2)) + return lighten_color(base_modes.BASE_MODES[base_mode]["color"], (2 - 1 / math.exp(0.1 * base_mode_count[base_mode]))) # Function: Maps "MODE", "PURPOSE", and "REPLACED_MODE" to colors. From 6630214a5fb2f0a6243694fa5851f58f654ffdac Mon Sep 17 00:00:00 2001 From: louisg1337 Date: Mon, 26 Aug 2024 17:16:17 -0400 Subject: [PATCH 06/39] Import and use emcommon's dedupe_colors function --- viz_scripts/scaffolding.py | 26 ++++++++++++-------------- 1 file changed, 12 insertions(+), 14 deletions(-) diff --git a/viz_scripts/scaffolding.py b/viz_scripts/scaffolding.py index 9efe6cb..ff9edd7 100644 --- a/viz_scripts/scaffolding.py +++ b/viz_scripts/scaffolding.py @@ -11,7 +11,7 @@ import emission.storage.timeseries.abstract_timeseries as esta import emission.storage.timeseries.tcquery as esttc import emission.core.wrapper.localdate as ecwl -import emcommon.diary.base_modes as base_modes +from emcommon.diary.base_modes import BASE_MODES, dedupe_colors # Module for pretty-printing outputs (e.g. head) to help users @@ -224,22 +224,14 @@ def lighten_color(hex_color, lightness): # Convert to hex return "#{:02x}{:02x}{:02x}".format(int(r*255), int(g*255), int(b*255)) -def get_color(base_mode, base_mode_count): - # Keep track of how many base modes we have seen so we can lighten the color if necessary - base_mode_count[base_mode] += 1 - if base_mode_count[base_mode] <= 1: - return base_modes.BASE_MODES[base_mode]["color"] - else: - return lighten_color(base_modes.BASE_MODES[base_mode]["color"], (2 - 1 / math.exp(0.1 * base_mode_count[base_mode]))) - - # Function: Maps "MODE", "PURPOSE", and "REPLACED_MODE" to colors. # Input: dynamic_labels, dic_re, and dic_pur # Output: Dictionary mapping between color with mode/purpose/sensed def mapping_color_labels(dynamic_labels, dic_re, dic_pur): sensed_values = ["WALKING", "BICYCLING", "IN_VEHICLE", "AIR_OR_HSR", "UNKNOWN", "OTHER", "Other"] + colors_mode = {} - base_mode_count = defaultdict(int) + if len(dynamic_labels) > 0: mode_values = list(mapping_labels(dynamic_labels, "MODE").values()) if "MODE" in dynamic_labels else [] replaced_mode_values = list(mapping_labels(dynamic_labels, "REPLACED_MODE").values()) if "REPLACED_MODE" in dynamic_labels else [] @@ -252,13 +244,19 @@ def mapping_color_labels(dynamic_labels, dic_re, dic_pur): for mode in dynamic_labels["MODE"] if mode["value"] == key } - colors_mode = {mode: get_color(translations_to_basemodes.get(mode, "UNKNOWN"), base_mode_count) for mode in combined_mode_values} + colors_mode = dedupe_colors([ + [mode, BASE_MODES[translations_to_basemodes.get(mode, "UNKNOWN")]['color']] + for mode in combined_mode_values + ]) else: combined_mode_values = (list(OrderedDict.fromkeys(dic_re.values())) + ['Other']) purpose_values = list(OrderedDict.fromkeys(dic_pur.values())) - colors_mode = {mode: (get_color(find_closest_key(mode, base_modes.BASE_MODES), base_mode_count)) for mode in combined_mode_values} + colors_mode = dedupe_colors([ + [mode, BASE_MODES[find_closest_key(mode, BASE_MODES)]['color']] + for mode in combined_mode_values + ]) colors_purpose = dict(zip(purpose_values, plt.cm.tab20.colors[:len(purpose_values)])) - colors_sensed = dict(zip(sensed_values, [base_modes.BASE_MODES[x.upper()]['color'] for x in sensed_values])) + colors_sensed = dict(zip(sensed_values, [BASE_MODES[x.upper()]['color'] for x in sensed_values])) return colors_mode, colors_purpose, colors_sensed From 998a725ec034b8ceb094547c5192800f0f374816 Mon Sep 17 00:00:00 2001 From: louisg1337 Date: Thu, 29 Aug 2024 18:27:57 -0400 Subject: [PATCH 07/39] Added support for emcommon default labels, but commented out for time being because of blocker. Removed unused imports and function. Changed dedupe colors to support new parameters. --- viz_scripts/generic_metrics.ipynb | 2 +- viz_scripts/scaffolding.py | 45 ++++++++++++++++++------------- 2 files changed, 27 insertions(+), 20 deletions(-) diff --git a/viz_scripts/generic_metrics.ipynb b/viz_scripts/generic_metrics.ipynb index 6ced8fc..8ad78dd 100644 --- a/viz_scripts/generic_metrics.ipynb +++ b/viz_scripts/generic_metrics.ipynb @@ -90,7 +90,7 @@ "metadata": {}, "outputs": [], "source": [ - "colors_mode, colors_purpose, colors_sensed = scaffolding.mapping_color_labels(dynamic_labels, dic_re, dic_pur)" + "colors_mode, colors_purpose, colors_sensed = await scaffolding.mapping_color_labels(dynamic_labels, dic_re, dic_pur)" ] }, { diff --git a/viz_scripts/scaffolding.py b/viz_scripts/scaffolding.py index ff9edd7..4510c34 100644 --- a/viz_scripts/scaffolding.py +++ b/viz_scripts/scaffolding.py @@ -5,13 +5,12 @@ from collections import defaultdict from collections import OrderedDict import difflib -import colorsys -import math import emission.storage.timeseries.abstract_timeseries as esta import emission.storage.timeseries.tcquery as esttc import emission.core.wrapper.localdate as ecwl from emcommon.diary.base_modes import BASE_MODES, dedupe_colors +from emcommon.util import read_json_resource # Module for pretty-printing outputs (e.g. head) to help users @@ -210,26 +209,34 @@ def find_closest_key(input_key, dictionary): return closest_matches[0] return "OTHER" -def lighten_color(hex_color, lightness): - # Cap lightness at 2 to avoid too light colors - lightness = min(2, lightness) - # Convert to RGB - r, g, b = int(hex_color[1:3], 16), int(hex_color[3:5], 16), int(hex_color[5:7], 16) - # Convert RGB to HLS - h, l, s = colorsys.rgb_to_hls(r/255, g/255, b/255) - # Modify lightness - l *= lightness - # Convert back to RGB - r, g, b = colorsys.hls_to_rgb(h, l, s) - # Convert to hex - return "#{:02x}{:02x}{:02x}".format(int(r*255), int(g*255), int(b*255)) - # Function: Maps "MODE", "PURPOSE", and "REPLACED_MODE" to colors. # Input: dynamic_labels, dic_re, and dic_pur # Output: Dictionary mapping between color with mode/purpose/sensed -def mapping_color_labels(dynamic_labels, dic_re, dic_pur): +async def mapping_color_labels(dynamic_labels, dic_re, dic_pur): sensed_values = ["WALKING", "BICYCLING", "IN_VEHICLE", "AIR_OR_HSR", "UNKNOWN", "OTHER", "Other"] + # labels = await read_json_resource("label-options.default.json") + # replaced_mode_values = [] + # if len(dynamic_labels) > 0: + # labels = dynamic_labels + # replaced_mode_values = list(mapping_labels(labels, "REPLACED_MODE").values()) if "REPLACED_MODE" in labels else [] + + # mode_values = list(mapping_labels(labels, "MODE").values()) if "MODE" in labels else [] + # purpose_values = list(mapping_labels(labels, "PURPOSE").values()) + ['Other'] if "PURPOSE" in labels else [] + # combined_mode_values = mode_values + replaced_mode_values + ['Other'] + + # # Translation to baseMode mapping so we can map directly to corresponding baseMode + # translations_to_basemodes = { + # labels["translations"]["en"][key]: mode.get("baseMode") or mode.get("base_mode") + # for key in labels["translations"]["en"].keys() + # for mode in labels["MODE"] + # if mode["value"] == key + # } + # colors_mode = dedupe_colors([ + # [mode, BASE_MODES[translations_to_basemodes.get(mode, "UNKNOWN")]['color']] + # for mode in combined_mode_values + # ]) + colors_mode = {} if len(dynamic_labels) > 0: @@ -247,14 +254,14 @@ def mapping_color_labels(dynamic_labels, dic_re, dic_pur): colors_mode = dedupe_colors([ [mode, BASE_MODES[translations_to_basemodes.get(mode, "UNKNOWN")]['color']] for mode in combined_mode_values - ]) + ], adjustment_range=[1,1.8]) else: combined_mode_values = (list(OrderedDict.fromkeys(dic_re.values())) + ['Other']) purpose_values = list(OrderedDict.fromkeys(dic_pur.values())) colors_mode = dedupe_colors([ [mode, BASE_MODES[find_closest_key(mode, BASE_MODES)]['color']] for mode in combined_mode_values - ]) + ], adjustment_range=[1,1.8]) colors_purpose = dict(zip(purpose_values, plt.cm.tab20.colors[:len(purpose_values)])) colors_sensed = dict(zip(sensed_values, [BASE_MODES[x.upper()]['color'] for x in sensed_values])) From ff703a9d656878e49c37dc66d4348d47d650e5f7 Mon Sep 17 00:00:00 2001 From: louisg1337 Date: Fri, 30 Aug 2024 13:20:18 -0400 Subject: [PATCH 08/39] Refactored plots and colors so they work off of internal values instead of display values. --- viz_scripts/generic_metrics.ipynb | 24 +++++------ viz_scripts/plots.py | 5 ++- viz_scripts/scaffolding.py | 72 +++++++++++-------------------- 3 files changed, 39 insertions(+), 62 deletions(-) diff --git a/viz_scripts/generic_metrics.ipynb b/viz_scripts/generic_metrics.ipynb index 8ad78dd..7951399 100644 --- a/viz_scripts/generic_metrics.ipynb +++ b/viz_scripts/generic_metrics.ipynb @@ -90,7 +90,7 @@ "metadata": {}, "outputs": [], "source": [ - "colors_mode, colors_purpose, colors_sensed = await scaffolding.mapping_color_labels(dynamic_labels, dic_re, dic_pur)" + "colors_mode, colors_purpose, colors_sensed, values_to_translations = await scaffolding.mapping_color_labels(dynamic_labels, dic_re, dic_pur)" ] }, { @@ -207,8 +207,8 @@ " # We will have text results corresponding to the axes for simplicity and consistency\n", " text_results = [[\"Unmodified Alt Text\", \"Unmodified HTML\"], [\"Unmodified Alt Text\", \"Unmodified HTML\"]]\n", " \n", - " plot_and_text_stacked_bar_chart(expanded_ct, lambda df: (df.groupby(\"Mode_confirm\").agg({distance_col: 'count'}).sort_values(by=distance_col, ascending=False)), \n", - " \"Labeled by user\\n\"+stacked_bar_quality_text_labeled, ax[0], text_results[0], colors_mode, debug_df)\n", + " plot_and_text_stacked_bar_chart(expanded_ct, lambda df: (df.groupby(\"mode_confirm\").agg({distance_col: 'count'}).sort_values(by=distance_col, ascending=False)), \n", + " \"Labeled by user\\n\"+stacked_bar_quality_text_labeled, ax[0], text_results[0], colors_mode, debug_df, values_to_translations)\n", " plot_and_text_stacked_bar_chart(expanded_ct_sensed, lambda df: (df.groupby(\"primary_mode\").agg({distance_col: 'count'}).sort_values(by=distance_col, ascending=False)), \n", " \"Sensed by OpenPATH\\n\"+stacked_bar_quality_text_sensed, ax[1], text_results[1], colors_sensed, debug_df_sensed)\n", " \n", @@ -258,8 +258,8 @@ " # Plot entries\n", " fig, ax = plt.subplots(nrows=1, ncols=1, figsize=(15,2*1), sharex=True) \n", " text_results = [\"Unmodified Alt Text\", \"Unmodified HTML\"]\n", - " plot_and_text_stacked_bar_chart(expanded_ct_commute, lambda df: df.groupby(\"Mode_confirm\").agg({distance_col: 'count'}).sort_values(by=distance_col, ascending=False), \n", - " \"Labeled by user\\n (Confirmed trips)\", ax, text_results, colors_mode, debug_df)\n", + " plot_and_text_stacked_bar_chart(expanded_ct_commute, lambda df: df.groupby(\"mode_confirm\").agg({distance_col: 'count'}).sort_values(by=distance_col, ascending=False), \n", + " \"Labeled by user\\n (Confirmed trips)\", ax, text_results, colors_mode, debug_df, values_to_translations)\n", " set_title_and_save(fig, text_results, plot_title, file_name)\n", "except (AttributeError, KeyError, pd.errors.UndefinedVariableError) as e:\n", " plt.clf()\n", @@ -335,7 +335,7 @@ "\n", " ## We do an existence check for the labeled df because we want to display the sensed value even if we don't have the labeled value\n", " ## but we don't need to have an existence check for sensed because in that case we will have no data to display\n", - " expanded_ct_u80 = expanded_ct.loc[(expanded_ct['distance'] <= cutoff)] if \"Mode_confirm\" in expanded_ct.columns else None\n", + " expanded_ct_u80 = expanded_ct.loc[(expanded_ct['distance'] <= cutoff)] if \"mode_confirm\" in expanded_ct.columns else None\n", " expanded_ct_sensed_u80 = expanded_ct_sensed.loc[(expanded_ct_sensed['distance'] <= cutoff)]\n", " sensed_u80_quality_text = f\"{len(expanded_ct_sensed_u80)} trips ({round(len(expanded_ct_sensed_u80)/len(expanded_ct_sensed)*100)}% of all trips)\\nfrom {scaffolding.unique_users(expanded_ct_sensed_u80)} {sensed_match.group(3)}\"\n", " labeled_u80_quality_text = f\"{len(expanded_ct_u80)} trips ({round(len(expanded_ct_u80)/len(expanded_ct)*100)}% of all labeled,\\n{round(len(expanded_ct_u80)/len(expanded_ct_sensed)*100)}% of all trips)\\nfrom {scaffolding.unique_users(expanded_ct_u80)} {sensed_match.group(3)}\" if \"Mode_confirm\" in expanded_ct.columns else \"0 labeled trips\"\n", @@ -343,8 +343,8 @@ " # Plot entries\n", " fig, ax = plt.subplots(nrows=2, ncols=1, figsize=(15,2*2), sharex=True)\n", " text_results = [[\"Unmodified Alt Text\", \"Unmodified HTML\"], [\"Unmodified Alt Text\", \"Unmodified HTML\"]]\n", - " plot_and_text_stacked_bar_chart(expanded_ct_u80, lambda df: df.groupby(\"Mode_confirm\").agg({distance_col: 'count'}).sort_values(by=distance_col, ascending=False), \n", - " \"Labeled by user\\n\"+labeled_u80_quality_text, ax[0], text_results[0], colors_mode, debug_df)\n", + " plot_and_text_stacked_bar_chart(expanded_ct_u80, lambda df: df.groupby(\"mode_confirm\").agg({distance_col: 'count'}).sort_values(by=distance_col, ascending=False), \n", + " \"Labeled by user\\n\"+labeled_u80_quality_text, ax[0], text_results[0], colors_mode, debug_df, values_to_translations)\n", " plot_and_text_stacked_bar_chart(expanded_ct_sensed_u80, lambda df: df.groupby(\"primary_mode\").agg({distance_col: 'count'}).sort_values(by=distance_col, ascending=False), \n", " \"Sensed by OpenPATH\\n\"+sensed_u80_quality_text, ax[1], text_results[1], colors_sensed, debug_df_sensed)\n", " set_title_and_save(fig, text_results, plot_title_no_quality, file_name)\n", @@ -383,8 +383,8 @@ " fig, ax = plt.subplots(nrows=2, ncols=1, figsize=(15,2*2), sharex=True)\n", " \n", " text_results = [[\"Unmodified Alt Text\", \"Unmodified HTML\"], [\"Unmodified Alt Text\", \"Unmodified HTML\"]]\n", - " plot_and_text_stacked_bar_chart(expanded_ct, lambda df: df.groupby(\"Mode_confirm\").agg({distance_col: 'sum'}).sort_values(by=distance_col, ascending=False), \n", - " \"Labeled by user\\n\"+stacked_bar_quality_text_labeled, ax[0], text_results[0], colors_mode, debug_df)\n", + " plot_and_text_stacked_bar_chart(expanded_ct, lambda df: df.groupby(\"mode_confirm\").agg({distance_col: 'sum'}).sort_values(by=distance_col, ascending=False), \n", + " \"Labeled by user\\n\"+stacked_bar_quality_text_labeled, ax[0], text_results[0], colors_mode, debug_df, values_to_translations)\n", " plot_and_text_stacked_bar_chart(expanded_ct_sensed, lambda df: df.groupby(\"primary_mode\").agg({distance_col: 'sum'}).sort_values(by=distance_col, ascending=False), \n", " \"Sensed by OpenPATH\\n\"+stacked_bar_quality_text_sensed, ax[1], text_results[1], colors_sensed, debug_df_sensed)\n", " set_title_and_save(fig, text_results, plot_title_no_quality, file_name) \n", @@ -426,8 +426,8 @@ " labeled_land_quality_text = f\"{len(labeled_land_trips_df)} trips ({round(len(labeled_land_trips_df)/len(expanded_ct)*100)}% of all labeled,\\n{round(len(labeled_land_trips_df)/len(expanded_ct_sensed)*100)}%) of all trips)\\nfrom {scaffolding.unique_users(labeled_land_trips_df)} {sensed_match.group(3)}\" if \"Mode_confirm\" in expanded_ct.columns else \"0 labeled trips\"\n", "\n", " fig, ax = plt.subplots(nrows=2, ncols=1, figsize=(15,2*2), sharex=True)\n", - " plot_and_text_stacked_bar_chart(labeled_land_trips_df, lambda df: df.groupby(\"Mode_confirm\").agg({distance_col: 'sum'}).sort_values(by=distance_col, ascending=False), \n", - " \"Labeled by user\\n\"+labeled_land_quality_text, ax[0], text_results[0], colors_mode, debug_df)\n", + " plot_and_text_stacked_bar_chart(labeled_land_trips_df, lambda df: df.groupby(\"mode_confirm\").agg({distance_col: 'sum'}).sort_values(by=distance_col, ascending=False), \n", + " \"Labeled by user\\n\"+labeled_land_quality_text, ax[0], text_results[0], colors_mode, debug_df, values_to_translations)\n", " plot_and_text_stacked_bar_chart(sensed_land_trips_df, lambda df: df.groupby(\"primary_mode\").agg({distance_col: 'sum'}).sort_values(by=distance_col, ascending=False), \n", " \"Sensed by OpenPATH\\n\"+sensed_land_quality_text, ax[1], text_results[1], colors_sensed, debug_df_sensed)\n", " set_title_and_save(fig, text_results, plot_title_no_quality, file_name) \n", diff --git a/viz_scripts/plots.py b/viz_scripts/plots.py index e7a9742..592f92a 100644 --- a/viz_scripts/plots.py +++ b/viz_scripts/plots.py @@ -94,7 +94,7 @@ def plot_and_text_error(e, ax, file_name): return alt_text, alt_html # Creates/ Appends single bar to the 100% Stacked Bar Chart -def plot_and_text_stacked_bar_chart(df, agg_fcn, bar_label, ax, text_result, colors, debug_df): +def plot_and_text_stacked_bar_chart(df, agg_fcn, bar_label, ax, text_result, colors, debug_df, values_to_translations={}): """ Inputs: df = Data frame corresponding to the bar in a stacked bar chart. It is expected to have three columns, which represent the 'label', 'value' @@ -128,7 +128,7 @@ def plot_and_text_stacked_bar_chart(df, agg_fcn, bar_label, ax, text_result, col mode_prop = long['Proportion'] mode_count = long['Value'] vals_str = [f'{y:.1f} %\n({x:.0f})' if y > 4 else '' for x, y in zip(mode_count, mode_prop)] - bar = ax.barh(y=bar_label, width=mode_prop, height=bar_height, left=bar_width, label=label, color=colors[label]) + bar = ax.barh(y=bar_label, width=mode_prop, height=bar_height, left=bar_width, label=values_to_translations.get(label, label), color=colors[label]) ax.bar_label(bar, label_type='center', labels=vals_str, rotation=90, fontsize=16) bar_width = [total + val for total, val in zip(bar_width, mode_prop)] else: @@ -144,6 +144,7 @@ def plot_and_text_stacked_bar_chart(df, agg_fcn, bar_label, ax, text_result, col text_result[0], text_result[1] = store_alt_text_and_html_stacked_bar_chart(df_all_entries, bar_label) print("After populating, %s" % text_result) except Exception as e: + print(e) # tb.print_exception(type(e), e, e.__traceback__) #ax.set_title("Insufficient data", loc="center") ax.set_ylabel(bar_label) diff --git a/viz_scripts/scaffolding.py b/viz_scripts/scaffolding.py index 4510c34..14cc7cd 100644 --- a/viz_scripts/scaffolding.py +++ b/viz_scripts/scaffolding.py @@ -212,60 +212,36 @@ def find_closest_key(input_key, dictionary): # Function: Maps "MODE", "PURPOSE", and "REPLACED_MODE" to colors. # Input: dynamic_labels, dic_re, and dic_pur # Output: Dictionary mapping between color with mode/purpose/sensed -async def mapping_color_labels(dynamic_labels, dic_re, dic_pur): +async def mapping_color_labels(dynamic_labels, dic_re, dic_pur, language="en"): + # Load default options from e-mission-common + labels = await read_json_resource("label-options.default.json") sensed_values = ["WALKING", "BICYCLING", "IN_VEHICLE", "AIR_OR_HSR", "UNKNOWN", "OTHER", "Other"] + replaced_mode_values = [] - # labels = await read_json_resource("label-options.default.json") - # replaced_mode_values = [] - # if len(dynamic_labels) > 0: - # labels = dynamic_labels - # replaced_mode_values = list(mapping_labels(labels, "REPLACED_MODE").values()) if "REPLACED_MODE" in labels else [] - - # mode_values = list(mapping_labels(labels, "MODE").values()) if "MODE" in labels else [] - # purpose_values = list(mapping_labels(labels, "PURPOSE").values()) + ['Other'] if "PURPOSE" in labels else [] - # combined_mode_values = mode_values + replaced_mode_values + ['Other'] - - # # Translation to baseMode mapping so we can map directly to corresponding baseMode - # translations_to_basemodes = { - # labels["translations"]["en"][key]: mode.get("baseMode") or mode.get("base_mode") - # for key in labels["translations"]["en"].keys() - # for mode in labels["MODE"] - # if mode["value"] == key - # } - # colors_mode = dedupe_colors([ - # [mode, BASE_MODES[translations_to_basemodes.get(mode, "UNKNOWN")]['color']] - # for mode in combined_mode_values - # ]) - - colors_mode = {} - + # If dynamic_labels are provided, then we will use the dynamic labels for mapping if len(dynamic_labels) > 0: - mode_values = list(mapping_labels(dynamic_labels, "MODE").values()) if "MODE" in dynamic_labels else [] - replaced_mode_values = list(mapping_labels(dynamic_labels, "REPLACED_MODE").values()) if "REPLACED_MODE" in dynamic_labels else [] - purpose_values = list(mapping_labels(dynamic_labels, "PURPOSE").values()) + ['Other'] if "PURPOSE" in dynamic_labels else [] - combined_mode_values = mode_values + replaced_mode_values + ['Other'] - # Translation to baseMode mapping so we can map directly to corresponding baseMode - translations_to_basemodes = { - dynamic_labels["translations"]["en"][key]: mode["baseMode"] - for key in dynamic_labels["translations"]["en"].keys() - for mode in dynamic_labels["MODE"] - if mode["value"] == key - } - colors_mode = dedupe_colors([ - [mode, BASE_MODES[translations_to_basemodes.get(mode, "UNKNOWN")]['color']] - for mode in combined_mode_values - ], adjustment_range=[1,1.8]) - else: - combined_mode_values = (list(OrderedDict.fromkeys(dic_re.values())) + ['Other']) - purpose_values = list(OrderedDict.fromkeys(dic_pur.values())) - colors_mode = dedupe_colors([ - [mode, BASE_MODES[find_closest_key(mode, BASE_MODES)]['color']] - for mode in combined_mode_values - ], adjustment_range=[1,1.8]) + labels = dynamic_labels + replaced_mode_values = list(mapping_labels(labels, "REPLACED_MODE").values()) if "REPLACED_MODE" in labels else [] + + # Load base mode values and purpose values + mode_values = [mode["value"] for mode in labels["MODE"]] + purpose_values = list(mapping_labels(labels, "PURPOSE").values()) + ['Other'] if "PURPOSE" in labels else [] + combined_mode_values = mode_values + replaced_mode_values + ['Other'] + + # Mapping between mode values and base_mode OR baseMode (backwards compatibility) + value_to_basemode = {mode["value"]: mode.get("base_mode", "baseMode") for mode in labels["MODE"]} + # Mapping between values and translations for display on plots + values_to_translations = {mode["value"]: labels["translations"][language][mode["value"]] for mode in labels["MODE"]} + + # Assign colors to mode, purpose, and sensed values + colors_mode = dedupe_colors([ + [mode, BASE_MODES[value_to_basemode.get(mode, "UNKNOWN")]['color']] + for mode in combined_mode_values + ], adjustment_range=[1,1.8]) colors_purpose = dict(zip(purpose_values, plt.cm.tab20.colors[:len(purpose_values)])) colors_sensed = dict(zip(sensed_values, [BASE_MODES[x.upper()]['color'] for x in sensed_values])) - return colors_mode, colors_purpose, colors_sensed + return colors_mode, colors_purpose, colors_sensed, values_to_translations # Function: Maps survey answers to colors. # Input: dictionary of raw and translated survey answers From 1808c698e6b8d63a10d5969bd64a79178771fe16 Mon Sep 17 00:00:00 2001 From: iantei Date: Wed, 4 Sep 2024 11:48:54 -0700 Subject: [PATCH 09/39] Initial commit. Changes copied from Integrate common base mode colors into public dashboard #143. --- .../environment36.dashboard.additions.yml | 1 + viz_scripts/generic_metrics.ipynb | 24 ++++----- viz_scripts/plots.py | 5 +- viz_scripts/scaffolding.py | 52 ++++++++++++++----- 4 files changed, 56 insertions(+), 26 deletions(-) diff --git a/viz_scripts/docker/environment36.dashboard.additions.yml b/viz_scripts/docker/environment36.dashboard.additions.yml index 59d26eb..df84f5e 100644 --- a/viz_scripts/docker/environment36.dashboard.additions.yml +++ b/viz_scripts/docker/environment36.dashboard.additions.yml @@ -7,3 +7,4 @@ dependencies: - pip: - nbparameterise==0.6 - devcron==0.4 + - git+https://github.com/JGreenlee/e-mission-common@master diff --git a/viz_scripts/generic_metrics.ipynb b/viz_scripts/generic_metrics.ipynb index 6ced8fc..7951399 100644 --- a/viz_scripts/generic_metrics.ipynb +++ b/viz_scripts/generic_metrics.ipynb @@ -90,7 +90,7 @@ "metadata": {}, "outputs": [], "source": [ - "colors_mode, colors_purpose, colors_sensed = scaffolding.mapping_color_labels(dynamic_labels, dic_re, dic_pur)" + "colors_mode, colors_purpose, colors_sensed, values_to_translations = await scaffolding.mapping_color_labels(dynamic_labels, dic_re, dic_pur)" ] }, { @@ -207,8 +207,8 @@ " # We will have text results corresponding to the axes for simplicity and consistency\n", " text_results = [[\"Unmodified Alt Text\", \"Unmodified HTML\"], [\"Unmodified Alt Text\", \"Unmodified HTML\"]]\n", " \n", - " plot_and_text_stacked_bar_chart(expanded_ct, lambda df: (df.groupby(\"Mode_confirm\").agg({distance_col: 'count'}).sort_values(by=distance_col, ascending=False)), \n", - " \"Labeled by user\\n\"+stacked_bar_quality_text_labeled, ax[0], text_results[0], colors_mode, debug_df)\n", + " plot_and_text_stacked_bar_chart(expanded_ct, lambda df: (df.groupby(\"mode_confirm\").agg({distance_col: 'count'}).sort_values(by=distance_col, ascending=False)), \n", + " \"Labeled by user\\n\"+stacked_bar_quality_text_labeled, ax[0], text_results[0], colors_mode, debug_df, values_to_translations)\n", " plot_and_text_stacked_bar_chart(expanded_ct_sensed, lambda df: (df.groupby(\"primary_mode\").agg({distance_col: 'count'}).sort_values(by=distance_col, ascending=False)), \n", " \"Sensed by OpenPATH\\n\"+stacked_bar_quality_text_sensed, ax[1], text_results[1], colors_sensed, debug_df_sensed)\n", " \n", @@ -258,8 +258,8 @@ " # Plot entries\n", " fig, ax = plt.subplots(nrows=1, ncols=1, figsize=(15,2*1), sharex=True) \n", " text_results = [\"Unmodified Alt Text\", \"Unmodified HTML\"]\n", - " plot_and_text_stacked_bar_chart(expanded_ct_commute, lambda df: df.groupby(\"Mode_confirm\").agg({distance_col: 'count'}).sort_values(by=distance_col, ascending=False), \n", - " \"Labeled by user\\n (Confirmed trips)\", ax, text_results, colors_mode, debug_df)\n", + " plot_and_text_stacked_bar_chart(expanded_ct_commute, lambda df: df.groupby(\"mode_confirm\").agg({distance_col: 'count'}).sort_values(by=distance_col, ascending=False), \n", + " \"Labeled by user\\n (Confirmed trips)\", ax, text_results, colors_mode, debug_df, values_to_translations)\n", " set_title_and_save(fig, text_results, plot_title, file_name)\n", "except (AttributeError, KeyError, pd.errors.UndefinedVariableError) as e:\n", " plt.clf()\n", @@ -335,7 +335,7 @@ "\n", " ## We do an existence check for the labeled df because we want to display the sensed value even if we don't have the labeled value\n", " ## but we don't need to have an existence check for sensed because in that case we will have no data to display\n", - " expanded_ct_u80 = expanded_ct.loc[(expanded_ct['distance'] <= cutoff)] if \"Mode_confirm\" in expanded_ct.columns else None\n", + " expanded_ct_u80 = expanded_ct.loc[(expanded_ct['distance'] <= cutoff)] if \"mode_confirm\" in expanded_ct.columns else None\n", " expanded_ct_sensed_u80 = expanded_ct_sensed.loc[(expanded_ct_sensed['distance'] <= cutoff)]\n", " sensed_u80_quality_text = f\"{len(expanded_ct_sensed_u80)} trips ({round(len(expanded_ct_sensed_u80)/len(expanded_ct_sensed)*100)}% of all trips)\\nfrom {scaffolding.unique_users(expanded_ct_sensed_u80)} {sensed_match.group(3)}\"\n", " labeled_u80_quality_text = f\"{len(expanded_ct_u80)} trips ({round(len(expanded_ct_u80)/len(expanded_ct)*100)}% of all labeled,\\n{round(len(expanded_ct_u80)/len(expanded_ct_sensed)*100)}% of all trips)\\nfrom {scaffolding.unique_users(expanded_ct_u80)} {sensed_match.group(3)}\" if \"Mode_confirm\" in expanded_ct.columns else \"0 labeled trips\"\n", @@ -343,8 +343,8 @@ " # Plot entries\n", " fig, ax = plt.subplots(nrows=2, ncols=1, figsize=(15,2*2), sharex=True)\n", " text_results = [[\"Unmodified Alt Text\", \"Unmodified HTML\"], [\"Unmodified Alt Text\", \"Unmodified HTML\"]]\n", - " plot_and_text_stacked_bar_chart(expanded_ct_u80, lambda df: df.groupby(\"Mode_confirm\").agg({distance_col: 'count'}).sort_values(by=distance_col, ascending=False), \n", - " \"Labeled by user\\n\"+labeled_u80_quality_text, ax[0], text_results[0], colors_mode, debug_df)\n", + " plot_and_text_stacked_bar_chart(expanded_ct_u80, lambda df: df.groupby(\"mode_confirm\").agg({distance_col: 'count'}).sort_values(by=distance_col, ascending=False), \n", + " \"Labeled by user\\n\"+labeled_u80_quality_text, ax[0], text_results[0], colors_mode, debug_df, values_to_translations)\n", " plot_and_text_stacked_bar_chart(expanded_ct_sensed_u80, lambda df: df.groupby(\"primary_mode\").agg({distance_col: 'count'}).sort_values(by=distance_col, ascending=False), \n", " \"Sensed by OpenPATH\\n\"+sensed_u80_quality_text, ax[1], text_results[1], colors_sensed, debug_df_sensed)\n", " set_title_and_save(fig, text_results, plot_title_no_quality, file_name)\n", @@ -383,8 +383,8 @@ " fig, ax = plt.subplots(nrows=2, ncols=1, figsize=(15,2*2), sharex=True)\n", " \n", " text_results = [[\"Unmodified Alt Text\", \"Unmodified HTML\"], [\"Unmodified Alt Text\", \"Unmodified HTML\"]]\n", - " plot_and_text_stacked_bar_chart(expanded_ct, lambda df: df.groupby(\"Mode_confirm\").agg({distance_col: 'sum'}).sort_values(by=distance_col, ascending=False), \n", - " \"Labeled by user\\n\"+stacked_bar_quality_text_labeled, ax[0], text_results[0], colors_mode, debug_df)\n", + " plot_and_text_stacked_bar_chart(expanded_ct, lambda df: df.groupby(\"mode_confirm\").agg({distance_col: 'sum'}).sort_values(by=distance_col, ascending=False), \n", + " \"Labeled by user\\n\"+stacked_bar_quality_text_labeled, ax[0], text_results[0], colors_mode, debug_df, values_to_translations)\n", " plot_and_text_stacked_bar_chart(expanded_ct_sensed, lambda df: df.groupby(\"primary_mode\").agg({distance_col: 'sum'}).sort_values(by=distance_col, ascending=False), \n", " \"Sensed by OpenPATH\\n\"+stacked_bar_quality_text_sensed, ax[1], text_results[1], colors_sensed, debug_df_sensed)\n", " set_title_and_save(fig, text_results, plot_title_no_quality, file_name) \n", @@ -426,8 +426,8 @@ " labeled_land_quality_text = f\"{len(labeled_land_trips_df)} trips ({round(len(labeled_land_trips_df)/len(expanded_ct)*100)}% of all labeled,\\n{round(len(labeled_land_trips_df)/len(expanded_ct_sensed)*100)}%) of all trips)\\nfrom {scaffolding.unique_users(labeled_land_trips_df)} {sensed_match.group(3)}\" if \"Mode_confirm\" in expanded_ct.columns else \"0 labeled trips\"\n", "\n", " fig, ax = plt.subplots(nrows=2, ncols=1, figsize=(15,2*2), sharex=True)\n", - " plot_and_text_stacked_bar_chart(labeled_land_trips_df, lambda df: df.groupby(\"Mode_confirm\").agg({distance_col: 'sum'}).sort_values(by=distance_col, ascending=False), \n", - " \"Labeled by user\\n\"+labeled_land_quality_text, ax[0], text_results[0], colors_mode, debug_df)\n", + " plot_and_text_stacked_bar_chart(labeled_land_trips_df, lambda df: df.groupby(\"mode_confirm\").agg({distance_col: 'sum'}).sort_values(by=distance_col, ascending=False), \n", + " \"Labeled by user\\n\"+labeled_land_quality_text, ax[0], text_results[0], colors_mode, debug_df, values_to_translations)\n", " plot_and_text_stacked_bar_chart(sensed_land_trips_df, lambda df: df.groupby(\"primary_mode\").agg({distance_col: 'sum'}).sort_values(by=distance_col, ascending=False), \n", " \"Sensed by OpenPATH\\n\"+sensed_land_quality_text, ax[1], text_results[1], colors_sensed, debug_df_sensed)\n", " set_title_and_save(fig, text_results, plot_title_no_quality, file_name) \n", diff --git a/viz_scripts/plots.py b/viz_scripts/plots.py index e7a9742..592f92a 100644 --- a/viz_scripts/plots.py +++ b/viz_scripts/plots.py @@ -94,7 +94,7 @@ def plot_and_text_error(e, ax, file_name): return alt_text, alt_html # Creates/ Appends single bar to the 100% Stacked Bar Chart -def plot_and_text_stacked_bar_chart(df, agg_fcn, bar_label, ax, text_result, colors, debug_df): +def plot_and_text_stacked_bar_chart(df, agg_fcn, bar_label, ax, text_result, colors, debug_df, values_to_translations={}): """ Inputs: df = Data frame corresponding to the bar in a stacked bar chart. It is expected to have three columns, which represent the 'label', 'value' @@ -128,7 +128,7 @@ def plot_and_text_stacked_bar_chart(df, agg_fcn, bar_label, ax, text_result, col mode_prop = long['Proportion'] mode_count = long['Value'] vals_str = [f'{y:.1f} %\n({x:.0f})' if y > 4 else '' for x, y in zip(mode_count, mode_prop)] - bar = ax.barh(y=bar_label, width=mode_prop, height=bar_height, left=bar_width, label=label, color=colors[label]) + bar = ax.barh(y=bar_label, width=mode_prop, height=bar_height, left=bar_width, label=values_to_translations.get(label, label), color=colors[label]) ax.bar_label(bar, label_type='center', labels=vals_str, rotation=90, fontsize=16) bar_width = [total + val for total, val in zip(bar_width, mode_prop)] else: @@ -144,6 +144,7 @@ def plot_and_text_stacked_bar_chart(df, agg_fcn, bar_label, ax, text_result, col text_result[0], text_result[1] = store_alt_text_and_html_stacked_bar_chart(df_all_entries, bar_label) print("After populating, %s" % text_result) except Exception as e: + print(e) # tb.print_exception(type(e), e, e.__traceback__) #ax.set_title("Insufficient data", loc="center") ax.set_ylabel(bar_label) diff --git a/viz_scripts/scaffolding.py b/viz_scripts/scaffolding.py index e2abc57..c67c94b 100644 --- a/viz_scripts/scaffolding.py +++ b/viz_scripts/scaffolding.py @@ -4,10 +4,13 @@ import sys from collections import defaultdict from collections import OrderedDict +import difflib import emission.storage.timeseries.abstract_timeseries as esta import emission.storage.timeseries.tcquery as esttc import emission.core.wrapper.localdate as ecwl +from emcommon.diary.base_modes import BASE_MODES, dedupe_colors +from emcommon.util import read_json_resource # Module for pretty-printing outputs (e.g. head) to help users # understand what is going on @@ -194,25 +197,50 @@ def translate_labels(labels): dic_mapping = translate_labels(dynamic_labels[label_type]) return dic_mapping +def find_closest_key(input_key, dictionary): + # Edge case + if " bike" in input_key.lower(): + return "BICYCLING" + input_key = input_key.split(",")[0].split("/")[0].replace("share","").upper() + keys = list(dictionary.keys()) + closest_matches = difflib.get_close_matches(input_key, keys, n=1) + if closest_matches: + return closest_matches[0] + return "OTHER" + # Function: Maps "MODE", "PURPOSE", and "REPLACED_MODE" to colors. # Input: dynamic_labels, dic_re, and dic_pur # Output: Dictionary mapping between color with mode/purpose/sensed -def mapping_color_labels(dynamic_labels, dic_re, dic_pur): +async def mapping_color_labels(dynamic_labels, dic_re, dic_pur, language="en"): + # Load default options from e-mission-common + labels = await read_json_resource("label-options.default.json") sensed_values = ["WALKING", "BICYCLING", "IN_VEHICLE", "AIR_OR_HSR", "UNKNOWN", "OTHER", "Other"] - if len(dynamic_labels) > 0: - mode_values = list(mapping_labels(dynamic_labels, "MODE").values()) if "MODE" in dynamic_labels else [] - replaced_mode_values = list(mapping_labels(dynamic_labels, "REPLACED_MODE").values()) if "REPLACED_MODE" in dynamic_labels else [] - purpose_values = list(mapping_labels(dynamic_labels, "PURPOSE").values()) + ['Other'] if "PURPOSE" in dynamic_labels else [] - combined_mode_values = mode_values + replaced_mode_values + ['Other'] - else: - combined_mode_values = (list(OrderedDict.fromkeys(dic_re.values())) + ['Other']) - purpose_values = list(OrderedDict.fromkeys(dic_pur.values())) + replaced_mode_values = [] - colors_mode = dict(zip(combined_mode_values, plt.cm.tab20.colors[:len(combined_mode_values)])) + # If dynamic_labels are provided, then we will use the dynamic labels for mapping + if len(dynamic_labels) > 0: + labels = dynamic_labels + replaced_mode_values = list(mapping_labels(labels, "REPLACED_MODE").values()) if "REPLACED_MODE" in labels else [] + + # Load base mode values and purpose values + mode_values = [mode["value"] for mode in labels["MODE"]] + purpose_values = list(mapping_labels(labels, "PURPOSE").values()) + ['Other'] if "PURPOSE" in labels else [] + combined_mode_values = mode_values + replaced_mode_values + ['Other'] + + # Mapping between mode values and base_mode OR baseMode (backwards compatibility) + value_to_basemode = {mode["value"]: mode.get("base_mode", "baseMode") for mode in labels["MODE"]} + # Mapping between values and translations for display on plots + values_to_translations = {mode["value"]: labels["translations"][language][mode["value"]] for mode in labels["MODE"]} + + # Assign colors to mode, purpose, and sensed values + colors_mode = dedupe_colors([ + [mode, BASE_MODES[value_to_basemode.get(mode, "UNKNOWN")]['color']] + for mode in combined_mode_values + ], adjustment_range=[1,1.8]) colors_purpose = dict(zip(purpose_values, plt.cm.tab20.colors[:len(purpose_values)])) - colors_sensed = dict(zip(sensed_values, plt.cm.tab20.colors[:len(sensed_values)])) + colors_sensed = dict(zip(sensed_values, [BASE_MODES[x.upper()]['color'] for x in sensed_values])) - return colors_mode, colors_purpose, colors_sensed + return colors_mode, colors_purpose, colors_sensed, values_to_translations # Function: Maps survey answers to colors. # Input: dictionary of raw and translated survey answers From 30948596844dd45533f4cb30c0fe0ef4b23b9467 Mon Sep 17 00:00:00 2001 From: iantei Date: Wed, 4 Sep 2024 13:40:10 -0700 Subject: [PATCH 10/39] Check for baseMode as key in dynamic_labels --- viz_scripts/scaffolding.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/viz_scripts/scaffolding.py b/viz_scripts/scaffolding.py index c67c94b..578947f 100644 --- a/viz_scripts/scaffolding.py +++ b/viz_scripts/scaffolding.py @@ -228,7 +228,7 @@ async def mapping_color_labels(dynamic_labels, dic_re, dic_pur, language="en"): combined_mode_values = mode_values + replaced_mode_values + ['Other'] # Mapping between mode values and base_mode OR baseMode (backwards compatibility) - value_to_basemode = {mode["value"]: mode.get("base_mode", "baseMode") for mode in labels["MODE"]} + value_to_basemode = {mode["value"]: mode.get("base_mode", mode.get("baseMode", "baseMode")) for mode in labels["MODE"]} # Mapping between values and translations for display on plots values_to_translations = {mode["value"]: labels["translations"][language][mode["value"]] for mode in labels["MODE"]} From 86f3798ecfa11784fc8d5e59494f465e201914e3 Mon Sep 17 00:00:00 2001 From: iantei Date: Wed, 4 Sep 2024 13:43:29 -0700 Subject: [PATCH 11/39] Update purpose-value mapping from label_options's Purpose's value instead of en translation mapping. --- viz_scripts/scaffolding.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/viz_scripts/scaffolding.py b/viz_scripts/scaffolding.py index 578947f..dd9912a 100644 --- a/viz_scripts/scaffolding.py +++ b/viz_scripts/scaffolding.py @@ -224,7 +224,7 @@ async def mapping_color_labels(dynamic_labels, dic_re, dic_pur, language="en"): # Load base mode values and purpose values mode_values = [mode["value"] for mode in labels["MODE"]] - purpose_values = list(mapping_labels(labels, "PURPOSE").values()) + ['Other'] if "PURPOSE" in labels else [] + purpose_values = [mode["value"] for mode in labels["PURPOSE"]] + ['Other'] combined_mode_values = mode_values + replaced_mode_values + ['Other'] # Mapping between mode values and base_mode OR baseMode (backwards compatibility) From e2cd09cad0a5c4b3559d8b7cc9366cffeb2ea25c Mon Sep 17 00:00:00 2001 From: iantei Date: Wed, 4 Sep 2024 13:44:35 -0700 Subject: [PATCH 12/39] Remove find_closest_key() since we are not using it anymore. --- viz_scripts/scaffolding.py | 11 ----------- 1 file changed, 11 deletions(-) diff --git a/viz_scripts/scaffolding.py b/viz_scripts/scaffolding.py index dd9912a..d1b1207 100644 --- a/viz_scripts/scaffolding.py +++ b/viz_scripts/scaffolding.py @@ -197,17 +197,6 @@ def translate_labels(labels): dic_mapping = translate_labels(dynamic_labels[label_type]) return dic_mapping -def find_closest_key(input_key, dictionary): - # Edge case - if " bike" in input_key.lower(): - return "BICYCLING" - input_key = input_key.split(",")[0].split("/")[0].replace("share","").upper() - keys = list(dictionary.keys()) - closest_matches = difflib.get_close_matches(input_key, keys, n=1) - if closest_matches: - return closest_matches[0] - return "OTHER" - # Function: Maps "MODE", "PURPOSE", and "REPLACED_MODE" to colors. # Input: dynamic_labels, dic_re, and dic_pur # Output: Dictionary mapping between color with mode/purpose/sensed From b07ffd15bf6e4409ee756778475e9099207bbd66 Mon Sep 17 00:00:00 2001 From: iantei Date: Wed, 4 Sep 2024 14:02:48 -0700 Subject: [PATCH 13/39] Remove dic_re, dic_pur as function parameter from mapping_color_labels. Introduce values_to_translations_purpose and return it. --- viz_scripts/scaffolding.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/viz_scripts/scaffolding.py b/viz_scripts/scaffolding.py index d1b1207..a220abc 100644 --- a/viz_scripts/scaffolding.py +++ b/viz_scripts/scaffolding.py @@ -200,7 +200,7 @@ def translate_labels(labels): # Function: Maps "MODE", "PURPOSE", and "REPLACED_MODE" to colors. # Input: dynamic_labels, dic_re, and dic_pur # Output: Dictionary mapping between color with mode/purpose/sensed -async def mapping_color_labels(dynamic_labels, dic_re, dic_pur, language="en"): +async def mapping_color_labels(dynamic_labels, language="en"): # Load default options from e-mission-common labels = await read_json_resource("label-options.default.json") sensed_values = ["WALKING", "BICYCLING", "IN_VEHICLE", "AIR_OR_HSR", "UNKNOWN", "OTHER", "Other"] @@ -218,8 +218,10 @@ async def mapping_color_labels(dynamic_labels, dic_re, dic_pur, language="en"): # Mapping between mode values and base_mode OR baseMode (backwards compatibility) value_to_basemode = {mode["value"]: mode.get("base_mode", mode.get("baseMode", "baseMode")) for mode in labels["MODE"]} - # Mapping between values and translations for display on plots - values_to_translations = {mode["value"]: labels["translations"][language][mode["value"]] for mode in labels["MODE"]} + # Mapping between values and translations for display on plots (for Mode) + values_to_translations_mode = {mode["value"]: labels["translations"][language][mode["value"]] for mode in labels["MODE"]} + # Mapping between values and translations for display on plots (for Purpose) + values_to_translations_purpose = {mode["value"]: labels["translations"][language][mode["value"]] for mode in labels["PURPOSE"]} # Assign colors to mode, purpose, and sensed values colors_mode = dedupe_colors([ @@ -229,7 +231,7 @@ async def mapping_color_labels(dynamic_labels, dic_re, dic_pur, language="en"): colors_purpose = dict(zip(purpose_values, plt.cm.tab20.colors[:len(purpose_values)])) colors_sensed = dict(zip(sensed_values, [BASE_MODES[x.upper()]['color'] for x in sensed_values])) - return colors_mode, colors_purpose, colors_sensed, values_to_translations + return colors_mode, colors_purpose, colors_sensed, values_to_translations_mode, values_to_translations_purpose # Function: Maps survey answers to colors. # Input: dictionary of raw and translated survey answers From 2e2bedc6e4ecacb2e9f271a60115d8f1cb5ca60e Mon Sep 17 00:00:00 2001 From: iantei Date: Wed, 4 Sep 2024 14:10:20 -0700 Subject: [PATCH 14/39] Update mapping_color_labels() to only take dynamic_labels as the function parameter. Use purpose_confirm instead of Trip_purpose to create plot_and_text_stacked_bar_chart for Number of trips for each purpose stacked bar chart. --- viz_scripts/generic_metrics.ipynb | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/viz_scripts/generic_metrics.ipynb b/viz_scripts/generic_metrics.ipynb index 7951399..06186b0 100644 --- a/viz_scripts/generic_metrics.ipynb +++ b/viz_scripts/generic_metrics.ipynb @@ -90,7 +90,7 @@ "metadata": {}, "outputs": [], "source": [ - "colors_mode, colors_purpose, colors_sensed, values_to_translations = await scaffolding.mapping_color_labels(dynamic_labels, dic_re, dic_pur)" + "colors_mode, colors_purpose, colors_sensed, values_to_translations, value_to_translations_purpose = await scaffolding.mapping_color_labels(dynamic_labels)" ] }, { @@ -291,8 +291,8 @@ "try:\n", " fig, ax = plt.subplots(nrows=1, ncols=1, figsize=(15,2*1), sharex=True)\n", " text_results = [\"Unmodified Alt Text\", \"Unmodified HTML\"]\n", - " plot_and_text_stacked_bar_chart(expanded_ct, lambda df: df.groupby(\"Trip_purpose\").agg({distance_col: 'count'}).sort_values(by=distance_col, ascending=False), \n", - " \"Labeled by user\\n\"+stacked_bar_quality_text_labeled, ax, text_results, colors_purpose, debug_df)\n", + " plot_and_text_stacked_bar_chart(expanded_ct, lambda df: df.groupby(\"purpose_confirm\").agg({distance_col: 'count'}).sort_values(by=distance_col, ascending=False), \n", + " \"Labeled by user\\n\"+stacked_bar_quality_text_labeled, ax, text_results, colors_purpose, debug_df, value_to_translations_purpose)\n", " set_title_and_save(fig, text_results, plot_title_no_quality, file_name)\n", "except (AttributeError, KeyError, pd.errors.UndefinedVariableError) as e:\n", " plt.clf()\n", From 26c39a80e87571bf5e39b49453088e289187fcec Mon Sep 17 00:00:00 2001 From: iantei Date: Tue, 10 Sep 2024 23:02:24 -0700 Subject: [PATCH 15/39] 1. Update load_viz_notebook_data() as async function so we can call await read_json_resource(). 2. Introduce merge_ column in the expanded_ct dataframe. 3. Update value_to_basemode default value to UNKNOWN. 4. Create new dictionary for colors_mode and colors_replaced. 5. Return translations for mode, purpose and replaced individually for legend values. --- viz_scripts/scaffolding.py | 47 ++++++++++++++++++++++++++------------ 1 file changed, 32 insertions(+), 15 deletions(-) diff --git a/viz_scripts/scaffolding.py b/viz_scripts/scaffolding.py index 32e046f..bcd85e9 100644 --- a/viz_scripts/scaffolding.py +++ b/viz_scripts/scaffolding.py @@ -112,7 +112,7 @@ def expand_userinputs(labeled_ct): unique_users = lambda df: len(df.user_id.unique()) if "user_id" in df.columns else 0 trip_label_count = lambda s, df: len(df[s].dropna()) if s in df.columns else 0 -def load_viz_notebook_data(year, month, program, study_type, dynamic_labels, dic_re, dic_pur=None, include_test_users=False): +async def load_viz_notebook_data(year, month, program, study_type, dynamic_labels, dic_re, dic_pur=None, include_test_users=False): """ Inputs: year/month/program/study_type = parameters from the visualization notebook dic_* = label mappings; if dic_pur is included it will be used to recode trip purpose @@ -137,16 +137,25 @@ def load_viz_notebook_data(year, month, program, study_type, dynamic_labels, dic if (len(dynamic_labels)): dic_mode_mapping = mapping_labels(dynamic_labels, "MODE") expanded_ct['Mode_confirm'] = expanded_ct['mode_confirm'].map(dic_mode_mapping) + labels = dynamic_labels else: expanded_ct['Mode_confirm'] = expanded_ct['mode_confirm'].map(dic_re) + labels = await read_json_resource("label-options.default.json") + # If the 'mode_confirm' is not available as the list of keys in the dynamic_labels or label_options.default.json, then, we should transform it as 'other' + mode_values = [item['value'] for item in labels['MODE']] + expanded_ct['merge_mode_confirm'] = expanded_ct['mode_confirm'].apply(lambda mode: 'other' if mode not in mode_values else mode) if study_type == 'program': # CASE 2 of https://github.com/e-mission/em-public-dashboard/issues/69#issuecomment-1256835867 if 'replaced_mode' in expanded_ct.columns: if (len(dynamic_labels)): dic_replaced_mapping = mapping_labels(dynamic_labels, "REPLACED_MODE") expanded_ct['Replaced_mode'] = expanded_ct['replaced_mode'].map(dic_replaced_mapping) + labels = dynamic_labels else: expanded_ct['Replaced_mode'] = expanded_ct['replaced_mode'].map(dic_re) + labels = await read_json_resource("label-options.default.json") + replaced_modes = [item['value'] for item in labels['REPLACED_MODE']] + expanded_ct['merge_replaced_mode'] = expanded_ct['replaced_mode'].apply(lambda mode: 'other' if mode not in replaced_modes else mode) else: print("This is a program, but no replaced modes found. Likely cold start case. Ignoring replaced mode mapping") else: @@ -156,10 +165,14 @@ def load_viz_notebook_data(year, month, program, study_type, dynamic_labels, dic # CASE 2 of https://github.com/e-mission/em-public-dashboard/issues/69#issuecomment-1256835867 if dic_pur is not None and "purpose_confirm" in expanded_ct.columns: if (len(dynamic_labels)): - dic_purpose_mapping = mapping_labels(dynamic_labels, "PURPOSE") - expanded_ct['Trip_purpose'] = expanded_ct['purpose_confirm'].map(dic_purpose_mapping) + dic_purpose_mapping = mapping_labels(dynamic_labels, "PURPOSE") + expanded_ct['Trip_purpose'] = expanded_ct['purpose_confirm'].map(dic_purpose_mapping) + labels = dynamic_labels else: expanded_ct['Trip_purpose'] = expanded_ct['purpose_confirm'].map(dic_pur) + labels = await read_json_resource("label-options.default.json") + purpose_values = [item['value'] for item in labels['PURPOSE']] + expanded_ct['merge_purpose_confirm'] = expanded_ct['purpose_confirm'].apply(lambda value: 'other' if value not in purpose_values else value) # Document data quality file_suffix = get_file_suffix(year, month, program) @@ -196,7 +209,7 @@ def translate_labels(labels): return defaultdict(lambda: 'Other', translation_mapping) dic_mapping = translate_labels(dynamic_labels[label_type]) return dic_mapping - + def find_closest_key(input_key, dictionary): # Edge case if " bike" in input_key.lower(): @@ -214,35 +227,39 @@ def find_closest_key(input_key, dictionary): async def mapping_color_labels(dynamic_labels, language="en"): # Load default options from e-mission-common labels = await read_json_resource("label-options.default.json") - sensed_values = ["WALKING", "BICYCLING", "IN_VEHICLE", "AIR_OR_HSR", "UNKNOWN", "OTHER", "Other"] - replaced_mode_values = [] + sensed_values = ["WALKING", "BICYCLING", "IN_VEHICLE", "AIR_OR_HSR", "UNKNOWN", "OTHER", "other"] # If dynamic_labels are provided, then we will use the dynamic labels for mapping if len(dynamic_labels) > 0: labels = dynamic_labels - replaced_mode_values = list(mapping_labels(labels, "REPLACED_MODE").values()) if "REPLACED_MODE" in labels else [] # Load base mode values and purpose values mode_values = [mode["value"] for mode in labels["MODE"]] - purpose_values = [mode["value"] for mode in labels["PURPOSE"]] + ['Other'] - combined_mode_values = mode_values + replaced_mode_values + ['Other'] + purpose_values = [mode["value"] for mode in labels["PURPOSE"]] + replaced_values = [mode["value"] for mode in labels["REPLACED_MODE"]] # Mapping between mode values and base_mode OR baseMode (backwards compatibility) - value_to_basemode = {mode["value"]: mode.get("base_mode", mode.get("baseMode", "baseMode")) for mode in labels["MODE"]} + value_to_basemode = {mode["value"]: mode.get("base_mode", mode.get("baseMode", "UNKNOWN")) for mode in labels["MODE"]} # Mapping between values and translations for display on plots (for Mode) - values_to_translations_mode = {mode["value"]: labels["translations"][language][mode["value"]] for mode in labels["MODE"]} + values_to_translations_mode = dict(mapping_labels(labels, "MODE")) # Mapping between values and translations for display on plots (for Purpose) - values_to_translations_purpose = {mode["value"]: labels["translations"][language][mode["value"]] for mode in labels["PURPOSE"]} + values_to_translations_purpose = dict(mapping_labels(labels, "PURPOSE")) + # Mapping between values and translations for display on plots (for Replaced mode) + values_to_translations_replaced = dict(mapping_labels(labels, "REPLACED_MODE")) - # Assign colors to mode, purpose, and sensed values + # Assign colors to mode, replaced, purpose, and sensed values colors_mode = dedupe_colors([ [mode, BASE_MODES[value_to_basemode.get(mode, "UNKNOWN")]['color']] - for mode in combined_mode_values + for mode in set(mode_values) + ], adjustment_range=[1,1.8]) + colors_replaced = dedupe_colors([ + [mode, BASE_MODES[value_to_basemode.get(mode, "UNKNOWN")]['color']] + for mode in set(replaced_values) ], adjustment_range=[1,1.8]) colors_purpose = dict(zip(purpose_values, plt.cm.tab20.colors[:len(purpose_values)])) colors_sensed = dict(zip(sensed_values, [BASE_MODES[x.upper()]['color'] for x in sensed_values])) - return colors_mode, colors_purpose, colors_sensed, values_to_translations_mode, values_to_translations_purpose + return colors_mode, colors_replaced, colors_purpose, colors_sensed, values_to_translations_mode, values_to_translations_purpose, values_to_translations_replaced # Function: Maps survey answers to colors. # Input: dictionary of raw and translated survey answers From 8c8b7ff9a66053bb09c78a94de79754c54089105 Mon Sep 17 00:00:00 2001 From: iantei Date: Tue, 10 Sep 2024 23:09:15 -0700 Subject: [PATCH 16/39] Update Other to other since we pass the internal mode instead of Display mode. --- viz_scripts/plots.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/viz_scripts/plots.py b/viz_scripts/plots.py index 592f92a..d9c2826 100644 --- a/viz_scripts/plots.py +++ b/viz_scripts/plots.py @@ -46,16 +46,16 @@ def merge_small_entries(labels, values): # This part if a bit tricky # We could have already had a non-zero other, and it could be small or large - if "Other" not in v2l_df.index: + if "other" not in v2l_df.index: # zero other will end up with misc_count if misc_count.vals > 0: - v2l_df.loc["Other"] = misc_count - elif "Other" in small_chunk.index: + v2l_df.loc["other"] = misc_count + elif "other" in small_chunk.index: # non-zero small other will already be in misc_count - v2l_df.loc["Other"] = misc_count + v2l_df.loc["other"] = misc_count else: # non-zero large other, will not already be in misc_count - v2l_df.loc["Other"] = v2l_df.loc["Other"] + misc_count + v2l_df.loc["other"] = v2l_df.loc["other"] + misc_count disp.display(v2l_df) From 1ef4c3b1817020b16ecd3ddd2c796d23b6f44aba Mon Sep 17 00:00:00 2001 From: iantei Date: Tue, 10 Sep 2024 23:15:40 -0700 Subject: [PATCH 17/39] Update mapping_color_labels() to use await,and update its return values. Update load_viz_notebook_data() to use await. Update use of merge_purpose_confirm over purpose_confirm as column name for Purpose related trip. --- viz_scripts/generic_metrics.ipynb | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/viz_scripts/generic_metrics.ipynb b/viz_scripts/generic_metrics.ipynb index 06186b0..78568de 100644 --- a/viz_scripts/generic_metrics.ipynb +++ b/viz_scripts/generic_metrics.ipynb @@ -90,7 +90,7 @@ "metadata": {}, "outputs": [], "source": [ - "colors_mode, colors_purpose, colors_sensed, values_to_translations, value_to_translations_purpose = await scaffolding.mapping_color_labels(dynamic_labels)" + "colors_mode, colors_replaced, colors_purpose, colors_sensed, values_to_translations, value_to_translations_purpose, values_to_translations_replaced = await scaffolding.mapping_color_labels(dynamic_labels)" ] }, { @@ -108,7 +108,7 @@ "metadata": {}, "outputs": [], "source": [ - "expanded_ct, file_suffix, quality_text, debug_df = scaffolding.load_viz_notebook_data(year,\n", + "expanded_ct, file_suffix, quality_text, debug_df = await scaffolding.load_viz_notebook_data(year,\n", " month,\n", " program,\n", " study_type,\n", @@ -291,7 +291,7 @@ "try:\n", " fig, ax = plt.subplots(nrows=1, ncols=1, figsize=(15,2*1), sharex=True)\n", " text_results = [\"Unmodified Alt Text\", \"Unmodified HTML\"]\n", - " plot_and_text_stacked_bar_chart(expanded_ct, lambda df: df.groupby(\"purpose_confirm\").agg({distance_col: 'count'}).sort_values(by=distance_col, ascending=False), \n", + " plot_and_text_stacked_bar_chart(expanded_ct, lambda df: df.groupby(\"merge_purpose_confirm\").agg({distance_col: 'count'}).sort_values(by=distance_col, ascending=False), \n", " \"Labeled by user\\n\"+stacked_bar_quality_text_labeled, ax, text_results, colors_purpose, debug_df, value_to_translations_purpose)\n", " set_title_and_save(fig, text_results, plot_title_no_quality, file_name)\n", "except (AttributeError, KeyError, pd.errors.UndefinedVariableError) as e:\n", From ac655281b007aa1820b6bf7b08fe86cb3ddea3cc Mon Sep 17 00:00:00 2001 From: iantei Date: Tue, 10 Sep 2024 23:22:18 -0700 Subject: [PATCH 18/39] Update mapping_colors_labels() to use await, and update return variable. Update load_viz_notebook_data() to use await. Update the column passed as Trip_purpose to merge_purpose_confirm and Replaced_mode to merge_replaced_mode for Stacked Bar Charts. --- viz_scripts/mode_specific_metrics.ipynb | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/viz_scripts/mode_specific_metrics.ipynb b/viz_scripts/mode_specific_metrics.ipynb index 0e2e63d..9040953 100644 --- a/viz_scripts/mode_specific_metrics.ipynb +++ b/viz_scripts/mode_specific_metrics.ipynb @@ -103,7 +103,7 @@ "metadata": {}, "outputs": [], "source": [ - "colors_mode, colors_purpose, colors_sensed = scaffolding.mapping_color_labels(dynamic_labels, dic_re, dic_pur)" + "colors_mode, colors_replaced, colors_purpose, colors_sensed, values_to_translations, value_to_translations_purpose, value_to_translations_replaced = await scaffolding.mapping_color_labels(dynamic_labels)" ] }, { @@ -121,7 +121,7 @@ "metadata": {}, "outputs": [], "source": [ - "expanded_ct, file_suffix, quality_text, debug_df = scaffolding.load_viz_notebook_data(year,\n", + "expanded_ct, file_suffix, quality_text, debug_df = await scaffolding.load_viz_notebook_data(year,\n", " month,\n", " program,\n", " study_type,\n", @@ -193,8 +193,8 @@ "try:\n", " fig, ax = plt.subplots(nrows=1, ncols=1, figsize=(15,2*1), sharex=True)\n", " text_results = [\"Unmodified Alt Text\", \"Unmodified HTML\"]\n", - " plot_and_text_stacked_bar_chart(data_eb, lambda df: df.groupby(\"Trip_purpose\").agg({distance_col: 'count'}).sort_values(by=distance_col, ascending=False),\n", - " f\"Labeled `{mode_of_interest}` by user\", ax, text_results, colors_purpose, debug_df)\n", + " plot_and_text_stacked_bar_chart(data_eb, lambda df: df.groupby(\"merge_purpose_confirm\").agg({distance_col: 'count'}).sort_values(by=distance_col, ascending=False),\n", + " f\"Labeled `{mode_of_interest}` by user\", ax, text_results, colors_purpose, debug_df, value_to_translations_purpose)\n", " plot_title = plot_title_no_quality + \"\\n\" + f\"For {mode_of_interest}: \" + quality_text\n", " set_title_and_save(fig, text_results, plot_title, file_name)\n", "except (AttributeError, KeyError, pd.errors.UndefinedVariableError) as e:\n", @@ -228,8 +228,8 @@ "try:\n", " fig, ax = plt.subplots(nrows=1, ncols=1, figsize=(15,2*1), sharex=True)\n", " text_results = [\"Unmodified Alt Text\", \"Unmodified HTML\"]\n", - " plot_and_text_stacked_bar_chart(data_eb, lambda df: df.groupby(\"Replaced_mode\").agg({distance_col: 'sum'}).sort_values(by=distance_col, ascending=False), \n", - " \"Labeled by user\\n (Trip distance)\", ax, text_results, colors_mode, debug_df)\n", + " plot_and_text_stacked_bar_chart(data_eb, lambda df: df.groupby(\"merge_replaced_mode\").agg({distance_col: 'sum'}).sort_values(by=distance_col, ascending=False), \n", + " \"Labeled by user\\n (Trip distance)\", ax, text_results, colors_replaced, debug_df, value_to_translations_replaced)\n", " plot_title = plot_title_no_quality + \"\\n\" + f\"For {mode_of_interest}: \" + quality_text\n", " set_title_and_save(fig, text_results, plot_title, file_name)\n", "except (AttributeError, KeyError, pd.errors.UndefinedVariableError) as e:\n", @@ -263,8 +263,8 @@ "try:\n", " fig, ax = plt.subplots(nrows=1, ncols=1, figsize=(15,2*1), sharex=True)\n", " text_results = [\"Unmodified Alt Text\", \"Unmodified HTML\"]\n", - " plot_and_text_stacked_bar_chart(data_eb, lambda df: df.groupby(\"Replaced_mode\").agg({distance_col: 'count'}).sort_values(by=distance_col, ascending=False), \n", - " f\"Labeled `{mode_of_interest}` by user\", ax, text_results, colors_mode, debug_df)\n", + " plot_and_text_stacked_bar_chart(data_eb, lambda df: df.groupby(\"merge_replaced_mode\").agg({distance_col: 'count'}).sort_values(by=distance_col, ascending=False), \n", + " f\"Labeled `{mode_of_interest}` by user\", ax, text_results, colors_replaced, debug_df, value_to_translations_replaced)\n", " plot_title = plot_title_no_quality + \"\\n\" + f\"For {mode_of_interest}: \" + quality_text\n", " set_title_and_save(fig, text_results, plot_title, file_name)\n", "except (AttributeError, KeyError, pd.errors.UndefinedVariableError) as e:\n", From fccc53d0c2c5ef353098b732a484f3e2b9a3af9a Mon Sep 17 00:00:00 2001 From: iantei Date: Tue, 10 Sep 2024 23:26:01 -0700 Subject: [PATCH 19/39] Change the mapping_color_labels() to use await, and update return variables. --- viz_scripts/survey_metrics.ipynb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/viz_scripts/survey_metrics.ipynb b/viz_scripts/survey_metrics.ipynb index a7d395d..5e3d18d 100644 --- a/viz_scripts/survey_metrics.ipynb +++ b/viz_scripts/survey_metrics.ipynb @@ -61,7 +61,7 @@ "label_units, short_label, label_units_lower, distance_col, weight_unit = scaffolding.get_units(use_imperial)\n", "\n", "# get color mappings\n", - "colors_mode, colors_purpose, colors_sensed = scaffolding.mapping_color_labels({}, {}, {}) #just need sensed" + "colors_mode, colors_replaced, colors_purpose, colors_sensed, values_to_translations, value_to_translations_purpose, values_to_translations_replaced = await scaffolding.mapping_color_labels({}) #just need sensed" ] }, { From a0fae22edc0c70646c396eea2dcb23c5dc46f76f Mon Sep 17 00:00:00 2001 From: iantei Date: Tue, 10 Sep 2024 23:48:30 -0700 Subject: [PATCH 20/39] Update use of merge_mode_confirm over mode_confirm. --- viz_scripts/generic_metrics.ipynb | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/viz_scripts/generic_metrics.ipynb b/viz_scripts/generic_metrics.ipynb index 78568de..6d51c1b 100644 --- a/viz_scripts/generic_metrics.ipynb +++ b/viz_scripts/generic_metrics.ipynb @@ -207,7 +207,7 @@ " # We will have text results corresponding to the axes for simplicity and consistency\n", " text_results = [[\"Unmodified Alt Text\", \"Unmodified HTML\"], [\"Unmodified Alt Text\", \"Unmodified HTML\"]]\n", " \n", - " plot_and_text_stacked_bar_chart(expanded_ct, lambda df: (df.groupby(\"mode_confirm\").agg({distance_col: 'count'}).sort_values(by=distance_col, ascending=False)), \n", + " plot_and_text_stacked_bar_chart(expanded_ct, lambda df: (df.groupby(\"merge_mode_confirm\").agg({distance_col: 'count'}).sort_values(by=distance_col, ascending=False)), \n", " \"Labeled by user\\n\"+stacked_bar_quality_text_labeled, ax[0], text_results[0], colors_mode, debug_df, values_to_translations)\n", " plot_and_text_stacked_bar_chart(expanded_ct_sensed, lambda df: (df.groupby(\"primary_mode\").agg({distance_col: 'count'}).sort_values(by=distance_col, ascending=False)), \n", " \"Sensed by OpenPATH\\n\"+stacked_bar_quality_text_sensed, ax[1], text_results[1], colors_sensed, debug_df_sensed)\n", @@ -258,7 +258,7 @@ " # Plot entries\n", " fig, ax = plt.subplots(nrows=1, ncols=1, figsize=(15,2*1), sharex=True) \n", " text_results = [\"Unmodified Alt Text\", \"Unmodified HTML\"]\n", - " plot_and_text_stacked_bar_chart(expanded_ct_commute, lambda df: df.groupby(\"mode_confirm\").agg({distance_col: 'count'}).sort_values(by=distance_col, ascending=False), \n", + " plot_and_text_stacked_bar_chart(expanded_ct_commute, lambda df: df.groupby(\"merge_mode_confirm\").agg({distance_col: 'count'}).sort_values(by=distance_col, ascending=False), \n", " \"Labeled by user\\n (Confirmed trips)\", ax, text_results, colors_mode, debug_df, values_to_translations)\n", " set_title_and_save(fig, text_results, plot_title, file_name)\n", "except (AttributeError, KeyError, pd.errors.UndefinedVariableError) as e:\n", @@ -335,7 +335,7 @@ "\n", " ## We do an existence check for the labeled df because we want to display the sensed value even if we don't have the labeled value\n", " ## but we don't need to have an existence check for sensed because in that case we will have no data to display\n", - " expanded_ct_u80 = expanded_ct.loc[(expanded_ct['distance'] <= cutoff)] if \"mode_confirm\" in expanded_ct.columns else None\n", + " expanded_ct_u80 = expanded_ct.loc[(expanded_ct['distance'] <= cutoff)] if \"merge_mode_confirm\" in expanded_ct.columns else None\n", " expanded_ct_sensed_u80 = expanded_ct_sensed.loc[(expanded_ct_sensed['distance'] <= cutoff)]\n", " sensed_u80_quality_text = f\"{len(expanded_ct_sensed_u80)} trips ({round(len(expanded_ct_sensed_u80)/len(expanded_ct_sensed)*100)}% of all trips)\\nfrom {scaffolding.unique_users(expanded_ct_sensed_u80)} {sensed_match.group(3)}\"\n", " labeled_u80_quality_text = f\"{len(expanded_ct_u80)} trips ({round(len(expanded_ct_u80)/len(expanded_ct)*100)}% of all labeled,\\n{round(len(expanded_ct_u80)/len(expanded_ct_sensed)*100)}% of all trips)\\nfrom {scaffolding.unique_users(expanded_ct_u80)} {sensed_match.group(3)}\" if \"Mode_confirm\" in expanded_ct.columns else \"0 labeled trips\"\n", @@ -343,7 +343,7 @@ " # Plot entries\n", " fig, ax = plt.subplots(nrows=2, ncols=1, figsize=(15,2*2), sharex=True)\n", " text_results = [[\"Unmodified Alt Text\", \"Unmodified HTML\"], [\"Unmodified Alt Text\", \"Unmodified HTML\"]]\n", - " plot_and_text_stacked_bar_chart(expanded_ct_u80, lambda df: df.groupby(\"mode_confirm\").agg({distance_col: 'count'}).sort_values(by=distance_col, ascending=False), \n", + " plot_and_text_stacked_bar_chart(expanded_ct_u80, lambda df: df.groupby(\"merge_mode_confirm\").agg({distance_col: 'count'}).sort_values(by=distance_col, ascending=False), \n", " \"Labeled by user\\n\"+labeled_u80_quality_text, ax[0], text_results[0], colors_mode, debug_df, values_to_translations)\n", " plot_and_text_stacked_bar_chart(expanded_ct_sensed_u80, lambda df: df.groupby(\"primary_mode\").agg({distance_col: 'count'}).sort_values(by=distance_col, ascending=False), \n", " \"Sensed by OpenPATH\\n\"+sensed_u80_quality_text, ax[1], text_results[1], colors_sensed, debug_df_sensed)\n", @@ -383,7 +383,7 @@ " fig, ax = plt.subplots(nrows=2, ncols=1, figsize=(15,2*2), sharex=True)\n", " \n", " text_results = [[\"Unmodified Alt Text\", \"Unmodified HTML\"], [\"Unmodified Alt Text\", \"Unmodified HTML\"]]\n", - " plot_and_text_stacked_bar_chart(expanded_ct, lambda df: df.groupby(\"mode_confirm\").agg({distance_col: 'sum'}).sort_values(by=distance_col, ascending=False), \n", + " plot_and_text_stacked_bar_chart(expanded_ct, lambda df: df.groupby(\"merge_mode_confirm\").agg({distance_col: 'sum'}).sort_values(by=distance_col, ascending=False), \n", " \"Labeled by user\\n\"+stacked_bar_quality_text_labeled, ax[0], text_results[0], colors_mode, debug_df, values_to_translations)\n", " plot_and_text_stacked_bar_chart(expanded_ct_sensed, lambda df: df.groupby(\"primary_mode\").agg({distance_col: 'sum'}).sort_values(by=distance_col, ascending=False), \n", " \"Sensed by OpenPATH\\n\"+stacked_bar_quality_text_sensed, ax[1], text_results[1], colors_sensed, debug_df_sensed)\n", @@ -426,7 +426,7 @@ " labeled_land_quality_text = f\"{len(labeled_land_trips_df)} trips ({round(len(labeled_land_trips_df)/len(expanded_ct)*100)}% of all labeled,\\n{round(len(labeled_land_trips_df)/len(expanded_ct_sensed)*100)}%) of all trips)\\nfrom {scaffolding.unique_users(labeled_land_trips_df)} {sensed_match.group(3)}\" if \"Mode_confirm\" in expanded_ct.columns else \"0 labeled trips\"\n", "\n", " fig, ax = plt.subplots(nrows=2, ncols=1, figsize=(15,2*2), sharex=True)\n", - " plot_and_text_stacked_bar_chart(labeled_land_trips_df, lambda df: df.groupby(\"mode_confirm\").agg({distance_col: 'sum'}).sort_values(by=distance_col, ascending=False), \n", + " plot_and_text_stacked_bar_chart(labeled_land_trips_df, lambda df: df.groupby(\"merge_mode_confirm\").agg({distance_col: 'sum'}).sort_values(by=distance_col, ascending=False), \n", " \"Labeled by user\\n\"+labeled_land_quality_text, ax[0], text_results[0], colors_mode, debug_df, values_to_translations)\n", " plot_and_text_stacked_bar_chart(sensed_land_trips_df, lambda df: df.groupby(\"primary_mode\").agg({distance_col: 'sum'}).sort_values(by=distance_col, ascending=False), \n", " \"Sensed by OpenPATH\\n\"+sensed_land_quality_text, ax[1], text_results[1], colors_sensed, debug_df_sensed)\n", From 12d479519d704be34570bb7fc41430f585625136 Mon Sep 17 00:00:00 2001 From: iantei Date: Wed, 11 Sep 2024 16:11:11 -0700 Subject: [PATCH 21/39] Update await to scaffolding.load_viz_notebook_data() --- viz_scripts/energy_calculations.ipynb | 2 +- viz_scripts/generic_timeseries.ipynb | 2 +- viz_scripts/mode_specific_timeseries.ipynb | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/viz_scripts/energy_calculations.ipynb b/viz_scripts/energy_calculations.ipynb index 4095929..f287e8f 100644 --- a/viz_scripts/energy_calculations.ipynb +++ b/viz_scripts/energy_calculations.ipynb @@ -106,7 +106,7 @@ }, "outputs": [], "source": [ - "expanded_ct, file_suffix, quality_text, debug_df = scaffolding.load_viz_notebook_data(year,\n", + "expanded_ct, file_suffix, quality_text, debug_df = await scaffolding.load_viz_notebook_data(year,\n", " month,\n", " program,\n", " study_type,\n", diff --git a/viz_scripts/generic_timeseries.ipynb b/viz_scripts/generic_timeseries.ipynb index 189afdf..63ef911 100644 --- a/viz_scripts/generic_timeseries.ipynb +++ b/viz_scripts/generic_timeseries.ipynb @@ -87,7 +87,7 @@ }, "outputs": [], "source": [ - "expanded_ct, file_suffix, quality_text, debug_df = scaffolding.load_viz_notebook_data(year,\n", + "expanded_ct, file_suffix, quality_text, debug_df = await scaffolding.load_viz_notebook_data(year,\n", " month,\n", " program,\n", " study_type,\n", diff --git a/viz_scripts/mode_specific_timeseries.ipynb b/viz_scripts/mode_specific_timeseries.ipynb index d696794..58ee56d 100644 --- a/viz_scripts/mode_specific_timeseries.ipynb +++ b/viz_scripts/mode_specific_timeseries.ipynb @@ -96,7 +96,7 @@ "metadata": {}, "outputs": [], "source": [ - "expanded_ct, file_suffix, quality_text, debug_df = scaffolding.load_viz_notebook_data(year,\n", + "expanded_ct, file_suffix, quality_text, debug_df = await scaffolding.load_viz_notebook_data(year,\n", " month,\n", " program,\n", " study_type,\n", From 51eb1da786a4c6c35478c48562e1921d0c527b7b Mon Sep 17 00:00:00 2001 From: iantei Date: Wed, 11 Sep 2024 16:13:44 -0700 Subject: [PATCH 22/39] Remove find_closest_key() since it's not being used anywhere. --- viz_scripts/scaffolding.py | 11 ----------- 1 file changed, 11 deletions(-) diff --git a/viz_scripts/scaffolding.py b/viz_scripts/scaffolding.py index bcd85e9..96d8394 100644 --- a/viz_scripts/scaffolding.py +++ b/viz_scripts/scaffolding.py @@ -210,17 +210,6 @@ def translate_labels(labels): dic_mapping = translate_labels(dynamic_labels[label_type]) return dic_mapping -def find_closest_key(input_key, dictionary): - # Edge case - if " bike" in input_key.lower(): - return "BICYCLING" - input_key = input_key.split(",")[0].split("/")[0].replace("share","").upper() - keys = list(dictionary.keys()) - closest_matches = difflib.get_close_matches(input_key, keys, n=1) - if closest_matches: - return closest_matches[0] - return "OTHER" - # Function: Maps "MODE", "PURPOSE", and "REPLACED_MODE" to colors. # Input: dynamic_labels, dic_re, and dic_pur # Output: Dictionary mapping between color with mode/purpose/sensed From a5180c4d563c355e05298fb416f71d15258ee264 Mon Sep 17 00:00:00 2001 From: iantei Date: Wed, 11 Sep 2024 16:16:20 -0700 Subject: [PATCH 23/39] Add await in front of scaffolding.mapping_color_labels() since it is now an async function. --- viz_scripts/generic_metrics_sensed.ipynb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/viz_scripts/generic_metrics_sensed.ipynb b/viz_scripts/generic_metrics_sensed.ipynb index 705c32f..2c937e1 100644 --- a/viz_scripts/generic_metrics_sensed.ipynb +++ b/viz_scripts/generic_metrics_sensed.ipynb @@ -96,7 +96,7 @@ " expanded_ct[\"primary_mode\"] = expanded_ct.ble_sensed_summary.apply(lambda md: max(md[\"distance\"], key=md[\"distance\"].get))\n", " unique_keys = expanded_ct.groupby(\"primary_mode\").agg({distance_col: \"count\"}).index\n", " print(unique_keys)\n", - " colors_mode, colors_purpose, colors_sensed = scaffolding.mapping_color_labels({}, dict(zip(unique_keys, unique_keys)), {})\n", + " colors_mode, colors_purpose, colors_sensed = await scaffolding.mapping_color_labels({}, dict(zip(unique_keys, unique_keys)), {})\n", " colors_sensed = colors_mode\n", "except ValueError as e:\n", " print(\"Got ValueError \", e)" From 78d105a85aeee11af7b017cf3df33e2468a792fb Mon Sep 17 00:00:00 2001 From: iantei Date: Thu, 12 Sep 2024 09:26:54 -0700 Subject: [PATCH 24/39] Add values_to_translations mapping for text and html table for mapping to translated Labels. --- viz_scripts/plots.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/viz_scripts/plots.py b/viz_scripts/plots.py index d9c2826..59bcd4f 100644 --- a/viz_scripts/plots.py +++ b/viz_scripts/plots.py @@ -141,7 +141,7 @@ def plot_and_text_stacked_bar_chart(df, agg_fcn, bar_label, ax, text_result, col # Fix for the error: RuntimeError("Unknown return type"), adding the below line to address as mentioned here https://github.com/matplotlib/matplotlib/issues/25625/ ax.set_xlim(right=ax.get_xlim()[1] + 1.0, auto=True) - text_result[0], text_result[1] = store_alt_text_and_html_stacked_bar_chart(df_all_entries, bar_label) + text_result[0], text_result[1] = store_alt_text_and_html_stacked_bar_chart(df_all_entries, bar_label, values_to_translations) print("After populating, %s" % text_result) except Exception as e: print(e) @@ -440,7 +440,7 @@ def access_alt_html(html_content, chart_name): return html_content # Appends bar information into into the alt_html -def store_alt_text_and_html_stacked_bar_chart(df, var_name): +def store_alt_text_and_html_stacked_bar_chart(df, var_name, values_to_translations): """ Inputs: df = dataframe combining columns as Trip Type, Label, Value, Proportion chart_name = name of the chart @@ -448,12 +448,12 @@ def store_alt_text_and_html_stacked_bar_chart(df, var_name): # Generate alt text file alt_text = f"\nStacked Bar of: {var_name}\n" for i in range(len(df)): - alt_text += f"{df['Label'].iloc[i]} is {df['Value'].iloc[i]}({df['Proportion'].iloc[i]}%).\n" + alt_text += f"{values_to_translations.get(df['Label'].iloc[i], df['Label'].iloc[i])} is {df['Value'].iloc[i]}({df['Proportion'].iloc[i]}%).\n" # Generate html table alt_html = "\n" for i in range(len(df)): - alt_html += f"{df['Label'].iloc[i]}{df['Value'].iloc[i]}{df['Proportion'].iloc[i]}%" + alt_html += f"{values_to_translations.get(df['Label'].iloc[i], df['Label'].iloc[i])}{df['Value'].iloc[i]}{df['Proportion'].iloc[i]}%" html_content = f"""

Trip Type: {var_name}

From e634d8e295bdccf1588a235a5df45ddbc561512c Mon Sep 17 00:00:00 2001 From: iantei Date: Thu, 12 Sep 2024 11:50:51 -0700 Subject: [PATCH 25/39] Use merge_mode_confirm column which has internal label, merged to other, which are unavailable on the label_options to filter out the mode of commute if it is not air. --- viz_scripts/generic_metrics.ipynb | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/viz_scripts/generic_metrics.ipynb b/viz_scripts/generic_metrics.ipynb index 6d51c1b..4800284 100644 --- a/viz_scripts/generic_metrics.ipynb +++ b/viz_scripts/generic_metrics.ipynb @@ -419,11 +419,11 @@ "try:\n", " ## We do an existence check for the labeled df because we want to display the sensed value even if we don't have the labeled value\n", " ## but we don't need to have an existence check for sensed because in that case we will have no data to display\n", - " labeled_land_trips_df = expanded_ct[expanded_ct['Mode_confirm'] != \"Airplane\"] if \"Mode_confirm\" in expanded_ct.columns else None\n", + " labeled_land_trips_df = expanded_ct[expanded_ct['merge_mode_confirm'] != \"air\"] if \"merge_mode_confirm\" in expanded_ct.columns else None\n", " sensed_land_trips_df = expanded_ct_sensed[expanded_ct_sensed['primary_mode'] != \"AIR_OR_HSR\"]\n", " \n", " sensed_land_quality_text = f\"{len(sensed_land_trips_df)} trips ({round(len(sensed_land_trips_df)/len(expanded_ct_sensed)*100)}% of all trips)\\nfrom {scaffolding.unique_users(sensed_land_trips_df)} {sensed_match.group(3)}\"\n", - " labeled_land_quality_text = f\"{len(labeled_land_trips_df)} trips ({round(len(labeled_land_trips_df)/len(expanded_ct)*100)}% of all labeled,\\n{round(len(labeled_land_trips_df)/len(expanded_ct_sensed)*100)}%) of all trips)\\nfrom {scaffolding.unique_users(labeled_land_trips_df)} {sensed_match.group(3)}\" if \"Mode_confirm\" in expanded_ct.columns else \"0 labeled trips\"\n", + " labeled_land_quality_text = f\"{len(labeled_land_trips_df)} trips ({round(len(labeled_land_trips_df)/len(expanded_ct)*100)}% of all labeled,\\n{round(len(labeled_land_trips_df)/len(expanded_ct_sensed)*100)}%) of all trips)\\nfrom {scaffolding.unique_users(labeled_land_trips_df)} {sensed_match.group(3)}\" if \"merge_mode_confirm\" in expanded_ct.columns else \"0 labeled trips\"\n", "\n", " fig, ax = plt.subplots(nrows=2, ncols=1, figsize=(15,2*2), sharex=True)\n", " plot_and_text_stacked_bar_chart(labeled_land_trips_df, lambda df: df.groupby(\"merge_mode_confirm\").agg({distance_col: 'sum'}).sort_values(by=distance_col, ascending=False), \n", From ed0c4c61ffb7cabd1976c5b9109e9cfccca75dd7 Mon Sep 17 00:00:00 2001 From: iantei Date: Thu, 12 Sep 2024 14:26:55 -0700 Subject: [PATCH 26/39] Handle cases when MODE, PURPOSE, or REPLACED_MODE is unavailable. --- viz_scripts/scaffolding.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/viz_scripts/scaffolding.py b/viz_scripts/scaffolding.py index 96d8394..ef208b1 100644 --- a/viz_scripts/scaffolding.py +++ b/viz_scripts/scaffolding.py @@ -207,7 +207,7 @@ def translate_labels(labels): translation = translations.get(value) translation_mapping[value] = translation return defaultdict(lambda: 'Other', translation_mapping) - dic_mapping = translate_labels(dynamic_labels[label_type]) + dic_mapping = translate_labels(dynamic_labels.get(label_type, '')) return dic_mapping # Function: Maps "MODE", "PURPOSE", and "REPLACED_MODE" to colors. @@ -222,10 +222,10 @@ async def mapping_color_labels(dynamic_labels, language="en"): if len(dynamic_labels) > 0: labels = dynamic_labels - # Load base mode values and purpose values - mode_values = [mode["value"] for mode in labels["MODE"]] - purpose_values = [mode["value"] for mode in labels["PURPOSE"]] - replaced_values = [mode["value"] for mode in labels["REPLACED_MODE"]] + # Load base mode values and purpose values + mode_values = [mode["value"] for mode in labels["MODE"]] if "MODE" in labels else [] + purpose_values = [mode["value"] for mode in labels["PURPOSE"]] if "PURPOSE" in labels else [] + replaced_values = [mode["value"] for mode in labels["REPLACED_MODE"]] if "REPLACED_MODE" in labels else [] # Mapping between mode values and base_mode OR baseMode (backwards compatibility) value_to_basemode = {mode["value"]: mode.get("base_mode", mode.get("baseMode", "UNKNOWN")) for mode in labels["MODE"]} From 75b4ce21662984c0cb67511a660cc6fb1a00a340 Mon Sep 17 00:00:00 2001 From: iantei Date: Sun, 15 Sep 2024 14:52:05 -0700 Subject: [PATCH 27/39] Split mapping_color_labels() to two functions - mapping_color_labels() for color label mapping, and translate_values_to_labels() for label to translations mapping --- viz_scripts/generic_metrics.ipynb | 3 ++- viz_scripts/mode_specific_metrics.ipynb | 3 ++- viz_scripts/scaffolding.py | 28 +++++++++++++++++-------- viz_scripts/survey_metrics.ipynb | 2 +- 4 files changed, 24 insertions(+), 12 deletions(-) diff --git a/viz_scripts/generic_metrics.ipynb b/viz_scripts/generic_metrics.ipynb index 4800284..a44e44a 100644 --- a/viz_scripts/generic_metrics.ipynb +++ b/viz_scripts/generic_metrics.ipynb @@ -90,7 +90,8 @@ "metadata": {}, "outputs": [], "source": [ - "colors_mode, colors_replaced, colors_purpose, colors_sensed, values_to_translations, value_to_translations_purpose, values_to_translations_replaced = await scaffolding.mapping_color_labels(dynamic_labels)" + "colors_mode, colors_replaced, colors_purpose, colors_sensed = await scaffolding.mapping_color_labels(dynamic_labels)\n", + "values_to_translations, value_to_translations_purpose, values_to_translations_replaced = await scaffolding.translate_values_to_labels(dynamic_labels)" ] }, { diff --git a/viz_scripts/mode_specific_metrics.ipynb b/viz_scripts/mode_specific_metrics.ipynb index 9040953..ae554b4 100644 --- a/viz_scripts/mode_specific_metrics.ipynb +++ b/viz_scripts/mode_specific_metrics.ipynb @@ -103,7 +103,8 @@ "metadata": {}, "outputs": [], "source": [ - "colors_mode, colors_replaced, colors_purpose, colors_sensed, values_to_translations, value_to_translations_purpose, value_to_translations_replaced = await scaffolding.mapping_color_labels(dynamic_labels)" + "colors_mode, colors_replaced, colors_purpose, colors_sensed = await scaffolding.mapping_color_labels(dynamic_labels)\n", + "values_to_translations, value_to_translations_purpose, value_to_translations_replaced = await scaffolding.translate_values_to_labels(dynamic_labels)" ] }, { diff --git a/viz_scripts/scaffolding.py b/viz_scripts/scaffolding.py index ef208b1..c1adb07 100644 --- a/viz_scripts/scaffolding.py +++ b/viz_scripts/scaffolding.py @@ -211,9 +211,9 @@ def translate_labels(labels): return dic_mapping # Function: Maps "MODE", "PURPOSE", and "REPLACED_MODE" to colors. -# Input: dynamic_labels, dic_re, and dic_pur +# Input: dynamic_labels # Output: Dictionary mapping between color with mode/purpose/sensed -async def mapping_color_labels(dynamic_labels, language="en"): +async def mapping_color_labels(dynamic_labels): # Load default options from e-mission-common labels = await read_json_resource("label-options.default.json") sensed_values = ["WALKING", "BICYCLING", "IN_VEHICLE", "AIR_OR_HSR", "UNKNOWN", "OTHER", "other"] @@ -229,12 +229,6 @@ async def mapping_color_labels(dynamic_labels, language="en"): # Mapping between mode values and base_mode OR baseMode (backwards compatibility) value_to_basemode = {mode["value"]: mode.get("base_mode", mode.get("baseMode", "UNKNOWN")) for mode in labels["MODE"]} - # Mapping between values and translations for display on plots (for Mode) - values_to_translations_mode = dict(mapping_labels(labels, "MODE")) - # Mapping between values and translations for display on plots (for Purpose) - values_to_translations_purpose = dict(mapping_labels(labels, "PURPOSE")) - # Mapping between values and translations for display on plots (for Replaced mode) - values_to_translations_replaced = dict(mapping_labels(labels, "REPLACED_MODE")) # Assign colors to mode, replaced, purpose, and sensed values colors_mode = dedupe_colors([ @@ -248,7 +242,23 @@ async def mapping_color_labels(dynamic_labels, language="en"): colors_purpose = dict(zip(purpose_values, plt.cm.tab20.colors[:len(purpose_values)])) colors_sensed = dict(zip(sensed_values, [BASE_MODES[x.upper()]['color'] for x in sensed_values])) - return colors_mode, colors_replaced, colors_purpose, colors_sensed, values_to_translations_mode, values_to_translations_purpose, values_to_translations_replaced + return colors_mode, colors_replaced, colors_purpose, colors_sensed + +async def translate_values_to_labels(dynamic_labels, language="en"): + # Load default options from e-mission-common + labels = await read_json_resource("label-options.default.json") + + # If dynamic_labels are provided, then we will use the dynamic labels for mapping + if len(dynamic_labels) > 0: + labels = dynamic_labels + # Mapping between values and translations for display on plots (for Mode) + values_to_translations_mode = dict(mapping_labels(labels, "MODE")) + # Mapping between values and translations for display on plots (for Purpose) + values_to_translations_purpose = dict(mapping_labels(labels, "PURPOSE")) + # Mapping between values and translations for display on plots (for Replaced mode) + values_to_translations_replaced = dict(mapping_labels(labels, "REPLACED_MODE")) + + return values_to_translations_mode, values_to_translations_purpose, values_to_translations_replaced # Function: Maps survey answers to colors. # Input: dictionary of raw and translated survey answers diff --git a/viz_scripts/survey_metrics.ipynb b/viz_scripts/survey_metrics.ipynb index 5e3d18d..3f8e62f 100644 --- a/viz_scripts/survey_metrics.ipynb +++ b/viz_scripts/survey_metrics.ipynb @@ -61,7 +61,7 @@ "label_units, short_label, label_units_lower, distance_col, weight_unit = scaffolding.get_units(use_imperial)\n", "\n", "# get color mappings\n", - "colors_mode, colors_replaced, colors_purpose, colors_sensed, values_to_translations, value_to_translations_purpose, values_to_translations_replaced = await scaffolding.mapping_color_labels({}) #just need sensed" + "colors_mode, colors_replaced, colors_purpose, colors_sensed = await scaffolding.mapping_color_labels({}) #just need sensed" ] }, { From 75d594cde41c67d4474e592f4c50d84c2c60d755 Mon Sep 17 00:00:00 2001 From: iantei Date: Mon, 16 Sep 2024 12:10:57 -0700 Subject: [PATCH 28/39] Handle INVALID sensed mode color mapping. Use dedupe to differentiate if there is both UNKNOWN and INVALID labels. --- viz_scripts/scaffolding.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/viz_scripts/scaffolding.py b/viz_scripts/scaffolding.py index 21cfbdc..36688f9 100644 --- a/viz_scripts/scaffolding.py +++ b/viz_scripts/scaffolding.py @@ -240,8 +240,10 @@ async def mapping_color_labels(dynamic_labels): for mode in set(replaced_values) ], adjustment_range=[1,1.8]) colors_purpose = dict(zip(purpose_values, plt.cm.tab20.colors[:len(purpose_values)])) - colors_sensed = dict(zip(sensed_values, [BASE_MODES[x.upper()]['color'] for x in sensed_values])) - + colors_sensed = dedupe_colors([ + [label, BASE_MODES[label.upper()]['color'] if label.upper() != 'INVALID' else BASE_MODES['UNKNOWN']['color']] + for label in sensed_values + ], adjustment_range=[1,1.8]) return colors_mode, colors_replaced, colors_purpose, colors_sensed async def translate_values_to_labels(dynamic_labels, language="en"): From f1176f6d6a8872a9bb84e465ecb41b9d31989302 Mon Sep 17 00:00:00 2001 From: iantei Date: Mon, 16 Sep 2024 13:40:15 -0700 Subject: [PATCH 29/39] Extract color mapping for BLE modes. Set the default values for dyanmic_labels as {} and unique_keys as []. --- viz_scripts/scaffolding.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/viz_scripts/scaffolding.py b/viz_scripts/scaffolding.py index 36688f9..c389d45 100644 --- a/viz_scripts/scaffolding.py +++ b/viz_scripts/scaffolding.py @@ -213,7 +213,7 @@ def translate_labels(labels): # Function: Maps "MODE", "PURPOSE", and "REPLACED_MODE" to colors. # Input: dynamic_labels # Output: Dictionary mapping between color with mode/purpose/sensed -async def mapping_color_labels(dynamic_labels): +async def mapping_color_labels(dynamic_labels = {}, unique_keys = []): # Load default options from e-mission-common labels = await read_json_resource("label-options.default.json") sensed_values = ["WALKING", "BICYCLING", "IN_VEHICLE", "AIR_OR_HSR", "UNKNOWN", "OTHER", "INVALID"] @@ -229,7 +229,6 @@ async def mapping_color_labels(dynamic_labels): # Mapping between mode values and base_mode OR baseMode (backwards compatibility) value_to_basemode = {mode["value"]: mode.get("base_mode", mode.get("baseMode", "UNKNOWN")) for mode in labels["MODE"]} - # Assign colors to mode, replaced, purpose, and sensed values colors_mode = dedupe_colors([ [mode, BASE_MODES[value_to_basemode.get(mode, "UNKNOWN")]['color']] @@ -244,7 +243,11 @@ async def mapping_color_labels(dynamic_labels): [label, BASE_MODES[label.upper()]['color'] if label.upper() != 'INVALID' else BASE_MODES['UNKNOWN']['color']] for label in sensed_values ], adjustment_range=[1,1.8]) - return colors_mode, colors_replaced, colors_purpose, colors_sensed + colors_ble = dedupe_colors([ + [label, BASE_MODES[label]['color']] + for label in set(unique_keys) + ], adjustment_range=[1,1.8]) + return colors_mode, colors_replaced, colors_purpose, colors_sensed, colors_ble async def translate_values_to_labels(dynamic_labels, language="en"): # Load default options from e-mission-common From 399cd5a2de98035e73c16cd33830ec28fccf5a63 Mon Sep 17 00:00:00 2001 From: iantei Date: Mon, 16 Sep 2024 13:44:37 -0700 Subject: [PATCH 30/39] Update mapping_color_labels() to return colors_ble as another return parameter. --- viz_scripts/generic_metrics.ipynb | 2 +- viz_scripts/mode_specific_metrics.ipynb | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/viz_scripts/generic_metrics.ipynb b/viz_scripts/generic_metrics.ipynb index a44e44a..fcbd892 100644 --- a/viz_scripts/generic_metrics.ipynb +++ b/viz_scripts/generic_metrics.ipynb @@ -90,7 +90,7 @@ "metadata": {}, "outputs": [], "source": [ - "colors_mode, colors_replaced, colors_purpose, colors_sensed = await scaffolding.mapping_color_labels(dynamic_labels)\n", + "colors_mode, colors_replaced, colors_purpose, colors_sensed, colors_ble = await scaffolding.mapping_color_labels(dynamic_labels)\n", "values_to_translations, value_to_translations_purpose, values_to_translations_replaced = await scaffolding.translate_values_to_labels(dynamic_labels)" ] }, diff --git a/viz_scripts/mode_specific_metrics.ipynb b/viz_scripts/mode_specific_metrics.ipynb index ae554b4..a8f1557 100644 --- a/viz_scripts/mode_specific_metrics.ipynb +++ b/viz_scripts/mode_specific_metrics.ipynb @@ -103,7 +103,7 @@ "metadata": {}, "outputs": [], "source": [ - "colors_mode, colors_replaced, colors_purpose, colors_sensed = await scaffolding.mapping_color_labels(dynamic_labels)\n", + "colors_mode, colors_replaced, colors_purpose, colors_sensed, colors_ble = await scaffolding.mapping_color_labels(dynamic_labels)\n", "values_to_translations, value_to_translations_purpose, value_to_translations_replaced = await scaffolding.translate_values_to_labels(dynamic_labels)" ] }, From 60f5af39cbeb1c2dd8655e08a7ce394274704f00 Mon Sep 17 00:00:00 2001 From: iantei Date: Mon, 16 Sep 2024 13:48:30 -0700 Subject: [PATCH 31/39] Update mapping_color_labels() for ble modes to pass only the unique_keys. Assign the colors_ble as colors_sensed. Update the mapping_color_labels to use await. --- viz_scripts/generic_metrics_sensed.ipynb | 4 ++-- viz_scripts/survey_metrics.ipynb | 6 +++--- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/viz_scripts/generic_metrics_sensed.ipynb b/viz_scripts/generic_metrics_sensed.ipynb index 2c937e1..dca74d2 100644 --- a/viz_scripts/generic_metrics_sensed.ipynb +++ b/viz_scripts/generic_metrics_sensed.ipynb @@ -96,8 +96,8 @@ " expanded_ct[\"primary_mode\"] = expanded_ct.ble_sensed_summary.apply(lambda md: max(md[\"distance\"], key=md[\"distance\"].get))\n", " unique_keys = expanded_ct.groupby(\"primary_mode\").agg({distance_col: \"count\"}).index\n", " print(unique_keys)\n", - " colors_mode, colors_purpose, colors_sensed = await scaffolding.mapping_color_labels({}, dict(zip(unique_keys, unique_keys)), {})\n", - " colors_sensed = colors_mode\n", + " colors_mode, colors_replaced, colors_purpose, colors_sensed, colors_ble = await scaffolding.mapping_color_labels(unique_keys) #Extract ble color mapping\n", + " colors_sensed = colors_ble\n", "except ValueError as e:\n", " print(\"Got ValueError \", e)" ] diff --git a/viz_scripts/survey_metrics.ipynb b/viz_scripts/survey_metrics.ipynb index 3f8e62f..cebe4d9 100644 --- a/viz_scripts/survey_metrics.ipynb +++ b/viz_scripts/survey_metrics.ipynb @@ -61,7 +61,7 @@ "label_units, short_label, label_units_lower, distance_col, weight_unit = scaffolding.get_units(use_imperial)\n", "\n", "# get color mappings\n", - "colors_mode, colors_replaced, colors_purpose, colors_sensed = await scaffolding.mapping_color_labels({}) #just need sensed" + "colors_mode, colors_replaced, colors_purpose, colors_sensed, colors_ble = await scaffolding.mapping_color_labels() #just need sensed" ] }, { @@ -113,8 +113,8 @@ " expanded_ct_sensed[\"primary_mode\"] = expanded_ct_sensed.ble_sensed_summary.apply(lambda md: max(md[\"distance\"], key=md[\"distance\"].get))\n", " unique_keys = expanded_ct_sensed.groupby(\"primary_mode\").agg({distance_col: \"count\"}).index\n", " print(unique_keys)\n", - " colors_mode, colors_purpose, colors_sensed = scaffolding.mapping_color_labels({}, dict(zip(unique_keys, unique_keys)), {})\n", - " colors_sensed = colors_mode" + " colors_mode, colors_replaced, colors_purpose, colors_sensed, colors_ble = await scaffolding.mapping_color_labels(unique_keys)\n", + " colors_sensed = colors_ble" ] }, { From 0e5d86948b47ac0ae84fb62eb0ce5385f0122c46 Mon Sep 17 00:00:00 2001 From: iantei Date: Mon, 16 Sep 2024 13:51:32 -0700 Subject: [PATCH 32/39] Add try-except block for ble processing for survey_metrics.ipynb like in generic_metrics_sensed.ipynb. --- viz_scripts/survey_metrics.ipynb | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/viz_scripts/survey_metrics.ipynb b/viz_scripts/survey_metrics.ipynb index cebe4d9..26fbf5f 100644 --- a/viz_scripts/survey_metrics.ipynb +++ b/viz_scripts/survey_metrics.ipynb @@ -108,13 +108,16 @@ "outputs": [], "source": [ "#if fleet, replace primary_mode with primary_ble_sensed mode\n", - "if bluetooth_only and 'ble_sensed_summary' in expanded_ct_sensed.columns:\n", - " expanded_ct_sensed = expanded_ct_sensed[expanded_ct_sensed['ble_sensed_summary'].notna()]\n", - " expanded_ct_sensed[\"primary_mode\"] = expanded_ct_sensed.ble_sensed_summary.apply(lambda md: max(md[\"distance\"], key=md[\"distance\"].get))\n", - " unique_keys = expanded_ct_sensed.groupby(\"primary_mode\").agg({distance_col: \"count\"}).index\n", - " print(unique_keys)\n", - " colors_mode, colors_replaced, colors_purpose, colors_sensed, colors_ble = await scaffolding.mapping_color_labels(unique_keys)\n", - " colors_sensed = colors_ble" + "try:\n", + " if bluetooth_only and 'ble_sensed_summary' in expanded_ct_sensed.columns:\n", + " expanded_ct_sensed = expanded_ct_sensed[expanded_ct_sensed['ble_sensed_summary'].notna()]\n", + " expanded_ct_sensed[\"primary_mode\"] = expanded_ct_sensed.ble_sensed_summary.apply(lambda md: max(md[\"distance\"], key=md[\"distance\"].get))\n", + " unique_keys = expanded_ct_sensed.groupby(\"primary_mode\").agg({distance_col: \"count\"}).index\n", + " print(unique_keys)\n", + " colors_mode, colors_replaced, colors_purpose, colors_sensed, colors_ble = await scaffolding.mapping_color_labels(unique_keys)\n", + " colors_sensed = colors_ble\n", + "except ValueError as e:\n", + " print(\"Got ValueError \", e)" ] }, { From b3a0391ad191de38147a767226f1421515f6a5ab Mon Sep 17 00:00:00 2001 From: "K. Shankari" Date: Wed, 18 Sep 2024 18:20:09 -0700 Subject: [PATCH 33/39] Remove e-mission-common here since it is pulled from the server --- viz_scripts/docker/environment36.dashboard.additions.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/viz_scripts/docker/environment36.dashboard.additions.yml b/viz_scripts/docker/environment36.dashboard.additions.yml index df84f5e..59d26eb 100644 --- a/viz_scripts/docker/environment36.dashboard.additions.yml +++ b/viz_scripts/docker/environment36.dashboard.additions.yml @@ -7,4 +7,3 @@ dependencies: - pip: - nbparameterise==0.6 - devcron==0.4 - - git+https://github.com/JGreenlee/e-mission-common@master From 0034d6f04fed44fa9668eef0c38fc6d268654a5f Mon Sep 17 00:00:00 2001 From: iantei Date: Thu, 19 Sep 2024 10:23:52 -0700 Subject: [PATCH 34/39] Aggregate a single place to chose labels from either dynamic_labels or label-options.default.json --- viz_scripts/scaffolding.py | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/viz_scripts/scaffolding.py b/viz_scripts/scaffolding.py index c389d45..4745e64 100644 --- a/viz_scripts/scaffolding.py +++ b/viz_scripts/scaffolding.py @@ -131,16 +131,21 @@ async def load_viz_notebook_data(year, month, program, study_type, dynamic_label if "distance" in expanded_ct.columns: unit_conversions(expanded_ct) + # Select the labels from dynamic_labels is available, + # else get it from emcommon/resources/label-options.default.json + if (len(dynamic_labels)): + labels = dynamic_labels + else: + labels = await read_json_resource("label-options.default.json") + # Map new mode labels with translations dictionary from dynamic_labels # CASE 2 of https://github.com/e-mission/em-public-dashboard/issues/69#issuecomment-1256835867 if "mode_confirm" in expanded_ct.columns: if (len(dynamic_labels)): dic_mode_mapping = mapping_labels(dynamic_labels, "MODE") expanded_ct['Mode_confirm'] = expanded_ct['mode_confirm'].map(dic_mode_mapping) - labels = dynamic_labels else: expanded_ct['Mode_confirm'] = expanded_ct['mode_confirm'].map(dic_re) - labels = await read_json_resource("label-options.default.json") # If the 'mode_confirm' is not available as the list of keys in the dynamic_labels or label_options.default.json, then, we should transform it as 'other' mode_values = [item['value'] for item in labels['MODE']] expanded_ct['merge_mode_confirm'] = expanded_ct['mode_confirm'].apply(lambda mode: 'other' if mode not in mode_values else mode) @@ -150,10 +155,8 @@ async def load_viz_notebook_data(year, month, program, study_type, dynamic_label if (len(dynamic_labels)): dic_replaced_mapping = mapping_labels(dynamic_labels, "REPLACED_MODE") expanded_ct['Replaced_mode'] = expanded_ct['replaced_mode'].map(dic_replaced_mapping) - labels = dynamic_labels else: expanded_ct['Replaced_mode'] = expanded_ct['replaced_mode'].map(dic_re) - labels = await read_json_resource("label-options.default.json") replaced_modes = [item['value'] for item in labels['REPLACED_MODE']] expanded_ct['merge_replaced_mode'] = expanded_ct['replaced_mode'].apply(lambda mode: 'other' if mode not in replaced_modes else mode) else: @@ -167,10 +170,8 @@ async def load_viz_notebook_data(year, month, program, study_type, dynamic_label if (len(dynamic_labels)): dic_purpose_mapping = mapping_labels(dynamic_labels, "PURPOSE") expanded_ct['Trip_purpose'] = expanded_ct['purpose_confirm'].map(dic_purpose_mapping) - labels = dynamic_labels else: expanded_ct['Trip_purpose'] = expanded_ct['purpose_confirm'].map(dic_pur) - labels = await read_json_resource("label-options.default.json") purpose_values = [item['value'] for item in labels['PURPOSE']] expanded_ct['merge_purpose_confirm'] = expanded_ct['purpose_confirm'].apply(lambda value: 'other' if value not in purpose_values else value) From afaed0930c5320dcb3d19b4250144ffb5f834001 Mon Sep 17 00:00:00 2001 From: iantei Date: Thu, 19 Sep 2024 11:21:55 -0700 Subject: [PATCH 35/39] Enforce coding standard for python as import X as x and then x.function. --- viz_scripts/scaffolding.py | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/viz_scripts/scaffolding.py b/viz_scripts/scaffolding.py index 4745e64..8fce054 100644 --- a/viz_scripts/scaffolding.py +++ b/viz_scripts/scaffolding.py @@ -9,8 +9,8 @@ import emission.storage.timeseries.abstract_timeseries as esta import emission.storage.timeseries.tcquery as esttc import emission.core.wrapper.localdate as ecwl -from emcommon.diary.base_modes import BASE_MODES, dedupe_colors -from emcommon.util import read_json_resource +import emcommon.diary.base_modes as edb +import emcommon.util as eu # Module for pretty-printing outputs (e.g. head) to help users # understand what is going on @@ -136,7 +136,7 @@ async def load_viz_notebook_data(year, month, program, study_type, dynamic_label if (len(dynamic_labels)): labels = dynamic_labels else: - labels = await read_json_resource("label-options.default.json") + labels = await eu.read_json_resource("label-options.default.json") # Map new mode labels with translations dictionary from dynamic_labels # CASE 2 of https://github.com/e-mission/em-public-dashboard/issues/69#issuecomment-1256835867 @@ -216,7 +216,7 @@ def translate_labels(labels): # Output: Dictionary mapping between color with mode/purpose/sensed async def mapping_color_labels(dynamic_labels = {}, unique_keys = []): # Load default options from e-mission-common - labels = await read_json_resource("label-options.default.json") + labels = await eu.read_json_resource("label-options.default.json") sensed_values = ["WALKING", "BICYCLING", "IN_VEHICLE", "AIR_OR_HSR", "UNKNOWN", "OTHER", "INVALID"] # If dynamic_labels are provided, then we will use the dynamic labels for mapping @@ -231,28 +231,28 @@ async def mapping_color_labels(dynamic_labels = {}, unique_keys = []): # Mapping between mode values and base_mode OR baseMode (backwards compatibility) value_to_basemode = {mode["value"]: mode.get("base_mode", mode.get("baseMode", "UNKNOWN")) for mode in labels["MODE"]} # Assign colors to mode, replaced, purpose, and sensed values - colors_mode = dedupe_colors([ - [mode, BASE_MODES[value_to_basemode.get(mode, "UNKNOWN")]['color']] + colors_mode = edb.dedupe_colors([ + [mode, edb.BASE_MODES[value_to_basemode.get(mode, "UNKNOWN")]['color']] for mode in set(mode_values) ], adjustment_range=[1,1.8]) - colors_replaced = dedupe_colors([ - [mode, BASE_MODES[value_to_basemode.get(mode, "UNKNOWN")]['color']] + colors_replaced = edb.dedupe_colors([ + [mode, edb.BASE_MODES[value_to_basemode.get(mode, "UNKNOWN")]['color']] for mode in set(replaced_values) ], adjustment_range=[1,1.8]) colors_purpose = dict(zip(purpose_values, plt.cm.tab20.colors[:len(purpose_values)])) - colors_sensed = dedupe_colors([ - [label, BASE_MODES[label.upper()]['color'] if label.upper() != 'INVALID' else BASE_MODES['UNKNOWN']['color']] + colors_sensed = edb.dedupe_colors([ + [label, edb.BASE_MODES[label.upper()]['color'] if label.upper() != 'INVALID' else edb.BASE_MODES['UNKNOWN']['color']] for label in sensed_values ], adjustment_range=[1,1.8]) - colors_ble = dedupe_colors([ - [label, BASE_MODES[label]['color']] + colors_ble = edb.dedupe_colors([ + [label, edb.BASE_MODES[label]['color']] for label in set(unique_keys) ], adjustment_range=[1,1.8]) return colors_mode, colors_replaced, colors_purpose, colors_sensed, colors_ble async def translate_values_to_labels(dynamic_labels, language="en"): # Load default options from e-mission-common - labels = await read_json_resource("label-options.default.json") + labels = await eu.read_json_resource("label-options.default.json") # If dynamic_labels are provided, then we will use the dynamic labels for mapping if len(dynamic_labels) > 0: From 9cefa33098e30595d2076a2f951eff398efa97b8 Mon Sep 17 00:00:00 2001 From: iantei Date: Thu, 19 Sep 2024 15:06:33 -0700 Subject: [PATCH 36/39] Rename the column in expaneded_ct in scaffolding from merge_** to **_w_other. --- viz_scripts/scaffolding.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/viz_scripts/scaffolding.py b/viz_scripts/scaffolding.py index 8fce054..c8c4498 100644 --- a/viz_scripts/scaffolding.py +++ b/viz_scripts/scaffolding.py @@ -148,7 +148,7 @@ async def load_viz_notebook_data(year, month, program, study_type, dynamic_label expanded_ct['Mode_confirm'] = expanded_ct['mode_confirm'].map(dic_re) # If the 'mode_confirm' is not available as the list of keys in the dynamic_labels or label_options.default.json, then, we should transform it as 'other' mode_values = [item['value'] for item in labels['MODE']] - expanded_ct['merge_mode_confirm'] = expanded_ct['mode_confirm'].apply(lambda mode: 'other' if mode not in mode_values else mode) + expanded_ct['mode_confirm_w_other'] = expanded_ct['mode_confirm'].apply(lambda mode: 'other' if mode not in mode_values else mode) if study_type == 'program': # CASE 2 of https://github.com/e-mission/em-public-dashboard/issues/69#issuecomment-1256835867 if 'replaced_mode' in expanded_ct.columns: @@ -158,7 +158,7 @@ async def load_viz_notebook_data(year, month, program, study_type, dynamic_label else: expanded_ct['Replaced_mode'] = expanded_ct['replaced_mode'].map(dic_re) replaced_modes = [item['value'] for item in labels['REPLACED_MODE']] - expanded_ct['merge_replaced_mode'] = expanded_ct['replaced_mode'].apply(lambda mode: 'other' if mode not in replaced_modes else mode) + expanded_ct['replaced_mode_w_other'] = expanded_ct['replaced_mode'].apply(lambda mode: 'other' if mode not in replaced_modes else mode) else: print("This is a program, but no replaced modes found. Likely cold start case. Ignoring replaced mode mapping") else: @@ -173,7 +173,7 @@ async def load_viz_notebook_data(year, month, program, study_type, dynamic_label else: expanded_ct['Trip_purpose'] = expanded_ct['purpose_confirm'].map(dic_pur) purpose_values = [item['value'] for item in labels['PURPOSE']] - expanded_ct['merge_purpose_confirm'] = expanded_ct['purpose_confirm'].apply(lambda value: 'other' if value not in purpose_values else value) + expanded_ct['purpose_confirm_w_other'] = expanded_ct['purpose_confirm'].apply(lambda value: 'other' if value not in purpose_values else value) # Document data quality file_suffix = get_file_suffix(year, month, program) From ec355caa1db3f53f600a653b9bd39151f3b2c1af Mon Sep 17 00:00:00 2001 From: iantei Date: Thu, 19 Sep 2024 15:16:46 -0700 Subject: [PATCH 37/39] Rename merge_** to **_w_other in generic_metrics and mode_specific_metrics notebook. --- viz_scripts/generic_metrics.ipynb | 18 +++++++++--------- viz_scripts/mode_specific_metrics.ipynb | 6 +++--- 2 files changed, 12 insertions(+), 12 deletions(-) diff --git a/viz_scripts/generic_metrics.ipynb b/viz_scripts/generic_metrics.ipynb index fcbd892..0e1f393 100644 --- a/viz_scripts/generic_metrics.ipynb +++ b/viz_scripts/generic_metrics.ipynb @@ -208,7 +208,7 @@ " # We will have text results corresponding to the axes for simplicity and consistency\n", " text_results = [[\"Unmodified Alt Text\", \"Unmodified HTML\"], [\"Unmodified Alt Text\", \"Unmodified HTML\"]]\n", " \n", - " plot_and_text_stacked_bar_chart(expanded_ct, lambda df: (df.groupby(\"merge_mode_confirm\").agg({distance_col: 'count'}).sort_values(by=distance_col, ascending=False)), \n", + " plot_and_text_stacked_bar_chart(expanded_ct, lambda df: (df.groupby(\"mode_confirm_w_other\").agg({distance_col: 'count'}).sort_values(by=distance_col, ascending=False)), \n", " \"Labeled by user\\n\"+stacked_bar_quality_text_labeled, ax[0], text_results[0], colors_mode, debug_df, values_to_translations)\n", " plot_and_text_stacked_bar_chart(expanded_ct_sensed, lambda df: (df.groupby(\"primary_mode\").agg({distance_col: 'count'}).sort_values(by=distance_col, ascending=False)), \n", " \"Sensed by OpenPATH\\n\"+stacked_bar_quality_text_sensed, ax[1], text_results[1], colors_sensed, debug_df_sensed)\n", @@ -259,7 +259,7 @@ " # Plot entries\n", " fig, ax = plt.subplots(nrows=1, ncols=1, figsize=(15,2*1), sharex=True) \n", " text_results = [\"Unmodified Alt Text\", \"Unmodified HTML\"]\n", - " plot_and_text_stacked_bar_chart(expanded_ct_commute, lambda df: df.groupby(\"merge_mode_confirm\").agg({distance_col: 'count'}).sort_values(by=distance_col, ascending=False), \n", + " plot_and_text_stacked_bar_chart(expanded_ct_commute, lambda df: df.groupby(\"mode_confirm_w_other\").agg({distance_col: 'count'}).sort_values(by=distance_col, ascending=False), \n", " \"Labeled by user\\n (Confirmed trips)\", ax, text_results, colors_mode, debug_df, values_to_translations)\n", " set_title_and_save(fig, text_results, plot_title, file_name)\n", "except (AttributeError, KeyError, pd.errors.UndefinedVariableError) as e:\n", @@ -292,7 +292,7 @@ "try:\n", " fig, ax = plt.subplots(nrows=1, ncols=1, figsize=(15,2*1), sharex=True)\n", " text_results = [\"Unmodified Alt Text\", \"Unmodified HTML\"]\n", - " plot_and_text_stacked_bar_chart(expanded_ct, lambda df: df.groupby(\"merge_purpose_confirm\").agg({distance_col: 'count'}).sort_values(by=distance_col, ascending=False), \n", + " plot_and_text_stacked_bar_chart(expanded_ct, lambda df: df.groupby(\"purpose_confirm_w_other\").agg({distance_col: 'count'}).sort_values(by=distance_col, ascending=False), \n", " \"Labeled by user\\n\"+stacked_bar_quality_text_labeled, ax, text_results, colors_purpose, debug_df, value_to_translations_purpose)\n", " set_title_and_save(fig, text_results, plot_title_no_quality, file_name)\n", "except (AttributeError, KeyError, pd.errors.UndefinedVariableError) as e:\n", @@ -336,7 +336,7 @@ "\n", " ## We do an existence check for the labeled df because we want to display the sensed value even if we don't have the labeled value\n", " ## but we don't need to have an existence check for sensed because in that case we will have no data to display\n", - " expanded_ct_u80 = expanded_ct.loc[(expanded_ct['distance'] <= cutoff)] if \"merge_mode_confirm\" in expanded_ct.columns else None\n", + " expanded_ct_u80 = expanded_ct.loc[(expanded_ct['distance'] <= cutoff)] if \"mode_confirm_w_other\" in expanded_ct.columns else None\n", " expanded_ct_sensed_u80 = expanded_ct_sensed.loc[(expanded_ct_sensed['distance'] <= cutoff)]\n", " sensed_u80_quality_text = f\"{len(expanded_ct_sensed_u80)} trips ({round(len(expanded_ct_sensed_u80)/len(expanded_ct_sensed)*100)}% of all trips)\\nfrom {scaffolding.unique_users(expanded_ct_sensed_u80)} {sensed_match.group(3)}\"\n", " labeled_u80_quality_text = f\"{len(expanded_ct_u80)} trips ({round(len(expanded_ct_u80)/len(expanded_ct)*100)}% of all labeled,\\n{round(len(expanded_ct_u80)/len(expanded_ct_sensed)*100)}% of all trips)\\nfrom {scaffolding.unique_users(expanded_ct_u80)} {sensed_match.group(3)}\" if \"Mode_confirm\" in expanded_ct.columns else \"0 labeled trips\"\n", @@ -344,7 +344,7 @@ " # Plot entries\n", " fig, ax = plt.subplots(nrows=2, ncols=1, figsize=(15,2*2), sharex=True)\n", " text_results = [[\"Unmodified Alt Text\", \"Unmodified HTML\"], [\"Unmodified Alt Text\", \"Unmodified HTML\"]]\n", - " plot_and_text_stacked_bar_chart(expanded_ct_u80, lambda df: df.groupby(\"merge_mode_confirm\").agg({distance_col: 'count'}).sort_values(by=distance_col, ascending=False), \n", + " plot_and_text_stacked_bar_chart(expanded_ct_u80, lambda df: df.groupby(\"mode_confirm_w_other\").agg({distance_col: 'count'}).sort_values(by=distance_col, ascending=False), \n", " \"Labeled by user\\n\"+labeled_u80_quality_text, ax[0], text_results[0], colors_mode, debug_df, values_to_translations)\n", " plot_and_text_stacked_bar_chart(expanded_ct_sensed_u80, lambda df: df.groupby(\"primary_mode\").agg({distance_col: 'count'}).sort_values(by=distance_col, ascending=False), \n", " \"Sensed by OpenPATH\\n\"+sensed_u80_quality_text, ax[1], text_results[1], colors_sensed, debug_df_sensed)\n", @@ -384,7 +384,7 @@ " fig, ax = plt.subplots(nrows=2, ncols=1, figsize=(15,2*2), sharex=True)\n", " \n", " text_results = [[\"Unmodified Alt Text\", \"Unmodified HTML\"], [\"Unmodified Alt Text\", \"Unmodified HTML\"]]\n", - " plot_and_text_stacked_bar_chart(expanded_ct, lambda df: df.groupby(\"merge_mode_confirm\").agg({distance_col: 'sum'}).sort_values(by=distance_col, ascending=False), \n", + " plot_and_text_stacked_bar_chart(expanded_ct, lambda df: df.groupby(\"mode_confirm_w_other\").agg({distance_col: 'sum'}).sort_values(by=distance_col, ascending=False), \n", " \"Labeled by user\\n\"+stacked_bar_quality_text_labeled, ax[0], text_results[0], colors_mode, debug_df, values_to_translations)\n", " plot_and_text_stacked_bar_chart(expanded_ct_sensed, lambda df: df.groupby(\"primary_mode\").agg({distance_col: 'sum'}).sort_values(by=distance_col, ascending=False), \n", " \"Sensed by OpenPATH\\n\"+stacked_bar_quality_text_sensed, ax[1], text_results[1], colors_sensed, debug_df_sensed)\n", @@ -420,14 +420,14 @@ "try:\n", " ## We do an existence check for the labeled df because we want to display the sensed value even if we don't have the labeled value\n", " ## but we don't need to have an existence check for sensed because in that case we will have no data to display\n", - " labeled_land_trips_df = expanded_ct[expanded_ct['merge_mode_confirm'] != \"air\"] if \"merge_mode_confirm\" in expanded_ct.columns else None\n", + " labeled_land_trips_df = expanded_ct[expanded_ct['mode_confirm_w_other'] != \"air\"] if \"mode_confirm_w_other\" in expanded_ct.columns else None\n", " sensed_land_trips_df = expanded_ct_sensed[expanded_ct_sensed['primary_mode'] != \"AIR_OR_HSR\"]\n", " \n", " sensed_land_quality_text = f\"{len(sensed_land_trips_df)} trips ({round(len(sensed_land_trips_df)/len(expanded_ct_sensed)*100)}% of all trips)\\nfrom {scaffolding.unique_users(sensed_land_trips_df)} {sensed_match.group(3)}\"\n", - " labeled_land_quality_text = f\"{len(labeled_land_trips_df)} trips ({round(len(labeled_land_trips_df)/len(expanded_ct)*100)}% of all labeled,\\n{round(len(labeled_land_trips_df)/len(expanded_ct_sensed)*100)}%) of all trips)\\nfrom {scaffolding.unique_users(labeled_land_trips_df)} {sensed_match.group(3)}\" if \"merge_mode_confirm\" in expanded_ct.columns else \"0 labeled trips\"\n", + " labeled_land_quality_text = f\"{len(labeled_land_trips_df)} trips ({round(len(labeled_land_trips_df)/len(expanded_ct)*100)}% of all labeled,\\n{round(len(labeled_land_trips_df)/len(expanded_ct_sensed)*100)}%) of all trips)\\nfrom {scaffolding.unique_users(labeled_land_trips_df)} {sensed_match.group(3)}\" if \"mode_confirm_w_other\" in expanded_ct.columns else \"0 labeled trips\"\n", "\n", " fig, ax = plt.subplots(nrows=2, ncols=1, figsize=(15,2*2), sharex=True)\n", - " plot_and_text_stacked_bar_chart(labeled_land_trips_df, lambda df: df.groupby(\"merge_mode_confirm\").agg({distance_col: 'sum'}).sort_values(by=distance_col, ascending=False), \n", + " plot_and_text_stacked_bar_chart(labeled_land_trips_df, lambda df: df.groupby(\"mode_confirm_w_other\").agg({distance_col: 'sum'}).sort_values(by=distance_col, ascending=False), \n", " \"Labeled by user\\n\"+labeled_land_quality_text, ax[0], text_results[0], colors_mode, debug_df, values_to_translations)\n", " plot_and_text_stacked_bar_chart(sensed_land_trips_df, lambda df: df.groupby(\"primary_mode\").agg({distance_col: 'sum'}).sort_values(by=distance_col, ascending=False), \n", " \"Sensed by OpenPATH\\n\"+sensed_land_quality_text, ax[1], text_results[1], colors_sensed, debug_df_sensed)\n", diff --git a/viz_scripts/mode_specific_metrics.ipynb b/viz_scripts/mode_specific_metrics.ipynb index a8f1557..dc62074 100644 --- a/viz_scripts/mode_specific_metrics.ipynb +++ b/viz_scripts/mode_specific_metrics.ipynb @@ -194,7 +194,7 @@ "try:\n", " fig, ax = plt.subplots(nrows=1, ncols=1, figsize=(15,2*1), sharex=True)\n", " text_results = [\"Unmodified Alt Text\", \"Unmodified HTML\"]\n", - " plot_and_text_stacked_bar_chart(data_eb, lambda df: df.groupby(\"merge_purpose_confirm\").agg({distance_col: 'count'}).sort_values(by=distance_col, ascending=False),\n", + " plot_and_text_stacked_bar_chart(data_eb, lambda df: df.groupby(\"purpose_confirm_w_other\").agg({distance_col: 'count'}).sort_values(by=distance_col, ascending=False),\n", " f\"Labeled `{mode_of_interest}` by user\", ax, text_results, colors_purpose, debug_df, value_to_translations_purpose)\n", " plot_title = plot_title_no_quality + \"\\n\" + f\"For {mode_of_interest}: \" + quality_text\n", " set_title_and_save(fig, text_results, plot_title, file_name)\n", @@ -229,7 +229,7 @@ "try:\n", " fig, ax = plt.subplots(nrows=1, ncols=1, figsize=(15,2*1), sharex=True)\n", " text_results = [\"Unmodified Alt Text\", \"Unmodified HTML\"]\n", - " plot_and_text_stacked_bar_chart(data_eb, lambda df: df.groupby(\"merge_replaced_mode\").agg({distance_col: 'sum'}).sort_values(by=distance_col, ascending=False), \n", + " plot_and_text_stacked_bar_chart(data_eb, lambda df: df.groupby(\"replaced_mode_w_other\").agg({distance_col: 'sum'}).sort_values(by=distance_col, ascending=False), \n", " \"Labeled by user\\n (Trip distance)\", ax, text_results, colors_replaced, debug_df, value_to_translations_replaced)\n", " plot_title = plot_title_no_quality + \"\\n\" + f\"For {mode_of_interest}: \" + quality_text\n", " set_title_and_save(fig, text_results, plot_title, file_name)\n", @@ -264,7 +264,7 @@ "try:\n", " fig, ax = plt.subplots(nrows=1, ncols=1, figsize=(15,2*1), sharex=True)\n", " text_results = [\"Unmodified Alt Text\", \"Unmodified HTML\"]\n", - " plot_and_text_stacked_bar_chart(data_eb, lambda df: df.groupby(\"merge_replaced_mode\").agg({distance_col: 'count'}).sort_values(by=distance_col, ascending=False), \n", + " plot_and_text_stacked_bar_chart(data_eb, lambda df: df.groupby(\"replaced_mode_w_other\").agg({distance_col: 'count'}).sort_values(by=distance_col, ascending=False), \n", " f\"Labeled `{mode_of_interest}` by user\", ax, text_results, colors_replaced, debug_df, value_to_translations_replaced)\n", " plot_title = plot_title_no_quality + \"\\n\" + f\"For {mode_of_interest}: \" + quality_text\n", " set_title_and_save(fig, text_results, plot_title, file_name)\n", From 821fe7d8200f8d0bca5fc28c323557b3bf0dc21d Mon Sep 17 00:00:00 2001 From: iantei Date: Thu, 19 Sep 2024 15:22:15 -0700 Subject: [PATCH 38/39] Remove unnecessary conversion of defaultdict to dict. --- viz_scripts/scaffolding.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/viz_scripts/scaffolding.py b/viz_scripts/scaffolding.py index c8c4498..50d3a53 100644 --- a/viz_scripts/scaffolding.py +++ b/viz_scripts/scaffolding.py @@ -258,11 +258,11 @@ async def translate_values_to_labels(dynamic_labels, language="en"): if len(dynamic_labels) > 0: labels = dynamic_labels # Mapping between values and translations for display on plots (for Mode) - values_to_translations_mode = dict(mapping_labels(labels, "MODE")) + values_to_translations_mode = mapping_labels(labels, "MODE") # Mapping between values and translations for display on plots (for Purpose) - values_to_translations_purpose = dict(mapping_labels(labels, "PURPOSE")) + values_to_translations_purpose = mapping_labels(labels, "PURPOSE") # Mapping between values and translations for display on plots (for Replaced mode) - values_to_translations_replaced = dict(mapping_labels(labels, "REPLACED_MODE")) + values_to_translations_replaced = mapping_labels(labels, "REPLACED_MODE") return values_to_translations_mode, values_to_translations_purpose, values_to_translations_replaced From ac07f67284fc1f249431f5aea645f39fed0382bb Mon Sep 17 00:00:00 2001 From: iantei Date: Thu, 19 Sep 2024 15:47:20 -0700 Subject: [PATCH 39/39] Rename edb for emcommon.diary.base.modes to emcdb. Rename eu for emcommon.util to emcu. --- viz_scripts/scaffolding.py | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/viz_scripts/scaffolding.py b/viz_scripts/scaffolding.py index 50d3a53..2c54bf6 100644 --- a/viz_scripts/scaffolding.py +++ b/viz_scripts/scaffolding.py @@ -9,8 +9,8 @@ import emission.storage.timeseries.abstract_timeseries as esta import emission.storage.timeseries.tcquery as esttc import emission.core.wrapper.localdate as ecwl -import emcommon.diary.base_modes as edb -import emcommon.util as eu +import emcommon.diary.base_modes as emcdb +import emcommon.util as emcu # Module for pretty-printing outputs (e.g. head) to help users # understand what is going on @@ -136,7 +136,7 @@ async def load_viz_notebook_data(year, month, program, study_type, dynamic_label if (len(dynamic_labels)): labels = dynamic_labels else: - labels = await eu.read_json_resource("label-options.default.json") + labels = await emcu.read_json_resource("label-options.default.json") # Map new mode labels with translations dictionary from dynamic_labels # CASE 2 of https://github.com/e-mission/em-public-dashboard/issues/69#issuecomment-1256835867 @@ -216,7 +216,7 @@ def translate_labels(labels): # Output: Dictionary mapping between color with mode/purpose/sensed async def mapping_color_labels(dynamic_labels = {}, unique_keys = []): # Load default options from e-mission-common - labels = await eu.read_json_resource("label-options.default.json") + labels = await emcu.read_json_resource("label-options.default.json") sensed_values = ["WALKING", "BICYCLING", "IN_VEHICLE", "AIR_OR_HSR", "UNKNOWN", "OTHER", "INVALID"] # If dynamic_labels are provided, then we will use the dynamic labels for mapping @@ -231,28 +231,28 @@ async def mapping_color_labels(dynamic_labels = {}, unique_keys = []): # Mapping between mode values and base_mode OR baseMode (backwards compatibility) value_to_basemode = {mode["value"]: mode.get("base_mode", mode.get("baseMode", "UNKNOWN")) for mode in labels["MODE"]} # Assign colors to mode, replaced, purpose, and sensed values - colors_mode = edb.dedupe_colors([ - [mode, edb.BASE_MODES[value_to_basemode.get(mode, "UNKNOWN")]['color']] + colors_mode = emcdb.dedupe_colors([ + [mode, emcdb.BASE_MODES[value_to_basemode.get(mode, "UNKNOWN")]['color']] for mode in set(mode_values) ], adjustment_range=[1,1.8]) - colors_replaced = edb.dedupe_colors([ - [mode, edb.BASE_MODES[value_to_basemode.get(mode, "UNKNOWN")]['color']] + colors_replaced = emcdb.dedupe_colors([ + [mode, emcdb.BASE_MODES[value_to_basemode.get(mode, "UNKNOWN")]['color']] for mode in set(replaced_values) ], adjustment_range=[1,1.8]) colors_purpose = dict(zip(purpose_values, plt.cm.tab20.colors[:len(purpose_values)])) - colors_sensed = edb.dedupe_colors([ - [label, edb.BASE_MODES[label.upper()]['color'] if label.upper() != 'INVALID' else edb.BASE_MODES['UNKNOWN']['color']] + colors_sensed = emcdb.dedupe_colors([ + [label, emcdb.BASE_MODES[label.upper()]['color'] if label.upper() != 'INVALID' else emcdb.BASE_MODES['UNKNOWN']['color']] for label in sensed_values ], adjustment_range=[1,1.8]) - colors_ble = edb.dedupe_colors([ - [label, edb.BASE_MODES[label]['color']] + colors_ble = emcdb.dedupe_colors([ + [label, emcdb.BASE_MODES[label]['color']] for label in set(unique_keys) ], adjustment_range=[1,1.8]) return colors_mode, colors_replaced, colors_purpose, colors_sensed, colors_ble async def translate_values_to_labels(dynamic_labels, language="en"): # Load default options from e-mission-common - labels = await eu.read_json_resource("label-options.default.json") + labels = await emcu.read_json_resource("label-options.default.json") # If dynamic_labels are provided, then we will use the dynamic labels for mapping if len(dynamic_labels) > 0: