From 34eae86533bc533ae1d4a934bff6bc2d43f55047 Mon Sep 17 00:00:00 2001 From: ananta Date: Thu, 7 Sep 2023 02:03:15 -0700 Subject: [PATCH 01/62] Changes on generic_metric and mode_specific_metrics in relation to Mode_confirm mapping with dynamic labels. --- viz_scripts/bin/generate_plots.py | 15 ++++-- viz_scripts/generic_metrics.ipynb | 14 +++-- viz_scripts/mode_specific_metrics.ipynb | 14 +++-- viz_scripts/plots.py | 69 +++---------------------- viz_scripts/scaffolding.py | 48 +++++++++++++---- 5 files changed, 75 insertions(+), 85 deletions(-) diff --git a/viz_scripts/bin/generate_plots.py b/viz_scripts/bin/generate_plots.py index 966e5fd..38928fa 100644 --- a/viz_scripts/bin/generate_plots.py +++ b/viz_scripts/bin/generate_plots.py @@ -33,14 +33,21 @@ else: dynamic_config = json.loads(r.text) print(f"Successfully downloaded config with version {dynamic_config['version']} "\ - f"for {dynamic_config['intro']['translated_text']['en']['deployment_name']} "\ - f"and data collection URL {dynamic_config['server']['connectUrl']}") + f"for {dynamic_config['intro']['translated_text']['en']['deployment_name']} ") if dynamic_config['intro']['program_or_study'] == 'program': mode_studied = dynamic_config['intro']['mode_studied'] else: mode_studied = None +# Check if the dynamic config contains dynamic labels 'label_options' +# Passing the boolean flag is not enough, bcos we need to trace through the dynamic_config - pass label_options URL. +if 'label_options' in dynamic_config: + has_dynamic_labels = True + dynamic_labels_url = dynamic_config['label_options'] +else: + has_dynamic_labels = False + if args.date is None: start_date = arrow.get(int(dynamic_config['intro']['start_year']), int(dynamic_config['intro']['start_month']), 1) @@ -69,7 +76,9 @@ def compute_for_date(month, year): program=args.program, study_type=dynamic_config['intro']['program_or_study'], mode_of_interest=mode_studied, - include_test_users=dynamic_config.get('metrics', {}).get('include_test_users', False)) + include_test_users=dynamic_config.get('metrics', {}).get('include_test_users', False), + has_dynamic_labels = has_dynamic_labels, + dynamic_labels_url = dynamic_labels_url) print(f"Running at {arrow.get()} with params {params}") diff --git a/viz_scripts/generic_metrics.ipynb b/viz_scripts/generic_metrics.ipynb index b59a6e4..9184ba3 100644 --- a/viz_scripts/generic_metrics.ipynb +++ b/viz_scripts/generic_metrics.ipynb @@ -25,12 +25,14 @@ "metadata": {}, "outputs": [], "source": [ - "year = 2020\n", - "month = 11\n", + "year = None\n", + "month = None\n", "program = \"default\"\n", "study_type = \"study\"\n", "mode_of_interest = None\n", - "include_test_users = False" + "include_test_users = False\n", + "has_dynamic_labels = True\n", + "dynamic_labels_url = \"https://raw.githubusercontent.com/e-mission/nrel-openpath-deploy-configs/main/label_options/example-program-label-options.json\"" ] }, { @@ -88,6 +90,8 @@ " month,\n", " program,\n", " study_type,\n", + " has_dynamic_labels,\n", + " dynamic_labels_url,\n", " dic_re,\n", " dic_pur=dic_pur,\n", " include_test_users=include_test_users)" @@ -295,7 +299,7 @@ " \n", " plot_title= plot_title_no_quality+\"\\n\"+quality_text\n", "\n", - " barplot_mode(data,x,y,plot_title,file_name)\n", + " barplot_mode(data,x,y,plot_title, expanded_ct['Mode_confirm'].dropna().unique().tolist(), file_name)\n", " alt_text = store_alt_text_bar(pd.DataFrame(data['Average (miles)'].values, data['Mode_confirm']), file_name, plot_title)\n", "except:\n", " generate_missing_plot(plot_title_no_quality,debug_df,file_name)\n", @@ -392,7 +396,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.7.12" + "version": "3.9.16" } }, "nbformat": 4, diff --git a/viz_scripts/mode_specific_metrics.ipynb b/viz_scripts/mode_specific_metrics.ipynb index 4e9da95..33db9b9 100644 --- a/viz_scripts/mode_specific_metrics.ipynb +++ b/viz_scripts/mode_specific_metrics.ipynb @@ -25,12 +25,14 @@ "metadata": {}, "outputs": [], "source": [ - "year = 2020\n", - "month = 11\n", + "year = None\n", + "month = None\n", "program = \"default\"\n", "study_type = \"program\"\n", "mode_of_interest = \"e-bike\"\n", - "include_test_users = False" + "include_test_users = False\n", + "has_dynamic_labels = True\n", + "dynamic_labels_url = \"https://raw.githubusercontent.com/e-mission/nrel-openpath-deploy-configs/main/label_options/example-program-label-options.json\"" ] }, { @@ -102,6 +104,8 @@ " month,\n", " program,\n", " study_type,\n", + " has_dynamic_labels,\n", + " dynamic_labels_url,\n", " dic_re,\n", " dic_pur=dic_pur,\n", " include_test_users=include_test_users)" @@ -266,7 +270,7 @@ " y2 = \"Count\"\n", "\n", " plot_title= plot_title_no_quality+\"\\n\"+quality_text\n", - " barplot_mode(data,x,y,plot_title,file_name)\n", + " barplot_mode(data,x,y,plot_title, expanded_ct['Mode_confirm'].dropna().unique().tolist(), file_name)\n", " alt_text = store_alt_text_bar(pd.DataFrame(data['Average (miles)'].values, data.Replaced_mode), file_name, plot_title)\n", "except:\n", " generate_missing_plot(plot_title_no_quality,debug_df,file_name)\n", @@ -365,7 +369,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.7.12" + "version": "3.9.16" } }, "nbformat": 4, diff --git a/viz_scripts/plots.py b/viz_scripts/plots.py index fa88b9b..9a3263d 100644 --- a/viz_scripts/plots.py +++ b/viz_scripts/plots.py @@ -51,29 +51,8 @@ def merge_small_entries(labels, values): return (v2l_df.index.to_list(),v2l_df.vals.to_list()) def pie_chart_mode(plot_title,labels,values,file_name): - all_labels= ['Gas Car, drove alone', - 'Bus', - 'Train', - 'Free Shuttle', - 'Taxi/Uber/Lyft', - 'Gas Car, with others', - 'Bikeshare', - 'Scooter share', - 'E-bike', - 'Walk', - 'Skate board', - 'Regular Bike', - 'Not a Trip', - 'No Travel', - 'Same Mode', - 'E-car, drove alone', - 'E-car, with others', - 'Air', - 'Other'] - - val2labeldf = pd.DataFrame({"labels": labels, "values": values}) - colours = dict(zip(all_labels, plt.cm.tab20.colors[:len(all_labels)])) + colors = dict(zip(labels, plt.cm.tab20.colors[:len(labels)])) fig, ax = plt.subplots(figsize=(10, 10), subplot_kw=dict(aspect="equal")) m_labels, m_values = merge_small_entries(labels, values) @@ -85,7 +64,7 @@ def func(pct, values): wedges, texts, autotexts = ax.pie(m_values, labels = m_labels, - colors=[colours[key] for key in labels], + colors=[colors[key] for key in labels], pctdistance=0.75, autopct= lambda pct: func(pct, values), textprops={'size': 23}) @@ -97,22 +76,8 @@ def func(pct, values): plt.show() def pie_chart_purpose(plot_title,labels,values,file_name): - labels_trip= ['Work', - 'Home', - 'Meal', - 'Shopping', - 'Personal/Medical', - 'Recreation/Exercise', - 'Transit transfer', - 'Pick-up/Drop off', - 'Entertainment/Social', - 'Other', - 'School', - 'Religious', - 'No travel', - 'not_a_trip'] - colours = dict(zip(labels_trip, plt.cm.tab20.colors[:len(labels_trip)])) + colors = dict(zip(labels, plt.cm.tab20.colors[:len(labels)])) fig, ax = plt.subplots(figsize=(10, 10), subplot_kw=dict(aspect="equal")) m_labels, m_values = merge_small_entries(labels, values) @@ -124,7 +89,7 @@ def func(pct, values): wedges, texts, autotexts = ax.pie(m_values, labels = m_labels, - colors=[colours[key] for key in labels], + colors=[colors[key] for key in labels], pctdistance=0.85, autopct=lambda pct: func(pct, values), textprops={'size': 23}) @@ -181,32 +146,12 @@ def energy_impact(x,y,color,plot_title,file_name): plt.legend(labels=objects, handles=patches, loc='upper right', borderaxespad=0, fontsize=15, frameon=True) plt.savefig(SAVE_DIR+file_name+".png", bbox_inches='tight') -def barplot_mode(data,x,y,plot_title,file_name): - all_labels= ['Gas Car, drove alone', - 'Bus', - 'Train', - 'Free Shuttle', - 'Taxi/Uber/Lyft', - 'Gas Car, with others', - 'Bikeshare', - 'Scooter share', - 'E-bike', - 'Walk', - 'Skate board', - 'Regular Bike', - 'Not a Trip', - 'No Travel', - 'Same Mode', - 'E-car, drove alone', - 'E-car, with others', - 'Air', - 'Other'] - - colours = dict(zip(all_labels, plt.cm.tab20.colors[:len(all_labels)])) +def barplot_mode(data,x,y,plot_title, labels, file_name): + colors = dict(zip(labels, plt.cm.tab20.colors[:len(labels)])) sns.set(font_scale=1.5) f = plt.subplots(figsize=(15, 6)) sns.set(style='whitegrid') - ax = sns.barplot(x=x, y=y, palette=colours,data=data, ci=None) + ax = sns.barplot(x=x, y=y, palette=colors,data=data, ci=None) plt.xlabel(x, fontsize=23) plt.ylabel(y, fontsize=23) plt.title(plot_title, fontsize=25) diff --git a/viz_scripts/scaffolding.py b/viz_scripts/scaffolding.py index 01b4158..7a1709f 100644 --- a/viz_scripts/scaffolding.py +++ b/viz_scripts/scaffolding.py @@ -1,6 +1,8 @@ import pandas as pd import numpy as np import sys +import requests +import json import emission.storage.timeseries.abstract_timeseries as esta import emission.storage.timeseries.tcquery as esttc @@ -106,7 +108,7 @@ def expand_userinputs(labeled_ct): unique_users = lambda df: len(df.user_id.unique()) if "user_id" in df.columns else 0 trip_label_count = lambda s, df: len(df[s].dropna()) if s in df.columns else 0 -def load_viz_notebook_data(year, month, program, study_type, dic_re, dic_pur=None, include_test_users=False): +def load_viz_notebook_data(year, month, program, study_type, has_dynamic_labels, dynamic_labels_url, dic_re, dic_pur=None, include_test_users=False): """ Inputs: year/month/program/study_type = parameters from the visualization notebook dic_* = label mappings; if dic_pur is included it will be used to recode trip purpose @@ -125,18 +127,44 @@ def load_viz_notebook_data(year, month, program, study_type, dic_re, dic_pur=Non if "distance" in expanded_ct.columns: unit_conversions(expanded_ct) - # Mapping new mode labels with dictionaries - # CASE 2 of https://github.com/e-mission/em-public-dashboard/issues/69#issuecomment-1256835867 - if "mode_confirm" in expanded_ct.columns: - expanded_ct['Mode_confirm']= expanded_ct['mode_confirm'].map(dic_re) - if study_type == 'program': + # Extract translations_data for dynamic_labels + if has_dynamic_labels: + print("This json has dynamic label") + req = requests.get(dynamic_labels_url) + if req.status_code != 200: + print("Unable to download dynamic_labels") + else: + print("dynamic labels download was successful.") + dynamic_labels = json.loads(req.text) + + # Extract translations section + translations_data = dynamic_labels["translations"]["en"] + + # Map new mode labels with translations_data from dynamic_labels # CASE 2 of https://github.com/e-mission/em-public-dashboard/issues/69#issuecomment-1256835867 - if 'replaced_mode' in expanded_ct.columns: - expanded_ct['Replaced_mode']= expanded_ct['replaced_mode'].map(dic_re) + if "mode_confirm" in expanded_ct.columns: + expanded_ct["Mode_confirm"] = expanded_ct["mode_confirm"].map(translations_data) + if study_type == 'program': + # CASE 2 of https://github.com/e-mission/em-public-dashboard/issues/69#issuecomment-1256835867 + if 'replaced_mode' in expanded_ct.columns: + expanded_ct['Replaced_mode']= expanded_ct['replaced_mode'].map(translations_data) + else: + print("This is a program, but no replaced modes found. Likely cold start case. Ignoring replaced mode mapping") else: - print("This is a program, but no replaced modes found. Likely cold start case. Ignoring replaced mode mapping") + print("This is a study, not expecting any replaced modes.") else: - print("This is a study, not expecting any replaced modes.") + # Mapping new mode labels with dictionaries + # CASE 2 of https://github.com/e-mission/em-public-dashboard/issues/69#issuecomment-1256835867 + if "mode_confirm" in expanded_ct.columns: + expanded_ct['Mode_confirm']= expanded_ct['mode_confirm'].map(dic_re) + if study_type == 'program': + # CASE 2 of https://github.com/e-mission/em-public-dashboard/issues/69#issuecomment-1256835867 + if 'replaced_mode' in expanded_ct.columns: + expanded_ct['Replaced_mode']= expanded_ct['replaced_mode'].map(dic_re) + else: + print("This is a program, but no replaced modes found. Likely cold start case. Ignoring replaced mode mapping") + else: + print("This is a study, not expecting any replaced modes.") # Trip purpose mapping # CASE 2 of https://github.com/e-mission/em-public-dashboard/issues/69#issuecomment-1256835867 From 176655a0b457a125544c1876ee5f5768efbe5943 Mon Sep 17 00:00:00 2001 From: iantei Date: Mon, 11 Sep 2023 11:48:05 -0700 Subject: [PATCH 02/62] Address Review comments. --- viz_scripts/bin/generate_plots.py | 18 ++++-- viz_scripts/energy_calculations.ipynb | 6 +- viz_scripts/generic_metrics.ipynb | 10 ++-- viz_scripts/generic_timeseries.ipynb | 6 +- viz_scripts/mode_specific_metrics.ipynb | 8 +-- viz_scripts/mode_specific_timeseries.ipynb | 6 +- viz_scripts/scaffolding.py | 66 +++++++++------------- 7 files changed, 60 insertions(+), 60 deletions(-) diff --git a/viz_scripts/bin/generate_plots.py b/viz_scripts/bin/generate_plots.py index 38928fa..1c05a34 100644 --- a/viz_scripts/bin/generate_plots.py +++ b/viz_scripts/bin/generate_plots.py @@ -33,7 +33,8 @@ else: dynamic_config = json.loads(r.text) print(f"Successfully downloaded config with version {dynamic_config['version']} "\ - f"for {dynamic_config['intro']['translated_text']['en']['deployment_name']} ") + f"for {dynamic_config['intro']['translated_text']['en']['deployment_name']} "\ + f"and data collection URL {dynamic_config['server']['connectUrl'] if 'server' in dynamic_config else 'default'}") if dynamic_config['intro']['program_or_study'] == 'program': mode_studied = dynamic_config['intro']['mode_studied'] @@ -43,10 +44,18 @@ # Check if the dynamic config contains dynamic labels 'label_options' # Passing the boolean flag is not enough, bcos we need to trace through the dynamic_config - pass label_options URL. if 'label_options' in dynamic_config: - has_dynamic_labels = True dynamic_labels_url = dynamic_config['label_options'] + +# Parse through the dynamic_labels_url: Create a new function for this +if dynamic_labels_url: + req = requests.get(dynamic_labels_url) + if req.status_code != 200: + print("Unable to download dynamic_labels, status code: {r.status_code}") + else: + print("Dynamic labels download was successful.") + dynamic_labels = json.loads(req.text) else: - has_dynamic_labels = False + print("Dynamic labels URL is unavailable.") if args.date is None: start_date = arrow.get(int(dynamic_config['intro']['start_year']), @@ -77,8 +86,7 @@ def compute_for_date(month, year): study_type=dynamic_config['intro']['program_or_study'], mode_of_interest=mode_studied, include_test_users=dynamic_config.get('metrics', {}).get('include_test_users', False), - has_dynamic_labels = has_dynamic_labels, - dynamic_labels_url = dynamic_labels_url) + dynamic_labels = dynamic_labels) print(f"Running at {arrow.get()} with params {params}") diff --git a/viz_scripts/energy_calculations.ipynb b/viz_scripts/energy_calculations.ipynb index a8efd6b..a4f01aa 100644 --- a/viz_scripts/energy_calculations.ipynb +++ b/viz_scripts/energy_calculations.ipynb @@ -30,7 +30,8 @@ "program = \"default\"\n", "study_type = \"program\"\n", "mode_of_interest = \"e-bike\"\n", - "include_test_users = False" + "include_test_users = False\n", + "dynamic_labels = { }" ] }, { @@ -107,6 +108,7 @@ " month,\n", " program,\n", " study_type,\n", + " dynamic_labels,\n", " dic_re,\n", " dic_pur=dic_pur,\n", " include_test_users=include_test_users)\n", @@ -252,7 +254,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.7.12" + "version": "3.9.16" } }, "nbformat": 4, diff --git a/viz_scripts/generic_metrics.ipynb b/viz_scripts/generic_metrics.ipynb index 9184ba3..750519a 100644 --- a/viz_scripts/generic_metrics.ipynb +++ b/viz_scripts/generic_metrics.ipynb @@ -25,14 +25,13 @@ "metadata": {}, "outputs": [], "source": [ - "year = None\n", - "month = None\n", + "year = 2020\n", + "month = 11\n", "program = \"default\"\n", "study_type = \"study\"\n", "mode_of_interest = None\n", "include_test_users = False\n", - "has_dynamic_labels = True\n", - "dynamic_labels_url = \"https://raw.githubusercontent.com/e-mission/nrel-openpath-deploy-configs/main/label_options/example-program-label-options.json\"" + "dynamic_labels = { }" ] }, { @@ -90,8 +89,7 @@ " month,\n", " program,\n", " study_type,\n", - " has_dynamic_labels,\n", - " dynamic_labels_url,\n", + " dynamic_labels,\n", " dic_re,\n", " dic_pur=dic_pur,\n", " include_test_users=include_test_users)" diff --git a/viz_scripts/generic_timeseries.ipynb b/viz_scripts/generic_timeseries.ipynb index ca012da..a35849c 100644 --- a/viz_scripts/generic_timeseries.ipynb +++ b/viz_scripts/generic_timeseries.ipynb @@ -27,7 +27,8 @@ "program = \"default\"\n", "study_type = \"study\"\n", "mode_of_interest = None\n", - "include_test_users = False" + "include_test_users = False\n", + "dynamic_labels = {}" ] }, { @@ -85,6 +86,7 @@ " month,\n", " program,\n", " study_type,\n", + " dynamic_labels,\n", " dic_re,\n", " include_test_users=include_test_users)\n", "expanded_ct = scaffolding.add_energy_labels(expanded_ct, df_ei, dic_fuel) if \"mode_confirm\" in expanded_ct.columns else expanded_ct" @@ -364,7 +366,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.7.12" + "version": "3.9.6" } }, "nbformat": 4, diff --git a/viz_scripts/mode_specific_metrics.ipynb b/viz_scripts/mode_specific_metrics.ipynb index 33db9b9..24434d1 100644 --- a/viz_scripts/mode_specific_metrics.ipynb +++ b/viz_scripts/mode_specific_metrics.ipynb @@ -25,13 +25,12 @@ "metadata": {}, "outputs": [], "source": [ - "year = None\n", - "month = None\n", + "year = 2020\n", + "month = 11\n", "program = \"default\"\n", "study_type = \"program\"\n", "mode_of_interest = \"e-bike\"\n", "include_test_users = False\n", - "has_dynamic_labels = True\n", "dynamic_labels_url = \"https://raw.githubusercontent.com/e-mission/nrel-openpath-deploy-configs/main/label_options/example-program-label-options.json\"" ] }, @@ -104,7 +103,6 @@ " month,\n", " program,\n", " study_type,\n", - " has_dynamic_labels,\n", " dynamic_labels_url,\n", " dic_re,\n", " dic_pur=dic_pur,\n", @@ -369,7 +367,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.9.16" + "version": "3.9.6" } }, "nbformat": 4, diff --git a/viz_scripts/mode_specific_timeseries.ipynb b/viz_scripts/mode_specific_timeseries.ipynb index a157bcc..aa4141e 100644 --- a/viz_scripts/mode_specific_timeseries.ipynb +++ b/viz_scripts/mode_specific_timeseries.ipynb @@ -27,7 +27,8 @@ "program = \"default\"\n", "study_type = \"program\"\n", "mode_of_interest = \"e-bike\"\n", - "include_test_users = False" + "include_test_users = False\n", + "dynamic_labels = { }" ] }, { @@ -95,6 +96,7 @@ " month,\n", " program,\n", " study_type,\n", + " dynamic_labels,\n", " dic_re,\n", " dic_pur=dic_pur,\n", " include_test_users=include_test_users)" @@ -483,7 +485,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.7.12" + "version": "3.9.6" } }, "nbformat": 4, diff --git a/viz_scripts/scaffolding.py b/viz_scripts/scaffolding.py index 7a1709f..e37443d 100644 --- a/viz_scripts/scaffolding.py +++ b/viz_scripts/scaffolding.py @@ -108,7 +108,12 @@ def expand_userinputs(labeled_ct): unique_users = lambda df: len(df.user_id.unique()) if "user_id" in df.columns else 0 trip_label_count = lambda s, df: len(df[s].dropna()) if s in df.columns else 0 -def load_viz_notebook_data(year, month, program, study_type, has_dynamic_labels, dynamic_labels_url, dic_re, dic_pur=None, include_test_users=False): +# Function to map: Selected an "old_mode" col from expanded_ct and +# map to the value of dic_translations into "new mode" col of expanded_ct +def update_expanded_ct(expanded_ct, new_mode, old_mode, dic_mapping): + expanded_ct[new_mode] = expanded_ct[old_mode].map(dic_mapping) + +def load_viz_notebook_data(year, month, program, study_type, dynamic_labels, dic_re, dic_pur=None, include_test_users=False): """ Inputs: year/month/program/study_type = parameters from the visualization notebook dic_* = label mappings; if dic_pur is included it will be used to recode trip purpose @@ -126,50 +131,35 @@ def load_viz_notebook_data(year, month, program, study_type, has_dynamic_labels, # CASE 2 of https://github.com/e-mission/em-public-dashboard/issues/69#issuecomment-1256835867 if "distance" in expanded_ct.columns: unit_conversions(expanded_ct) - - # Extract translations_data for dynamic_labels - if has_dynamic_labels: - print("This json has dynamic label") - req = requests.get(dynamic_labels_url) - if req.status_code != 200: - print("Unable to download dynamic_labels") - else: - print("dynamic labels download was successful.") - dynamic_labels = json.loads(req.text) - - # Extract translations section - translations_data = dynamic_labels["translations"]["en"] - - # Map new mode labels with translations_data from dynamic_labels - # CASE 2 of https://github.com/e-mission/em-public-dashboard/issues/69#issuecomment-1256835867 - if "mode_confirm" in expanded_ct.columns: - expanded_ct["Mode_confirm"] = expanded_ct["mode_confirm"].map(translations_data) - if study_type == 'program': - # CASE 2 of https://github.com/e-mission/em-public-dashboard/issues/69#issuecomment-1256835867 - if 'replaced_mode' in expanded_ct.columns: - expanded_ct['Replaced_mode']= expanded_ct['replaced_mode'].map(translations_data) - else: - print("This is a program, but no replaced modes found. Likely cold start case. Ignoring replaced mode mapping") - else: - print("This is a study, not expecting any replaced modes.") + + # Extract translations key + dic_translations = dict() + if "translations" in dynamic_labels and "en" in dynamic_labels["translations"]: + dic_translations = dynamic_labels["translations"]["en"] + + # Select the mapping based on availability of dynamic_labels + if dic_translations: + dic_mapping = dic_translations else: - # Mapping new mode labels with dictionaries + dic_mapping = dic_re + + # Map new mode labels with translations dictionary from dynamic_labels + # CASE 2 of https://github.com/e-mission/em-public-dashboard/issues/69#issuecomment-1256835867 + if "mode_confirm" in expanded_ct.columns: + update_expanded_ct(expanded_ct, "Mode_confirm", "mode_confirm", dic_mapping) + if study_type == 'program': # CASE 2 of https://github.com/e-mission/em-public-dashboard/issues/69#issuecomment-1256835867 - if "mode_confirm" in expanded_ct.columns: - expanded_ct['Mode_confirm']= expanded_ct['mode_confirm'].map(dic_re) - if study_type == 'program': - # CASE 2 of https://github.com/e-mission/em-public-dashboard/issues/69#issuecomment-1256835867 - if 'replaced_mode' in expanded_ct.columns: - expanded_ct['Replaced_mode']= expanded_ct['replaced_mode'].map(dic_re) - else: - print("This is a program, but no replaced modes found. Likely cold start case. Ignoring replaced mode mapping") + if 'replaced_mode' in expanded_ct.columns: + update_expanded_ct(expanded_ct, "Replaced_mode", "replaced_mode", dic_mapping) else: - print("This is a study, not expecting any replaced modes.") + print("This is a program, but no replaced modes found. Likely cold start case. Ignoring replaced mode mapping") + else: + print("This is a study, not expecting any replaced modes.") # Trip purpose mapping # CASE 2 of https://github.com/e-mission/em-public-dashboard/issues/69#issuecomment-1256835867 if dic_pur is not None and "purpose_confirm" in expanded_ct.columns: - expanded_ct['Trip_purpose']= expanded_ct['purpose_confirm'].map(dic_pur) + update_expanded_ct(expanded_ct, "Trip_purpose", "purpose_confirm", dic_pur) # Document data quality file_suffix = get_file_suffix(year, month, program) From f04e5968ffab1a748c67ac229e7e3b5496f905f6 Mon Sep 17 00:00:00 2001 From: iantei Date: Mon, 11 Sep 2023 19:54:22 -0700 Subject: [PATCH 03/62] Update generate_plots.py & scaffolding.py --- viz_scripts/bin/generate_plots.py | 21 ++++++++++++--------- viz_scripts/scaffolding.py | 11 +++-------- 2 files changed, 15 insertions(+), 17 deletions(-) diff --git a/viz_scripts/bin/generate_plots.py b/viz_scripts/bin/generate_plots.py index 1c05a34..2069b5f 100644 --- a/viz_scripts/bin/generate_plots.py +++ b/viz_scripts/bin/generate_plots.py @@ -42,20 +42,22 @@ mode_studied = None # Check if the dynamic config contains dynamic labels 'label_options' -# Passing the boolean flag is not enough, bcos we need to trace through the dynamic_config - pass label_options URL. +# Parse through the dynamic_labels_url: +dynamic_labels = { } if 'label_options' in dynamic_config: dynamic_labels_url = dynamic_config['label_options'] -# Parse through the dynamic_labels_url: Create a new function for this -if dynamic_labels_url: - req = requests.get(dynamic_labels_url) - if req.status_code != 200: - print("Unable to download dynamic_labels, status code: {r.status_code}") + if dynamic_labels_url: + req = requests.get(dynamic_labels_url) + if req.status_code != 200: + print(f"Unable to download dynamic_labels, status code: {req.status_code}") + else: + print("Dynamic labels download was successful.") + dynamic_labels = json.loads(req.text) else: - print("Dynamic labels download was successful.") - dynamic_labels = json.loads(req.text) + print("Dynamic labels URL is unavailable.") else: - print("Dynamic labels URL is unavailable.") + print("Dynamic labels are not available.") if args.date is None: start_date = arrow.get(int(dynamic_config['intro']['start_year']), @@ -86,6 +88,7 @@ def compute_for_date(month, year): study_type=dynamic_config['intro']['program_or_study'], mode_of_interest=mode_studied, include_test_users=dynamic_config.get('metrics', {}).get('include_test_users', False), + sensed_algo_prefix=dynamic_config.get('metrics', {}).get('sensed_algo_prefix', "cleaned"), dynamic_labels = dynamic_labels) print(f"Running at {arrow.get()} with params {params}") diff --git a/viz_scripts/scaffolding.py b/viz_scripts/scaffolding.py index e37443d..b2ff640 100644 --- a/viz_scripts/scaffolding.py +++ b/viz_scripts/scaffolding.py @@ -108,11 +108,6 @@ def expand_userinputs(labeled_ct): unique_users = lambda df: len(df.user_id.unique()) if "user_id" in df.columns else 0 trip_label_count = lambda s, df: len(df[s].dropna()) if s in df.columns else 0 -# Function to map: Selected an "old_mode" col from expanded_ct and -# map to the value of dic_translations into "new mode" col of expanded_ct -def update_expanded_ct(expanded_ct, new_mode, old_mode, dic_mapping): - expanded_ct[new_mode] = expanded_ct[old_mode].map(dic_mapping) - def load_viz_notebook_data(year, month, program, study_type, dynamic_labels, dic_re, dic_pur=None, include_test_users=False): """ Inputs: year/month/program/study_type = parameters from the visualization notebook @@ -146,11 +141,11 @@ def load_viz_notebook_data(year, month, program, study_type, dynamic_labels, dic # Map new mode labels with translations dictionary from dynamic_labels # CASE 2 of https://github.com/e-mission/em-public-dashboard/issues/69#issuecomment-1256835867 if "mode_confirm" in expanded_ct.columns: - update_expanded_ct(expanded_ct, "Mode_confirm", "mode_confirm", dic_mapping) + expanded_ct['Mode_confirm'] = expanded_ct['mode_confirm'].map(dic_mapping) if study_type == 'program': # CASE 2 of https://github.com/e-mission/em-public-dashboard/issues/69#issuecomment-1256835867 if 'replaced_mode' in expanded_ct.columns: - update_expanded_ct(expanded_ct, "Replaced_mode", "replaced_mode", dic_mapping) + expanded_ct['Replaced_mode'] = expanded_ct['replaced_mode'].map(dic_mapping) else: print("This is a program, but no replaced modes found. Likely cold start case. Ignoring replaced mode mapping") else: @@ -159,7 +154,7 @@ def load_viz_notebook_data(year, month, program, study_type, dynamic_labels, dic # Trip purpose mapping # CASE 2 of https://github.com/e-mission/em-public-dashboard/issues/69#issuecomment-1256835867 if dic_pur is not None and "purpose_confirm" in expanded_ct.columns: - update_expanded_ct(expanded_ct, "Trip_purpose", "purpose_confirm", dic_pur) + expanded_ct['Trip_purpose'] = expanded_ctp['purpose_confirm'].map(dic_pur) # Document data quality file_suffix = get_file_suffix(year, month, program) From 3182a0bab05ab9864e0228a65b4991f4cf6e18d7 Mon Sep 17 00:00:00 2001 From: iantei Date: Mon, 11 Sep 2023 20:00:50 -0700 Subject: [PATCH 04/62] Fix merge conflict. --- viz_scripts/bin/generate_plots.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/viz_scripts/bin/generate_plots.py b/viz_scripts/bin/generate_plots.py index 2069b5f..4ab661f 100644 --- a/viz_scripts/bin/generate_plots.py +++ b/viz_scripts/bin/generate_plots.py @@ -88,8 +88,9 @@ def compute_for_date(month, year): study_type=dynamic_config['intro']['program_or_study'], mode_of_interest=mode_studied, include_test_users=dynamic_config.get('metrics', {}).get('include_test_users', False), - sensed_algo_prefix=dynamic_config.get('metrics', {}).get('sensed_algo_prefix', "cleaned"), - dynamic_labels = dynamic_labels) + dynamic_labels = dynamic_labels, + sensed_algo_prefix=dynamic_config.get('metrics', {}).get('sensed_algo_prefix', "cleaned")) + print(f"Running at {arrow.get()} with params {params}") From 43d974b4f25d2f86f41dc02c884e0a40b6c2bd16 Mon Sep 17 00:00:00 2001 From: iantei Date: Mon, 11 Sep 2023 22:57:26 -0700 Subject: [PATCH 05/62] Fix scaffolding, updated mode_specific_metrics.ipynb. --- viz_scripts/mode_specific_metrics.ipynb | 4 ++-- viz_scripts/scaffolding.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/viz_scripts/mode_specific_metrics.ipynb b/viz_scripts/mode_specific_metrics.ipynb index 24434d1..a26bf08 100644 --- a/viz_scripts/mode_specific_metrics.ipynb +++ b/viz_scripts/mode_specific_metrics.ipynb @@ -31,7 +31,7 @@ "study_type = \"program\"\n", "mode_of_interest = \"e-bike\"\n", "include_test_users = False\n", - "dynamic_labels_url = \"https://raw.githubusercontent.com/e-mission/nrel-openpath-deploy-configs/main/label_options/example-program-label-options.json\"" + "dynamic_labels = { }" ] }, { @@ -103,7 +103,7 @@ " month,\n", " program,\n", " study_type,\n", - " dynamic_labels_url,\n", + " dynamic_labels,\n", " dic_re,\n", " dic_pur=dic_pur,\n", " include_test_users=include_test_users)" diff --git a/viz_scripts/scaffolding.py b/viz_scripts/scaffolding.py index b2ff640..0739fc5 100644 --- a/viz_scripts/scaffolding.py +++ b/viz_scripts/scaffolding.py @@ -154,7 +154,7 @@ def load_viz_notebook_data(year, month, program, study_type, dynamic_labels, dic # Trip purpose mapping # CASE 2 of https://github.com/e-mission/em-public-dashboard/issues/69#issuecomment-1256835867 if dic_pur is not None and "purpose_confirm" in expanded_ct.columns: - expanded_ct['Trip_purpose'] = expanded_ctp['purpose_confirm'].map(dic_pur) + expanded_ct['Trip_purpose'] = expanded_ct['purpose_confirm'].map(dic_pur) # Document data quality file_suffix = get_file_suffix(year, month, program) From 7343606668d01e1f75a553bdfdb5be9274b395bd Mon Sep 17 00:00:00 2001 From: iantei Date: Mon, 11 Sep 2023 23:18:25 -0700 Subject: [PATCH 06/62] Revert version changes to 3.7.12. --- viz_scripts/energy_calculations.ipynb | 2 +- viz_scripts/generic_metrics.ipynb | 2 +- viz_scripts/generic_timeseries.ipynb | 2 +- viz_scripts/mode_specific_metrics.ipynb | 2 +- viz_scripts/mode_specific_timeseries.ipynb | 2 +- 5 files changed, 5 insertions(+), 5 deletions(-) diff --git a/viz_scripts/energy_calculations.ipynb b/viz_scripts/energy_calculations.ipynb index a4f01aa..7d0f4bc 100644 --- a/viz_scripts/energy_calculations.ipynb +++ b/viz_scripts/energy_calculations.ipynb @@ -254,7 +254,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.9.16" + "version": "3.7.12" } }, "nbformat": 4, diff --git a/viz_scripts/generic_metrics.ipynb b/viz_scripts/generic_metrics.ipynb index 750519a..7415978 100644 --- a/viz_scripts/generic_metrics.ipynb +++ b/viz_scripts/generic_metrics.ipynb @@ -394,7 +394,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.9.16" + "version": "3.7.12" } }, "nbformat": 4, diff --git a/viz_scripts/generic_timeseries.ipynb b/viz_scripts/generic_timeseries.ipynb index a35849c..47acf9e 100644 --- a/viz_scripts/generic_timeseries.ipynb +++ b/viz_scripts/generic_timeseries.ipynb @@ -366,7 +366,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.9.6" + "version": "3.7.12" } }, "nbformat": 4, diff --git a/viz_scripts/mode_specific_metrics.ipynb b/viz_scripts/mode_specific_metrics.ipynb index a26bf08..754e1f9 100644 --- a/viz_scripts/mode_specific_metrics.ipynb +++ b/viz_scripts/mode_specific_metrics.ipynb @@ -367,7 +367,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.9.6" + "version": "3.7.12" } }, "nbformat": 4, diff --git a/viz_scripts/mode_specific_timeseries.ipynb b/viz_scripts/mode_specific_timeseries.ipynb index aa4141e..e935b48 100644 --- a/viz_scripts/mode_specific_timeseries.ipynb +++ b/viz_scripts/mode_specific_timeseries.ipynb @@ -485,7 +485,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.9.6" + "version": "3.7.12" } }, "nbformat": 4, From 080f4f4d128b429928573453f0bc93b7126b671b Mon Sep 17 00:00:00 2001 From: iantei Date: Mon, 11 Sep 2023 23:32:05 -0700 Subject: [PATCH 07/62] Update initialization of dynamic_labels from { } to json object referenced from example-study-label-options.json --- viz_scripts/bin/generate_plots.py | 110 +++++++++++++++++++++++++++++- 1 file changed, 109 insertions(+), 1 deletion(-) diff --git a/viz_scripts/bin/generate_plots.py b/viz_scripts/bin/generate_plots.py index 4ab661f..14162a9 100644 --- a/viz_scripts/bin/generate_plots.py +++ b/viz_scripts/bin/generate_plots.py @@ -41,9 +41,117 @@ else: mode_studied = None +# Default value of dynamic_labels referenced from +# https://github.com/e-mission/nrel-openpath-deploy-configs/blob/main/label_options/example-study-label-options.json +dynamic_labels = { + "MODE": [ + {"value":"walk", "baseMode":"WALKING", "met_equivalent":"WALKING", "kgCo2PerKm": 0}, + {"value":"e-bike", "baseMode":"E_BIKE", "met": {"ALL": {"range": [0, -1], "mets": 4.9}}, "kgCo2PerKm": 0.00728}, + {"value":"bike", "baseMode":"BICYCLING", "met_equivalent":"BICYCLING", "kgCo2PerKm": 0}, + {"value":"bikeshare", "baseMode":"BICYCLING", "met_equivalent":"BICYCLING", "kgCo2PerKm": 0}, + {"value":"scootershare", "baseMode":"E_SCOOTER", "met_equivalent":"IN_VEHICLE", "kgCo2PerKm": 0.00894}, + {"value":"drove_alone", "baseMode":"CAR", "met_equivalent":"IN_VEHICLE", "kgCo2PerKm": 0.22031}, + {"value":"shared_ride", "baseMode":"CAR", "met_equivalent":"IN_VEHICLE", "kgCo2PerKm": 0.11015}, + {"value":"e_car_drove_alone", "baseMode":"E_CAR", "met_equivalent":"IN_VEHICLE", "kgCo2PerKm": 0.08216}, + {"value":"e_car_shared_ride", "baseMode":"E_CAR", "met_equivalent":"IN_VEHICLE", "kgCo2PerKm": 0.04108}, + {"value":"moped", "baseMode":"MOPED", "met_equivalent":"IN_VEHICLE", "kgCo2PerKm": 0.05555}, + {"value":"taxi", "baseMode":"TAXI", "met_equivalent":"IN_VEHICLE", "kgCo2PerKm": 0.30741}, + {"value":"bus", "baseMode":"BUS", "met_equivalent":"IN_VEHICLE", "kgCo2PerKm": 0.20727}, + {"value":"train", "baseMode":"TRAIN", "met_equivalent":"IN_VEHICLE", "kgCo2PerKm": 0.12256}, + {"value":"free_shuttle", "baseMode":"BUS", "met_equivalent":"IN_VEHICLE", "kgCo2PerKm": 0.20727}, + {"value":"air", "baseMode":"AIR", "met_equivalent":"IN_VEHICLE", "kgCo2PerKm": 0.09975}, + {"value":"not_a_trip", "baseMode":"UNKNOWN", "met_equivalent":"UNKNOWN", "kgCo2PerKm": 0}, + {"value":"other", "baseMode":"OTHER", "met_equivalent":"UNKNOWN", "kgCo2PerKm": 0} + ], + "PURPOSE": [ + {"value":"home"}, + {"value":"work"}, + {"value":"at_work"}, + {"value":"school"}, + {"value":"transit_transfer"}, + {"value":"shopping"}, + {"value":"meal"}, + {"value":"pick_drop_person"}, + {"value":"pick_drop_item"}, + {"value":"personal_med"}, + {"value":"access_recreation"}, + {"value":"exercise"}, + {"value":"entertainment"}, + {"value":"religious"}, + {"value":"other"} + ], + "translations": { + "en": { + "walk": "Walk", + "e-bike": "E-bike", + "bike": "Regular Bike", + "bikeshare": "Bikeshare", + "scootershare": "Scooter share", + "drove_alone": "Gas Car Drove Alone", + "shared_ride": "Gas Car Shared Ride", + "e_car_drove_alone": "E-Car Drove Alone", + "e_car_shared_ride": "E-Car Shared Ride", + "moped": "Moped", + "taxi": "Taxi/Uber/Lyft", + "bus": "Bus", + "train": "Train", + "free_shuttle": "Free Shuttle", + "air": "Air", + "not_a_trip": "Not a trip", + "home": "Home", + "work": "To Work", + "at_work": "At Work", + "school": "School", + "transit_transfer": "Transit transfer", + "shopping": "Shopping", + "meal": "Meal", + "pick_drop_person": "Pick-up/ Drop off Person", + "pick_drop_item": "Pick-up/ Drop off Item", + "personal_med": "Personal/ Medical", + "access_recreation": "Access Recreation", + "exercise": "Recreation/ Exercise", + "entertainment": "Entertainment/ Social", + "religious": "Religious", + "other": "Other" + }, + "es": { + "walk": "Caminando", + "e-bike": "e-bicicleta", + "bike": "Bicicleta", + "bikeshare": "Bicicleta compartida", + "scootershare": "Motoneta compartida", + "drove_alone": "Coche de Gas, Condujo solo", + "shared_ride": "Coche de Gas, Condujo con otros", + "e_car_drove_alone": "e-coche, Condujo solo", + "e_car_shared_ride": "e-coche, Condujo con ontras", + "moped": "Ciclomotor", + "taxi": "Taxi/Uber/Lyft", + "bus": "Autobús", + "train": "Tren", + "free_shuttle": "Colectivo gratuito", + "air": "Avión", + "not_a_trip": "No es un viaje", + "home": "Inicio", + "work": "Trabajo", + "at_work": "En el trabajo", + "school": "Escuela", + "transit_transfer": "Transbordo", + "shopping": "Compras", + "meal": "Comida", + "pick_drop_person": "Recoger/ Entregar Individuo", + "pick_drop_item": "Recoger/ Entregar Objeto", + "personal_med": "Personal/ Médico", + "access_recreation": "Acceder a Recreación", + "exercise": "Recreación/ Ejercicio", + "entertainment": "Entretenimiento/ Social", + "religious": "Religioso", + "other": "Otros" + } + } +} + # Check if the dynamic config contains dynamic labels 'label_options' # Parse through the dynamic_labels_url: -dynamic_labels = { } if 'label_options' in dynamic_config: dynamic_labels_url = dynamic_config['label_options'] From 7d8b2d604135597cfdc6781e832197693ac8929b Mon Sep 17 00:00:00 2001 From: iantei Date: Thu, 14 Sep 2023 03:56:49 -0700 Subject: [PATCH 08/62] Default initialize dynamic_labels to { }. --- viz_scripts/bin/generate_plots.py | 111 +----------------------------- 1 file changed, 3 insertions(+), 108 deletions(-) diff --git a/viz_scripts/bin/generate_plots.py b/viz_scripts/bin/generate_plots.py index 14162a9..8851e20 100644 --- a/viz_scripts/bin/generate_plots.py +++ b/viz_scripts/bin/generate_plots.py @@ -41,114 +41,9 @@ else: mode_studied = None -# Default value of dynamic_labels referenced from +# dynamic_labels can be referenced from # https://github.com/e-mission/nrel-openpath-deploy-configs/blob/main/label_options/example-study-label-options.json -dynamic_labels = { - "MODE": [ - {"value":"walk", "baseMode":"WALKING", "met_equivalent":"WALKING", "kgCo2PerKm": 0}, - {"value":"e-bike", "baseMode":"E_BIKE", "met": {"ALL": {"range": [0, -1], "mets": 4.9}}, "kgCo2PerKm": 0.00728}, - {"value":"bike", "baseMode":"BICYCLING", "met_equivalent":"BICYCLING", "kgCo2PerKm": 0}, - {"value":"bikeshare", "baseMode":"BICYCLING", "met_equivalent":"BICYCLING", "kgCo2PerKm": 0}, - {"value":"scootershare", "baseMode":"E_SCOOTER", "met_equivalent":"IN_VEHICLE", "kgCo2PerKm": 0.00894}, - {"value":"drove_alone", "baseMode":"CAR", "met_equivalent":"IN_VEHICLE", "kgCo2PerKm": 0.22031}, - {"value":"shared_ride", "baseMode":"CAR", "met_equivalent":"IN_VEHICLE", "kgCo2PerKm": 0.11015}, - {"value":"e_car_drove_alone", "baseMode":"E_CAR", "met_equivalent":"IN_VEHICLE", "kgCo2PerKm": 0.08216}, - {"value":"e_car_shared_ride", "baseMode":"E_CAR", "met_equivalent":"IN_VEHICLE", "kgCo2PerKm": 0.04108}, - {"value":"moped", "baseMode":"MOPED", "met_equivalent":"IN_VEHICLE", "kgCo2PerKm": 0.05555}, - {"value":"taxi", "baseMode":"TAXI", "met_equivalent":"IN_VEHICLE", "kgCo2PerKm": 0.30741}, - {"value":"bus", "baseMode":"BUS", "met_equivalent":"IN_VEHICLE", "kgCo2PerKm": 0.20727}, - {"value":"train", "baseMode":"TRAIN", "met_equivalent":"IN_VEHICLE", "kgCo2PerKm": 0.12256}, - {"value":"free_shuttle", "baseMode":"BUS", "met_equivalent":"IN_VEHICLE", "kgCo2PerKm": 0.20727}, - {"value":"air", "baseMode":"AIR", "met_equivalent":"IN_VEHICLE", "kgCo2PerKm": 0.09975}, - {"value":"not_a_trip", "baseMode":"UNKNOWN", "met_equivalent":"UNKNOWN", "kgCo2PerKm": 0}, - {"value":"other", "baseMode":"OTHER", "met_equivalent":"UNKNOWN", "kgCo2PerKm": 0} - ], - "PURPOSE": [ - {"value":"home"}, - {"value":"work"}, - {"value":"at_work"}, - {"value":"school"}, - {"value":"transit_transfer"}, - {"value":"shopping"}, - {"value":"meal"}, - {"value":"pick_drop_person"}, - {"value":"pick_drop_item"}, - {"value":"personal_med"}, - {"value":"access_recreation"}, - {"value":"exercise"}, - {"value":"entertainment"}, - {"value":"religious"}, - {"value":"other"} - ], - "translations": { - "en": { - "walk": "Walk", - "e-bike": "E-bike", - "bike": "Regular Bike", - "bikeshare": "Bikeshare", - "scootershare": "Scooter share", - "drove_alone": "Gas Car Drove Alone", - "shared_ride": "Gas Car Shared Ride", - "e_car_drove_alone": "E-Car Drove Alone", - "e_car_shared_ride": "E-Car Shared Ride", - "moped": "Moped", - "taxi": "Taxi/Uber/Lyft", - "bus": "Bus", - "train": "Train", - "free_shuttle": "Free Shuttle", - "air": "Air", - "not_a_trip": "Not a trip", - "home": "Home", - "work": "To Work", - "at_work": "At Work", - "school": "School", - "transit_transfer": "Transit transfer", - "shopping": "Shopping", - "meal": "Meal", - "pick_drop_person": "Pick-up/ Drop off Person", - "pick_drop_item": "Pick-up/ Drop off Item", - "personal_med": "Personal/ Medical", - "access_recreation": "Access Recreation", - "exercise": "Recreation/ Exercise", - "entertainment": "Entertainment/ Social", - "religious": "Religious", - "other": "Other" - }, - "es": { - "walk": "Caminando", - "e-bike": "e-bicicleta", - "bike": "Bicicleta", - "bikeshare": "Bicicleta compartida", - "scootershare": "Motoneta compartida", - "drove_alone": "Coche de Gas, Condujo solo", - "shared_ride": "Coche de Gas, Condujo con otros", - "e_car_drove_alone": "e-coche, Condujo solo", - "e_car_shared_ride": "e-coche, Condujo con ontras", - "moped": "Ciclomotor", - "taxi": "Taxi/Uber/Lyft", - "bus": "Autobús", - "train": "Tren", - "free_shuttle": "Colectivo gratuito", - "air": "Avión", - "not_a_trip": "No es un viaje", - "home": "Inicio", - "work": "Trabajo", - "at_work": "En el trabajo", - "school": "Escuela", - "transit_transfer": "Transbordo", - "shopping": "Compras", - "meal": "Comida", - "pick_drop_person": "Recoger/ Entregar Individuo", - "pick_drop_item": "Recoger/ Entregar Objeto", - "personal_med": "Personal/ Médico", - "access_recreation": "Acceder a Recreación", - "exercise": "Recreación/ Ejercicio", - "entertainment": "Entretenimiento/ Social", - "religious": "Religioso", - "other": "Otros" - } - } -} +dynamic_labels = { } # Check if the dynamic config contains dynamic labels 'label_options' # Parse through the dynamic_labels_url: @@ -160,8 +55,8 @@ if req.status_code != 200: print(f"Unable to download dynamic_labels, status code: {req.status_code}") else: - print("Dynamic labels download was successful.") dynamic_labels = json.loads(req.text) + print("Dynamic labels download was successful.") else: print("Dynamic labels URL is unavailable.") else: From fea6365d38db402d2684cea8d6ef7f448fcbc9b1 Mon Sep 17 00:00:00 2001 From: iantei Date: Sun, 17 Sep 2023 10:49:54 -0700 Subject: [PATCH 09/62] Add an environment variable in Docker Compose to pass dynamic config. --- docker-compose.dev.yml | 1 + viz_scripts/bin/generate_plots.py | 4 +++- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/docker-compose.dev.yml b/docker-compose.dev.yml index fdbdbb8..77a35d4 100644 --- a/docker-compose.dev.yml +++ b/docker-compose.dev.yml @@ -27,6 +27,7 @@ services: - WEB_SERVER_HOST=0.0.0.0 - CRON_MODE= - STUDY_CONFIG=stage-program + - DYNAMIC_CONFIG=dev-emulator-study ports: # ipynb in numbers - "47962:8888" diff --git a/viz_scripts/bin/generate_plots.py b/viz_scripts/bin/generate_plots.py index 8851e20..87756fa 100644 --- a/viz_scripts/bin/generate_plots.py +++ b/viz_scripts/bin/generate_plots.py @@ -14,6 +14,7 @@ # Full list is at # https://github.com/e-mission/nrel-openpath-deploy-configs/tree/main/configs STUDY_CONFIG = os.getenv('STUDY_CONFIG', "stage-program") +DYNAMIC_CONFIG = os.getenv('DYNAMIC_CONFIG', "dev-emulator-study") parser = argparse.ArgumentParser(prog="generate_metrics") parser.add_argument("plot_notebook", help="the notebook the generates the plot") @@ -24,7 +25,8 @@ args = parser.parse_args() # Read and use parameters from the unified config file on the e-mission Github page -download_url = "https://raw.githubusercontent.com/e-mission/nrel-openpath-deploy-configs/main/configs/" + STUDY_CONFIG + ".nrel-op.json" +# download_url = "https://raw.githubusercontent.com/e-mission/nrel-openpath-deploy-configs/main/configs/" + STUDY_CONFIG + ".nrel-op.json" +download_url = "https://raw.githubusercontent.com/e-mission/nrel-openpath-deploy-configs/main/configs/" + DYNAMIC_CONFIG + ".nrel-op.json" print("About to download config from %s" % download_url) r = requests.get(download_url) if r.status_code is not 200: From b04096b1fc120c55266f872f1c14d6093a6eeb91 Mon Sep 17 00:00:00 2001 From: iantei Date: Mon, 18 Sep 2023 10:56:21 -0700 Subject: [PATCH 10/62] Chang in docker-compose for STUDY_CONFIG to dev-emulator-study. Reverted changes for DYNAMIC_CONFIG back to STUDY_CONFIG from generate_plots.py --- docker-compose.dev.yml | 3 +-- viz_scripts/bin/generate_plots.py | 4 +--- 2 files changed, 2 insertions(+), 5 deletions(-) diff --git a/docker-compose.dev.yml b/docker-compose.dev.yml index 77a35d4..9cbc4de 100644 --- a/docker-compose.dev.yml +++ b/docker-compose.dev.yml @@ -26,8 +26,7 @@ services: - DB_HOST=db - WEB_SERVER_HOST=0.0.0.0 - CRON_MODE= - - STUDY_CONFIG=stage-program - - DYNAMIC_CONFIG=dev-emulator-study + - STUDY_CONFIG=dev-emulator-study ports: # ipynb in numbers - "47962:8888" diff --git a/viz_scripts/bin/generate_plots.py b/viz_scripts/bin/generate_plots.py index 87756fa..8851e20 100644 --- a/viz_scripts/bin/generate_plots.py +++ b/viz_scripts/bin/generate_plots.py @@ -14,7 +14,6 @@ # Full list is at # https://github.com/e-mission/nrel-openpath-deploy-configs/tree/main/configs STUDY_CONFIG = os.getenv('STUDY_CONFIG', "stage-program") -DYNAMIC_CONFIG = os.getenv('DYNAMIC_CONFIG', "dev-emulator-study") parser = argparse.ArgumentParser(prog="generate_metrics") parser.add_argument("plot_notebook", help="the notebook the generates the plot") @@ -25,8 +24,7 @@ args = parser.parse_args() # Read and use parameters from the unified config file on the e-mission Github page -# download_url = "https://raw.githubusercontent.com/e-mission/nrel-openpath-deploy-configs/main/configs/" + STUDY_CONFIG + ".nrel-op.json" -download_url = "https://raw.githubusercontent.com/e-mission/nrel-openpath-deploy-configs/main/configs/" + DYNAMIC_CONFIG + ".nrel-op.json" +download_url = "https://raw.githubusercontent.com/e-mission/nrel-openpath-deploy-configs/main/configs/" + STUDY_CONFIG + ".nrel-op.json" print("About to download config from %s" % download_url) r = requests.get(download_url) if r.status_code is not 200: From 5a70ecb1454e8456cadcf01484a4a62fca342a0d Mon Sep 17 00:00:00 2001 From: iantei Date: Tue, 19 Sep 2023 10:37:06 -0700 Subject: [PATCH 11/62] Change back docker-compose.dev.yml: STUDY_CONFIG to stage-program. --- docker-compose.dev.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker-compose.dev.yml b/docker-compose.dev.yml index 9cbc4de..fdbdbb8 100644 --- a/docker-compose.dev.yml +++ b/docker-compose.dev.yml @@ -26,7 +26,7 @@ services: - DB_HOST=db - WEB_SERVER_HOST=0.0.0.0 - CRON_MODE= - - STUDY_CONFIG=dev-emulator-study + - STUDY_CONFIG=stage-program ports: # ipynb in numbers - "47962:8888" From 6e80c7fd17e4de4da2950fb406c0cec61e1b6610 Mon Sep 17 00:00:00 2001 From: iantei Date: Fri, 22 Sep 2023 15:36:05 -0700 Subject: [PATCH 12/62] Replace Mode_confirm with Replaced_mode for barplot_mode() in mode_specific_metrics.ipynb --- viz_scripts/mode_specific_metrics.ipynb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/viz_scripts/mode_specific_metrics.ipynb b/viz_scripts/mode_specific_metrics.ipynb index 754e1f9..abebc19 100644 --- a/viz_scripts/mode_specific_metrics.ipynb +++ b/viz_scripts/mode_specific_metrics.ipynb @@ -268,7 +268,7 @@ " y2 = \"Count\"\n", "\n", " plot_title= plot_title_no_quality+\"\\n\"+quality_text\n", - " barplot_mode(data,x,y,plot_title, expanded_ct['Mode_confirm'].dropna().unique().tolist(), file_name)\n", + " barplot_mode(data,x,y,plot_title, expanded_ct['Replaced_mode'].dropna().unique().tolist(), file_name)\n", " alt_text = store_alt_text_bar(pd.DataFrame(data['Average (miles)'].values, data.Replaced_mode), file_name, plot_title)\n", "except:\n", " generate_missing_plot(plot_title_no_quality,debug_df,file_name)\n", From 30c1dd4bfbfa4792d8a02844f68d075d686ebc95 Mon Sep 17 00:00:00 2001 From: iantei Date: Mon, 25 Sep 2023 08:50:36 -0700 Subject: [PATCH 13/62] 1. Remove extraneous whitespace from generate_plots.py. 2. Revert refactoring colour -> color in plots.py --- viz_scripts/bin/generate_plots.py | 1 - viz_scripts/plots.py | 12 ++++++------ 2 files changed, 6 insertions(+), 7 deletions(-) diff --git a/viz_scripts/bin/generate_plots.py b/viz_scripts/bin/generate_plots.py index 8851e20..69f906d 100644 --- a/viz_scripts/bin/generate_plots.py +++ b/viz_scripts/bin/generate_plots.py @@ -94,7 +94,6 @@ def compute_for_date(month, year): dynamic_labels = dynamic_labels, sensed_algo_prefix=dynamic_config.get('metrics', {}).get('sensed_algo_prefix', "cleaned")) - print(f"Running at {arrow.get()} with params {params}") # Make a notebook object with these definitions diff --git a/viz_scripts/plots.py b/viz_scripts/plots.py index e18af93..9168e9b 100644 --- a/viz_scripts/plots.py +++ b/viz_scripts/plots.py @@ -59,14 +59,14 @@ def format_pct(pct, values): def pie_chart_mode(plot_title,labels,values,file_name): - colors = dict(zip(labels, plt.cm.tab20.colors[:len(labels)])) + colours = dict(zip(labels, plt.cm.tab20.colors[:len(labels)])) fig, ax = plt.subplots(figsize=(10, 10), subplot_kw=dict(aspect="equal")) m_labels, m_values = merge_small_entries(labels, values) wedges, texts, autotexts = ax.pie(m_values, labels = m_labels, - colors=[colors[key] for key in labels], + colors=[colours[key] for key in labels], pctdistance=0.75, autopct= lambda pct: format_pct(pct, values), textprops={'size': 23}) @@ -107,7 +107,7 @@ def pie_chart_sensed_mode(plot_title,labels,values,file_name): def pie_chart_purpose(plot_title,labels,values,file_name): - colors = dict(zip(labels, plt.cm.tab20.colors[:len(labels)])) + colours = dict(zip(labels, plt.cm.tab20.colors[:len(labels)])) fig, ax = plt.subplots(figsize=(10, 10), subplot_kw=dict(aspect="equal")) m_labels, m_values = merge_small_entries(labels, values) @@ -119,7 +119,7 @@ def func(pct, values): wedges, texts, autotexts = ax.pie(m_values, labels = m_labels, - colors=[colors[key] for key in labels], + colors=[colours[key] for key in labels], pctdistance=0.85, autopct=lambda pct: func(pct, values), textprops={'size': 23}) @@ -177,11 +177,11 @@ def energy_impact(x,y,color,plot_title,file_name): plt.savefig(SAVE_DIR+file_name+".png", bbox_inches='tight') def barplot_mode(data,x,y,plot_title, labels, file_name): - colors = dict(zip(labels, plt.cm.tab20.colors[:len(labels)])) + colours = dict(zip(labels, plt.cm.tab20.colors[:len(labels)])) sns.set(font_scale=1.5) f = plt.subplots(figsize=(15, 6)) sns.set(style='whitegrid') - ax = sns.barplot(x=x, y=y, palette=colors,data=data, ci=None) + ax = sns.barplot(x=x, y=y, palette=colours,data=data, ci=None) plt.xlabel(x, fontsize=23) plt.ylabel(y, fontsize=23) plt.title(plot_title, fontsize=25) From 86f81331cd88783e244eba2c21b014ba85004df6 Mon Sep 17 00:00:00 2001 From: iantei Date: Mon, 25 Sep 2023 08:54:39 -0700 Subject: [PATCH 14/62] Remove extraneous whitespace from line 96 in generate_plots.py --- viz_scripts/bin/generate_plots.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/viz_scripts/bin/generate_plots.py b/viz_scripts/bin/generate_plots.py index 69f906d..b30edb6 100644 --- a/viz_scripts/bin/generate_plots.py +++ b/viz_scripts/bin/generate_plots.py @@ -93,7 +93,7 @@ def compute_for_date(month, year): include_test_users=dynamic_config.get('metrics', {}).get('include_test_users', False), dynamic_labels = dynamic_labels, sensed_algo_prefix=dynamic_config.get('metrics', {}).get('sensed_algo_prefix', "cleaned")) - + print(f"Running at {arrow.get()} with params {params}") # Make a notebook object with these definitions From 845dd478733673b3defb8cee79b0c29ace2a605f Mon Sep 17 00:00:00 2001 From: iantei Date: Tue, 26 Sep 2023 13:05:35 -0700 Subject: [PATCH 15/62] Introduce defaultdic for dic_translations with Other in analogy to dic_re in scaffolding.py. --- viz_scripts/scaffolding.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/viz_scripts/scaffolding.py b/viz_scripts/scaffolding.py index beb6b9e..ad1f5d7 100644 --- a/viz_scripts/scaffolding.py +++ b/viz_scripts/scaffolding.py @@ -1,8 +1,7 @@ import pandas as pd import numpy as np import sys -import requests -import json +from collections import defaultdict import emission.storage.timeseries.abstract_timeseries as esta import emission.storage.timeseries.tcquery as esttc @@ -129,8 +128,10 @@ def load_viz_notebook_data(year, month, program, study_type, dynamic_labels, dic # Extract translations key dic_translations = dict() + if "translations" in dynamic_labels and "en" in dynamic_labels["translations"]: dic_translations = dynamic_labels["translations"]["en"] + dic_translations = defaultdict(lambda: 'Other', dic_translations) # Select the mapping based on availability of dynamic_labels if dic_translations: From af9d3fcc5eaa4c0becb539acf2760a7730266504 Mon Sep 17 00:00:00 2001 From: iantei Date: Thu, 28 Sep 2023 09:05:05 -0700 Subject: [PATCH 16/62] Handle initialization of mode_count_interest. Introduce try-except block to handle raise RuntimeError error. --- viz_scripts/mode_specific_timeseries.ipynb | 18 ++++++++++++++---- 1 file changed, 14 insertions(+), 4 deletions(-) diff --git a/viz_scripts/mode_specific_timeseries.ipynb b/viz_scripts/mode_specific_timeseries.ipynb index e935b48..c29bbc1 100644 --- a/viz_scripts/mode_specific_timeseries.ipynb +++ b/viz_scripts/mode_specific_timeseries.ipynb @@ -157,6 +157,8 @@ }, "outputs": [], "source": [ + "mode_counts_interest = pd.DataFrame()\n", + "\n", "if len(expanded_ct) > 0:\n", " # Get the count of unique users that were active on each given date\n", " active_users = pd.DataFrame(data.groupby(['date_time'], as_index=False)['user_id'].nunique())\n", @@ -180,10 +182,18 @@ " mode_counts_interest = mode_counts[mode_counts['mode_confirm']==mode_of_interest].copy()\n", " mode_distance_interest = mode_distance[mode_distance['mode_confirm']==mode_of_interest].copy()\n", " \n", - " # Mapping new mode labels with dictionaries\n", - " mode_counts['Mode_confirm'] = mode_counts['mode_confirm'].map(dic_re)\n", - " mode_counts_interest['Mode_confirm'] = mode_counts_interest['mode_confirm'].map(dic_re)\n", - " mode_distance_interest['Mode_confirm'] = mode_distance_interest['mode_confirm'].map(dic_re)" + " try:\n", + " if len(mode_counts_interest) == 0:\n", + " # force error generation so that we will go into the \"missing\" data code path\n", + " raise RuntimeError(f\"No {mode_of_interest} trips found\")\n", + "\n", + " # Mapping new mode labels with dictionaries\n", + " mode_counts['Mode_confirm'] = mode_counts['mode_confirm'].map(dic_re)\n", + " mode_counts_interest['Mode_confirm'] = mode_counts_interest['mode_confirm'].map(dic_re)\n", + " mode_distance_interest['Mode_confirm'] = mode_distance_interest['mode_confirm'].map(dic_re)\n", + " except:\n", + " print (f\"No {mode_of_interest} trips found\")\n", + " " ] }, { From a868e4cfaf94c3f8543254dabb50aff788fedfd2 Mon Sep 17 00:00:00 2001 From: iantei Date: Thu, 28 Sep 2023 10:10:50 -0700 Subject: [PATCH 17/62] Move quality_text code into the if block in the above cell. --- viz_scripts/mode_specific_timeseries.ipynb | 29 +++++----------------- 1 file changed, 6 insertions(+), 23 deletions(-) diff --git a/viz_scripts/mode_specific_timeseries.ipynb b/viz_scripts/mode_specific_timeseries.ipynb index c29bbc1..a4eef6d 100644 --- a/viz_scripts/mode_specific_timeseries.ipynb +++ b/viz_scripts/mode_specific_timeseries.ipynb @@ -157,8 +157,6 @@ }, "outputs": [], "source": [ - "mode_counts_interest = pd.DataFrame()\n", - "\n", "if len(expanded_ct) > 0:\n", " # Get the count of unique users that were active on each given date\n", " active_users = pd.DataFrame(data.groupby(['date_time'], as_index=False)['user_id'].nunique())\n", @@ -181,30 +179,15 @@ " # This is the mode specific part\n", " mode_counts_interest = mode_counts[mode_counts['mode_confirm']==mode_of_interest].copy()\n", " mode_distance_interest = mode_distance[mode_distance['mode_confirm']==mode_of_interest].copy()\n", - " \n", - " try:\n", - " if len(mode_counts_interest) == 0:\n", - " # force error generation so that we will go into the \"missing\" data code path\n", - " raise RuntimeError(f\"No {mode_of_interest} trips found\")\n", - "\n", - " # Mapping new mode labels with dictionaries\n", - " mode_counts['Mode_confirm'] = mode_counts['mode_confirm'].map(dic_re)\n", - " mode_counts_interest['Mode_confirm'] = mode_counts_interest['mode_confirm'].map(dic_re)\n", - " mode_distance_interest['Mode_confirm'] = mode_distance_interest['mode_confirm'].map(dic_re)\n", - " except:\n", - " print (f\"No {mode_of_interest} trips found\")\n", + "\n", + " # Mapping new mode labels with dictionaries\n", + " mode_counts['Mode_confirm'] = mode_counts['mode_confirm'].map(dic_re)\n", + " mode_counts_interest['Mode_confirm'] = mode_counts_interest['mode_confirm'].map(dic_re)\n", + " mode_distance_interest['Mode_confirm'] = mode_distance_interest['mode_confirm'].map(dic_re)\n", + " quality_text = scaffolding.get_quality_text(expanded_ct, mode_counts_interest, mode_of_interest, include_test_users)\n", " " ] }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "quality_text = scaffolding.get_quality_text(expanded_ct, mode_counts_interest, mode_of_interest, include_test_users)" - ] - }, { "cell_type": "markdown", "metadata": {}, From 54e213bd854ee236da9818ff7091be13c997a065 Mon Sep 17 00:00:00 2001 From: iantei Date: Thu, 28 Sep 2023 19:52:50 -0700 Subject: [PATCH 18/62] Update log messages for dynamic_labels_url and label_options unavailable cases. --- viz_scripts/bin/generate_plots.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/viz_scripts/bin/generate_plots.py b/viz_scripts/bin/generate_plots.py index b30edb6..cf6d33e 100644 --- a/viz_scripts/bin/generate_plots.py +++ b/viz_scripts/bin/generate_plots.py @@ -58,9 +58,9 @@ dynamic_labels = json.loads(req.text) print("Dynamic labels download was successful.") else: - print("Dynamic labels URL is unavailable.") + print(f"dynamic_labels_url is unavailable for {dynamic_config['label_options']}") else: - print("Dynamic labels are not available.") + print(f"label_options is unavailable for the dynamic_config.") if args.date is None: start_date = arrow.get(int(dynamic_config['intro']['start_year']), From 81407fc6c61c70b004d8aa05278bb26a6e846991 Mon Sep 17 00:00:00 2001 From: iantei Date: Wed, 4 Oct 2023 21:35:39 -0700 Subject: [PATCH 19/62] Update scaffolding to introduce compute_CO2_impact_kg() computation when dynamic label is available. --- viz_scripts/scaffolding.py | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/viz_scripts/scaffolding.py b/viz_scripts/scaffolding.py index b723f40..1b0c35e 100644 --- a/viz_scripts/scaffolding.py +++ b/viz_scripts/scaffolding.py @@ -343,7 +343,6 @@ def CO2_footprint_lb(df, distance, col): df[col+'_lb_CO2'] = np.select(conditions_col, values_col) return df - def CO2_impact_lb(df,distance): if 'Mode_confirm_lb_CO2' not in df.columns: print("Mode confirm footprint not found, computing before impact") @@ -351,3 +350,18 @@ def CO2_impact_lb(df,distance): df = CO2_footprint_lb(df, distance, "Replaced_mode") df['CO2_Impact(lb)'] = round((df['Replaced_mode_lb_CO2'] - df['Mode_confirm_lb_CO2']),3) return df + +def compute_CO2_impact_kg(expanded_ct, dynamic_labels): + + convert_miles_to_kilometer = 1.60934 # 1 Miles = 1.60934 KM + + dic_mode_kgCO2PerKm = {mode["value"]: mode["kgCo2PerKm"] for mode in dynamic_labels["MODE"]} + + if "mode_confirm" in expanded_ct.columns: + expanded_ct['Mode_confirm_kg_CO2'] = expanded_ct['mode_confirm'].map(dic_mode_kgCO2PerKm) + + if "replaced_mode" in expanded_ct.columns: + expanded_ct['Replaced_mode_kg_CO2'] = expanded_ct['replaced_mode'].map(dic_mode_kgCO2PerKm) + + expanded_ct['CO2_Impact(kg)'] = round ( (expanded_ct['distance_miles'] * convert_miles_to_kilometer ) * ( expanded_ct['Replaced_mode_kg_CO2'] - expanded_ct['Mode_confirm_kg_CO2']),3) + return expanded_ct From d4fb9bc3acb643bfb82e61ba57faca449c6613fc Mon Sep 17 00:00:00 2001 From: iantei Date: Wed, 4 Oct 2023 22:07:23 -0700 Subject: [PATCH 20/62] 1. Change the plots.py X-axis to showcase CO2 Impact in kg. 2. In scaffolding.py update conversion from lb to kg for final return of CO2_Impact(kg). 3. Update energy_calculations to call new function compute_CO2_impact_kg when dynamic label is available for direct kgCO2 computation. --- viz_scripts/energy_calculations.ipynb | 22 +++++++++++++--------- viz_scripts/plots.py | 2 +- viz_scripts/scaffolding.py | 9 ++++++--- 3 files changed, 20 insertions(+), 13 deletions(-) diff --git a/viz_scripts/energy_calculations.ipynb b/viz_scripts/energy_calculations.ipynb index 7d0f4bc..2298c4b 100644 --- a/viz_scripts/energy_calculations.ipynb +++ b/viz_scripts/energy_calculations.ipynb @@ -112,8 +112,12 @@ " dic_re,\n", " dic_pur=dic_pur,\n", " include_test_users=include_test_users)\n", + "\n", "# CASE 1 of https://github.com/e-mission/em-public-dashboard/issues/69#issuecomment-1256835867 \n", - "expanded_ct = scaffolding.add_energy_impact(expanded_ct, df_ei, dic_fuel) if len(expanded_ct) > 0 else expanded_ct" + "if len(dynamic_labels) > 0:\n", + " expanded_ct = scaffolding.compute_CO2_impact_kg(expanded_ct, dynamic_labels) if len(expanded_ct) > 0 else expanded_ct\n", + "else:\n", + " expanded_ct = scaffolding.add_energy_impact(expanded_ct, df_ei, dic_fuel) if len(expanded_ct) > 0 else expanded_ct" ] }, { @@ -214,22 +218,22 @@ "metadata": {}, "outputs": [], "source": [ - "plot_title_no_quality=f\"Sketch of Total Pounds of CO2 Emissions of {mode_of_interest} trips\"\n", + "plot_title_no_quality=f\"Sketch of Total Kilograms of CO2 Emissions of {mode_of_interest} trips\"\n", "file_name =f'sketch_CO2impact_{mode_of_interest}%s' % file_suffix\n", "\n", "try:\n", - " ebco2=data_eb.groupby('Replaced_mode').agg({'CO2_Impact(lb)': ['sum', 'mean']},)\n", - " ebco2.columns = ['total_lb_CO2_emissions', 'average_lb_CO2_emission']\n", + " ebco2=data_eb.groupby('Replaced_mode').agg({'CO2_Impact(kg)': ['sum', 'mean']},)\n", + " ebco2.columns = ['total_kg_CO2_emissions', 'average_kg_CO2_emission']\n", " ebco2 = ebco2.reset_index()\n", - " ebco2 = ebco2.sort_values(by=['total_lb_CO2_emissions'], ascending=False)\n", - " ebco2['boolean'] = ebco2['total_lb_CO2_emissions'] > 0\n", - " net_CO2_emissions = round(sum(ebco2['total_lb_CO2_emissions']), 2)\n", + " ebco2 = ebco2.sort_values(by=['total_kg_CO2_emissions'], ascending=False)\n", + " ebco2['boolean'] = ebco2['total_kg_CO2_emissions'] > 0\n", + " net_CO2_emissions = round(sum(ebco2['total_kg_CO2_emissions']), 2)\n", "\n", - " x = ebco2['total_lb_CO2_emissions']\n", + " x = ebco2['total_kg_CO2_emissions']\n", " y = ebco2['Replaced_mode']\n", " color = ebco2['boolean']\n", "\n", - " plot_title= plot_title_no_quality+f\"\\n Contribution by replaced mode towards a total of {net_CO2_emissions}(lb CO2 Emissions )\\n\"+quality_text\n", + " plot_title= plot_title_no_quality+f\"\\n Contribution by replaced mode towards a total of {net_CO2_emissions}(kg CO2 Emissions )\\n\"+quality_text\n", " CO2_impact(x,y,color,plot_title,file_name)\n", " alt_text = store_alt_text_bar(pd.DataFrame(x.values,y), file_name, plot_title)\n", "except:\n", diff --git a/viz_scripts/plots.py b/viz_scripts/plots.py index 9168e9b..6f2bf29 100644 --- a/viz_scripts/plots.py +++ b/viz_scripts/plots.py @@ -256,7 +256,7 @@ def CO2_impact(x,y,color,plot_title,file_name): width = 0.8 ax = x.plot(kind='barh',width=width, color=color) ax.set_title(plot_title, fontsize=18) - ax.set_xlabel('CO2 Emissions (lb)', fontsize=18) + ax.set_xlabel('CO2 Emissions (kg)', fontsize=18) ax.set_ylabel('Replaced Mode',fontsize=18) ax.set_yticklabels(y_labels) ax.xaxis.set_tick_params(labelsize=15) diff --git a/viz_scripts/scaffolding.py b/viz_scripts/scaffolding.py index 1b0c35e..63c92c4 100644 --- a/viz_scripts/scaffolding.py +++ b/viz_scripts/scaffolding.py @@ -344,16 +344,19 @@ def CO2_footprint_lb(df, distance, col): return df def CO2_impact_lb(df,distance): + conversion_lb_to_kilogram = 0.453592 # 1 lb = 0.453592 KG if 'Mode_confirm_lb_CO2' not in df.columns: print("Mode confirm footprint not found, computing before impact") df = CO2_footprint_lb(df, distance, "Mode_confirm") df = CO2_footprint_lb(df, distance, "Replaced_mode") - df['CO2_Impact(lb)'] = round((df['Replaced_mode_lb_CO2'] - df['Mode_confirm_lb_CO2']),3) + + # Convert the CO2_Impact to be represented in kilogram + df['CO2_Impact(kg)'] = round((df['Replaced_mode_lb_CO2'] - df['Mode_confirm_lb_CO2']) * conversion_lb_to_kilogram, 3) return df def compute_CO2_impact_kg(expanded_ct, dynamic_labels): - convert_miles_to_kilometer = 1.60934 # 1 Miles = 1.60934 KM + conversion_miles_to_kilometer = 1.60934 # 1 Miles = 1.60934 KM dic_mode_kgCO2PerKm = {mode["value"]: mode["kgCo2PerKm"] for mode in dynamic_labels["MODE"]} @@ -363,5 +366,5 @@ def compute_CO2_impact_kg(expanded_ct, dynamic_labels): if "replaced_mode" in expanded_ct.columns: expanded_ct['Replaced_mode_kg_CO2'] = expanded_ct['replaced_mode'].map(dic_mode_kgCO2PerKm) - expanded_ct['CO2_Impact(kg)'] = round ( (expanded_ct['distance_miles'] * convert_miles_to_kilometer ) * ( expanded_ct['Replaced_mode_kg_CO2'] - expanded_ct['Mode_confirm_kg_CO2']),3) + expanded_ct['CO2_Impact(kg)'] = round ( (expanded_ct['distance_miles'] * conversion_miles_to_kilometer ) * ( expanded_ct['Replaced_mode_kg_CO2'] - expanded_ct['Mode_confirm_kg_CO2']), 3) return expanded_ct From 42990a2d1af9f6defd1dd47b4183512163ada7d3 Mon Sep 17 00:00:00 2001 From: iantei Date: Wed, 4 Oct 2023 23:18:07 -0700 Subject: [PATCH 21/62] Move the compute_CO2_impact_kg() call to scaffolding, such that energy_impact is computed regardless. --- viz_scripts/energy_calculations.ipynb | 6 +----- viz_scripts/scaffolding.py | 8 ++++++-- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/viz_scripts/energy_calculations.ipynb b/viz_scripts/energy_calculations.ipynb index 2298c4b..dd4ad6f 100644 --- a/viz_scripts/energy_calculations.ipynb +++ b/viz_scripts/energy_calculations.ipynb @@ -113,11 +113,7 @@ " dic_pur=dic_pur,\n", " include_test_users=include_test_users)\n", "\n", - "# CASE 1 of https://github.com/e-mission/em-public-dashboard/issues/69#issuecomment-1256835867 \n", - "if len(dynamic_labels) > 0:\n", - " expanded_ct = scaffolding.compute_CO2_impact_kg(expanded_ct, dynamic_labels) if len(expanded_ct) > 0 else expanded_ct\n", - "else:\n", - " expanded_ct = scaffolding.add_energy_impact(expanded_ct, df_ei, dic_fuel) if len(expanded_ct) > 0 else expanded_ct" + "expanded_ct = scaffolding.add_energy_impact(expanded_ct, df_ei, dic_fuel, dynamic_labels) if len(expanded_ct) > 0 else expanded_ct" ] }, { diff --git a/viz_scripts/scaffolding.py b/viz_scripts/scaffolding.py index 63c92c4..48fa9a4 100644 --- a/viz_scripts/scaffolding.py +++ b/viz_scripts/scaffolding.py @@ -222,7 +222,7 @@ def add_energy_labels(expanded_ct, df_ei, dic_fuel): expanded_ct = CO2_footprint_lb(expanded_ct, 'distance_miles', 'Mode_confirm') return expanded_ct -def add_energy_impact(expanded_ct, df_ei, dic_fuel): +def add_energy_impact(expanded_ct, df_ei, dic_fuel, dynamic_labels): # Let's first calculate everything for the mode confirm # And then calculate everything for the replaced mode expanded_ct = add_energy_labels(expanded_ct, df_ei, dic_fuel) @@ -230,7 +230,11 @@ def add_energy_impact(expanded_ct, df_ei, dic_fuel): expanded_ct = energy_intensity(expanded_ct, df_ei, 'Replaced_mode') # and then compute the impacts expanded_ct = energy_impact_kWH(expanded_ct, 'distance_miles') - expanded_ct = CO2_impact_lb(expanded_ct, 'distance_miles') + + if (len(dynamic_labels) > 0): + expanded_ct = compute_CO2_impact_kg(expanded_ct, dynamic_labels) + else: + expanded_ct = CO2_impact_lb(expanded_ct, 'distance_miles') return expanded_ct def get_quality_text(before_df, after_df, mode_of_interest=None, include_test_users=False): From 2dd360f7e810a9858f8e4e170178d81da5c4cfb4 Mon Sep 17 00:00:00 2001 From: iantei Date: Thu, 5 Oct 2023 22:57:21 -0700 Subject: [PATCH 22/62] Changes to understand difference between default mapping and dynamic label computation for CO2 Emissions. --- viz_scripts/energy_calculations.ipynb | 43 +++++++++++++++++++++++++-- viz_scripts/scaffolding.py | 20 ++++++++----- 2 files changed, 54 insertions(+), 9 deletions(-) diff --git a/viz_scripts/energy_calculations.ipynb b/viz_scripts/energy_calculations.ipynb index dd4ad6f..b6b4c1f 100644 --- a/viz_scripts/energy_calculations.ipynb +++ b/viz_scripts/energy_calculations.ipynb @@ -25,8 +25,8 @@ "metadata": {}, "outputs": [], "source": [ - "year = 2020\n", - "month = 11\n", + "year = None\n", + "month = None\n", "program = \"default\"\n", "study_type = \"program\"\n", "mode_of_interest = \"e-bike\"\n", @@ -217,6 +217,42 @@ "plot_title_no_quality=f\"Sketch of Total Kilograms of CO2 Emissions of {mode_of_interest} trips\"\n", "file_name =f'sketch_CO2impact_{mode_of_interest}%s' % file_suffix\n", "\n", + "filtered_taxi_data = data_eb[ data_eb['Replaced_mode'] == \"Taxi/Uber/Lyft\" ]\n", + "filtered_bus_data = data_eb[ data_eb['Replaced_mode'] == \"Bus\" ]\n", + "filtered_notravel_data = data_eb[ data_eb['Replaced_mode'] == \"No Travel\" ]\n", + "filtered_freeshuttle_data = data_eb[ data_eb['Replaced_mode'] == \"Free Shuttle\" ]\n", + "filtered_walk_data = data_eb[ data_eb['Replaced_mode'] == \"Walk\" ]\n", + "filtered_GasCarShared_data = data_eb[ data_eb['Replaced_mode'] == \"Gas Car, with others\" ]\n", + "\n", + "selected_columns = ['distance', 'Replaced_mode_kg_CO2', 'Mode_confirm_kg_CO2', \"replaced_mode\", \"mode_confirm\"]\n", + "\n", + "print(\"With Default mapping:\")\n", + "print(\"\")\n", + "\n", + "print(\"Walk Data:\") \n", + "print(str(filtered_walk_data[selected_columns].head()))\n", + "print(\" \")\n", + "\n", + "print(\"No Travel Data:\") \n", + "print(str(filtered_notravel_data[selected_columns].head()))\n", + "print(\" \")\n", + "\n", + "print(\"Gas Car Shared Data:\") \n", + "print(str(filtered_GasCarShared_data[selected_columns].head()))\n", + "print(\" \")\n", + "\n", + "print(\"Free Shuttle Data:\") \n", + "print(str(filtered_freeshuttle_data[selected_columns].head()))\n", + "print(\" \")\n", + "\n", + "print(\"Bus Data:\") \n", + "print(str(filtered_bus_data[selected_columns].head()))\n", + "print(\" \")\n", + "\n", + "print(\"Taxi/Uber/Lyft Data:\") \n", + "print(str(filtered_taxi_data[selected_columns].head()))\n", + "print(\" \")\n", + "\n", "try:\n", " ebco2=data_eb.groupby('Replaced_mode').agg({'CO2_Impact(kg)': ['sum', 'mean']},)\n", " ebco2.columns = ['total_kg_CO2_emissions', 'average_kg_CO2_emission']\n", @@ -232,6 +268,9 @@ " plot_title= plot_title_no_quality+f\"\\n Contribution by replaced mode towards a total of {net_CO2_emissions}(kg CO2 Emissions )\\n\"+quality_text\n", " CO2_impact(x,y,color,plot_title,file_name)\n", " alt_text = store_alt_text_bar(pd.DataFrame(x.values,y), file_name, plot_title)\n", + " \n", + " print(\"Total Kg CO2 Emissions\")\n", + " print(ebco2['total_kg_CO2_emissions'])\n", "except:\n", " generate_missing_plot(plot_title_no_quality,debug_df,file_name)\n", " alt_text = store_alt_text_missing(debug_df, file_name, plot_title_no_quality)" diff --git a/viz_scripts/scaffolding.py b/viz_scripts/scaffolding.py index 48fa9a4..641aa23 100644 --- a/viz_scripts/scaffolding.py +++ b/viz_scripts/scaffolding.py @@ -335,16 +335,20 @@ def CO2_footprint_lb(df, distance, col): distance = distance in miles col = Replaced_mode or Mode_confirm """ + + conversion_lb_to_kilogram = 0.453592 # 1 lb = 0.453592 KG + conditions_col = [(df[col+'_fuel'] =='gasoline'), (df[col+'_fuel'] == 'diesel'), (df[col+'_fuel'] == 'electric')] - - gasoline_col = (df[distance]*df['ei_'+col]*0.000001)* df['CO2_'+col] - diesel_col = (df[distance]*df['ei_'+col]*0.000001)* df['CO2_'+col] - electric_col = (((df[distance]*df['ei_'+col])+df['ei_trip_'+col])*0.001)*df['CO2_'+col] + + gasoline_col = (df['ei_'+col]*0.000001)* df['CO2_'+col] + diesel_col = (df['ei_'+col]*0.000001)* df['CO2_'+col] + electric_col = (((df['ei_'+col])+df['ei_trip_'+col])*0.001)*df['CO2_'+col] values_col = [gasoline_col,diesel_col,electric_col] df[col+'_lb_CO2'] = np.select(conditions_col, values_col) + df[col+'_kg_CO2'] = df[col+'_lb_CO2'] * conversion_lb_to_kilogram return df def CO2_impact_lb(df,distance): @@ -355,20 +359,22 @@ def CO2_impact_lb(df,distance): df = CO2_footprint_lb(df, distance, "Replaced_mode") # Convert the CO2_Impact to be represented in kilogram - df['CO2_Impact(kg)'] = round((df['Replaced_mode_lb_CO2'] - df['Mode_confirm_lb_CO2']) * conversion_lb_to_kilogram, 3) + df['CO2_Impact(kg)'] = round(df[distance] * (df['Replaced_mode_lb_CO2'] - df['Mode_confirm_lb_CO2']) * conversion_lb_to_kilogram, 5) return df def compute_CO2_impact_kg(expanded_ct, dynamic_labels): - conversion_miles_to_kilometer = 1.60934 # 1 Miles = 1.60934 KM + conversion_meter_to_kilometer = 0.001 # 1 meter = 0.001 kilometer dic_mode_kgCO2PerKm = {mode["value"]: mode["kgCo2PerKm"] for mode in dynamic_labels["MODE"]} if "mode_confirm" in expanded_ct.columns: expanded_ct['Mode_confirm_kg_CO2'] = expanded_ct['mode_confirm'].map(dic_mode_kgCO2PerKm) + expanded_ct['Mode_confirm_kg_CO2'] = expanded_ct['Mode_confirm_kg_CO2'].fillna(0) if "replaced_mode" in expanded_ct.columns: expanded_ct['Replaced_mode_kg_CO2'] = expanded_ct['replaced_mode'].map(dic_mode_kgCO2PerKm) + expanded_ct['Replaced_mode_kg_CO2'] = expanded_ct['Replaced_mode_kg_CO2'].fillna(0) - expanded_ct['CO2_Impact(kg)'] = round ( (expanded_ct['distance_miles'] * conversion_miles_to_kilometer ) * ( expanded_ct['Replaced_mode_kg_CO2'] - expanded_ct['Mode_confirm_kg_CO2']), 3) + expanded_ct['CO2_Impact(kg)'] = round ( (expanded_ct['distance'] * conversion_meter_to_kilometer ) * ( expanded_ct['Replaced_mode_kg_CO2'] - expanded_ct['Mode_confirm_kg_CO2']), 5) return expanded_ct From 0aeb70c7ba01dff692da57fabb620e899ca6f97c Mon Sep 17 00:00:00 2001 From: iantei Date: Sat, 7 Oct 2023 15:49:29 -0700 Subject: [PATCH 23/62] Update scaffolding to create different dictionary mappings for MODE, REPLACED, and PURPOSE on the basis of the dynamic labels. --- viz_scripts/scaffolding.py | 39 ++++++++++++++++++++++++-------------- 1 file changed, 25 insertions(+), 14 deletions(-) diff --git a/viz_scripts/scaffolding.py b/viz_scripts/scaffolding.py index 641aa23..5a40b6e 100644 --- a/viz_scripts/scaffolding.py +++ b/viz_scripts/scaffolding.py @@ -126,27 +126,35 @@ def load_viz_notebook_data(year, month, program, study_type, dynamic_labels, dic if "distance" in expanded_ct.columns: unit_conversions(expanded_ct) - # Extract translations key - dic_translations = dict() - if "translations" in dynamic_labels and "en" in dynamic_labels["translations"]: - dic_translations = dynamic_labels["translations"]["en"] - dic_translations = defaultdict(lambda: 'Other', dic_translations) + translations = dynamic_labels["translations"]["en"] + + def translate_labels(labels): + translation_mapping = {} + for label in labels: + value = label["value"] + translation = translations.get(value) + translation_mapping[value] = translation + return defaultdict(lambda: 'Other', translation_mapping) + + dic_mode_mapping = translate_labels(dynamic_labels["MODE"]) + dic_replaced_mapping = translate_labels(dynamic_labels["REPLACED_MODE"]) + dic_purpose_mapping = translate_labels(dynamic_labels["PURPOSE"]) - # Select the mapping based on availability of dynamic_labels - if dic_translations: - dic_mapping = dic_translations - else: - dic_mapping = dic_re - # Map new mode labels with translations dictionary from dynamic_labels # CASE 2 of https://github.com/e-mission/em-public-dashboard/issues/69#issuecomment-1256835867 if "mode_confirm" in expanded_ct.columns: - expanded_ct['Mode_confirm'] = expanded_ct['mode_confirm'].map(dic_mapping) + if (len(dynamic_labels)): + expanded_ct['Mode_confirm'] = expanded_ct['mode_confirm'].map(dic_mode_mapping) + else: + expanded_ct['Mode_confirm'] = expanded_ct['mode_confirm'].map(dic_re) if study_type == 'program': # CASE 2 of https://github.com/e-mission/em-public-dashboard/issues/69#issuecomment-1256835867 if 'replaced_mode' in expanded_ct.columns: - expanded_ct['Replaced_mode'] = expanded_ct['replaced_mode'].map(dic_mapping) + if (len(dynamic_labels)): + expanded_ct['Replaced_mode'] = expanded_ct['replaced_mode'].map(dic_replaced_mapping) + else: + expanded_ct['Replaced_mode'] = expanded_ct['replaced_mode'].map(dic_re) else: print("This is a program, but no replaced modes found. Likely cold start case. Ignoring replaced mode mapping") else: @@ -155,7 +163,10 @@ def load_viz_notebook_data(year, month, program, study_type, dynamic_labels, dic # Trip purpose mapping # CASE 2 of https://github.com/e-mission/em-public-dashboard/issues/69#issuecomment-1256835867 if dic_pur is not None and "purpose_confirm" in expanded_ct.columns: - expanded_ct['Trip_purpose'] = expanded_ct['purpose_confirm'].map(dic_pur) + if (len(dynamic_labels)): + expanded_ct['Trip_purpose'] = expanded_ct['purpose_confirm'].map(dic_purpose_mapping) + else: + expanded_ct['Trip_purpose'] = expanded_ct['purpose_confirm'].map(dic_pur) # Document data quality file_suffix = get_file_suffix(year, month, program) From aae0d2a11f9c5d948e42bcc82c2c7b0c532badac Mon Sep 17 00:00:00 2001 From: iantei Date: Sat, 7 Oct 2023 16:01:02 -0700 Subject: [PATCH 24/62] Update Trip_purpose to To Work instead of Work whenever dynamic label is available. --- viz_scripts/generic_metrics.ipynb | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/viz_scripts/generic_metrics.ipynb b/viz_scripts/generic_metrics.ipynb index 7415978..5273b34 100644 --- a/viz_scripts/generic_metrics.ipynb +++ b/viz_scripts/generic_metrics.ipynb @@ -147,15 +147,19 @@ "file_name= 'ntrips_commute_mode_confirm%s' % file_suffix\n", "\n", "try:\n", - " labels_mc = expanded_ct.query(\"Trip_purpose == 'Work'\").Mode_confirm.value_counts(dropna=True).keys().tolist()\n", - " values_mc = expanded_ct.query(\"Trip_purpose == 'Work'\").Mode_confirm.value_counts(dropna=True).tolist()\n", - " commute_quality_text = scaffolding.get_quality_text(expanded_ct, expanded_ct.query(\"Trip_purpose == 'Work'\"), \"commute\", include_test_users)\n", + " if (len(dynamic_labels)):\n", + " trip_purpose_query = \"Trip_purpose == 'To Work'\"\n", + " else:\n", + " trip_purpose_query = \"Trip_purpose == 'Work'\"\n", + "\n", + " labels_mc = expanded_ct.query(trip_purpose_query).Mode_confirm.value_counts(dropna=True).keys().tolist()\n", + " values_mc = expanded_ct.query(trip_purpose_query).Mode_confirm.value_counts(dropna=True).tolist()\n", + " commute_quality_text = scaffolding.get_quality_text(expanded_ct, expanded_ct.query(trip_purpose_query), \"commute\", include_test_users)\n", " plot_title= plot_title_no_quality+\"\\n\"+commute_quality_text\n", " pie_chart_mode(plot_title,labels_mc,values_mc,file_name)\n", " alt_text = store_alt_text_pie(pd.DataFrame(values_mc, labels_mc), file_name, plot_title)\n", - " print(expanded_ct.query(\"Trip_purpose == 'Work'\").Mode_confirm.value_counts(dropna=True))\n", "except:\n", - " debug_df.loc[\"Commute_trips\"] = len(expanded_ct.query(\"Trip_purpose == 'Work'\")) if \"Trip_purpose\" in expanded_ct.columns else 0\n", + " debug_df.loc[\"Commute_trips\"] = len(expanded_ct.query(trip_purpose_query)) if \"Trip_purpose\" in expanded_ct.columns else 0\n", " generate_missing_plot(plot_title_no_quality,debug_df,file_name)\n", " alt_text = store_alt_text_missing(debug_df, file_name, plot_title_no_quality)" ] From f2f0a453056d1656efca4a7bc660335da9feb09d Mon Sep 17 00:00:00 2001 From: iantei Date: Mon, 9 Oct 2023 13:29:14 -0700 Subject: [PATCH 25/62] Update scaffolding.py CO2_footprint_lb() to utilize the distance while computing for different fuel types so that Mode_confirm_lb_CO2 and Replaced_mode_lb_CO2 is properly computed. The overall CO2_Impact(lb) computation remains the same. The representation in Energy Emission notebook is same. --- viz_scripts/scaffolding.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/viz_scripts/scaffolding.py b/viz_scripts/scaffolding.py index 5a40b6e..5a6d672 100644 --- a/viz_scripts/scaffolding.py +++ b/viz_scripts/scaffolding.py @@ -353,9 +353,9 @@ def CO2_footprint_lb(df, distance, col): (df[col+'_fuel'] == 'diesel'), (df[col+'_fuel'] == 'electric')] - gasoline_col = (df['ei_'+col]*0.000001)* df['CO2_'+col] - diesel_col = (df['ei_'+col]*0.000001)* df['CO2_'+col] - electric_col = (((df['ei_'+col])+df['ei_trip_'+col])*0.001)*df['CO2_'+col] + gasoline_col = (df[distance]*df['ei_'+col]*0.000001)* df['CO2_'+col] + diesel_col = (df[distance]*df['ei_'+col]*0.000001)* df['CO2_'+col] + electric_col = (((df[distance]*df['ei_'+col])+df['ei_trip_'+col])*0.001)*df['CO2_'+col] values_col = [gasoline_col,diesel_col,electric_col] df[col+'_lb_CO2'] = np.select(conditions_col, values_col) @@ -368,9 +368,10 @@ def CO2_impact_lb(df,distance): print("Mode confirm footprint not found, computing before impact") df = CO2_footprint_lb(df, distance, "Mode_confirm") df = CO2_footprint_lb(df, distance, "Replaced_mode") + df['CO2_Impact(lb)'] = round((df['Replaced_mode_lb_CO2'] - df['Mode_confirm_lb_CO2']), 5) # Convert the CO2_Impact to be represented in kilogram - df['CO2_Impact(kg)'] = round(df[distance] * (df['Replaced_mode_lb_CO2'] - df['Mode_confirm_lb_CO2']) * conversion_lb_to_kilogram, 5) + df['CO2_Impact(kg)'] = round((df['Replaced_mode_kg_CO2'] - df['Mode_confirm_kg_CO2']), 5) return df def compute_CO2_impact_kg(expanded_ct, dynamic_labels): From fecec3949723d6b7977e775512bfbaed0028304b Mon Sep 17 00:00:00 2001 From: iantei Date: Mon, 9 Oct 2023 19:14:29 -0700 Subject: [PATCH 26/62] Update changes for generic_timeseries. 1. Inside scaffolding.py introduce new func compute_CO2_footprint_kg() to compute Mode_confirm_kg_CO2 metric only. Update distance computation to be reflected for expanded_ct['Mode_confirm_kg_CO2']. Updated generic_timeseries to showcase metrics in kg and lb for all cases- default mapping and dynamic labels. Added dynamic_labels in call to add_energy_labels. --- viz_scripts/generic_timeseries.ipynb | 79 +++++++++++++++++++--------- viz_scripts/scaffolding.py | 31 ++++++++--- 2 files changed, 78 insertions(+), 32 deletions(-) diff --git a/viz_scripts/generic_timeseries.ipynb b/viz_scripts/generic_timeseries.ipynb index 47acf9e..26eaf35 100644 --- a/viz_scripts/generic_timeseries.ipynb +++ b/viz_scripts/generic_timeseries.ipynb @@ -89,7 +89,8 @@ " dynamic_labels,\n", " dic_re,\n", " include_test_users=include_test_users)\n", - "expanded_ct = scaffolding.add_energy_labels(expanded_ct, df_ei, dic_fuel) if \"mode_confirm\" in expanded_ct.columns else expanded_ct" + "\n", + "expanded_ct = scaffolding.add_energy_labels(expanded_ct, df_ei, dic_fuel, dynamic_labels) if \"mode_confirm\" in expanded_ct.columns else expanded_ct" ] }, { @@ -106,8 +107,9 @@ "outputs": [], "source": [ "# Get timestamp from known year/month/day aggregated to days\n", - "sel_cols_no_label_dep = ['user_id','start_local_dt_year','start_local_dt_month','start_local_dt_day','distance_miles']\n", - "sel_cols_with_label_dep = sel_cols_no_label_dep + ['Mode_confirm','Mode_confirm_EI(kWH)','Mode_confirm_lb_CO2']\n", + "sel_cols_no_label_dep = ['user_id','start_local_dt_year','start_local_dt_month','start_local_dt_day','distance']\n", + "sel_cols_with_label_dep = sel_cols_no_label_dep + ['Mode_confirm','Mode_confirm_EI(kWH)','Mode_confirm_kg_CO2']\n", + "\n", "if len(expanded_ct) == 0:\n", " data = expanded_ct.copy()\n", "elif \"Mode_confirm\" not in expanded_ct.columns:\n", @@ -125,8 +127,22 @@ " data.date_time = pd.Categorical(data.date_time)\n", " \n", " if \"Mode_confirm\" in expanded_ct.columns:\n", - " data.Mode_confirm = pd.Categorical(data.Mode_confirm, ordered=True, categories=np.unique(list(dic_re.values())))\n", + " if (len(dynamic_labels) > 0):\n", + " if \"translations\" in dynamic_labels and \"en\" in dynamic_labels[\"translations\"]:\n", + " translations = dynamic_labels[\"translations\"][\"en\"]\n", + "\n", + " def translate_labels(labels):\n", + " translation_mapping = {}\n", + " for label in labels:\n", + " value = label[\"value\"]\n", + " translation = translations.get(value)\n", + " translation_mapping[value] = translation\n", + " return defaultdict(lambda: 'Other', translation_mapping)\n", "\n", + " dic_mode_mapping = translate_labels(dynamic_labels[\"MODE\"])\n", + " data.Mode_confirm = pd.Categorical(data.Mode_confirm, ordered=True, categories=np.unique(list(dic_mode_mapping.values())))\n", + " else:\n", + " data.Mode_confirm = pd.Categorical(data.Mode_confirm, ordered=True, categories=np.unique(list(dic_re.values())))\n", "data.head()" ] }, @@ -149,25 +165,25 @@ " mode_counts.rename(columns={'size':'trip_count'}, inplace=True)\n", "\n", " # Sum daily distance traveled for each mode\n", - " mode_distance = data.groupby(['user_id','date_time','Mode_confirm'], as_index=False)[['distance_miles']].sum()\n", - " mode_distance.rename(columns={'sum':'distance_miles'}, inplace=True)\n", - " mode_distance['distance_miles'] = mode_distance['distance_miles'].fillna(0)\n", + " mode_distance = data.groupby(['user_id','date_time','Mode_confirm'], as_index=False)[['distance']].sum()\n", + " mode_distance.rename(columns={'sum':'distance'}, inplace=True)\n", + " mode_distance['distance'] = mode_distance['distance'].fillna(0)\n", "\n", " # Sum daily emissions for each user\n", - " emissions = data.groupby(['user_id','date_time'], as_index=False)[['Mode_confirm_lb_CO2', 'distance_miles']].sum()\n", - " emissions['Mode_confirm_lb_CO2'] = emissions['Mode_confirm_lb_CO2'].fillna(0)\n", - " emissions['distance_miles'] = emissions['Mode_confirm_lb_CO2'].fillna(0)\n", + " emissions = data.groupby(['user_id','date_time'], as_index=False)[['Mode_confirm_kg_CO2', 'distance']].sum()\n", + " emissions['Mode_confirm_kg_CO2'] = emissions['Mode_confirm_kg_CO2'].fillna(0)\n", + " emissions['distance'] = emissions['Mode_confirm_kg_CO2'].fillna(0)\n", "\n", " # Sum daily energy for each user\n", - " energy = data.groupby(['user_id','date_time'], as_index=False)[['Mode_confirm_EI(kWH)', 'distance_miles']].sum()\n", + " energy = data.groupby(['user_id','date_time'], as_index=False)[['Mode_confirm_EI(kWH)', 'distance']].sum()\n", " energy['Mode_confirm_EI(kWH)'] = energy['Mode_confirm_EI(kWH)'].fillna(0)\n", - " energy['distance_miles'] = energy['Mode_confirm_EI(kWH)'].fillna(0)\n", + " energy['distance'] = energy['Mode_confirm_EI(kWH)'].fillna(0)\n", "\n", " # Add 7-day rolling avg smoothing to better see trends\n", " mode_counts['trip_count_smooth'] = mode_counts.groupby(['user_id','Mode_confirm'])['trip_count'].apply(lambda x: x.rolling(7,1).mean())\n", - " mode_distance['distance_miles_smooth'] = mode_distance.groupby(['user_id','Mode_confirm'])['distance_miles'].apply(lambda x: x.rolling(7,1).mean())\n", - " emissions['distance_miles_smooth'] = emissions.groupby(['user_id'])['distance_miles'].apply(lambda x: x.rolling(7,1).mean())\n", - " energy['distance_miles_smooth'] = energy.groupby(['user_id'])['distance_miles'].apply(lambda x: x.rolling(7,1).mean())" + " mode_distance['distance_smooth'] = mode_distance.groupby(['user_id','Mode_confirm'])['distance'].apply(lambda x: x.rolling(7,1).mean())\n", + " emissions['distance_smooth'] = emissions.groupby(['user_id'])['distance'].apply(lambda x: x.rolling(7,1).mean())\n", + " energy['distance_smooth'] = energy.groupby(['user_id'])['distance'].apply(lambda x: x.rolling(7,1).mean())" ] }, { @@ -195,12 +211,14 @@ "\n", "try:\n", " # Emissions per week across all users (net impact)\n", - " plot_data = emissions.groupby(['date_time'], as_index=False)['Mode_confirm_lb_CO2'].agg(['sum'])\n", + " plot_data = emissions.groupby(['date_time'], as_index=False)['Mode_confirm_kg_CO2'].agg(['sum'])\n", + " total_sum = plot_data['sum'].sum()\n", + "\n", " plot_data = plot_data.merge(active_users, on='date_time')\n", " plot_data['sum'] = plot_data['sum'] / plot_data['active_users']\n", "\n", " plot_title= plot_title_no_quality+\"\\n\"+quality_text\n", - " ylab = 'Emissions (lb CO2/day/user)'\n", + " ylab = 'Emissions (kg CO2/day/user)'\n", " timeseries_plot(plot_data['date_time'], plot_data['sum'], plot_title, ylab, file_name)\n", " alt_text = store_alt_text_timeseries(plot_data, file_name, plot_title)\n", "except:\n", @@ -258,12 +276,12 @@ "try:\n", " # Emissions per mile per day across all users (travel efficiency)\n", " # Note that the energy plot will be identical to this one since scale factor is divided out\n", - " emissions['CO2_per_mile'] = emissions['Mode_confirm_lb_CO2'] / emissions['distance_miles_smooth']\n", - " emissions['CO2_per_mile'] = emissions['CO2_per_mile'].fillna(0)\n", - " plot_data = emissions.groupby(['date_time'])['CO2_per_mile'].agg(['mean']).reset_index()\n", + " emissions['CO2_per_km'] = emissions['Mode_confirm_kg_CO2'] / emissions['distance_smooth']\n", + " emissions['CO2_per_km'] = emissions['CO2_per_km'].fillna(0)\n", + " plot_data = emissions.groupby(['date_time'])['CO2_per_km'].agg(['mean']).reset_index()\n", "\n", " plot_title= plot_title_no_quality+\"\\n\"+quality_text\n", - " ylab = 'Emissions (lb CO2/mile/day)'\n", + " ylab = 'Emissions (kg CO2/km/day)'\n", " timeseries_plot(plot_data['date_time'], plot_data['mean'], plot_title, ylab, file_name)\n", " alt_text = store_alt_text_timeseries(plot_data, file_name, plot_title)\n", "except:\n", @@ -335,10 +353,21 @@ " plot_data = plot_data.merge(total_trips, on='date_time')\n", " plot_data['trip_proportion'] = plot_data['trip_count_smooth_x'] / plot_data['trip_count_smooth_y']\n", "\n", - " # Re-establish categorical variable to not include Other and Non-trips\n", - " plot_data = plot_data[~plot_data['Mode_confirm'].isin(['Not a Trip','Other'])]\n", - " plot_data.Mode_confirm = pd.Categorical(plot_data.Mode_confirm, ordered=True, categories=np.unique(list(dic_re.values())))\n", - "\n", + " if (len(dynamic_labels)):\n", + " mode_to_en = {}\n", + " for mode_values in dynamic_labels[\"MODE\"]:\n", + " mode_value = mode_values[\"value\"]\n", + " en_translation = dynamic_labels[\"translations\"][\"en\"].get(mode_value, \"Translation not available\")\n", + " mode_to_en[mode_value] = en_translation\n", + " \n", + " # Re-establish categorical variable to not include Other and Non-trips\n", + " plot_data = plot_data[~plot_data['Mode_confirm'].isin(['Not a Trip','Other'])]\n", + " plot_data.Mode_confirm = pd.Categorical(plot_data.Mode_confirm, ordered=True, categories=np.unique(list(mode_to_en.values())))\n", + " else:\n", + " # Re-establish categorical variable to not include Other and Non-trips\n", + " plot_data = plot_data[~plot_data['Mode_confirm'].isin(['Not a Trip','Other'])]\n", + " plot_data.Mode_confirm = pd.Categorical(plot_data.Mode_confirm, ordered=True, categories=np.unique(list(dic_re.values())))\n", + " \n", " plot_title= plot_title_no_quality+\"\\n\"+quality_text\n", " ylab = 'Proportion of All Trips'\n", " legend_title = 'Confirmed Mode'\n", diff --git a/viz_scripts/scaffolding.py b/viz_scripts/scaffolding.py index 5a6d672..4b24e38 100644 --- a/viz_scripts/scaffolding.py +++ b/viz_scripts/scaffolding.py @@ -222,7 +222,7 @@ def load_viz_notebook_sensor_inference_data(year, month, program, include_test_u return expanded_ct, file_suffix, quality_text, debug_df -def add_energy_labels(expanded_ct, df_ei, dic_fuel): +def add_energy_labels(expanded_ct, df_ei, dic_fuel, dynamic_labels): """ Inputs: expanded_ct = dataframe of trips that has had Mode_confirm and Replaced_mode added dic/df_* = label mappings for energy impact and fuel @@ -230,13 +230,17 @@ def add_energy_labels(expanded_ct, df_ei, dic_fuel): expanded_ct['Mode_confirm_fuel']= expanded_ct['Mode_confirm'].map(dic_fuel) expanded_ct = energy_intensity(expanded_ct, df_ei, 'Mode_confirm') expanded_ct = energy_footprint_kWH(expanded_ct, 'distance_miles', 'Mode_confirm') - expanded_ct = CO2_footprint_lb(expanded_ct, 'distance_miles', 'Mode_confirm') + + if (len(dynamic_labels) > 0): + expanded_ct = compute_CO2_footprint_kg(expanded_ct, dynamic_labels) + else: + expanded_ct = CO2_footprint_lb(expanded_ct, 'distance_miles', 'Mode_confirm') return expanded_ct def add_energy_impact(expanded_ct, df_ei, dic_fuel, dynamic_labels): # Let's first calculate everything for the mode confirm # And then calculate everything for the replaced mode - expanded_ct = add_energy_labels(expanded_ct, df_ei, dic_fuel) + expanded_ct = add_energy_labels(expanded_ct, df_ei, dic_fuel, dynamic_labels) expanded_ct['Replaced_mode_fuel']= expanded_ct['Replaced_mode'].map(dic_fuel) expanded_ct = energy_intensity(expanded_ct, df_ei, 'Replaced_mode') # and then compute the impacts @@ -374,19 +378,32 @@ def CO2_impact_lb(df,distance): df['CO2_Impact(kg)'] = round((df['Replaced_mode_kg_CO2'] - df['Mode_confirm_kg_CO2']), 5) return df -def compute_CO2_impact_kg(expanded_ct, dynamic_labels): +def compute_CO2_footprint_kg(expanded_ct, dynamic_labels): + conversion_meter_to_kilometer = 0.001 + dic_mode_kgCO2PerKm = {mode["value"]: mode["kgCo2PerKm"] for mode in dynamic_labels["MODE"]} + + if "mode_confirm" in expanded_ct.columns: + expanded_ct['Mode_confirm_kg_CO2'] = ((expanded_ct['distance'] * conversion_meter_to_kilometer )) * (expanded_ct['mode_confirm'].map(dic_mode_kgCO2PerKm)) + expanded_ct['Mode_confirm_kg_CO2'] = expanded_ct['Mode_confirm_kg_CO2'].fillna(0) + + return expanded_ct + +def compute_CO2_impact_kg(expanded_ct, dynamic_labels): conversion_meter_to_kilometer = 0.001 # 1 meter = 0.001 kilometer + conversion_kg_to_lb = 2.20462 # 1 kg = 2.20462 dic_mode_kgCO2PerKm = {mode["value"]: mode["kgCo2PerKm"] for mode in dynamic_labels["MODE"]} if "mode_confirm" in expanded_ct.columns: - expanded_ct['Mode_confirm_kg_CO2'] = expanded_ct['mode_confirm'].map(dic_mode_kgCO2PerKm) + expanded_ct['Mode_confirm_kg_CO2'] = ((expanded_ct['distance'] * conversion_meter_to_kilometer )) * (expanded_ct['mode_confirm'].map(dic_mode_kgCO2PerKm)) expanded_ct['Mode_confirm_kg_CO2'] = expanded_ct['Mode_confirm_kg_CO2'].fillna(0) if "replaced_mode" in expanded_ct.columns: - expanded_ct['Replaced_mode_kg_CO2'] = expanded_ct['replaced_mode'].map(dic_mode_kgCO2PerKm) + expanded_ct['Replaced_mode_kg_CO2'] = (expanded_ct['distance'] * conversion_meter_to_kilometer ) * (expanded_ct['replaced_mode'].map(dic_mode_kgCO2PerKm)) expanded_ct['Replaced_mode_kg_CO2'] = expanded_ct['Replaced_mode_kg_CO2'].fillna(0) - expanded_ct['CO2_Impact(kg)'] = round ( (expanded_ct['distance'] * conversion_meter_to_kilometer ) * ( expanded_ct['Replaced_mode_kg_CO2'] - expanded_ct['Mode_confirm_kg_CO2']), 5) + expanded_ct['CO2_Impact(kg)'] = round ((expanded_ct['Replaced_mode_kg_CO2'] - expanded_ct['Mode_confirm_kg_CO2']), 5) + + expanded_ct['CO2_Impact(lb)'] = expanded_ct['CO2_Impact(kg)'] * conversion_kg_to_lb return expanded_ct From 5cf84bcc9717af8c2666984a4ad49132cfe8c554 Mon Sep 17 00:00:00 2001 From: iantei Date: Thu, 12 Oct 2023 12:08:25 -0700 Subject: [PATCH 27/62] Create new metrics_study and metrics_program options html files without energy metrics options. --- .../metrics_program_withoutEnergyMetrics.html | 36 +++++++++++++++++++ .../metrics_study_withoutEnergyMetrics.html | 22 ++++++++++++ 2 files changed, 58 insertions(+) create mode 100644 frontend/metrics_program_withoutEnergyMetrics.html create mode 100644 frontend/metrics_study_withoutEnergyMetrics.html diff --git a/frontend/metrics_program_withoutEnergyMetrics.html b/frontend/metrics_program_withoutEnergyMetrics.html new file mode 100644 index 0000000..29e3501 --- /dev/null +++ b/frontend/metrics_program_withoutEnergyMetrics.html @@ -0,0 +1,36 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/frontend/metrics_study_withoutEnergyMetrics.html b/frontend/metrics_study_withoutEnergyMetrics.html new file mode 100644 index 0000000..190a946 --- /dev/null +++ b/frontend/metrics_study_withoutEnergyMetrics.html @@ -0,0 +1,22 @@ + + + + + + + + + + + + + + + + + + + + + + From 5a8c9d6d56f9443667f63c7095c55303362d7c59 Mon Sep 17 00:00:00 2001 From: iantei Date: Thu, 12 Oct 2023 12:09:30 -0700 Subject: [PATCH 28/62] Update index.html to use different metrics_ options html when dynamic label is available. --- frontend/index.html | 70 ++++++++++++++++++++++++++++++++++++--------- 1 file changed, 57 insertions(+), 13 deletions(-) diff --git a/frontend/index.html b/frontend/index.html index 80671f7..f7cea02 100644 --- a/frontend/index.html +++ b/frontend/index.html @@ -374,8 +374,14 @@ const STUDY_CONFIG = _getStudyName(window.location); $.getJSON('https://raw.githubusercontent.com/e-mission/nrel-openpath-deploy-configs/main/configs/' + STUDY_CONFIG + '.nrel-op.json', function(data) { // Load list of plots corresponding to study/program - if (data.intro.program_or_study=='program') { - $.get("metrics_program.html", function(response) { + if (data.intro.program_or_study=='program') + { + // Note: We're disabling energy metrics on public dashboard when dynamic labels are available. + // TODO: Remove the if (data.label_options) in future when energy computation is handled properly. + if (data.label_options) + { + console.log("Dynamic Label is available for: " + STUDY_CONFIG) + $.get("metrics_program_withoutEnergyMetrics.html", function(response) { const configuredResponse = response.replaceAll("${data.intro.mode_studied}", data.intro.mode_studied); $('#metric').append(configuredResponse); addPreconfiguredMetrics([ @@ -387,18 +393,56 @@ `ntrips_${data.intro.mode_studied}_per_weekday`, `sketch_CO2impact_${data.intro.mode_studied}` ]); - }); - } else { - $('#metric').load('metrics_study.html'); - addPreconfiguredMetrics([ - "ntrips_mode_confirm", - "miles_mode_confirm", - "ntrips_sensed_mode", - "miles_sensed_mode", - "ntrips_purpose", - "ntrips_sensed_per_weekday", - "ts_emissions_user" + }); + } + else + { + console.log("Dynamic Label is unavailable for: " + STUDY_CONFIG) + $.get("metrics_program.html", function(response) { + const configuredResponse = response.replaceAll("${data.intro.mode_studied}", data.intro.mode_studied); + $('#metric').append(configuredResponse); + addPreconfiguredMetrics([ + "ntrips_mode_confirm", + "miles_mode_confirm", + "ntrips_sensed_mode", + "miles_sensed_mode", + `ntrips_${data.intro.mode_studied}_purpose`, + `ntrips_${data.intro.mode_studied}_per_weekday`, + `sketch_CO2impact_${data.intro.mode_studied}` + ]); + }); + } + } + else // CASE: STUDY + { + if (data.label_options) + { + console.log("Dynamic Label is available for: " + STUDY_CONFIG) + $('#metric').load('metrics_study_withoutEnergyMetrics.html'); + addPreconfiguredMetrics([ + "ntrips_mode_confirm", + "miles_mode_confirm", + "ntrips_sensed_mode", + "miles_sensed_mode", + "ntrips_purpose", + "ntrips_sensed_per_weekday", + "ts_emissions_user" ]); + } + else + { + console.log("Dynamic Label is unavailable for: " + STUDY_CONFIG) + $('#metric').load('metrics_study.html'); + addPreconfiguredMetrics([ + "ntrips_mode_confirm", + "miles_mode_confirm", + "ntrips_sensed_mode", + "miles_sensed_mode", + "ntrips_purpose", + "ntrips_sensed_per_weekday", + "ts_emissions_user" + ]); + } }; // Figure out all months since start date to present var date_options = ""; From 8fefdea4b30eebc66e12e0703deb9c4ce744951a Mon Sep 17 00:00:00 2001 From: iantei Date: Thu, 12 Oct 2023 14:05:46 -0700 Subject: [PATCH 29/62] Remove redundant check for dynamic_labels_urls. Update the message to show the STUDY_CONFIG associated with the success/failure message. --- viz_scripts/bin/generate_plots.py | 15 ++++++--------- 1 file changed, 6 insertions(+), 9 deletions(-) diff --git a/viz_scripts/bin/generate_plots.py b/viz_scripts/bin/generate_plots.py index cf6d33e..6198d7f 100644 --- a/viz_scripts/bin/generate_plots.py +++ b/viz_scripts/bin/generate_plots.py @@ -50,17 +50,14 @@ if 'label_options' in dynamic_config: dynamic_labels_url = dynamic_config['label_options'] - if dynamic_labels_url: - req = requests.get(dynamic_labels_url) - if req.status_code != 200: - print(f"Unable to download dynamic_labels, status code: {req.status_code}") - else: - dynamic_labels = json.loads(req.text) - print("Dynamic labels download was successful.") + req = requests.get(dynamic_labels_url) + if req.status_code != 200: + print(f"Unable to download dynamic_labels_url, status code: {req.status_code} for {STUDY_CONFIG}") else: - print(f"dynamic_labels_url is unavailable for {dynamic_config['label_options']}") + dynamic_labels = json.loads(req.text) + print(f"Dynamic labels download was successful for nrel-openpath-deploy-configs: {STUDY_CONFIG}" ) else: - print(f"label_options is unavailable for the dynamic_config.") + print(f"label_options is unavailable for the dynamic_config in {STUDY_CONFIG}") if args.date is None: start_date = arrow.get(int(dynamic_config['intro']['start_year']), From a2c191028bce74aad7a717b7d3f6571cf3652a53 Mon Sep 17 00:00:00 2001 From: iantei Date: Thu, 12 Oct 2023 14:11:03 -0700 Subject: [PATCH 30/62] Remove print statements used to evaluate the difference in energy calculations while using dynamic labels and default mapping (from auxillary_files/energy_intensity). --- viz_scripts/energy_calculations.ipynb | 42 ++------------------------- 1 file changed, 2 insertions(+), 40 deletions(-) diff --git a/viz_scripts/energy_calculations.ipynb b/viz_scripts/energy_calculations.ipynb index b6b4c1f..abca667 100644 --- a/viz_scripts/energy_calculations.ipynb +++ b/viz_scripts/energy_calculations.ipynb @@ -25,8 +25,8 @@ "metadata": {}, "outputs": [], "source": [ - "year = None\n", - "month = None\n", + "year = 2020\n", + "month = 11\n", "program = \"default\"\n", "study_type = \"program\"\n", "mode_of_interest = \"e-bike\"\n", @@ -217,42 +217,6 @@ "plot_title_no_quality=f\"Sketch of Total Kilograms of CO2 Emissions of {mode_of_interest} trips\"\n", "file_name =f'sketch_CO2impact_{mode_of_interest}%s' % file_suffix\n", "\n", - "filtered_taxi_data = data_eb[ data_eb['Replaced_mode'] == \"Taxi/Uber/Lyft\" ]\n", - "filtered_bus_data = data_eb[ data_eb['Replaced_mode'] == \"Bus\" ]\n", - "filtered_notravel_data = data_eb[ data_eb['Replaced_mode'] == \"No Travel\" ]\n", - "filtered_freeshuttle_data = data_eb[ data_eb['Replaced_mode'] == \"Free Shuttle\" ]\n", - "filtered_walk_data = data_eb[ data_eb['Replaced_mode'] == \"Walk\" ]\n", - "filtered_GasCarShared_data = data_eb[ data_eb['Replaced_mode'] == \"Gas Car, with others\" ]\n", - "\n", - "selected_columns = ['distance', 'Replaced_mode_kg_CO2', 'Mode_confirm_kg_CO2', \"replaced_mode\", \"mode_confirm\"]\n", - "\n", - "print(\"With Default mapping:\")\n", - "print(\"\")\n", - "\n", - "print(\"Walk Data:\") \n", - "print(str(filtered_walk_data[selected_columns].head()))\n", - "print(\" \")\n", - "\n", - "print(\"No Travel Data:\") \n", - "print(str(filtered_notravel_data[selected_columns].head()))\n", - "print(\" \")\n", - "\n", - "print(\"Gas Car Shared Data:\") \n", - "print(str(filtered_GasCarShared_data[selected_columns].head()))\n", - "print(\" \")\n", - "\n", - "print(\"Free Shuttle Data:\") \n", - "print(str(filtered_freeshuttle_data[selected_columns].head()))\n", - "print(\" \")\n", - "\n", - "print(\"Bus Data:\") \n", - "print(str(filtered_bus_data[selected_columns].head()))\n", - "print(\" \")\n", - "\n", - "print(\"Taxi/Uber/Lyft Data:\") \n", - "print(str(filtered_taxi_data[selected_columns].head()))\n", - "print(\" \")\n", - "\n", "try:\n", " ebco2=data_eb.groupby('Replaced_mode').agg({'CO2_Impact(kg)': ['sum', 'mean']},)\n", " ebco2.columns = ['total_kg_CO2_emissions', 'average_kg_CO2_emission']\n", @@ -269,8 +233,6 @@ " CO2_impact(x,y,color,plot_title,file_name)\n", " alt_text = store_alt_text_bar(pd.DataFrame(x.values,y), file_name, plot_title)\n", " \n", - " print(\"Total Kg CO2 Emissions\")\n", - " print(ebco2['total_kg_CO2_emissions'])\n", "except:\n", " generate_missing_plot(plot_title_no_quality,debug_df,file_name)\n", " alt_text = store_alt_text_missing(debug_df, file_name, plot_title_no_quality)" From a42b30838db89a6dbb81cd0f3aa8ae565b307dc3 Mon Sep 17 00:00:00 2001 From: iantei Date: Thu, 12 Oct 2023 22:51:49 -0700 Subject: [PATCH 31/62] Update the mode mapping in accordance to MODE for Daily Mode Share in generic_timeseries.ipynb. --- viz_scripts/generic_timeseries.ipynb | 34 ++++++++++++++++------------ 1 file changed, 19 insertions(+), 15 deletions(-) diff --git a/viz_scripts/generic_timeseries.ipynb b/viz_scripts/generic_timeseries.ipynb index 26eaf35..d9e4a8e 100644 --- a/viz_scripts/generic_timeseries.ipynb +++ b/viz_scripts/generic_timeseries.ipynb @@ -28,7 +28,7 @@ "study_type = \"study\"\n", "mode_of_interest = None\n", "include_test_users = False\n", - "dynamic_labels = {}" + "dynamic_labels = { }" ] }, { @@ -159,7 +159,7 @@ " active_users = pd.DataFrame(data.groupby(['date_time'], as_index=False)['user_id'].nunique())\n", " active_users.rename(columns={'user_id':'active_users'}, inplace=True)\n", "\n", - " if \"Mode_confirm\" in expanded_ct.columns:\n", + " if \"Mode_confirm\" in expanded_ct.columns: \n", " # Count the number of trips for each confirmed mode\n", " mode_counts = data.groupby(['user_id','date_time','Mode_confirm'], as_index=False).size()\n", " mode_counts.rename(columns={'size':'trip_count'}, inplace=True)\n", @@ -352,22 +352,26 @@ " total_trips = plot_data.groupby(['date_time'], as_index=False).sum()\n", " plot_data = plot_data.merge(total_trips, on='date_time')\n", " plot_data['trip_proportion'] = plot_data['trip_count_smooth_x'] / plot_data['trip_count_smooth_y']\n", + " \n", + " # Re-establish categorical variable to not include Other and Non-trips\n", + " plot_data = plot_data[~plot_data['Mode_confirm'].isin(['Not a Trip','Other'])]\n", + "\n", + " if (len(dynamic_labels) > 0):\n", + " if \"translations\" in dynamic_labels and \"en\" in dynamic_labels[\"translations\"]:\n", + " translations = dynamic_labels[\"translations\"][\"en\"]\n", "\n", - " if (len(dynamic_labels)):\n", - " mode_to_en = {}\n", - " for mode_values in dynamic_labels[\"MODE\"]:\n", - " mode_value = mode_values[\"value\"]\n", - " en_translation = dynamic_labels[\"translations\"][\"en\"].get(mode_value, \"Translation not available\")\n", - " mode_to_en[mode_value] = en_translation\n", - " \n", - " # Re-establish categorical variable to not include Other and Non-trips\n", - " plot_data = plot_data[~plot_data['Mode_confirm'].isin(['Not a Trip','Other'])]\n", - " plot_data.Mode_confirm = pd.Categorical(plot_data.Mode_confirm, ordered=True, categories=np.unique(list(mode_to_en.values())))\n", + " def translate_labels(labels):\n", + " translation_mapping = {}\n", + " for label in labels:\n", + " value = label[\"value\"]\n", + " translation = translations.get(value)\n", + " translation_mapping[value] = translation\n", + " return defaultdict(lambda: 'Other', translation_mapping)\n", + "\n", + " dic_mode_mapping = translate_labels(dynamic_labels[\"MODE\"])\n", + " plot_data.Mode_confirm = pd.Categorical(plot_data.Mode_confirm, ordered=True, categories=np.unique(list(dic_mode_mapping.values()))) \n", " else:\n", - " # Re-establish categorical variable to not include Other and Non-trips\n", - " plot_data = plot_data[~plot_data['Mode_confirm'].isin(['Not a Trip','Other'])]\n", " plot_data.Mode_confirm = pd.Categorical(plot_data.Mode_confirm, ordered=True, categories=np.unique(list(dic_re.values())))\n", - " \n", " plot_title= plot_title_no_quality+\"\\n\"+quality_text\n", " ylab = 'Proportion of All Trips'\n", " legend_title = 'Confirmed Mode'\n", From 6f13c039e416c59f87a82c166144fba420e2b020 Mon Sep 17 00:00:00 2001 From: iantei Date: Thu, 19 Oct 2023 09:53:24 -0700 Subject: [PATCH 32/62] Refactor the code to load the config and corresponding metrics.html page., --- frontend/index.html | 50 ++++++++++++++------------------------------- 1 file changed, 15 insertions(+), 35 deletions(-) diff --git a/frontend/index.html b/frontend/index.html index f7cea02..b5a7100 100644 --- a/frontend/index.html +++ b/frontend/index.html @@ -380,25 +380,15 @@ // TODO: Remove the if (data.label_options) in future when energy computation is handled properly. if (data.label_options) { + load_file = "metrics_program_withoutEnergyMetrics.html" console.log("Dynamic Label is available for: " + STUDY_CONFIG) - $.get("metrics_program_withoutEnergyMetrics.html", function(response) { - const configuredResponse = response.replaceAll("${data.intro.mode_studied}", data.intro.mode_studied); - $('#metric').append(configuredResponse); - addPreconfiguredMetrics([ - "ntrips_mode_confirm", - "miles_mode_confirm", - "ntrips_sensed_mode", - "miles_sensed_mode", - `ntrips_${data.intro.mode_studied}_purpose`, - `ntrips_${data.intro.mode_studied}_per_weekday`, - `sketch_CO2impact_${data.intro.mode_studied}` - ]); - }); } else { + load_file = "metrics_program_withoutEnergyMetrics.html" console.log("Dynamic Label is unavailable for: " + STUDY_CONFIG) - $.get("metrics_program.html", function(response) { + } + $.get(load_file, function(response) { const configuredResponse = response.replaceAll("${data.intro.mode_studied}", data.intro.mode_studied); $('#metric').append(configuredResponse); addPreconfiguredMetrics([ @@ -411,7 +401,6 @@ `sketch_CO2impact_${data.intro.mode_studied}` ]); }); - } } else // CASE: STUDY { @@ -419,31 +408,22 @@ { console.log("Dynamic Label is available for: " + STUDY_CONFIG) $('#metric').load('metrics_study_withoutEnergyMetrics.html'); - addPreconfiguredMetrics([ - "ntrips_mode_confirm", - "miles_mode_confirm", - "ntrips_sensed_mode", - "miles_sensed_mode", - "ntrips_purpose", - "ntrips_sensed_per_weekday", - "ts_emissions_user" - ]); } else { console.log("Dynamic Label is unavailable for: " + STUDY_CONFIG) $('#metric').load('metrics_study.html'); - addPreconfiguredMetrics([ - "ntrips_mode_confirm", - "miles_mode_confirm", - "ntrips_sensed_mode", - "miles_sensed_mode", - "ntrips_purpose", - "ntrips_sensed_per_weekday", - "ts_emissions_user" - ]); - } - }; + }; + addPreconfiguredMetrics([ + "ntrips_mode_confirm", + "miles_mode_confirm", + "ntrips_sensed_mode", + "miles_sensed_mode", + "ntrips_purpose", + "ntrips_sensed_per_weekday", + "ts_emissions_user" + ]); + } // Figure out all months since start date to present var date_options = ""; const dates = getMonthList( From 32864c66cede316bdab1b211a5192d3192e91070 Mon Sep 17 00:00:00 2001 From: iantei Date: Thu, 19 Oct 2023 10:18:42 -0700 Subject: [PATCH 33/62] Reinstate the comment for energy_calculations.ipynb. Removed extranneous space for generic_timeseries.ipynb. Removed extranneous space for scaffolding.py. Reverted the bumped round off value to 3. This had been bumped up to 5, to make a better comparative study between the CO2 emission computation from energy_intensity.csv and label_options. Since, the comparative changes have been reverted. This has been reverted to pre-existing round-off of 3 places. --- viz_scripts/energy_calculations.ipynb | 1 + viz_scripts/generic_timeseries.ipynb | 4 ++-- viz_scripts/scaffolding.py | 7 +++---- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/viz_scripts/energy_calculations.ipynb b/viz_scripts/energy_calculations.ipynb index abca667..c9a9461 100644 --- a/viz_scripts/energy_calculations.ipynb +++ b/viz_scripts/energy_calculations.ipynb @@ -112,6 +112,7 @@ " dic_re,\n", " dic_pur=dic_pur,\n", " include_test_users=include_test_users)\n", + "# CASE 1 of https://github.com/e-mission/em-public-dashboard/issues/69#issuecomment-1256835867 \n", "\n", "expanded_ct = scaffolding.add_energy_impact(expanded_ct, df_ei, dic_fuel, dynamic_labels) if len(expanded_ct) > 0 else expanded_ct" ] diff --git a/viz_scripts/generic_timeseries.ipynb b/viz_scripts/generic_timeseries.ipynb index d9e4a8e..b32a4d0 100644 --- a/viz_scripts/generic_timeseries.ipynb +++ b/viz_scripts/generic_timeseries.ipynb @@ -28,7 +28,7 @@ "study_type = \"study\"\n", "mode_of_interest = None\n", "include_test_users = False\n", - "dynamic_labels = { }" + "dynamic_labels = {}" ] }, { @@ -159,7 +159,7 @@ " active_users = pd.DataFrame(data.groupby(['date_time'], as_index=False)['user_id'].nunique())\n", " active_users.rename(columns={'user_id':'active_users'}, inplace=True)\n", "\n", - " if \"Mode_confirm\" in expanded_ct.columns: \n", + " if \"Mode_confirm\" in expanded_ct.columns:\n", " # Count the number of trips for each confirmed mode\n", " mode_counts = data.groupby(['user_id','date_time','Mode_confirm'], as_index=False).size()\n", " mode_counts.rename(columns={'size':'trip_count'}, inplace=True)\n", diff --git a/viz_scripts/scaffolding.py b/viz_scripts/scaffolding.py index 4b24e38..9a2a8b2 100644 --- a/viz_scripts/scaffolding.py +++ b/viz_scripts/scaffolding.py @@ -356,7 +356,6 @@ def CO2_footprint_lb(df, distance, col): conditions_col = [(df[col+'_fuel'] =='gasoline'), (df[col+'_fuel'] == 'diesel'), (df[col+'_fuel'] == 'electric')] - gasoline_col = (df[distance]*df['ei_'+col]*0.000001)* df['CO2_'+col] diesel_col = (df[distance]*df['ei_'+col]*0.000001)* df['CO2_'+col] electric_col = (((df[distance]*df['ei_'+col])+df['ei_trip_'+col])*0.001)*df['CO2_'+col] @@ -372,10 +371,10 @@ def CO2_impact_lb(df,distance): print("Mode confirm footprint not found, computing before impact") df = CO2_footprint_lb(df, distance, "Mode_confirm") df = CO2_footprint_lb(df, distance, "Replaced_mode") - df['CO2_Impact(lb)'] = round((df['Replaced_mode_lb_CO2'] - df['Mode_confirm_lb_CO2']), 5) + df['CO2_Impact(lb)'] = round((df['Replaced_mode_lb_CO2'] - df['Mode_confirm_lb_CO2']), 3) # Convert the CO2_Impact to be represented in kilogram - df['CO2_Impact(kg)'] = round((df['Replaced_mode_kg_CO2'] - df['Mode_confirm_kg_CO2']), 5) + df['CO2_Impact(kg)'] = round((df['Replaced_mode_kg_CO2'] - df['Mode_confirm_kg_CO2']), 3) return df def compute_CO2_footprint_kg(expanded_ct, dynamic_labels): @@ -403,7 +402,7 @@ def compute_CO2_impact_kg(expanded_ct, dynamic_labels): expanded_ct['Replaced_mode_kg_CO2'] = (expanded_ct['distance'] * conversion_meter_to_kilometer ) * (expanded_ct['replaced_mode'].map(dic_mode_kgCO2PerKm)) expanded_ct['Replaced_mode_kg_CO2'] = expanded_ct['Replaced_mode_kg_CO2'].fillna(0) - expanded_ct['CO2_Impact(kg)'] = round ((expanded_ct['Replaced_mode_kg_CO2'] - expanded_ct['Mode_confirm_kg_CO2']), 5) + expanded_ct['CO2_Impact(kg)'] = round ((expanded_ct['Replaced_mode_kg_CO2'] - expanded_ct['Mode_confirm_kg_CO2']), 3) expanded_ct['CO2_Impact(lb)'] = expanded_ct['CO2_Impact(kg)'] * conversion_kg_to_lb return expanded_ct From b3a92a1cf0c5d9ea96085043745216c74a6a2494 Mon Sep 17 00:00:00 2001 From: iantei Date: Thu, 19 Oct 2023 10:30:28 -0700 Subject: [PATCH 34/62] Update data.label_options to dynamic_labels to make it coherent across different notebook and generate_plots.py. --- frontend/index.html | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/frontend/index.html b/frontend/index.html index b5a7100..b2aa3c0 100644 --- a/frontend/index.html +++ b/frontend/index.html @@ -374,19 +374,20 @@ const STUDY_CONFIG = _getStudyName(window.location); $.getJSON('https://raw.githubusercontent.com/e-mission/nrel-openpath-deploy-configs/main/configs/' + STUDY_CONFIG + '.nrel-op.json', function(data) { // Load list of plots corresponding to study/program + dynamic_labels = data.label_options if (data.intro.program_or_study=='program') { // Note: We're disabling energy metrics on public dashboard when dynamic labels are available. // TODO: Remove the if (data.label_options) in future when energy computation is handled properly. - if (data.label_options) + if (dynamic_labels) { load_file = "metrics_program_withoutEnergyMetrics.html" - console.log("Dynamic Label is available for: " + STUDY_CONFIG) + console.log("Dynamic Labels are available for: " + STUDY_CONFIG) } else { load_file = "metrics_program_withoutEnergyMetrics.html" - console.log("Dynamic Label is unavailable for: " + STUDY_CONFIG) + console.log("Dynamic Labels are unavailable for: " + STUDY_CONFIG) } $.get(load_file, function(response) { const configuredResponse = response.replaceAll("${data.intro.mode_studied}", data.intro.mode_studied); @@ -404,14 +405,14 @@ } else // CASE: STUDY { - if (data.label_options) + if (dynamic_labels) { - console.log("Dynamic Label is available for: " + STUDY_CONFIG) + console.log("Dynamic Labels are available for: " + STUDY_CONFIG) $('#metric').load('metrics_study_withoutEnergyMetrics.html'); } else { - console.log("Dynamic Label is unavailable for: " + STUDY_CONFIG) + console.log("Dynamic Labels are unavailable for: " + STUDY_CONFIG) $('#metric').load('metrics_study.html'); }; addPreconfiguredMetrics([ From 1628f5af69e61394c73091ef29a4641caefe71b1 Mon Sep 17 00:00:00 2001 From: iantei Date: Thu, 19 Oct 2023 11:31:56 -0700 Subject: [PATCH 35/62] Created a new function inside scaffolding.py, mapping_labels(dynamic_labels, label_type). This replaces the same/similar logic written across different notebooks. Notably called from generic_metrics notebook - load_viz_notebook_data(), and other calls from generic_timeseries. Updated generic_timeseries.ipynb to call the scaffolding.mapping_labels(dynamic_labels, label_type) to remove redundancy of same code. --- viz_scripts/generic_timeseries.ipynb | 28 +++--------------- viz_scripts/scaffolding.py | 43 +++++++++++++++++++--------- 2 files changed, 33 insertions(+), 38 deletions(-) diff --git a/viz_scripts/generic_timeseries.ipynb b/viz_scripts/generic_timeseries.ipynb index b32a4d0..9c14a8c 100644 --- a/viz_scripts/generic_timeseries.ipynb +++ b/viz_scripts/generic_timeseries.ipynb @@ -128,18 +128,8 @@ " \n", " if \"Mode_confirm\" in expanded_ct.columns:\n", " if (len(dynamic_labels) > 0):\n", - " if \"translations\" in dynamic_labels and \"en\" in dynamic_labels[\"translations\"]:\n", - " translations = dynamic_labels[\"translations\"][\"en\"]\n", - "\n", - " def translate_labels(labels):\n", - " translation_mapping = {}\n", - " for label in labels:\n", - " value = label[\"value\"]\n", - " translation = translations.get(value)\n", - " translation_mapping[value] = translation\n", - " return defaultdict(lambda: 'Other', translation_mapping)\n", - "\n", - " dic_mode_mapping = translate_labels(dynamic_labels[\"MODE\"])\n", + " label_type = \"MODE\" \n", + " dic_mode_mapping = scaffolding.mapping_labels(dynamic_labels, label_type)\n", " data.Mode_confirm = pd.Categorical(data.Mode_confirm, ordered=True, categories=np.unique(list(dic_mode_mapping.values())))\n", " else:\n", " data.Mode_confirm = pd.Categorical(data.Mode_confirm, ordered=True, categories=np.unique(list(dic_re.values())))\n", @@ -357,18 +347,8 @@ " plot_data = plot_data[~plot_data['Mode_confirm'].isin(['Not a Trip','Other'])]\n", "\n", " if (len(dynamic_labels) > 0):\n", - " if \"translations\" in dynamic_labels and \"en\" in dynamic_labels[\"translations\"]:\n", - " translations = dynamic_labels[\"translations\"][\"en\"]\n", - "\n", - " def translate_labels(labels):\n", - " translation_mapping = {}\n", - " for label in labels:\n", - " value = label[\"value\"]\n", - " translation = translations.get(value)\n", - " translation_mapping[value] = translation\n", - " return defaultdict(lambda: 'Other', translation_mapping)\n", - "\n", - " dic_mode_mapping = translate_labels(dynamic_labels[\"MODE\"])\n", + " label_type = \"MODE\"\n", + " dic_mode_mapping = scaffolding.mapping_labels(dynamic_labels, label_type)\n", " plot_data.Mode_confirm = pd.Categorical(plot_data.Mode_confirm, ordered=True, categories=np.unique(list(dic_mode_mapping.values()))) \n", " else:\n", " plot_data.Mode_confirm = pd.Categorical(plot_data.Mode_confirm, ordered=True, categories=np.unique(list(dic_re.values())))\n", diff --git a/viz_scripts/scaffolding.py b/viz_scripts/scaffolding.py index 9a2a8b2..e408da2 100644 --- a/viz_scripts/scaffolding.py +++ b/viz_scripts/scaffolding.py @@ -126,20 +126,10 @@ def load_viz_notebook_data(year, month, program, study_type, dynamic_labels, dic if "distance" in expanded_ct.columns: unit_conversions(expanded_ct) - if "translations" in dynamic_labels and "en" in dynamic_labels["translations"]: - translations = dynamic_labels["translations"]["en"] - - def translate_labels(labels): - translation_mapping = {} - for label in labels: - value = label["value"] - translation = translations.get(value) - translation_mapping[value] = translation - return defaultdict(lambda: 'Other', translation_mapping) - - dic_mode_mapping = translate_labels(dynamic_labels["MODE"]) - dic_replaced_mapping = translate_labels(dynamic_labels["REPLACED_MODE"]) - dic_purpose_mapping = translate_labels(dynamic_labels["PURPOSE"]) + if dynamic_labels: + dic_mode_mapping = mapping_labels(dynamic_labels, "MODE") + dic_replaced_mapping = mapping_labels(dynamic_labels, "REPLACED_MODE") + dic_purpose_mapping = mapping_labels(dynamic_labels, "PURPOSE") # Map new mode labels with translations dictionary from dynamic_labels # CASE 2 of https://github.com/e-mission/em-public-dashboard/issues/69#issuecomment-1256835867 @@ -186,6 +176,31 @@ def translate_labels(labels): return expanded_ct, file_suffix, quality_text, debug_df +# Function to map the "MODE", "REPLACED_MODE", "PURPOSE" to respective en-translations +# Input: dynamic_labels, label_type: MODE, REPLACED_MODE, PURPOSE +# Return: Dictionary mapping between the label type and its english translation. +def mapping_labels(dynamic_labels, label_type): + if "translations" in dynamic_labels and "en" in dynamic_labels["translations"]: + translations = dynamic_labels["translations"]["en"] + dic_mapping = dict() + + def translate_labels(labels): + translation_mapping = {} + for label in labels: + value = label["value"] + translation = translations.get(value) + translation_mapping[value] = translation + return defaultdict(lambda: 'Other', translation_mapping) + + if (label_type == "MODE"): + dic_mapping = translate_labels(dynamic_labels[label_type]) + elif (label_type == "REPLACED_MODE"): + dic_mapping = translate_labels(dynamic_labels[label_type]) + elif (label_type == "PURPOSE"): + dic_mapping = translate_labels(dynamic_labels[label_type]) + + return dic_mapping + def load_viz_notebook_sensor_inference_data(year, month, program, include_test_users=False, sensed_algo_prefix="cleaned"): """ Inputs: year/month/program = parameters from the visualization notebook From f3edc61fde3562e2b76de0f98675e9c72de823d1 Mon Sep 17 00:00:00 2001 From: iantei Date: Thu, 19 Oct 2023 11:56:58 -0700 Subject: [PATCH 36/62] Moving the mapping_labels() call inside the conditional check for each label_type. This way, we would not be looking for REPLACED_MODE for study in label_options. --- viz_scripts/scaffolding.py | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/viz_scripts/scaffolding.py b/viz_scripts/scaffolding.py index e408da2..ba9064d 100644 --- a/viz_scripts/scaffolding.py +++ b/viz_scripts/scaffolding.py @@ -126,15 +126,11 @@ def load_viz_notebook_data(year, month, program, study_type, dynamic_labels, dic if "distance" in expanded_ct.columns: unit_conversions(expanded_ct) - if dynamic_labels: - dic_mode_mapping = mapping_labels(dynamic_labels, "MODE") - dic_replaced_mapping = mapping_labels(dynamic_labels, "REPLACED_MODE") - dic_purpose_mapping = mapping_labels(dynamic_labels, "PURPOSE") - # Map new mode labels with translations dictionary from dynamic_labels # CASE 2 of https://github.com/e-mission/em-public-dashboard/issues/69#issuecomment-1256835867 if "mode_confirm" in expanded_ct.columns: if (len(dynamic_labels)): + dic_mode_mapping = mapping_labels(dynamic_labels, "MODE") expanded_ct['Mode_confirm'] = expanded_ct['mode_confirm'].map(dic_mode_mapping) else: expanded_ct['Mode_confirm'] = expanded_ct['mode_confirm'].map(dic_re) @@ -142,6 +138,7 @@ def load_viz_notebook_data(year, month, program, study_type, dynamic_labels, dic # CASE 2 of https://github.com/e-mission/em-public-dashboard/issues/69#issuecomment-1256835867 if 'replaced_mode' in expanded_ct.columns: if (len(dynamic_labels)): + dic_replaced_mapping = mapping_labels(dynamic_labels, "REPLACED_MODE") expanded_ct['Replaced_mode'] = expanded_ct['replaced_mode'].map(dic_replaced_mapping) else: expanded_ct['Replaced_mode'] = expanded_ct['replaced_mode'].map(dic_re) @@ -154,6 +151,7 @@ def load_viz_notebook_data(year, month, program, study_type, dynamic_labels, dic # CASE 2 of https://github.com/e-mission/em-public-dashboard/issues/69#issuecomment-1256835867 if dic_pur is not None and "purpose_confirm" in expanded_ct.columns: if (len(dynamic_labels)): + dic_purpose_mapping = mapping_labels(dynamic_labels, "PURPOSE") expanded_ct['Trip_purpose'] = expanded_ct['purpose_confirm'].map(dic_purpose_mapping) else: expanded_ct['Trip_purpose'] = expanded_ct['purpose_confirm'].map(dic_pur) From e7c17a183d7c0ceed55faf30a138309d072d7f56 Mon Sep 17 00:00:00 2001 From: iantei Date: Thu, 19 Oct 2023 14:12:27 -0700 Subject: [PATCH 37/62] Update the purpose mapping for Work to select from scaffolding.mapping_labels() instead of hardcoding it with translation from work to To Work. --- viz_scripts/generic_metrics.ipynb | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/viz_scripts/generic_metrics.ipynb b/viz_scripts/generic_metrics.ipynb index 5273b34..abd5431 100644 --- a/viz_scripts/generic_metrics.ipynb +++ b/viz_scripts/generic_metrics.ipynb @@ -31,7 +31,7 @@ "study_type = \"study\"\n", "mode_of_interest = None\n", "include_test_users = False\n", - "dynamic_labels = { }" + "dynamic_labels = {}" ] }, { @@ -148,7 +148,9 @@ "\n", "try:\n", " if (len(dynamic_labels)):\n", - " trip_purpose_query = \"Trip_purpose == 'To Work'\"\n", + " purpose_map_label = scaffolding.mapping_labels(dynamic_labels, \"PURPOSE\")\n", + " translation_work = purpose_map_label['work']\n", + " trip_purpose_query = f\"Trip_purpose == '{translation_work}'\"\n", " else:\n", " trip_purpose_query = \"Trip_purpose == 'Work'\"\n", "\n", From c114d389e935ea239b4755bfa43d8871373264d8 Mon Sep 17 00:00:00 2001 From: iantei Date: Tue, 24 Oct 2023 12:55:43 -0700 Subject: [PATCH 38/62] Update the Timeseries of emissions per miles TO Timeseries of emissions per kilometer. We are computing in kg and kilometer for generic_timeseries and energy_calculations. --- frontend/Dockerfile.dev | 0 frontend/metrics_program.html | 2 +- frontend/metrics_program_withoutEnergyMetrics.html | 2 +- frontend/metrics_study.html | 2 +- frontend/metrics_study_withoutEnergyMetrics.html | 2 +- 5 files changed, 4 insertions(+), 4 deletions(-) create mode 100644 frontend/Dockerfile.dev diff --git a/frontend/Dockerfile.dev b/frontend/Dockerfile.dev new file mode 100644 index 0000000..e69de29 diff --git a/frontend/metrics_program.html b/frontend/metrics_program.html index 5e7d643..8460278 100644 --- a/frontend/metrics_program.html +++ b/frontend/metrics_program.html @@ -17,7 +17,7 @@ - + diff --git a/frontend/metrics_program_withoutEnergyMetrics.html b/frontend/metrics_program_withoutEnergyMetrics.html index 29e3501..a208819 100644 --- a/frontend/metrics_program_withoutEnergyMetrics.html +++ b/frontend/metrics_program_withoutEnergyMetrics.html @@ -17,7 +17,7 @@ - + diff --git a/frontend/metrics_study.html b/frontend/metrics_study.html index fc1c675..614d1a2 100644 --- a/frontend/metrics_study.html +++ b/frontend/metrics_study.html @@ -17,6 +17,6 @@ - + diff --git a/frontend/metrics_study_withoutEnergyMetrics.html b/frontend/metrics_study_withoutEnergyMetrics.html index 190a946..ba0b6ad 100644 --- a/frontend/metrics_study_withoutEnergyMetrics.html +++ b/frontend/metrics_study_withoutEnergyMetrics.html @@ -17,6 +17,6 @@ - + From b26207d3d29ae400d5073480331dcfd9af872316 Mon Sep 17 00:00:00 2001 From: iantei Date: Thu, 26 Oct 2023 20:34:30 -0700 Subject: [PATCH 39/62] Update plots.py CO2_impact to take x and y Labels from the notebook. Update energy_calculations notebook to display Imperic way for non dynamic config, and Metric way for dynamic config. Update scaffolding to compute the CO2 emissions as lbs/mile and kg/km. Both computatoin of default mapping and dynamic config mapping will have lbs/mile and kg/km CO2 emissions. --- viz_scripts/energy_calculations.ipynb | 70 +++++++++++++++++++-------- viz_scripts/plots.py | 6 +-- viz_scripts/scaffolding.py | 16 ++++-- 3 files changed, 64 insertions(+), 28 deletions(-) diff --git a/viz_scripts/energy_calculations.ipynb b/viz_scripts/energy_calculations.ipynb index c9a9461..4675d84 100644 --- a/viz_scripts/energy_calculations.ipynb +++ b/viz_scripts/energy_calculations.ipynb @@ -31,7 +31,7 @@ "study_type = \"program\"\n", "mode_of_interest = \"e-bike\"\n", "include_test_users = False\n", - "dynamic_labels = { }" + "dynamic_labels = {}" ] }, { @@ -215,28 +215,58 @@ "metadata": {}, "outputs": [], "source": [ - "plot_title_no_quality=f\"Sketch of Total Kilograms of CO2 Emissions of {mode_of_interest} trips\"\n", - "file_name =f'sketch_CO2impact_{mode_of_interest}%s' % file_suffix\n", + "# Case: label_options is present on the config file\n", + "if len(dynamic_labels) > 0:\n", + " plot_title_no_quality=f\"Sketch of Total Kilograms of CO2 Emissions of {mode_of_interest} trips\"\n", + " file_name =f'sketch_CO2impact_{mode_of_interest}%s' % file_suffix\n", "\n", - "try:\n", - " ebco2=data_eb.groupby('Replaced_mode').agg({'CO2_Impact(kg)': ['sum', 'mean']},)\n", - " ebco2.columns = ['total_kg_CO2_emissions', 'average_kg_CO2_emission']\n", - " ebco2 = ebco2.reset_index()\n", - " ebco2 = ebco2.sort_values(by=['total_kg_CO2_emissions'], ascending=False)\n", - " ebco2['boolean'] = ebco2['total_kg_CO2_emissions'] > 0\n", - " net_CO2_emissions = round(sum(ebco2['total_kg_CO2_emissions']), 2)\n", + " try:\n", + " ebco2=data_eb.groupby('Replaced_mode').agg({'CO2_Impact(kg)': ['sum', 'mean']},)\n", + " ebco2.columns = ['total_kg_CO2_emissions', 'average_kg_CO2_emission']\n", + " ebco2 = ebco2.reset_index()\n", + " ebco2 = ebco2.sort_values(by=['total_kg_CO2_emissions'], ascending=False)\n", + " ebco2['boolean'] = ebco2['total_kg_CO2_emissions'] > 0\n", + " net_CO2_emissions = round(sum(ebco2['total_kg_CO2_emissions']), 2)\n", "\n", - " x = ebco2['total_kg_CO2_emissions']\n", - " y = ebco2['Replaced_mode']\n", - " color = ebco2['boolean']\n", + " x = ebco2['total_kg_CO2_emissions']\n", + " y = ebco2['Replaced_mode']\n", + " xLabel = 'CO2 Emissions (kg)'\n", + " yLabel = 'Replaced Mode'\n", + " color = ebco2['boolean']\n", "\n", - " plot_title= plot_title_no_quality+f\"\\n Contribution by replaced mode towards a total of {net_CO2_emissions}(kg CO2 Emissions )\\n\"+quality_text\n", - " CO2_impact(x,y,color,plot_title,file_name)\n", - " alt_text = store_alt_text_bar(pd.DataFrame(x.values,y), file_name, plot_title)\n", - " \n", - "except:\n", - " generate_missing_plot(plot_title_no_quality,debug_df,file_name)\n", - " alt_text = store_alt_text_missing(debug_df, file_name, plot_title_no_quality)" + " plot_title= plot_title_no_quality+f\"\\n Contribution by replaced mode towards a total of {net_CO2_emissions}(kg CO2 Emissions )\\n\"+quality_text\n", + " CO2_impact(x,y,color,plot_title, xLabel,yLabel,file_name)\n", + " alt_text = store_alt_text_bar(pd.DataFrame(x.values,y), file_name, plot_title)\n", + " \n", + " except:\n", + " generate_missing_plot(plot_title_no_quality,debug_df,file_name)\n", + " alt_text = store_alt_text_missing(debug_df, file_name, plot_title_no_quality)\n", + "# case: when label_options is absent from the config file\n", + "else: \n", + " plot_title_no_quality=f\"Sketch of Total pounds of CO2 Emissions of {mode_of_interest} trips\"\n", + " file_name =f'sketch_CO2impact_{mode_of_interest}%s' % file_suffix\n", + "\n", + " try:\n", + " ebco2=data_eb.groupby('Replaced_mode').agg({'CO2_Impact(lb)': ['sum', 'mean']},)\n", + " ebco2.columns = ['total_lb_CO2_emissions', 'average_lb_CO2_emission']\n", + " ebco2 = ebco2.reset_index()\n", + " ebco2 = ebco2.sort_values(by=['total_lb_CO2_emissions'], ascending=False)\n", + " ebco2['boolean'] = ebco2['total_lb_CO2_emissions'] > 0\n", + " net_CO2_emissions = round(sum(ebco2['total_lb_CO2_emissions']), 2)\n", + "\n", + " x = ebco2['total_lb_CO2_emissions']\n", + " y = ebco2['Replaced_mode']\n", + " xLabel = 'CO2 Emissions (lb)'\n", + " yLabel = 'Replaced Mode'\n", + " color = ebco2['boolean']\n", + "\n", + " plot_title= plot_title_no_quality+f\"\\n Contribution by replaced mode towards a total of {net_CO2_emissions}(lb CO2 Emissions )\\n\"+quality_text\n", + " CO2_impact(x,y,color,plot_title,xLabel,yLabel,file_name)\n", + " alt_text = store_alt_text_bar(pd.DataFrame(x.values,y), file_name, plot_title)\n", + " \n", + " except:\n", + " generate_missing_plot(plot_title_no_quality,debug_df,file_name)\n", + " alt_text = store_alt_text_missing(debug_df, file_name, plot_title_no_quality)" ] } ], diff --git a/viz_scripts/plots.py b/viz_scripts/plots.py index 6f2bf29..c27e94f 100644 --- a/viz_scripts/plots.py +++ b/viz_scripts/plots.py @@ -247,7 +247,7 @@ def barplot_day(data,x,y,plot_title,file_name): plt.text(0,-(data[y].max())/8,f"Last updated {arrow.get()}", fontsize=10) plt.savefig(SAVE_DIR+file_name+".png", bbox_inches='tight') -def CO2_impact(x,y,color,plot_title,file_name): +def CO2_impact(x,y,color,plot_title, xLabel, yLabel, file_name): color = color.map({True: 'green', False: 'red'}) objects = ('CO2 Reduction', 'CO2 Increase') @@ -256,8 +256,8 @@ def CO2_impact(x,y,color,plot_title,file_name): width = 0.8 ax = x.plot(kind='barh',width=width, color=color) ax.set_title(plot_title, fontsize=18) - ax.set_xlabel('CO2 Emissions (kg)', fontsize=18) - ax.set_ylabel('Replaced Mode',fontsize=18) + ax.set_xlabel(xLabel, fontsize=18) + ax.set_ylabel(yLabel,fontsize=18) ax.set_yticklabels(y_labels) ax.xaxis.set_tick_params(labelsize=15) ax.yaxis.set_tick_params(labelsize=15) diff --git a/viz_scripts/scaffolding.py b/viz_scripts/scaffolding.py index ba9064d..d289742 100644 --- a/viz_scripts/scaffolding.py +++ b/viz_scripts/scaffolding.py @@ -364,7 +364,8 @@ def CO2_footprint_lb(df, distance, col): col = Replaced_mode or Mode_confirm """ - conversion_lb_to_kilogram = 0.453592 # 1 lb = 0.453592 KG + conversion_lb_to_kilogram = 0.453592 # 1 lb = 0.453592 kg + conversion_mile_to_kilometer = 1.60934 # 1 mile = 1.60934 km conditions_col = [(df[col+'_fuel'] =='gasoline'), (df[col+'_fuel'] == 'diesel'), @@ -375,7 +376,7 @@ def CO2_footprint_lb(df, distance, col): values_col = [gasoline_col,diesel_col,electric_col] df[col+'_lb_CO2'] = np.select(conditions_col, values_col) - df[col+'_kg_CO2'] = df[col+'_lb_CO2'] * conversion_lb_to_kilogram + df[col+'_kg_CO2'] = df[col+'_lb_CO2'] * conversion_lb_to_kilogram / conversion_mile_to_kilometer return df def CO2_impact_lb(df,distance): @@ -392,30 +393,35 @@ def CO2_impact_lb(df,distance): def compute_CO2_footprint_kg(expanded_ct, dynamic_labels): conversion_meter_to_kilometer = 0.001 + conversion_kilogram_to_lbs = 2.20462 + conversion_kilometer_to_mile = 0.621371 dic_mode_kgCO2PerKm = {mode["value"]: mode["kgCo2PerKm"] for mode in dynamic_labels["MODE"]} if "mode_confirm" in expanded_ct.columns: expanded_ct['Mode_confirm_kg_CO2'] = ((expanded_ct['distance'] * conversion_meter_to_kilometer )) * (expanded_ct['mode_confirm'].map(dic_mode_kgCO2PerKm)) expanded_ct['Mode_confirm_kg_CO2'] = expanded_ct['Mode_confirm_kg_CO2'].fillna(0) + expanded_ct['Mode_confirm_lb_CO2'] = expanded_ct['Mode_confirm_kg_CO2'] * conversion_kilogram_to_lbs / conversion_kilometer_to_mile return expanded_ct def compute_CO2_impact_kg(expanded_ct, dynamic_labels): conversion_meter_to_kilometer = 0.001 # 1 meter = 0.001 kilometer - conversion_kg_to_lb = 2.20462 # 1 kg = 2.20462 + conversion_kilogram_to_lbs = 2.20462 + conversion_kilometer_to_mile = 0.621371 dic_mode_kgCO2PerKm = {mode["value"]: mode["kgCo2PerKm"] for mode in dynamic_labels["MODE"]} if "mode_confirm" in expanded_ct.columns: expanded_ct['Mode_confirm_kg_CO2'] = ((expanded_ct['distance'] * conversion_meter_to_kilometer )) * (expanded_ct['mode_confirm'].map(dic_mode_kgCO2PerKm)) expanded_ct['Mode_confirm_kg_CO2'] = expanded_ct['Mode_confirm_kg_CO2'].fillna(0) + expanded_ct['Mode_confirm_lb_CO2'] = expanded_ct['Mode_confirm_kg_CO2'] * conversion_kilogram_to_lbs / conversion_kilometer_to_mile if "replaced_mode" in expanded_ct.columns: expanded_ct['Replaced_mode_kg_CO2'] = (expanded_ct['distance'] * conversion_meter_to_kilometer ) * (expanded_ct['replaced_mode'].map(dic_mode_kgCO2PerKm)) expanded_ct['Replaced_mode_kg_CO2'] = expanded_ct['Replaced_mode_kg_CO2'].fillna(0) + expanded_ct['Replaced_mode_lb_CO2'] = expanded_ct['Replaced_mode_kg_CO2'] * conversion_kilogram_to_lbs / conversion_kilometer_to_mile expanded_ct['CO2_Impact(kg)'] = round ((expanded_ct['Replaced_mode_kg_CO2'] - expanded_ct['Mode_confirm_kg_CO2']), 3) - - expanded_ct['CO2_Impact(lb)'] = expanded_ct['CO2_Impact(kg)'] * conversion_kg_to_lb + expanded_ct['CO2_Impact(lb)'] = round ((expanded_ct['Replaced_mode_lb_CO2'] - expanded_ct['Mode_confirm_lb_CO2']), 3) return expanded_ct From 14a3666f62b66f8937fdff527a192f744a59c32f Mon Sep 17 00:00:00 2001 From: iantei Date: Thu, 26 Oct 2023 21:05:20 -0700 Subject: [PATCH 40/62] Update generic_timeseries to show the CO2 emissions in Imperial system for default mapping, and in Metric system for dynamic config. --- viz_scripts/generic_timeseries.ipynb | 177 ++++++++++++++++++--------- 1 file changed, 121 insertions(+), 56 deletions(-) diff --git a/viz_scripts/generic_timeseries.ipynb b/viz_scripts/generic_timeseries.ipynb index 9c14a8c..c4e1be1 100644 --- a/viz_scripts/generic_timeseries.ipynb +++ b/viz_scripts/generic_timeseries.ipynb @@ -106,9 +106,14 @@ "metadata": {}, "outputs": [], "source": [ - "# Get timestamp from known year/month/day aggregated to days\n", - "sel_cols_no_label_dep = ['user_id','start_local_dt_year','start_local_dt_month','start_local_dt_day','distance']\n", - "sel_cols_with_label_dep = sel_cols_no_label_dep + ['Mode_confirm','Mode_confirm_EI(kWH)','Mode_confirm_kg_CO2']\n", + "if len(dynamic_labels) > 0:\n", + " # Get timestamp from known year/month/day aggregated to days\n", + " sel_cols_no_label_dep = ['user_id','start_local_dt_year','start_local_dt_month','start_local_dt_day','distance']\n", + " sel_cols_with_label_dep = sel_cols_no_label_dep + ['Mode_confirm','Mode_confirm_EI(kWH)','Mode_confirm_kg_CO2']\n", + "else:\n", + " # Get timestamp from known year/month/day aggregated to days\n", + " sel_cols_no_label_dep = ['user_id','start_local_dt_year','start_local_dt_month','start_local_dt_day','distance_miles']\n", + " sel_cols_with_label_dep = sel_cols_no_label_dep + ['Mode_confirm','Mode_confirm_EI(kWH)','Mode_confirm_lb_CO2']\n", "\n", "if len(expanded_ct) == 0:\n", " data = expanded_ct.copy()\n", @@ -128,8 +133,7 @@ " \n", " if \"Mode_confirm\" in expanded_ct.columns:\n", " if (len(dynamic_labels) > 0):\n", - " label_type = \"MODE\" \n", - " dic_mode_mapping = scaffolding.mapping_labels(dynamic_labels, label_type)\n", + " dic_mode_mapping = scaffolding.mapping_labels(dynamic_labels, \"MODE\")\n", " data.Mode_confirm = pd.Categorical(data.Mode_confirm, ordered=True, categories=np.unique(list(dic_mode_mapping.values())))\n", " else:\n", " data.Mode_confirm = pd.Categorical(data.Mode_confirm, ordered=True, categories=np.unique(list(dic_re.values())))\n", @@ -154,26 +158,48 @@ " mode_counts = data.groupby(['user_id','date_time','Mode_confirm'], as_index=False).size()\n", " mode_counts.rename(columns={'size':'trip_count'}, inplace=True)\n", "\n", - " # Sum daily distance traveled for each mode\n", - " mode_distance = data.groupby(['user_id','date_time','Mode_confirm'], as_index=False)[['distance']].sum()\n", - " mode_distance.rename(columns={'sum':'distance'}, inplace=True)\n", - " mode_distance['distance'] = mode_distance['distance'].fillna(0)\n", - "\n", - " # Sum daily emissions for each user\n", - " emissions = data.groupby(['user_id','date_time'], as_index=False)[['Mode_confirm_kg_CO2', 'distance']].sum()\n", - " emissions['Mode_confirm_kg_CO2'] = emissions['Mode_confirm_kg_CO2'].fillna(0)\n", - " emissions['distance'] = emissions['Mode_confirm_kg_CO2'].fillna(0)\n", - "\n", - " # Sum daily energy for each user\n", - " energy = data.groupby(['user_id','date_time'], as_index=False)[['Mode_confirm_EI(kWH)', 'distance']].sum()\n", - " energy['Mode_confirm_EI(kWH)'] = energy['Mode_confirm_EI(kWH)'].fillna(0)\n", - " energy['distance'] = energy['Mode_confirm_EI(kWH)'].fillna(0)\n", - "\n", - " # Add 7-day rolling avg smoothing to better see trends\n", - " mode_counts['trip_count_smooth'] = mode_counts.groupby(['user_id','Mode_confirm'])['trip_count'].apply(lambda x: x.rolling(7,1).mean())\n", - " mode_distance['distance_smooth'] = mode_distance.groupby(['user_id','Mode_confirm'])['distance'].apply(lambda x: x.rolling(7,1).mean())\n", - " emissions['distance_smooth'] = emissions.groupby(['user_id'])['distance'].apply(lambda x: x.rolling(7,1).mean())\n", - " energy['distance_smooth'] = energy.groupby(['user_id'])['distance'].apply(lambda x: x.rolling(7,1).mean())" + " if (len(dynamic_labels)) > 0:\n", + " # Sum daily distance traveled for each mode\n", + " mode_distance = data.groupby(['user_id','date_time','Mode_confirm'], as_index=False)[['distance']].sum()\n", + " mode_distance.rename(columns={'sum':'distance'}, inplace=True)\n", + " mode_distance['distance'] = mode_distance['distance'].fillna(0)\n", + "\n", + " # Sum daily emissions for each user\n", + " emissions = data.groupby(['user_id','date_time'], as_index=False)[['Mode_confirm_kg_CO2', 'distance']].sum()\n", + " emissions['Mode_confirm_kg_CO2'] = emissions['Mode_confirm_kg_CO2'].fillna(0)\n", + " emissions['distance'] = emissions['Mode_confirm_kg_CO2'].fillna(0)\n", + "\n", + " # Sum daily energy for each user\n", + " energy = data.groupby(['user_id','date_time'], as_index=False)[['Mode_confirm_EI(kWH)', 'distance']].sum()\n", + " energy['Mode_confirm_EI(kWH)'] = energy['Mode_confirm_EI(kWH)'].fillna(0)\n", + " energy['distance'] = energy['Mode_confirm_EI(kWH)'].fillna(0)\n", + "\n", + " # Add 7-day rolling avg smoothing to better see trends\n", + " mode_counts['trip_count_smooth'] = mode_counts.groupby(['user_id','Mode_confirm'])['trip_count'].apply(lambda x: x.rolling(7,1).mean())\n", + " mode_distance['distance_smooth'] = mode_distance.groupby(['user_id','Mode_confirm'])['distance'].apply(lambda x: x.rolling(7,1).mean())\n", + " emissions['distance_smooth'] = emissions.groupby(['user_id'])['distance'].apply(lambda x: x.rolling(7,1).mean())\n", + " energy['distance_smooth'] = energy.groupby(['user_id'])['distance'].apply(lambda x: x.rolling(7,1).mean())\n", + " else:\n", + " # Sum daily distance traveled for each mode\n", + " mode_distance = data.groupby(['user_id','date_time','Mode_confirm'], as_index=False)[['distance_miles']].sum()\n", + " mode_distance.rename(columns={'sum':'distance_miles'}, inplace=True)\n", + " mode_distance['distance_miles'] = mode_distance['distance_miles'].fillna(0)\n", + "\n", + " # Sum daily emissions for each user\n", + " emissions = data.groupby(['user_id','date_time'], as_index=False)[['Mode_confirm_lb_CO2', 'distance_miles']].sum()\n", + " emissions['Mode_confirm_lb_CO2'] = emissions['Mode_confirm_lb_CO2'].fillna(0)\n", + " emissions['distance_miles'] = emissions['Mode_confirm_lb_CO2'].fillna(0)\n", + "\n", + " # Sum daily energy for each user\n", + " energy = data.groupby(['user_id','date_time'], as_index=False)[['Mode_confirm_EI(kWH)', 'distance_miles']].sum()\n", + " energy['Mode_confirm_EI(kWH)'] = energy['Mode_confirm_EI(kWH)'].fillna(0)\n", + " energy['distance_miles'] = energy['Mode_confirm_EI(kWH)'].fillna(0)\n", + "\n", + " # Add 7-day rolling avg smoothing to better see trends\n", + " mode_counts['trip_count_smooth'] = mode_counts.groupby(['user_id','Mode_confirm'])['trip_count'].apply(lambda x: x.rolling(7,1).mean())\n", + " mode_distance['distance_miles_smooth'] = mode_distance.groupby(['user_id','Mode_confirm'])['distance_miles'].apply(lambda x: x.rolling(7,1).mean())\n", + " emissions['distance_miles_smooth'] = emissions.groupby(['user_id'])['distance_miles'].apply(lambda x: x.rolling(7,1).mean())\n", + " energy['distance_miles_smooth'] = energy.groupby(['user_id'])['distance_miles'].apply(lambda x: x.rolling(7,1).mean())" ] }, { @@ -196,24 +222,44 @@ "metadata": {}, "outputs": [], "source": [ - "plot_title_no_quality = 'Net Daily Emissions (All Users, excluding air)'\n", - "file_name = \"ts_emissions_user%s\"%file_suffix\n", + "if (len(dynamic_labels)) > 0:\n", + " plot_title_no_quality = 'Net Daily Emissions (All Users, excluding air)'\n", + " file_name = \"ts_emissions_user%s\"%file_suffix\n", "\n", - "try:\n", - " # Emissions per week across all users (net impact)\n", - " plot_data = emissions.groupby(['date_time'], as_index=False)['Mode_confirm_kg_CO2'].agg(['sum'])\n", - " total_sum = plot_data['sum'].sum()\n", + " try:\n", + " # Emissions per week across all users (net impact)\n", + " plot_data = emissions.groupby(['date_time'], as_index=False)['Mode_confirm_kg_CO2'].agg(['sum'])\n", + " total_sum = plot_data['sum'].sum()\n", "\n", - " plot_data = plot_data.merge(active_users, on='date_time')\n", - " plot_data['sum'] = plot_data['sum'] / plot_data['active_users']\n", + " plot_data = plot_data.merge(active_users, on='date_time')\n", + " plot_data['sum'] = plot_data['sum'] / plot_data['active_users']\n", "\n", - " plot_title= plot_title_no_quality+\"\\n\"+quality_text\n", - " ylab = 'Emissions (kg CO2/day/user)'\n", - " timeseries_plot(plot_data['date_time'], plot_data['sum'], plot_title, ylab, file_name)\n", - " alt_text = store_alt_text_timeseries(plot_data, file_name, plot_title)\n", - "except:\n", - " generate_missing_plot(plot_title_no_quality,debug_df,file_name)\n", - " alt_text = store_alt_text_missing(debug_df, file_name, plot_title_no_quality)" + " plot_title= plot_title_no_quality+\"\\n\"+quality_text\n", + " ylab = 'Emissions (kg CO2/day/user)'\n", + " timeseries_plot(plot_data['date_time'], plot_data['sum'], plot_title, ylab, file_name)\n", + " alt_text = store_alt_text_timeseries(plot_data, file_name, plot_title)\n", + " except:\n", + " generate_missing_plot(plot_title_no_quality,debug_df,file_name)\n", + " alt_text = store_alt_text_missing(debug_df, file_name, plot_title_no_quality)\n", + "else:\n", + " plot_title_no_quality = 'Net Daily Emissions (All Users, excluding air)'\n", + " file_name = \"ts_emissions_user%s\"%file_suffix\n", + "\n", + " try:\n", + " # Emissions per week across all users (net impact)\n", + " plot_data = emissions.groupby(['date_time'], as_index=False)['Mode_confirm_lb_CO2'].agg(['sum'])\n", + " total_sum = plot_data['sum'].sum()\n", + "\n", + " plot_data = plot_data.merge(active_users, on='date_time')\n", + " plot_data['sum'] = plot_data['sum'] / plot_data['active_users']\n", + "\n", + " plot_title= plot_title_no_quality+\"\\n\"+quality_text\n", + " ylab = 'Emissions (lb CO2/day/user)'\n", + " timeseries_plot(plot_data['date_time'], plot_data['sum'], plot_title, ylab, file_name)\n", + " alt_text = store_alt_text_timeseries(plot_data, file_name, plot_title)\n", + " except:\n", + " generate_missing_plot(plot_title_no_quality,debug_df,file_name)\n", + " alt_text = store_alt_text_missing(debug_df, file_name, plot_title_no_quality)" ] }, { @@ -251,7 +297,7 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "### Emissions per mile per day" + "### Emissions per mile/kilometer per day" ] }, { @@ -260,23 +306,42 @@ "metadata": {}, "outputs": [], "source": [ - "plot_title_no_quality = 'Average Daily Emission Rate (All Users, excluding air)'\n", - "file_name = \"ts_emissions_vmt%s\"%file_suffix\n", + "if (len(dynamic_labels) > 0):\n", + " plot_title_no_quality = 'Average Daily Emission Rate (All Users, excluding air)'\n", + " file_name = \"ts_emissions_vmt%s\"%file_suffix\n", "\n", - "try:\n", - " # Emissions per mile per day across all users (travel efficiency)\n", - " # Note that the energy plot will be identical to this one since scale factor is divided out\n", - " emissions['CO2_per_km'] = emissions['Mode_confirm_kg_CO2'] / emissions['distance_smooth']\n", - " emissions['CO2_per_km'] = emissions['CO2_per_km'].fillna(0)\n", - " plot_data = emissions.groupby(['date_time'])['CO2_per_km'].agg(['mean']).reset_index()\n", + " try:\n", + " # Emissions per kilometer per day across all users (travel efficiency)\n", + " # Note that the energy plot will be identical to this one since scale factor is divided out\n", + " emissions['CO2_per_km'] = emissions['Mode_confirm_kg_CO2'] / emissions['distance_smooth']\n", + " emissions['CO2_per_km'] = emissions['CO2_per_km'].fillna(0)\n", + " plot_data = emissions.groupby(['date_time'])['CO2_per_km'].agg(['mean']).reset_index()\n", "\n", - " plot_title= plot_title_no_quality+\"\\n\"+quality_text\n", - " ylab = 'Emissions (kg CO2/km/day)'\n", - " timeseries_plot(plot_data['date_time'], plot_data['mean'], plot_title, ylab, file_name)\n", - " alt_text = store_alt_text_timeseries(plot_data, file_name, plot_title)\n", - "except:\n", - " generate_missing_plot(plot_title_no_quality,debug_df,file_name)\n", - " alt_text = store_alt_text_missing(debug_df, file_name, plot_title_no_quality) " + " plot_title= plot_title_no_quality+\"\\n\"+quality_text\n", + " ylab = 'Emissions (kg CO2/km/day)'\n", + " timeseries_plot(plot_data['date_time'], plot_data['mean'], plot_title, ylab, file_name)\n", + " alt_text = store_alt_text_timeseries(plot_data, file_name, plot_title)\n", + " except:\n", + " generate_missing_plot(plot_title_no_quality,debug_df,file_name)\n", + " alt_text = store_alt_text_missing(debug_df, file_name, plot_title_no_quality)\n", + "else:\n", + " plot_title_no_quality = 'Average Daily Emission Rate (All Users, excluding air)'\n", + " file_name = \"ts_emissions_vmt%s\"%file_suffix\n", + "\n", + " try:\n", + " # Emissions per kilometer per day across all users (travel efficiency)\n", + " # Note that the energy plot will be identical to this one since scale factor is divided out\n", + " emissions['CO2_per_mile'] = emissions['Mode_confirm_lb_CO2'] / emissions['distance_miles_smooth']\n", + " emissions['CO2_per_mile'] = emissions['CO2_per_mile'].fillna(0)\n", + " plot_data = emissions.groupby(['date_time'])['CO2_per_mile'].agg(['mean']).reset_index()\n", + "\n", + " plot_title= plot_title_no_quality+\"\\n\"+quality_text\n", + " ylab = 'Emissions (lb CO2/mile/day)'\n", + " timeseries_plot(plot_data['date_time'], plot_data['mean'], plot_title, ylab, file_name)\n", + " alt_text = store_alt_text_timeseries(plot_data, file_name, plot_title)\n", + " except:\n", + " generate_missing_plot(plot_title_no_quality,debug_df,file_name)\n", + " alt_text = store_alt_text_missing(debug_df, file_name, plot_title_no_quality)" ] }, { From 4c177d78c22aa0f2009ffa89efd48a3dfc54414c Mon Sep 17 00:00:00 2001 From: iantei Date: Fri, 27 Oct 2023 08:18:57 -0700 Subject: [PATCH 41/62] Added print statements to understand the difference in CO2 emissions for default mapping and one with dynamic config. --- viz_scripts/energy_calculations.ipynb | 154 ++++++++++++++++++++++++-- 1 file changed, 147 insertions(+), 7 deletions(-) diff --git a/viz_scripts/energy_calculations.ipynb b/viz_scripts/energy_calculations.ipynb index 4675d84..d4bb7db 100644 --- a/viz_scripts/energy_calculations.ipynb +++ b/viz_scripts/energy_calculations.ipynb @@ -238,32 +238,172 @@ " CO2_impact(x,y,color,plot_title, xLabel,yLabel,file_name)\n", " alt_text = store_alt_text_bar(pd.DataFrame(x.values,y), file_name, plot_title)\n", " \n", + " filtered_taxi_data = data_eb[data_eb['Replaced_mode'] == \"Taxi/Uber/Lyft\"]\n", + " filtered_bus_data = data_eb[data_eb['Replaced_mode'] == \"Bus\"]\n", + " filtered_notravel_data = data_eb[data_eb['Replaced_mode'] == \"No travel\"]\n", + " filtered_freeshuttle_data = data_eb[data_eb['Replaced_mode'] == \"Free Shuttle\"]\n", + " filtered_walk_data = data_eb[data_eb['Replaced_mode'] == \"Walk\"]\n", + " filtered_GasCarShared_data = data_eb[data_eb['Replaced_mode'] == \"Gas Car Shared Ride\"]\n", + "\n", + " selected_columns = ['distance', 'Replaced_mode_kg_CO2', 'Mode_confirm_kg_CO2', \"replaced_mode\", \"mode_confirm\"]\n", + "\n", + " print(\"With Dynamic config mapping:\")\n", + " print(\"\\n\")\n", + "\n", + " print(\"Walk Data:\")\n", + " print(str(filtered_walk_data[selected_columns].head()))\n", + " print(\"\\n\")\n", + "\n", + " print(\"No Travel Data:\")\n", + " print(str(filtered_notravel_data[selected_columns].head()))\n", + " print(\"\\n\")\n", + "\n", + " print(\"Gas Car Shared Data:\")\n", + " print(str(filtered_GasCarShared_data[selected_columns].head()))\n", + " print(\"\\n\")\n", + "\n", + " print(\"Free Shuttle Data:\")\n", + " print(str(filtered_freeshuttle_data[selected_columns].head()))\n", + " print(\"\\n\")\n", + "\n", + " print(\"Bus Data:\")\n", + " print(str(filtered_bus_data[selected_columns].head()))\n", + " print(\"\\n\")\n", + " \n", + " print(\"Taxi/Uber/Lyft Data:\")\n", + " print(str(filtered_taxi_data[selected_columns].head()))\n", + " print(\"\\n\")\n", + " \n", + " print(\"Total Kg CO2 Emissions\")\n", + " print(ebco2['total_kg_CO2_emissions'])\n", + " \n", " except:\n", " generate_missing_plot(plot_title_no_quality,debug_df,file_name)\n", " alt_text = store_alt_text_missing(debug_df, file_name, plot_title_no_quality)\n", "# case: when label_options is absent from the config file\n", + "# else: \n", + "# plot_title_no_quality=f\"Sketch of Total pounds of CO2 Emissions of {mode_of_interest} trips\"\n", + "# file_name =f'sketch_CO2impact_{mode_of_interest}%s' % file_suffix\n", + "\n", + "# try:\n", + "# ebco2=data_eb.groupby('Replaced_mode').agg({'CO2_Impact(lb)': ['sum', 'mean']},)\n", + "# ebco2.columns = ['total_lb_CO2_emissions', 'average_lb_CO2_emission']\n", + "# ebco2 = ebco2.reset_index()\n", + "# ebco2 = ebco2.sort_values(by=['total_lb_CO2_emissions'], ascending=False)\n", + "# ebco2['boolean'] = ebco2['total_lb_CO2_emissions'] > 0\n", + "# net_CO2_emissions = round(sum(ebco2['total_lb_CO2_emissions']), 2)\n", + "\n", + "# x = ebco2['total_lb_CO2_emissions']\n", + "# y = ebco2['Replaced_mode']\n", + "# xLabel = 'CO2 Emissions (lb)'\n", + "# yLabel = 'Replaced Mode'\n", + "# color = ebco2['boolean']\n", + "\n", + "# plot_title= plot_title_no_quality+f\"\\n Contribution by replaced mode towards a total of {net_CO2_emissions}(lb CO2 Emissions )\\n\"+quality_text\n", + "# CO2_impact(x,y,color,plot_title,xLabel,yLabel,file_name)\n", + "# alt_text = store_alt_text_bar(pd.DataFrame(x.values,y), file_name, plot_title)\n", + "\n", + "# filtered_taxi_data = data_eb[data_eb['Replaced_mode'] == \"Taxi/Uber/Lyft\"]\n", + "# filtered_bus_data = data_eb[data_eb['Replaced_mode'] == \"Bus\"]\n", + "# filtered_notravel_data = data_eb[data_eb['Replaced_mode'] == \"No Travel\"]\n", + "# filtered_freeshuttle_data = data_eb[data_eb['Replaced_mode'] == \"Free Shuttle\"]\n", + "# filtered_walk_data = data_eb[data_eb['Replaced_mode'] == \"Walk\"]\n", + "# filtered_GasCarShared_data = data_eb[data_eb['Replaced_mode'] == \"Gas Car, with others\"]\n", + "\n", + "# selected_columns = ['distance', 'Replaced_mode_kg_CO2', 'Mode_confirm_kg_CO2', \"replaced_mode\", \"mode_confirm\"]\n", + "\n", + "# print(\"With Default mapping:\")\n", + "# print(\"\\n\")\n", + "\n", + "# print(\"Walk Data:\")\n", + "# print(str(filtered_walk_data[selected_columns].head()))\n", + "# print(\"\\n\")\n", + "\n", + "# print(\"No Travel Data:\")\n", + "# print(str(filtered_notravel_data[selected_columns].head()))\n", + "# print(\"\\n\")\n", + "\n", + "# print(\"Gas Car Shared Data:\")\n", + "# print(str(filtered_GasCarShared_data[selected_columns].head()))\n", + "# print(\"\\n\")\n", + "\n", + "# print(\"Free Shuttle Data:\")\n", + "# print(str(filtered_freeshuttle_data[selected_columns].head()))\n", + "# print(\"\\n\")\n", + "\n", + "# print(\"Bus Data:\")\n", + "# print(str(filtered_bus_data[selected_columns].head()))\n", + "# print(\"\\n\")\n", + " \n", + "# print(\"Taxi/Uber/Lyft Data:\")\n", + "# print(str(filtered_taxi_data[selected_columns].head()))\n", + "# print(\"\\n\")\n", + " \n", + "# print(\"Total lbs CO2 Emissions\")\n", + "# print(ebco2['total_lb_CO2_emissions'])\n", + "# except:\n", + "# generate_missing_plot(plot_title_no_quality,debug_df,file_name)\n", + "# alt_text = store_alt_text_missing(debug_df, file_name, plot_title_no_quality)\n", "else: \n", " plot_title_no_quality=f\"Sketch of Total pounds of CO2 Emissions of {mode_of_interest} trips\"\n", " file_name =f'sketch_CO2impact_{mode_of_interest}%s' % file_suffix\n", "\n", " try:\n", - " ebco2=data_eb.groupby('Replaced_mode').agg({'CO2_Impact(lb)': ['sum', 'mean']},)\n", - " ebco2.columns = ['total_lb_CO2_emissions', 'average_lb_CO2_emission']\n", + " ebco2=data_eb.groupby('Replaced_mode').agg({'CO2_Impact(kg)': ['sum', 'mean']},)\n", + " ebco2.columns = ['total_kg_CO2_emissions', 'average_kg_CO2_emission']\n", " ebco2 = ebco2.reset_index()\n", - " ebco2 = ebco2.sort_values(by=['total_lb_CO2_emissions'], ascending=False)\n", - " ebco2['boolean'] = ebco2['total_lb_CO2_emissions'] > 0\n", - " net_CO2_emissions = round(sum(ebco2['total_lb_CO2_emissions']), 2)\n", + " ebco2 = ebco2.sort_values(by=['total_kg_CO2_emissions'], ascending=False)\n", + " ebco2['boolean'] = ebco2['total_kg_CO2_emissions'] > 0\n", + " net_CO2_emissions = round(sum(ebco2['total_kg_CO2_emissions']), 2)\n", "\n", - " x = ebco2['total_lb_CO2_emissions']\n", + " x = ebco2['total_kg_CO2_emissions']\n", " y = ebco2['Replaced_mode']\n", - " xLabel = 'CO2 Emissions (lb)'\n", + " xLabel = 'CO2 Emissions (kg)'\n", " yLabel = 'Replaced Mode'\n", " color = ebco2['boolean']\n", "\n", " plot_title= plot_title_no_quality+f\"\\n Contribution by replaced mode towards a total of {net_CO2_emissions}(lb CO2 Emissions )\\n\"+quality_text\n", " CO2_impact(x,y,color,plot_title,xLabel,yLabel,file_name)\n", " alt_text = store_alt_text_bar(pd.DataFrame(x.values,y), file_name, plot_title)\n", + "\n", + " filtered_taxi_data = data_eb[data_eb['Replaced_mode'] == \"Taxi/Uber/Lyft\"]\n", + " filtered_bus_data = data_eb[data_eb['Replaced_mode'] == \"Bus\"]\n", + " filtered_notravel_data = data_eb[data_eb['Replaced_mode'] == \"No Travel\"]\n", + " filtered_freeshuttle_data = data_eb[data_eb['Replaced_mode'] == \"Free Shuttle\"]\n", + " filtered_walk_data = data_eb[data_eb['Replaced_mode'] == \"Walk\"]\n", + " filtered_GasCarShared_data = data_eb[data_eb['Replaced_mode'] == \"Gas Car, with others\"]\n", + "\n", + " selected_columns = ['distance_miles', 'Replaced_mode_kg_CO2', 'Mode_confirm_kg_CO2', \"replaced_mode\", \"mode_confirm\"]\n", + "\n", + " print(\"With Default mapping:\")\n", + " print(\"\\n\")\n", + "\n", + " print(\"Walk Data:\")\n", + " print(str(filtered_walk_data[selected_columns].head()))\n", + " print(\"\\n\")\n", + "\n", + " print(\"No Travel Data:\")\n", + " print(str(filtered_notravel_data[selected_columns].head()))\n", + " print(\"\\n\")\n", + "\n", + " print(\"Gas Car Shared Data:\")\n", + " print(str(filtered_GasCarShared_data[selected_columns].head()))\n", + " print(\"\\n\")\n", + "\n", + " print(\"Free Shuttle Data:\")\n", + " print(str(filtered_freeshuttle_data[selected_columns].head()))\n", + " print(\"\\n\")\n", + "\n", + " print(\"Bus Data:\")\n", + " print(str(filtered_bus_data[selected_columns].head()))\n", + " print(\"\\n\")\n", + " \n", + " print(\"Taxi/Uber/Lyft Data:\")\n", + " print(str(filtered_taxi_data[selected_columns].head()))\n", + " print(\"\\n\")\n", " \n", + " print(\"Total Kg CO2 Emissions\")\n", + " print(ebco2['total_kg_CO2_emissions'])\n", " except:\n", " generate_missing_plot(plot_title_no_quality,debug_df,file_name)\n", " alt_text = store_alt_text_missing(debug_df, file_name, plot_title_no_quality)" From a8f369786fd05ec23ffadc3ba3b760c88f23fb6b Mon Sep 17 00:00:00 2001 From: iantei Date: Fri, 27 Oct 2023 08:31:40 -0700 Subject: [PATCH 42/62] Renamed 1. compute_CO2_impact_kg to compute_CO2_impact_dynamic 2. compute_CO2_footprint_kg to compute_footprint_dynamic 3. compute_CO2_impact_lb to compute_CO2_impact_default 4. compute_CO2_footprint_lb to compute_CO2_footprint_default. --- viz_scripts/scaffolding.py | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/viz_scripts/scaffolding.py b/viz_scripts/scaffolding.py index d289742..12f4fb2 100644 --- a/viz_scripts/scaffolding.py +++ b/viz_scripts/scaffolding.py @@ -245,9 +245,9 @@ def add_energy_labels(expanded_ct, df_ei, dic_fuel, dynamic_labels): expanded_ct = energy_footprint_kWH(expanded_ct, 'distance_miles', 'Mode_confirm') if (len(dynamic_labels) > 0): - expanded_ct = compute_CO2_footprint_kg(expanded_ct, dynamic_labels) + expanded_ct = compute_CO2_footprint_dynamic(expanded_ct, dynamic_labels) else: - expanded_ct = CO2_footprint_lb(expanded_ct, 'distance_miles', 'Mode_confirm') + expanded_ct = CO2_footprint_default(expanded_ct, 'distance_miles', 'Mode_confirm') return expanded_ct def add_energy_impact(expanded_ct, df_ei, dic_fuel, dynamic_labels): @@ -260,9 +260,9 @@ def add_energy_impact(expanded_ct, df_ei, dic_fuel, dynamic_labels): expanded_ct = energy_impact_kWH(expanded_ct, 'distance_miles') if (len(dynamic_labels) > 0): - expanded_ct = compute_CO2_impact_kg(expanded_ct, dynamic_labels) + expanded_ct = compute_CO2_impact_dynamic(expanded_ct, dynamic_labels) else: - expanded_ct = CO2_impact_lb(expanded_ct, 'distance_miles') + expanded_ct = CO2_impact_default(expanded_ct, 'distance_miles') return expanded_ct def get_quality_text(before_df, after_df, mode_of_interest=None, include_test_users=False): @@ -357,7 +357,7 @@ def energy_impact_kWH(df,distance): df['Energy_Impact(kWH)'] = round((df['Replaced_mode_EI(kWH)'] - df['Mode_confirm_EI(kWH)']),3) return df -def CO2_footprint_lb(df, distance, col): +def CO2_footprint_default(df, distance, col): """ Inputs: df = dataframe with data distance = distance in miles @@ -379,19 +379,19 @@ def CO2_footprint_lb(df, distance, col): df[col+'_kg_CO2'] = df[col+'_lb_CO2'] * conversion_lb_to_kilogram / conversion_mile_to_kilometer return df -def CO2_impact_lb(df,distance): +def CO2_impact_default(df,distance): conversion_lb_to_kilogram = 0.453592 # 1 lb = 0.453592 KG if 'Mode_confirm_lb_CO2' not in df.columns: print("Mode confirm footprint not found, computing before impact") - df = CO2_footprint_lb(df, distance, "Mode_confirm") - df = CO2_footprint_lb(df, distance, "Replaced_mode") + df = CO2_footprint_default(df, distance, "Mode_confirm") + df = CO2_footprint_default(df, distance, "Replaced_mode") df['CO2_Impact(lb)'] = round((df['Replaced_mode_lb_CO2'] - df['Mode_confirm_lb_CO2']), 3) # Convert the CO2_Impact to be represented in kilogram df['CO2_Impact(kg)'] = round((df['Replaced_mode_kg_CO2'] - df['Mode_confirm_kg_CO2']), 3) return df -def compute_CO2_footprint_kg(expanded_ct, dynamic_labels): +def compute_CO2_footprint_dynamic(expanded_ct, dynamic_labels): conversion_meter_to_kilometer = 0.001 conversion_kilogram_to_lbs = 2.20462 conversion_kilometer_to_mile = 0.621371 @@ -405,7 +405,7 @@ def compute_CO2_footprint_kg(expanded_ct, dynamic_labels): return expanded_ct -def compute_CO2_impact_kg(expanded_ct, dynamic_labels): +def compute_CO2_impact_dynamic(expanded_ct, dynamic_labels): conversion_meter_to_kilometer = 0.001 # 1 meter = 0.001 kilometer conversion_kilogram_to_lbs = 2.20462 conversion_kilometer_to_mile = 0.621371 From e7f804e30c688fbdbaface5056dfab045211b3c9 Mon Sep 17 00:00:00 2001 From: iantei Date: Fri, 27 Oct 2023 08:35:28 -0700 Subject: [PATCH 43/62] Removed the unnecessary file unde frontend/Dockerfile.dev. --- frontend/Dockerfile.dev | 0 1 file changed, 0 insertions(+), 0 deletions(-) delete mode 100644 frontend/Dockerfile.dev diff --git a/frontend/Dockerfile.dev b/frontend/Dockerfile.dev deleted file mode 100644 index e69de29..0000000 From 23e5773157ada1ef50bf60a148416d9606e3dd43 Mon Sep 17 00:00:00 2001 From: iantei Date: Fri, 27 Oct 2023 08:39:39 -0700 Subject: [PATCH 44/62] Showing Timeseries of emissions per mile for default mapping. --- frontend/metrics_program.html | 2 +- frontend/metrics_study.html | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/frontend/metrics_program.html b/frontend/metrics_program.html index 8460278..5e7d643 100644 --- a/frontend/metrics_program.html +++ b/frontend/metrics_program.html @@ -17,7 +17,7 @@ - + diff --git a/frontend/metrics_study.html b/frontend/metrics_study.html index 614d1a2..fc1c675 100644 --- a/frontend/metrics_study.html +++ b/frontend/metrics_study.html @@ -17,6 +17,6 @@ - + From 6c469a517052edd00d308dcafd385fa463ffe528 Mon Sep 17 00:00:00 2001 From: iantei Date: Fri, 27 Oct 2023 08:50:49 -0700 Subject: [PATCH 45/62] Remove extranneous whitespaces from energy_calculations and generic_timeseries notebook. --- viz_scripts/energy_calculations.ipynb | 1 - viz_scripts/generic_timeseries.ipynb | 1 - 2 files changed, 2 deletions(-) diff --git a/viz_scripts/energy_calculations.ipynb b/viz_scripts/energy_calculations.ipynb index d4bb7db..ba170f7 100644 --- a/viz_scripts/energy_calculations.ipynb +++ b/viz_scripts/energy_calculations.ipynb @@ -113,7 +113,6 @@ " dic_pur=dic_pur,\n", " include_test_users=include_test_users)\n", "# CASE 1 of https://github.com/e-mission/em-public-dashboard/issues/69#issuecomment-1256835867 \n", - "\n", "expanded_ct = scaffolding.add_energy_impact(expanded_ct, df_ei, dic_fuel, dynamic_labels) if len(expanded_ct) > 0 else expanded_ct" ] }, diff --git a/viz_scripts/generic_timeseries.ipynb b/viz_scripts/generic_timeseries.ipynb index c4e1be1..6b6e53c 100644 --- a/viz_scripts/generic_timeseries.ipynb +++ b/viz_scripts/generic_timeseries.ipynb @@ -89,7 +89,6 @@ " dynamic_labels,\n", " dic_re,\n", " include_test_users=include_test_users)\n", - "\n", "expanded_ct = scaffolding.add_energy_labels(expanded_ct, df_ei, dic_fuel, dynamic_labels) if \"mode_confirm\" in expanded_ct.columns else expanded_ct" ] }, From a18c549d878564db82b1f7af040fefd4b615bc44 Mon Sep 17 00:00:00 2001 From: iantei Date: Fri, 27 Oct 2023 08:58:19 -0700 Subject: [PATCH 46/62] Remove whitespace from generic_timeseries.ipynb --- viz_scripts/generic_timeseries.ipynb | 1 - 1 file changed, 1 deletion(-) diff --git a/viz_scripts/generic_timeseries.ipynb b/viz_scripts/generic_timeseries.ipynb index 6b6e53c..c9697df 100644 --- a/viz_scripts/generic_timeseries.ipynb +++ b/viz_scripts/generic_timeseries.ipynb @@ -406,7 +406,6 @@ " total_trips = plot_data.groupby(['date_time'], as_index=False).sum()\n", " plot_data = plot_data.merge(total_trips, on='date_time')\n", " plot_data['trip_proportion'] = plot_data['trip_count_smooth_x'] / plot_data['trip_count_smooth_y']\n", - " \n", " # Re-establish categorical variable to not include Other and Non-trips\n", " plot_data = plot_data[~plot_data['Mode_confirm'].isin(['Not a Trip','Other'])]\n", "\n", From 88b899b01e7debae899115d23a7df8b7dcb0d3a4 Mon Sep 17 00:00:00 2001 From: iantei Date: Mon, 30 Oct 2023 09:51:39 -0700 Subject: [PATCH 47/62] Fixing the scaffolding calculation for computation of conversion to kg_CO2 from lb_CO2 and vice-versa in both dynamic config and default mapping case. --- viz_scripts/scaffolding.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/viz_scripts/scaffolding.py b/viz_scripts/scaffolding.py index 12f4fb2..1be3d6e 100644 --- a/viz_scripts/scaffolding.py +++ b/viz_scripts/scaffolding.py @@ -376,7 +376,7 @@ def CO2_footprint_default(df, distance, col): values_col = [gasoline_col,diesel_col,electric_col] df[col+'_lb_CO2'] = np.select(conditions_col, values_col) - df[col+'_kg_CO2'] = df[col+'_lb_CO2'] * conversion_lb_to_kilogram / conversion_mile_to_kilometer + df[col+'_kg_CO2'] = df[col+'_lb_CO2'] * conversion_lb_to_kilogram return df def CO2_impact_default(df,distance): @@ -401,7 +401,7 @@ def compute_CO2_footprint_dynamic(expanded_ct, dynamic_labels): if "mode_confirm" in expanded_ct.columns: expanded_ct['Mode_confirm_kg_CO2'] = ((expanded_ct['distance'] * conversion_meter_to_kilometer )) * (expanded_ct['mode_confirm'].map(dic_mode_kgCO2PerKm)) expanded_ct['Mode_confirm_kg_CO2'] = expanded_ct['Mode_confirm_kg_CO2'].fillna(0) - expanded_ct['Mode_confirm_lb_CO2'] = expanded_ct['Mode_confirm_kg_CO2'] * conversion_kilogram_to_lbs / conversion_kilometer_to_mile + expanded_ct['Mode_confirm_lb_CO2'] = expanded_ct['Mode_confirm_kg_CO2'] * conversion_kilogram_to_lbs return expanded_ct @@ -413,14 +413,15 @@ def compute_CO2_impact_dynamic(expanded_ct, dynamic_labels): dic_mode_kgCO2PerKm = {mode["value"]: mode["kgCo2PerKm"] for mode in dynamic_labels["MODE"]} if "mode_confirm" in expanded_ct.columns: + # The expanded_ct['Mode_confirm_kg_CO2'] is CO2 emission in terms of [distance in kms * CO2 emission in kgCO2 per km = kg of CO2] expanded_ct['Mode_confirm_kg_CO2'] = ((expanded_ct['distance'] * conversion_meter_to_kilometer )) * (expanded_ct['mode_confirm'].map(dic_mode_kgCO2PerKm)) expanded_ct['Mode_confirm_kg_CO2'] = expanded_ct['Mode_confirm_kg_CO2'].fillna(0) - expanded_ct['Mode_confirm_lb_CO2'] = expanded_ct['Mode_confirm_kg_CO2'] * conversion_kilogram_to_lbs / conversion_kilometer_to_mile + expanded_ct['Mode_confirm_lb_CO2'] = expanded_ct['Mode_confirm_kg_CO2'] * conversion_kilogram_to_lbs if "replaced_mode" in expanded_ct.columns: expanded_ct['Replaced_mode_kg_CO2'] = (expanded_ct['distance'] * conversion_meter_to_kilometer ) * (expanded_ct['replaced_mode'].map(dic_mode_kgCO2PerKm)) expanded_ct['Replaced_mode_kg_CO2'] = expanded_ct['Replaced_mode_kg_CO2'].fillna(0) - expanded_ct['Replaced_mode_lb_CO2'] = expanded_ct['Replaced_mode_kg_CO2'] * conversion_kilogram_to_lbs / conversion_kilometer_to_mile + expanded_ct['Replaced_mode_lb_CO2'] = expanded_ct['Replaced_mode_kg_CO2'] * conversion_kilogram_to_lbs expanded_ct['CO2_Impact(kg)'] = round ((expanded_ct['Replaced_mode_kg_CO2'] - expanded_ct['Mode_confirm_kg_CO2']), 3) expanded_ct['CO2_Impact(lb)'] = round ((expanded_ct['Replaced_mode_lb_CO2'] - expanded_ct['Mode_confirm_lb_CO2']), 3) From 9134465fc2ab48a7284f940552a1135ca05f28ea Mon Sep 17 00:00:00 2001 From: iantei Date: Mon, 30 Oct 2023 11:56:53 -0700 Subject: [PATCH 48/62] Add new function in scaffolding.py to print the computation of CO2 emissions. This will be called from energy_calculations notebook. --- viz_scripts/scaffolding.py | 51 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 51 insertions(+) diff --git a/viz_scripts/scaffolding.py b/viz_scripts/scaffolding.py index 1be3d6e..399fe3d 100644 --- a/viz_scripts/scaffolding.py +++ b/viz_scripts/scaffolding.py @@ -426,3 +426,54 @@ def compute_CO2_impact_dynamic(expanded_ct, dynamic_labels): expanded_ct['CO2_Impact(kg)'] = round ((expanded_ct['Replaced_mode_kg_CO2'] - expanded_ct['Mode_confirm_kg_CO2']), 3) expanded_ct['CO2_Impact(lb)'] = round ((expanded_ct['Replaced_mode_lb_CO2'] - expanded_ct['Mode_confirm_lb_CO2']), 3) return expanded_ct + +def print_CO2_emission_calculations(data_eb, ebco2_lb, ebco2_kg, dynamic_labels): + + filtered_taxi_data = data_eb[data_eb['Replaced_mode'] == "Taxi/Uber/Lyft"] + filtered_bus_data = data_eb[data_eb['Replaced_mode'] == "Bus"] + filtered_freeshuttle_data = data_eb[data_eb['Replaced_mode'] == "Free Shuttle"] + filtered_walk_data = data_eb[data_eb['Replaced_mode'] == "Walk"] + + # Handling different cases of Replaced mode translations + if len(dynamic_labels) > 0: + filtered_GasCarShared_data = data_eb[data_eb['Replaced_mode'] == "Gas Car Shared Ride"] + filtered_notravel_data = data_eb[data_eb['Replaced_mode'] == "No travel"] + print("With Dynamic Config:") + print("\n") + else: + filtered_GasCarShared_data = data_eb[data_eb['Replaced_mode'] == "Gas Car, with others"] + filtered_notravel_data = data_eb[data_eb['Replaced_mode'] == "No Travel"] + print("With Default mapping:") + print("\n") + + selected_columns = ['distance', 'Replaced_mode_kg_CO2', 'Replaced_mode_lb_CO2', 'Mode_confirm_kg_CO2','Mode_confirm_lb_CO2', "replaced_mode", "mode_confirm"] + + print("Walk Data:") + print(str(filtered_walk_data[selected_columns].head())) + print("\n") + + print("No Travel Data:") + print(str(filtered_notravel_data[selected_columns].head())) + print("\n") + + print("Gas Car Shared Data:") + print(str(filtered_GasCarShared_data[selected_columns].head())) + print("\n") + + print("Free Shuttle Data:") + print(str(filtered_freeshuttle_data[selected_columns].head())) + print("\n") + + print("Bus Data:") + print(str(filtered_bus_data[selected_columns].head())) + print("\n") + + print("Taxi/Uber/Lyft Data:") + print(str(filtered_taxi_data[selected_columns].head())) + print("\n") + + combined_df = pd.concat([ebco2_lb['total_lb_CO2_emissions'], ebco2_kg['total_kg_CO2_emissions']], axis=1) + combined_df.columns = ['Total CO2 Emissions (lb)', 'Total CO2 Emissions (kg)'] + + print("CO2 Emissions:") + print(combined_df) From 5229d7e5a210b1c362c6034486eb6f8b592be3ba Mon Sep 17 00:00:00 2001 From: iantei Date: Mon, 30 Oct 2023 12:08:31 -0700 Subject: [PATCH 49/62] Added is_debug_mode flag, which by default is set to False. When it is set to True, then the print statements associated with the CO2 emissions computation will be called from scaffolding.py print_CO2_emission_calculations(). Removed the duplicate code about print statements, and moved into scaffolding.py. --- viz_scripts/energy_calculations.ipynb | 168 ++++---------------------- 1 file changed, 22 insertions(+), 146 deletions(-) diff --git a/viz_scripts/energy_calculations.ipynb b/viz_scripts/energy_calculations.ipynb index ba170f7..fdece19 100644 --- a/viz_scripts/energy_calculations.ipynb +++ b/viz_scripts/energy_calculations.ipynb @@ -31,6 +31,7 @@ "study_type = \"program\"\n", "mode_of_interest = \"e-bike\"\n", "include_test_users = False\n", + "is_debug_mode = True\n", "dynamic_labels = {}" ] }, @@ -237,127 +238,33 @@ " CO2_impact(x,y,color,plot_title, xLabel,yLabel,file_name)\n", " alt_text = store_alt_text_bar(pd.DataFrame(x.values,y), file_name, plot_title)\n", " \n", - " filtered_taxi_data = data_eb[data_eb['Replaced_mode'] == \"Taxi/Uber/Lyft\"]\n", - " filtered_bus_data = data_eb[data_eb['Replaced_mode'] == \"Bus\"]\n", - " filtered_notravel_data = data_eb[data_eb['Replaced_mode'] == \"No travel\"]\n", - " filtered_freeshuttle_data = data_eb[data_eb['Replaced_mode'] == \"Free Shuttle\"]\n", - " filtered_walk_data = data_eb[data_eb['Replaced_mode'] == \"Walk\"]\n", - " filtered_GasCarShared_data = data_eb[data_eb['Replaced_mode'] == \"Gas Car Shared Ride\"]\n", - "\n", - " selected_columns = ['distance', 'Replaced_mode_kg_CO2', 'Mode_confirm_kg_CO2', \"replaced_mode\", \"mode_confirm\"]\n", - "\n", - " print(\"With Dynamic config mapping:\")\n", - " print(\"\\n\")\n", - "\n", - " print(\"Walk Data:\")\n", - " print(str(filtered_walk_data[selected_columns].head()))\n", - " print(\"\\n\")\n", - "\n", - " print(\"No Travel Data:\")\n", - " print(str(filtered_notravel_data[selected_columns].head()))\n", - " print(\"\\n\")\n", - "\n", - " print(\"Gas Car Shared Data:\")\n", - " print(str(filtered_GasCarShared_data[selected_columns].head()))\n", - " print(\"\\n\")\n", - "\n", - " print(\"Free Shuttle Data:\")\n", - " print(str(filtered_freeshuttle_data[selected_columns].head()))\n", - " print(\"\\n\")\n", - "\n", - " print(\"Bus Data:\")\n", - " print(str(filtered_bus_data[selected_columns].head()))\n", - " print(\"\\n\")\n", - " \n", - " print(\"Taxi/Uber/Lyft Data:\")\n", - " print(str(filtered_taxi_data[selected_columns].head()))\n", - " print(\"\\n\")\n", - " \n", - " print(\"Total Kg CO2 Emissions\")\n", - " print(ebco2['total_kg_CO2_emissions'])\n", + " if (is_debug_mode == True):\n", + " ebco2_lb=data_eb.groupby('Replaced_mode').agg({'CO2_Impact(lb)': ['sum', 'mean']},)\n", + " ebco2_lb.columns = ['total_lb_CO2_emissions', 'average_lb_CO2_emission']\n", + " ebco2_lb = ebco2_lb.reset_index()\n", + " ebco2_lb = ebco2_lb.sort_values(by=['total_lb_CO2_emissions'], ascending=False)\n", + " \n", + " scaffolding.print_CO2_emission_calculations(data_eb, ebco2_lb, ebco2, dynamic_labels)\n", " \n", " except:\n", " generate_missing_plot(plot_title_no_quality,debug_df,file_name)\n", " alt_text = store_alt_text_missing(debug_df, file_name, plot_title_no_quality)\n", "# case: when label_options is absent from the config file\n", - "# else: \n", - "# plot_title_no_quality=f\"Sketch of Total pounds of CO2 Emissions of {mode_of_interest} trips\"\n", - "# file_name =f'sketch_CO2impact_{mode_of_interest}%s' % file_suffix\n", - "\n", - "# try:\n", - "# ebco2=data_eb.groupby('Replaced_mode').agg({'CO2_Impact(lb)': ['sum', 'mean']},)\n", - "# ebco2.columns = ['total_lb_CO2_emissions', 'average_lb_CO2_emission']\n", - "# ebco2 = ebco2.reset_index()\n", - "# ebco2 = ebco2.sort_values(by=['total_lb_CO2_emissions'], ascending=False)\n", - "# ebco2['boolean'] = ebco2['total_lb_CO2_emissions'] > 0\n", - "# net_CO2_emissions = round(sum(ebco2['total_lb_CO2_emissions']), 2)\n", - "\n", - "# x = ebco2['total_lb_CO2_emissions']\n", - "# y = ebco2['Replaced_mode']\n", - "# xLabel = 'CO2 Emissions (lb)'\n", - "# yLabel = 'Replaced Mode'\n", - "# color = ebco2['boolean']\n", - "\n", - "# plot_title= plot_title_no_quality+f\"\\n Contribution by replaced mode towards a total of {net_CO2_emissions}(lb CO2 Emissions )\\n\"+quality_text\n", - "# CO2_impact(x,y,color,plot_title,xLabel,yLabel,file_name)\n", - "# alt_text = store_alt_text_bar(pd.DataFrame(x.values,y), file_name, plot_title)\n", - "\n", - "# filtered_taxi_data = data_eb[data_eb['Replaced_mode'] == \"Taxi/Uber/Lyft\"]\n", - "# filtered_bus_data = data_eb[data_eb['Replaced_mode'] == \"Bus\"]\n", - "# filtered_notravel_data = data_eb[data_eb['Replaced_mode'] == \"No Travel\"]\n", - "# filtered_freeshuttle_data = data_eb[data_eb['Replaced_mode'] == \"Free Shuttle\"]\n", - "# filtered_walk_data = data_eb[data_eb['Replaced_mode'] == \"Walk\"]\n", - "# filtered_GasCarShared_data = data_eb[data_eb['Replaced_mode'] == \"Gas Car, with others\"]\n", - "\n", - "# selected_columns = ['distance', 'Replaced_mode_kg_CO2', 'Mode_confirm_kg_CO2', \"replaced_mode\", \"mode_confirm\"]\n", - "\n", - "# print(\"With Default mapping:\")\n", - "# print(\"\\n\")\n", - "\n", - "# print(\"Walk Data:\")\n", - "# print(str(filtered_walk_data[selected_columns].head()))\n", - "# print(\"\\n\")\n", - "\n", - "# print(\"No Travel Data:\")\n", - "# print(str(filtered_notravel_data[selected_columns].head()))\n", - "# print(\"\\n\")\n", - "\n", - "# print(\"Gas Car Shared Data:\")\n", - "# print(str(filtered_GasCarShared_data[selected_columns].head()))\n", - "# print(\"\\n\")\n", - "\n", - "# print(\"Free Shuttle Data:\")\n", - "# print(str(filtered_freeshuttle_data[selected_columns].head()))\n", - "# print(\"\\n\")\n", - "\n", - "# print(\"Bus Data:\")\n", - "# print(str(filtered_bus_data[selected_columns].head()))\n", - "# print(\"\\n\")\n", - " \n", - "# print(\"Taxi/Uber/Lyft Data:\")\n", - "# print(str(filtered_taxi_data[selected_columns].head()))\n", - "# print(\"\\n\")\n", - " \n", - "# print(\"Total lbs CO2 Emissions\")\n", - "# print(ebco2['total_lb_CO2_emissions'])\n", - "# except:\n", - "# generate_missing_plot(plot_title_no_quality,debug_df,file_name)\n", - "# alt_text = store_alt_text_missing(debug_df, file_name, plot_title_no_quality)\n", "else: \n", " plot_title_no_quality=f\"Sketch of Total pounds of CO2 Emissions of {mode_of_interest} trips\"\n", " file_name =f'sketch_CO2impact_{mode_of_interest}%s' % file_suffix\n", "\n", " try:\n", - " ebco2=data_eb.groupby('Replaced_mode').agg({'CO2_Impact(kg)': ['sum', 'mean']},)\n", - " ebco2.columns = ['total_kg_CO2_emissions', 'average_kg_CO2_emission']\n", + " ebco2=data_eb.groupby('Replaced_mode').agg({'CO2_Impact(lb)': ['sum', 'mean']},)\n", + " ebco2.columns = ['total_lb_CO2_emissions', 'average_lb_CO2_emission']\n", " ebco2 = ebco2.reset_index()\n", - " ebco2 = ebco2.sort_values(by=['total_kg_CO2_emissions'], ascending=False)\n", - " ebco2['boolean'] = ebco2['total_kg_CO2_emissions'] > 0\n", - " net_CO2_emissions = round(sum(ebco2['total_kg_CO2_emissions']), 2)\n", + " ebco2 = ebco2.sort_values(by=['total_lb_CO2_emissions'], ascending=False)\n", + " ebco2['boolean'] = ebco2['total_lb_CO2_emissions'] > 0\n", + " net_CO2_emissions = round(sum(ebco2['total_lb_CO2_emissions']), 2)\n", "\n", - " x = ebco2['total_kg_CO2_emissions']\n", + " x = ebco2['total_lb_CO2_emissions']\n", " y = ebco2['Replaced_mode']\n", - " xLabel = 'CO2 Emissions (kg)'\n", + " xLabel = 'CO2 Emissions (lb)'\n", " yLabel = 'Replaced Mode'\n", " color = ebco2['boolean']\n", "\n", @@ -365,44 +272,13 @@ " CO2_impact(x,y,color,plot_title,xLabel,yLabel,file_name)\n", " alt_text = store_alt_text_bar(pd.DataFrame(x.values,y), file_name, plot_title)\n", "\n", - " filtered_taxi_data = data_eb[data_eb['Replaced_mode'] == \"Taxi/Uber/Lyft\"]\n", - " filtered_bus_data = data_eb[data_eb['Replaced_mode'] == \"Bus\"]\n", - " filtered_notravel_data = data_eb[data_eb['Replaced_mode'] == \"No Travel\"]\n", - " filtered_freeshuttle_data = data_eb[data_eb['Replaced_mode'] == \"Free Shuttle\"]\n", - " filtered_walk_data = data_eb[data_eb['Replaced_mode'] == \"Walk\"]\n", - " filtered_GasCarShared_data = data_eb[data_eb['Replaced_mode'] == \"Gas Car, with others\"]\n", - "\n", - " selected_columns = ['distance_miles', 'Replaced_mode_kg_CO2', 'Mode_confirm_kg_CO2', \"replaced_mode\", \"mode_confirm\"]\n", - "\n", - " print(\"With Default mapping:\")\n", - " print(\"\\n\")\n", - "\n", - " print(\"Walk Data:\")\n", - " print(str(filtered_walk_data[selected_columns].head()))\n", - " print(\"\\n\")\n", - "\n", - " print(\"No Travel Data:\")\n", - " print(str(filtered_notravel_data[selected_columns].head()))\n", - " print(\"\\n\")\n", - "\n", - " print(\"Gas Car Shared Data:\")\n", - " print(str(filtered_GasCarShared_data[selected_columns].head()))\n", - " print(\"\\n\")\n", - "\n", - " print(\"Free Shuttle Data:\")\n", - " print(str(filtered_freeshuttle_data[selected_columns].head()))\n", - " print(\"\\n\")\n", - "\n", - " print(\"Bus Data:\")\n", - " print(str(filtered_bus_data[selected_columns].head()))\n", - " print(\"\\n\")\n", - " \n", - " print(\"Taxi/Uber/Lyft Data:\")\n", - " print(str(filtered_taxi_data[selected_columns].head()))\n", - " print(\"\\n\")\n", - " \n", - " print(\"Total Kg CO2 Emissions\")\n", - " print(ebco2['total_kg_CO2_emissions'])\n", + " if (is_debug_mode == True):\n", + " ebco2_kg=data_eb.groupby('Replaced_mode').agg({'CO2_Impact(kg)': ['sum', 'mean']},)\n", + " ebco2_kg.columns = ['total_kg_CO2_emissions', 'average_kg_CO2_emission']\n", + " ebco2_kg = ebco2_kg.reset_index()\n", + " ebco2_kg = ebco2_kg.sort_values(by=['total_kg_CO2_emissions'], ascending=False)\n", + " \n", + " scaffolding.print_CO2_emission_calculations(data_eb, ebco2, ebco2_kg, dynamic_labels)\n", " except:\n", " generate_missing_plot(plot_title_no_quality,debug_df,file_name)\n", " alt_text = store_alt_text_missing(debug_df, file_name, plot_title_no_quality)" From efbd31f593045f4c3f581e2bdd45bfffed399cc1 Mon Sep 17 00:00:00 2001 From: iantei Date: Mon, 30 Oct 2023 20:16:36 -0700 Subject: [PATCH 50/62] Removed energy_impact computation for Dynamic config case. --- viz_scripts/generic_timeseries.ipynb | 34 ++++++++++++---------------- 1 file changed, 15 insertions(+), 19 deletions(-) diff --git a/viz_scripts/generic_timeseries.ipynb b/viz_scripts/generic_timeseries.ipynb index c9697df..a728771 100644 --- a/viz_scripts/generic_timeseries.ipynb +++ b/viz_scripts/generic_timeseries.ipynb @@ -168,11 +168,6 @@ " emissions['Mode_confirm_kg_CO2'] = emissions['Mode_confirm_kg_CO2'].fillna(0)\n", " emissions['distance'] = emissions['Mode_confirm_kg_CO2'].fillna(0)\n", "\n", - " # Sum daily energy for each user\n", - " energy = data.groupby(['user_id','date_time'], as_index=False)[['Mode_confirm_EI(kWH)', 'distance']].sum()\n", - " energy['Mode_confirm_EI(kWH)'] = energy['Mode_confirm_EI(kWH)'].fillna(0)\n", - " energy['distance'] = energy['Mode_confirm_EI(kWH)'].fillna(0)\n", - "\n", " # Add 7-day rolling avg smoothing to better see trends\n", " mode_counts['trip_count_smooth'] = mode_counts.groupby(['user_id','Mode_confirm'])['trip_count'].apply(lambda x: x.rolling(7,1).mean())\n", " mode_distance['distance_smooth'] = mode_distance.groupby(['user_id','Mode_confirm'])['distance'].apply(lambda x: x.rolling(7,1).mean())\n", @@ -274,22 +269,23 @@ "metadata": {}, "outputs": [], "source": [ - "plot_title_no_quality = 'Net Daily Energy (All Users, excluding air)'\n", - "file_name = \"ts_energy_user%s\"%file_suffix\n", + "if (len(dynamic_labels) == 0):\n", + " plot_title_no_quality = 'Net Daily Energy (All Users, excluding air)'\n", + " file_name = \"ts_energy_user%s\"%file_suffix\n", "\n", - "try:\n", - " # Energy per week across all users (net impact)\n", - " plot_data = energy.groupby(['date_time'], as_index=False)['Mode_confirm_EI(kWH)'].agg(['sum'])\n", - " plot_data = plot_data.merge(active_users, on='date_time')\n", - " plot_data['sum'] = plot_data['sum'] / plot_data['active_users']\n", + " try:\n", + " # Energy per week across all users (net impact)\n", + " plot_data = energy.groupby(['date_time'], as_index=False)['Mode_confirm_EI(kWH)'].agg(['sum'])\n", + " plot_data = plot_data.merge(active_users, on='date_time')\n", + " plot_data['sum'] = plot_data['sum'] / plot_data['active_users']\n", "\n", - " plot_title= plot_title_no_quality+\"\\n\"+quality_text\n", - " ylab = 'Energy (kWH/day/user)'\n", - " timeseries_plot(plot_data['date_time'], plot_data['sum'], plot_title, ylab, file_name)\n", - " alt_text = store_alt_text_timeseries(plot_data, file_name, plot_title)\n", - "except:\n", - " generate_missing_plot(plot_title_no_quality,debug_df,file_name)\n", - " alt_text = store_alt_text_missing(debug_df, file_name, plot_title_no_quality)" + " plot_title= plot_title_no_quality+\"\\n\"+quality_text\n", + " ylab = 'Energy (kWH/day/user)'\n", + " timeseries_plot(plot_data['date_time'], plot_data['sum'], plot_title, ylab, file_name)\n", + " alt_text = store_alt_text_timeseries(plot_data, file_name, plot_title)\n", + " except:\n", + " generate_missing_plot(plot_title_no_quality,debug_df,file_name)\n", + " alt_text = store_alt_text_missing(debug_df, file_name, plot_title_no_quality)" ] }, { From 3c8823ec67b4ad3016b1f024dd967ea430e16aa8 Mon Sep 17 00:00:00 2001 From: iantei Date: Mon, 30 Oct 2023 20:21:30 -0700 Subject: [PATCH 51/62] Update distace_miles column in the selected_columns in scaffolding.py for print_CO2_emission_calculations() function. --- viz_scripts/scaffolding.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/viz_scripts/scaffolding.py b/viz_scripts/scaffolding.py index 399fe3d..4171020 100644 --- a/viz_scripts/scaffolding.py +++ b/viz_scripts/scaffolding.py @@ -321,7 +321,6 @@ def energy_intensity(trip_df,mode_intensity_df,col): mode_intensity_df = dataframe with energy/cost/time factors col = the column for which we want to map the intensity """ - mode_intensity_df = mode_intensity_df.copy() mode_intensity_df[col] = mode_intensity_df['mode'] dic_ei_factor = dict(zip(mode_intensity_df[col],mode_intensity_df['energy_intensity_factor'])) @@ -446,7 +445,7 @@ def print_CO2_emission_calculations(data_eb, ebco2_lb, ebco2_kg, dynamic_labels) print("With Default mapping:") print("\n") - selected_columns = ['distance', 'Replaced_mode_kg_CO2', 'Replaced_mode_lb_CO2', 'Mode_confirm_kg_CO2','Mode_confirm_lb_CO2', "replaced_mode", "mode_confirm"] + selected_columns = ['distance','distance_miles', 'Replaced_mode_kg_CO2', 'Replaced_mode_lb_CO2', 'Mode_confirm_kg_CO2','Mode_confirm_lb_CO2', "replaced_mode", "mode_confirm"] print("Walk Data:") print(str(filtered_walk_data[selected_columns].head())) From adb525868552f110ce8ac6328816debc209c2756 Mon Sep 17 00:00:00 2001 From: iantei Date: Mon, 30 Oct 2023 21:51:32 -0700 Subject: [PATCH 52/62] Setting the default is_debug_mode flag to False. This will disable call to print the CO2 emission calculation. --- viz_scripts/energy_calculations.ipynb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/viz_scripts/energy_calculations.ipynb b/viz_scripts/energy_calculations.ipynb index fdece19..306eb4a 100644 --- a/viz_scripts/energy_calculations.ipynb +++ b/viz_scripts/energy_calculations.ipynb @@ -31,7 +31,7 @@ "study_type = \"program\"\n", "mode_of_interest = \"e-bike\"\n", "include_test_users = False\n", - "is_debug_mode = True\n", + "is_debug_mode = False\n", "dynamic_labels = {}" ] }, From b858b539df1d5b2fe8024750c6253d00121c1d80 Mon Sep 17 00:00:00 2001 From: iantei Date: Thu, 2 Nov 2023 14:35:03 -0700 Subject: [PATCH 53/62] Removed redundant if/else case for different label_type inside mapping_labels() function. Refactored compute_CO2_impact_dynamic() and compute_CO2_footprint_dynamic() to compute the CO2 emission in compute_CO2_footprint_dynamic() function and differentiate between Mode_confirm and Replaced_mode CO2 emission in compute_CO2_impact_dynamic(). --- viz_scripts/scaffolding.py | 45 +++++++++++--------------------------- 1 file changed, 13 insertions(+), 32 deletions(-) diff --git a/viz_scripts/scaffolding.py b/viz_scripts/scaffolding.py index 4171020..675d439 100644 --- a/viz_scripts/scaffolding.py +++ b/viz_scripts/scaffolding.py @@ -189,14 +189,7 @@ def translate_labels(labels): translation = translations.get(value) translation_mapping[value] = translation return defaultdict(lambda: 'Other', translation_mapping) - - if (label_type == "MODE"): - dic_mapping = translate_labels(dynamic_labels[label_type]) - elif (label_type == "REPLACED_MODE"): - dic_mapping = translate_labels(dynamic_labels[label_type]) - elif (label_type == "PURPOSE"): - dic_mapping = translate_labels(dynamic_labels[label_type]) - + dic_mapping = translate_labels(dynamic_labels[label_type]) return dic_mapping def load_viz_notebook_sensor_inference_data(year, month, program, include_test_users=False, sensed_algo_prefix="cleaned"): @@ -245,7 +238,7 @@ def add_energy_labels(expanded_ct, df_ei, dic_fuel, dynamic_labels): expanded_ct = energy_footprint_kWH(expanded_ct, 'distance_miles', 'Mode_confirm') if (len(dynamic_labels) > 0): - expanded_ct = compute_CO2_footprint_dynamic(expanded_ct, dynamic_labels) + expanded_ct = compute_CO2_footprint_dynamic(expanded_ct, dynamic_labels, 'Mode_confirm') else: expanded_ct = CO2_footprint_default(expanded_ct, 'distance_miles', 'Mode_confirm') return expanded_ct @@ -379,7 +372,6 @@ def CO2_footprint_default(df, distance, col): return df def CO2_impact_default(df,distance): - conversion_lb_to_kilogram = 0.453592 # 1 lb = 0.453592 KG if 'Mode_confirm_lb_CO2' not in df.columns: print("Mode confirm footprint not found, computing before impact") df = CO2_footprint_default(df, distance, "Mode_confirm") @@ -390,39 +382,28 @@ def CO2_impact_default(df,distance): df['CO2_Impact(kg)'] = round((df['Replaced_mode_kg_CO2'] - df['Mode_confirm_kg_CO2']), 3) return df -def compute_CO2_footprint_dynamic(expanded_ct, dynamic_labels): +def compute_CO2_footprint_dynamic(expanded_ct, dynamic_labels, label_type): conversion_meter_to_kilometer = 0.001 conversion_kilogram_to_lbs = 2.20462 conversion_kilometer_to_mile = 0.621371 dic_mode_kgCO2PerKm = {mode["value"]: mode["kgCo2PerKm"] for mode in dynamic_labels["MODE"]} - if "mode_confirm" in expanded_ct.columns: - expanded_ct['Mode_confirm_kg_CO2'] = ((expanded_ct['distance'] * conversion_meter_to_kilometer )) * (expanded_ct['mode_confirm'].map(dic_mode_kgCO2PerKm)) - expanded_ct['Mode_confirm_kg_CO2'] = expanded_ct['Mode_confirm_kg_CO2'].fillna(0) - expanded_ct['Mode_confirm_lb_CO2'] = expanded_ct['Mode_confirm_kg_CO2'] * conversion_kilogram_to_lbs + if label_type.lower() in expanded_ct.columns: + # The expanded_ct['Mode_confirm_kg_CO2'] is CO2 emission in terms of [distance in kms * CO2 emission in kgCO2 per km = kg of CO2] + expanded_ct[label_type+'_kg_CO2'] = ((expanded_ct['distance'] * conversion_meter_to_kilometer )) * (expanded_ct[label_type.lower()].map(dic_mode_kgCO2PerKm)) + expanded_ct[label_type+'_kg_CO2'] = expanded_ct[label_type+'_kg_CO2'].fillna(0) + expanded_ct[label_type+'_lb_CO2'] = expanded_ct[label_type+'_kg_CO2'] * conversion_kilogram_to_lbs return expanded_ct def compute_CO2_impact_dynamic(expanded_ct, dynamic_labels): - conversion_meter_to_kilometer = 0.001 # 1 meter = 0.001 kilometer - conversion_kilogram_to_lbs = 2.20462 - conversion_kilometer_to_mile = 0.621371 - - dic_mode_kgCO2PerKm = {mode["value"]: mode["kgCo2PerKm"] for mode in dynamic_labels["MODE"]} - - if "mode_confirm" in expanded_ct.columns: - # The expanded_ct['Mode_confirm_kg_CO2'] is CO2 emission in terms of [distance in kms * CO2 emission in kgCO2 per km = kg of CO2] - expanded_ct['Mode_confirm_kg_CO2'] = ((expanded_ct['distance'] * conversion_meter_to_kilometer )) * (expanded_ct['mode_confirm'].map(dic_mode_kgCO2PerKm)) - expanded_ct['Mode_confirm_kg_CO2'] = expanded_ct['Mode_confirm_kg_CO2'].fillna(0) - expanded_ct['Mode_confirm_lb_CO2'] = expanded_ct['Mode_confirm_kg_CO2'] * conversion_kilogram_to_lbs - - if "replaced_mode" in expanded_ct.columns: - expanded_ct['Replaced_mode_kg_CO2'] = (expanded_ct['distance'] * conversion_meter_to_kilometer ) * (expanded_ct['replaced_mode'].map(dic_mode_kgCO2PerKm)) - expanded_ct['Replaced_mode_kg_CO2'] = expanded_ct['Replaced_mode_kg_CO2'].fillna(0) - expanded_ct['Replaced_mode_lb_CO2'] = expanded_ct['Replaced_mode_kg_CO2'] * conversion_kilogram_to_lbs + if 'Mode_confirm_kg_CO2' not in expanded_ct.columns: + print("Mode confirm footprint not found, computing before impact.") + compute_CO2_footprint_dynamic(expanded_ct, dynamic_labels, "Mode_confirm") + compute_CO2_footprint_dynamic(expanded_ct, dynamic_labels, "Replaced_mode") - expanded_ct['CO2_Impact(kg)'] = round ((expanded_ct['Replaced_mode_kg_CO2'] - expanded_ct['Mode_confirm_kg_CO2']), 3) + expanded_ct['CO2_Impact(kg)'] = round ((expanded_ct['Replaced_mode_kg_CO2'] - expanded_ct['Mode_confirm_kg_CO2']), 3) expanded_ct['CO2_Impact(lb)'] = round ((expanded_ct['Replaced_mode_lb_CO2'] - expanded_ct['Mode_confirm_lb_CO2']), 3) return expanded_ct From 3ee1bbb6f2ba35a65c55b574e34d78e45b99ab15 Mon Sep 17 00:00:00 2001 From: iantei Date: Thu, 2 Nov 2023 14:39:28 -0700 Subject: [PATCH 54/62] Energy_calculations.ipynb added justification for computation of total_lb_CO2 emission in Dynamic Label case, and total_kg_CO2_emission in default config case. --- viz_scripts/energy_calculations.ipynb | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/viz_scripts/energy_calculations.ipynb b/viz_scripts/energy_calculations.ipynb index 306eb4a..d9e55ea 100644 --- a/viz_scripts/energy_calculations.ipynb +++ b/viz_scripts/energy_calculations.ipynb @@ -215,7 +215,7 @@ "metadata": {}, "outputs": [], "source": [ - "# Case: label_options is present on the config file\n", + "# Case: label_options is present on the config file\n", "if len(dynamic_labels) > 0:\n", " plot_title_no_quality=f\"Sketch of Total Kilograms of CO2 Emissions of {mode_of_interest} trips\"\n", " file_name =f'sketch_CO2impact_{mode_of_interest}%s' % file_suffix\n", @@ -239,6 +239,9 @@ " alt_text = store_alt_text_bar(pd.DataFrame(x.values,y), file_name, plot_title)\n", " \n", " if (is_debug_mode == True):\n", + " # Currently, we're only displaying the CO2 emission in kg for Dynamic Config case in the chart.\n", + " # Computing CO2 emission in Imperial system i.e. in lb, so it's easy for comparision and validate the changes from kg to lb is justified.\n", + " # We're displaying both total_lb_CO2_emission and total_kg_CO2_emission calculation in debugging mode.\n", " ebco2_lb=data_eb.groupby('Replaced_mode').agg({'CO2_Impact(lb)': ['sum', 'mean']},)\n", " ebco2_lb.columns = ['total_lb_CO2_emissions', 'average_lb_CO2_emission']\n", " ebco2_lb = ebco2_lb.reset_index()\n", @@ -273,6 +276,9 @@ " alt_text = store_alt_text_bar(pd.DataFrame(x.values,y), file_name, plot_title)\n", "\n", " if (is_debug_mode == True):\n", + " # Currently, we're only displaying the CO2 emission in lb for Default Mapping case.\n", + " # Computing CO2 emission in Imperial system i.e. in kg, so it's easy for comparision and validate the changes from lb to kg is justified.\n", + " # We're displaying both total_lb_CO2_emission and total_kg_CO2_emission calculatoin in debugging mode.\n", " ebco2_kg=data_eb.groupby('Replaced_mode').agg({'CO2_Impact(kg)': ['sum', 'mean']},)\n", " ebco2_kg.columns = ['total_kg_CO2_emissions', 'average_kg_CO2_emission']\n", " ebco2_kg = ebco2_kg.reset_index()\n", From 5cbce255b751850d1b63b99f79e2ee7c518fc918 Mon Sep 17 00:00:00 2001 From: iantei Date: Tue, 7 Nov 2023 10:25:38 -0700 Subject: [PATCH 55/62] Refactor both energy_calculations and generic_timeseries notebook to avoid code duplication for dynamic labels and default mapping cases. --- viz_scripts/energy_calculations.ipynb | 86 ++++++----------- viz_scripts/generic_timeseries.ipynb | 128 +++++++++----------------- 2 files changed, 69 insertions(+), 145 deletions(-) diff --git a/viz_scripts/energy_calculations.ipynb b/viz_scripts/energy_calculations.ipynb index d9e55ea..5e90ee5 100644 --- a/viz_scripts/energy_calculations.ipynb +++ b/viz_scripts/energy_calculations.ipynb @@ -215,79 +215,47 @@ "metadata": {}, "outputs": [], "source": [ - "# Case: label_options is present on the config file\n", - "if len(dynamic_labels) > 0:\n", - " plot_title_no_quality=f\"Sketch of Total Kilograms of CO2 Emissions of {mode_of_interest} trips\"\n", - " file_name =f'sketch_CO2impact_{mode_of_interest}%s' % file_suffix\n", + "def ebco2_compute(data_eb, weight_unit):\n", + " ebco2=data_eb.groupby('Replaced_mode').agg({f'CO2_Impact({weight_unit})': ['sum', 'mean']},)\n", + " ebco2.columns = [f'total_{weight_unit}_CO2_emissions', f'average_{weight_unit}_CO2_emission']\n", + " ebco2 = ebco2.reset_index()\n", + " ebco2 = ebco2.sort_values(by=[f'total_{weight_unit}_CO2_emissions'], ascending=False)\n", + " return ebco2\n", "\n", + "def display_total_CO2_emissions(data_eb, weight_unit, weight_unit_debug, debug_df, mode_of_interest):\n", " try:\n", - " ebco2=data_eb.groupby('Replaced_mode').agg({'CO2_Impact(kg)': ['sum', 'mean']},)\n", - " ebco2.columns = ['total_kg_CO2_emissions', 'average_kg_CO2_emission']\n", - " ebco2 = ebco2.reset_index()\n", - " ebco2 = ebco2.sort_values(by=['total_kg_CO2_emissions'], ascending=False)\n", - " ebco2['boolean'] = ebco2['total_kg_CO2_emissions'] > 0\n", - " net_CO2_emissions = round(sum(ebco2['total_kg_CO2_emissions']), 2)\n", + " plot_title_no_quality=f\"Sketch of Total {weight_unit} of CO2 Emissions of {mode_of_interest} trips\"\n", + " file_name =f'sketch_CO2impact_{mode_of_interest}%s' % file_suffix\n", + " \n", + " ebco2 = ebco2_compute(data_eb, weight_unit)\n", + " net_CO2_emissions = round(sum(ebco2[f'total_{weight_unit}_CO2_emissions']), 2)\n", + " ebco2['boolean'] = ebco2[f'total_{weight_unit}_CO2_emissions'] > 0\n", "\n", - " x = ebco2['total_kg_CO2_emissions']\n", + " x = ebco2[f'total_{weight_unit}_CO2_emissions']\n", " y = ebco2['Replaced_mode']\n", - " xLabel = 'CO2 Emissions (kg)'\n", + " xLabel = f'CO2 Emissions ({weight_unit})'\n", " yLabel = 'Replaced Mode'\n", " color = ebco2['boolean']\n", "\n", - " plot_title= plot_title_no_quality+f\"\\n Contribution by replaced mode towards a total of {net_CO2_emissions}(kg CO2 Emissions )\\n\"+quality_text\n", + " plot_title= plot_title_no_quality+f\"\\n Contribution by replaced mode towards a total of {net_CO2_emissions}({weight_unit} CO2 Emissions )\\n\"+quality_text\n", " CO2_impact(x,y,color,plot_title, xLabel,yLabel,file_name)\n", " alt_text = store_alt_text_bar(pd.DataFrame(x.values,y), file_name, plot_title)\n", - " \n", + "\n", + " # For debugging: Computing and displaying Total CO2 emissions in both Metric(kg) and Imperial(lb) unit system for easy comparison.\n", " if (is_debug_mode == True):\n", - " # Currently, we're only displaying the CO2 emission in kg for Dynamic Config case in the chart.\n", - " # Computing CO2 emission in Imperial system i.e. in lb, so it's easy for comparision and validate the changes from kg to lb is justified.\n", - " # We're displaying both total_lb_CO2_emission and total_kg_CO2_emission calculation in debugging mode.\n", - " ebco2_lb=data_eb.groupby('Replaced_mode').agg({'CO2_Impact(lb)': ['sum', 'mean']},)\n", - " ebco2_lb.columns = ['total_lb_CO2_emissions', 'average_lb_CO2_emission']\n", - " ebco2_lb = ebco2_lb.reset_index()\n", - " ebco2_lb = ebco2_lb.sort_values(by=['total_lb_CO2_emissions'], ascending=False)\n", - " \n", - " scaffolding.print_CO2_emission_calculations(data_eb, ebco2_lb, ebco2, dynamic_labels)\n", - " \n", + " ebco2_debug = ebco2_compute(data_eb, weight_unit_debug)\n", + " if (weight_unit == 'kg'):\n", + " scaffolding.print_CO2_emission_calculations(data_eb, ebco2_debug, ebco2, dynamic_labels)\n", + " else:\n", + " scaffolding.print_CO2_emission_calculations(data_eb, ebco2, ebco2_debug, dynamic_labels)\n", " except:\n", " generate_missing_plot(plot_title_no_quality,debug_df,file_name)\n", " alt_text = store_alt_text_missing(debug_df, file_name, plot_title_no_quality)\n", - "# case: when label_options is absent from the config file\n", - "else: \n", - " plot_title_no_quality=f\"Sketch of Total pounds of CO2 Emissions of {mode_of_interest} trips\"\n", - " file_name =f'sketch_CO2impact_{mode_of_interest}%s' % file_suffix\n", - "\n", - " try:\n", - " ebco2=data_eb.groupby('Replaced_mode').agg({'CO2_Impact(lb)': ['sum', 'mean']},)\n", - " ebco2.columns = ['total_lb_CO2_emissions', 'average_lb_CO2_emission']\n", - " ebco2 = ebco2.reset_index()\n", - " ebco2 = ebco2.sort_values(by=['total_lb_CO2_emissions'], ascending=False)\n", - " ebco2['boolean'] = ebco2['total_lb_CO2_emissions'] > 0\n", - " net_CO2_emissions = round(sum(ebco2['total_lb_CO2_emissions']), 2)\n", - "\n", - " x = ebco2['total_lb_CO2_emissions']\n", - " y = ebco2['Replaced_mode']\n", - " xLabel = 'CO2 Emissions (lb)'\n", - " yLabel = 'Replaced Mode'\n", - " color = ebco2['boolean']\n", - "\n", - " plot_title= plot_title_no_quality+f\"\\n Contribution by replaced mode towards a total of {net_CO2_emissions}(lb CO2 Emissions )\\n\"+quality_text\n", - " CO2_impact(x,y,color,plot_title,xLabel,yLabel,file_name)\n", - " alt_text = store_alt_text_bar(pd.DataFrame(x.values,y), file_name, plot_title)\n", "\n", - " if (is_debug_mode == True):\n", - " # Currently, we're only displaying the CO2 emission in lb for Default Mapping case.\n", - " # Computing CO2 emission in Imperial system i.e. in kg, so it's easy for comparision and validate the changes from lb to kg is justified.\n", - " # We're displaying both total_lb_CO2_emission and total_kg_CO2_emission calculatoin in debugging mode.\n", - " ebco2_kg=data_eb.groupby('Replaced_mode').agg({'CO2_Impact(kg)': ['sum', 'mean']},)\n", - " ebco2_kg.columns = ['total_kg_CO2_emissions', 'average_kg_CO2_emission']\n", - " ebco2_kg = ebco2_kg.reset_index()\n", - " ebco2_kg = ebco2_kg.sort_values(by=['total_kg_CO2_emissions'], ascending=False)\n", - " \n", - " scaffolding.print_CO2_emission_calculations(data_eb, ebco2, ebco2_kg, dynamic_labels)\n", - " except:\n", - " generate_missing_plot(plot_title_no_quality,debug_df,file_name)\n", - " alt_text = store_alt_text_missing(debug_df, file_name, plot_title_no_quality)" + "if (len(dynamic_labels) > 0):\n", + " display_total_CO2_emissions(data_eb,'kg','lb', debug_df, mode_of_interest)\n", + "else:\n", + " display_total_CO2_emissions(data_eb,'lb','kg', debug_df, mode_of_interest)" ] } ], diff --git a/viz_scripts/generic_timeseries.ipynb b/viz_scripts/generic_timeseries.ipynb index a728771..f311c2c 100644 --- a/viz_scripts/generic_timeseries.ipynb +++ b/viz_scripts/generic_timeseries.ipynb @@ -147,6 +147,25 @@ }, "outputs": [], "source": [ + "def compute_daily_metrics(data, weight_unit, distance_unit):\n", + " mode_distance = data.groupby(['user_id','date_time','Mode_confirm'], as_index=False)[[f'{distance_unit}']].sum()\n", + " mode_distance.rename(columns={'sum':f'{distance_unit}'}, inplace=True)\n", + " mode_distance[f'{distance_unit}'] = mode_distance[f'{distance_unit}'].fillna(0)\n", + " \n", + " emissions = data.groupby(['user_id','date_time'], as_index=False)[[f'Mode_confirm_{weight_unit}_CO2', f'{distance_unit}']].sum()\n", + " emissions[f'Mode_confirm_{weight_unit}_CO2'] = emissions[f'Mode_confirm_{weight_unit}_CO2'].fillna(0)\n", + " emissions[f'{distance_unit}'] = emissions[f'Mode_confirm_{weight_unit}_CO2'].fillna(0)\n", + " \n", + " energy = data.groupby(['user_id','date_time'], as_index=False)[['Mode_confirm_EI(kWH)', f'{distance_unit}']].sum()\n", + " energy['Mode_confirm_EI(kWH)'] = energy['Mode_confirm_EI(kWH)'].fillna(0)\n", + " energy[f'{distance_unit}'] = energy['Mode_confirm_EI(kWH)'].fillna(0)\n", + " \n", + " mode_counts['trip_count_smooth'] = mode_counts.groupby(['user_id','Mode_confirm'])['trip_count'].apply(lambda x: x.rolling(7,1).mean())\n", + " mode_distance[f'{distance_unit}smooth'] = mode_distance.groupby(['user_id','Mode_confirm'])[f'{distance_unit}'].apply(lambda x: x.rolling(7,1).mean())\n", + " emissions[f'{distance_unit}_smooth'] = emissions.groupby(['user_id'])[f'{distance_unit}'].apply(lambda x: x.rolling(7,1).mean())\n", + " energy[f'{distance_unit}_smooth'] = energy.groupby(['user_id'])[f'{distance_unit}'].apply(lambda x: x.rolling(7,1).mean())\n", + " return mode_counts, mode_distance, emissions, energy\n", + "\n", "if len(expanded_ct) > 0:\n", " # Get the count of unique users that were active on each given date\n", " active_users = pd.DataFrame(data.groupby(['date_time'], as_index=False)['user_id'].nunique())\n", @@ -158,42 +177,9 @@ " mode_counts.rename(columns={'size':'trip_count'}, inplace=True)\n", "\n", " if (len(dynamic_labels)) > 0:\n", - " # Sum daily distance traveled for each mode\n", - " mode_distance = data.groupby(['user_id','date_time','Mode_confirm'], as_index=False)[['distance']].sum()\n", - " mode_distance.rename(columns={'sum':'distance'}, inplace=True)\n", - " mode_distance['distance'] = mode_distance['distance'].fillna(0)\n", - "\n", - " # Sum daily emissions for each user\n", - " emissions = data.groupby(['user_id','date_time'], as_index=False)[['Mode_confirm_kg_CO2', 'distance']].sum()\n", - " emissions['Mode_confirm_kg_CO2'] = emissions['Mode_confirm_kg_CO2'].fillna(0)\n", - " emissions['distance'] = emissions['Mode_confirm_kg_CO2'].fillna(0)\n", - "\n", - " # Add 7-day rolling avg smoothing to better see trends\n", - " mode_counts['trip_count_smooth'] = mode_counts.groupby(['user_id','Mode_confirm'])['trip_count'].apply(lambda x: x.rolling(7,1).mean())\n", - " mode_distance['distance_smooth'] = mode_distance.groupby(['user_id','Mode_confirm'])['distance'].apply(lambda x: x.rolling(7,1).mean())\n", - " emissions['distance_smooth'] = emissions.groupby(['user_id'])['distance'].apply(lambda x: x.rolling(7,1).mean())\n", - " energy['distance_smooth'] = energy.groupby(['user_id'])['distance'].apply(lambda x: x.rolling(7,1).mean())\n", + " mode_counts, mode_distance, emissions, energy = compute_daily_metrics(data, weight_unit = 'kg', distance_unit = 'distance')\n", " else:\n", - " # Sum daily distance traveled for each mode\n", - " mode_distance = data.groupby(['user_id','date_time','Mode_confirm'], as_index=False)[['distance_miles']].sum()\n", - " mode_distance.rename(columns={'sum':'distance_miles'}, inplace=True)\n", - " mode_distance['distance_miles'] = mode_distance['distance_miles'].fillna(0)\n", - "\n", - " # Sum daily emissions for each user\n", - " emissions = data.groupby(['user_id','date_time'], as_index=False)[['Mode_confirm_lb_CO2', 'distance_miles']].sum()\n", - " emissions['Mode_confirm_lb_CO2'] = emissions['Mode_confirm_lb_CO2'].fillna(0)\n", - " emissions['distance_miles'] = emissions['Mode_confirm_lb_CO2'].fillna(0)\n", - "\n", - " # Sum daily energy for each user\n", - " energy = data.groupby(['user_id','date_time'], as_index=False)[['Mode_confirm_EI(kWH)', 'distance_miles']].sum()\n", - " energy['Mode_confirm_EI(kWH)'] = energy['Mode_confirm_EI(kWH)'].fillna(0)\n", - " energy['distance_miles'] = energy['Mode_confirm_EI(kWH)'].fillna(0)\n", - "\n", - " # Add 7-day rolling avg smoothing to better see trends\n", - " mode_counts['trip_count_smooth'] = mode_counts.groupby(['user_id','Mode_confirm'])['trip_count'].apply(lambda x: x.rolling(7,1).mean())\n", - " mode_distance['distance_miles_smooth'] = mode_distance.groupby(['user_id','Mode_confirm'])['distance_miles'].apply(lambda x: x.rolling(7,1).mean())\n", - " emissions['distance_miles_smooth'] = emissions.groupby(['user_id'])['distance_miles'].apply(lambda x: x.rolling(7,1).mean())\n", - " energy['distance_miles_smooth'] = energy.groupby(['user_id'])['distance_miles'].apply(lambda x: x.rolling(7,1).mean())" + " mode_counts, mode_distance, emissions, energy = compute_daily_metrics( data, weight_unit = 'lb', distance_unit = 'distance_miles')" ] }, { @@ -216,44 +202,29 @@ "metadata": {}, "outputs": [], "source": [ - "if (len(dynamic_labels)) > 0:\n", - " plot_title_no_quality = 'Net Daily Emissions (All Users, excluding air)'\n", - " file_name = \"ts_emissions_user%s\"%file_suffix\n", + "plot_title_no_quality = 'Net Daily Emissions (All Users, excluding air)'\n", + "file_name = \"ts_emissions_user%s\"%file_suffix\n", "\n", + "def emission_per_week(emissions, unit, debug_df, active_users):\n", " try:\n", " # Emissions per week across all users (net impact)\n", - " plot_data = emissions.groupby(['date_time'], as_index=False)['Mode_confirm_kg_CO2'].agg(['sum'])\n", + " plot_data = emissions.groupby(['date_time'], as_index=False)[f'Mode_confirm_{unit}_CO2'].agg(['sum'])\n", " total_sum = plot_data['sum'].sum()\n", - "\n", " plot_data = plot_data.merge(active_users, on='date_time')\n", " plot_data['sum'] = plot_data['sum'] / plot_data['active_users']\n", "\n", " plot_title= plot_title_no_quality+\"\\n\"+quality_text\n", - " ylab = 'Emissions (kg CO2/day/user)'\n", + " ylab = f'Emissions ({unit} CO2/day/user)'\n", " timeseries_plot(plot_data['date_time'], plot_data['sum'], plot_title, ylab, file_name)\n", " alt_text = store_alt_text_timeseries(plot_data, file_name, plot_title)\n", " except:\n", " generate_missing_plot(plot_title_no_quality,debug_df,file_name)\n", " alt_text = store_alt_text_missing(debug_df, file_name, plot_title_no_quality)\n", - "else:\n", - " plot_title_no_quality = 'Net Daily Emissions (All Users, excluding air)'\n", - " file_name = \"ts_emissions_user%s\"%file_suffix\n", - "\n", - " try:\n", - " # Emissions per week across all users (net impact)\n", - " plot_data = emissions.groupby(['date_time'], as_index=False)['Mode_confirm_lb_CO2'].agg(['sum'])\n", - " total_sum = plot_data['sum'].sum()\n", "\n", - " plot_data = plot_data.merge(active_users, on='date_time')\n", - " plot_data['sum'] = plot_data['sum'] / plot_data['active_users']\n", - "\n", - " plot_title= plot_title_no_quality+\"\\n\"+quality_text\n", - " ylab = 'Emissions (lb CO2/day/user)'\n", - " timeseries_plot(plot_data['date_time'], plot_data['sum'], plot_title, ylab, file_name)\n", - " alt_text = store_alt_text_timeseries(plot_data, file_name, plot_title)\n", - " except:\n", - " generate_missing_plot(plot_title_no_quality,debug_df,file_name)\n", - " alt_text = store_alt_text_missing(debug_df, file_name, plot_title_no_quality)" + "if (len(dynamic_labels)) > 0:\n", + " emission_per_week(emissions, 'kg', debug_df, active_users)\n", + "else:\n", + " emission_per_week(emissions, 'lb', debug_df, active_users)" ] }, { @@ -301,42 +272,28 @@ "metadata": {}, "outputs": [], "source": [ - "if (len(dynamic_labels) > 0):\n", - " plot_title_no_quality = 'Average Daily Emission Rate (All Users, excluding air)'\n", - " file_name = \"ts_emissions_vmt%s\"%file_suffix\n", - "\n", + "plot_title_no_quality = 'Average Daily Emission Rate (All Users, excluding air)'\n", + "file_name = \"ts_emissions_vmt%s\"%file_suffix\n", + "def emissions_per_distance_day(emissions, weight_unit, distance_unit, distance_unit_smooth, debug_df):\n", " try:\n", " # Emissions per kilometer per day across all users (travel efficiency)\n", " # Note that the energy plot will be identical to this one since scale factor is divided out\n", - " emissions['CO2_per_km'] = emissions['Mode_confirm_kg_CO2'] / emissions['distance_smooth']\n", - " emissions['CO2_per_km'] = emissions['CO2_per_km'].fillna(0)\n", - " plot_data = emissions.groupby(['date_time'])['CO2_per_km'].agg(['mean']).reset_index()\n", + " emissions[f'CO2_per_{distance_unit}'] = emissions[f'Mode_confirm_{weight_unit}_CO2'] / emissions[f'{distance_unit_smooth}']\n", + " emissions[f'CO2_per_{distance_unit}'] = emissions[f'CO2_per_{distance_unit}'].fillna(0)\n", + " plot_data = emissions.groupby(['date_time'])[f'CO2_per_{distance_unit}'].agg(['mean']).reset_index()\n", "\n", " plot_title= plot_title_no_quality+\"\\n\"+quality_text\n", - " ylab = 'Emissions (kg CO2/km/day)'\n", + " ylab = f'Emissions ({weight_unit} CO2/{distance_unit}/day)'\n", " timeseries_plot(plot_data['date_time'], plot_data['mean'], plot_title, ylab, file_name)\n", " alt_text = store_alt_text_timeseries(plot_data, file_name, plot_title)\n", " except:\n", " generate_missing_plot(plot_title_no_quality,debug_df,file_name)\n", " alt_text = store_alt_text_missing(debug_df, file_name, plot_title_no_quality)\n", - "else:\n", - " plot_title_no_quality = 'Average Daily Emission Rate (All Users, excluding air)'\n", - " file_name = \"ts_emissions_vmt%s\"%file_suffix\n", "\n", - " try:\n", - " # Emissions per kilometer per day across all users (travel efficiency)\n", - " # Note that the energy plot will be identical to this one since scale factor is divided out\n", - " emissions['CO2_per_mile'] = emissions['Mode_confirm_lb_CO2'] / emissions['distance_miles_smooth']\n", - " emissions['CO2_per_mile'] = emissions['CO2_per_mile'].fillna(0)\n", - " plot_data = emissions.groupby(['date_time'])['CO2_per_mile'].agg(['mean']).reset_index()\n", - "\n", - " plot_title= plot_title_no_quality+\"\\n\"+quality_text\n", - " ylab = 'Emissions (lb CO2/mile/day)'\n", - " timeseries_plot(plot_data['date_time'], plot_data['mean'], plot_title, ylab, file_name)\n", - " alt_text = store_alt_text_timeseries(plot_data, file_name, plot_title)\n", - " except:\n", - " generate_missing_plot(plot_title_no_quality,debug_df,file_name)\n", - " alt_text = store_alt_text_missing(debug_df, file_name, plot_title_no_quality)" + "if (len(dynamic_labels) > 0):\n", + " emissions_per_distance_day(emissions, 'kg', 'km', 'distance_smooth', debug_df)\n", + "else:\n", + " emissions_per_distance_day(emissions, 'lb', 'mile','distance_miles_smooth', debug_df)" ] }, { @@ -406,8 +363,7 @@ " plot_data = plot_data[~plot_data['Mode_confirm'].isin(['Not a Trip','Other'])]\n", "\n", " if (len(dynamic_labels) > 0):\n", - " label_type = \"MODE\"\n", - " dic_mode_mapping = scaffolding.mapping_labels(dynamic_labels, label_type)\n", + " dic_mode_mapping = scaffolding.mapping_labels(dynamic_labels, label_type = \"MODE\")\n", " plot_data.Mode_confirm = pd.Categorical(plot_data.Mode_confirm, ordered=True, categories=np.unique(list(dic_mode_mapping.values()))) \n", " else:\n", " plot_data.Mode_confirm = pd.Categorical(plot_data.Mode_confirm, ordered=True, categories=np.unique(list(dic_re.values())))\n", From dd645138020f49c4782ef8e52515c16f1297b7d1 Mon Sep 17 00:00:00 2001 From: iantei Date: Tue, 7 Nov 2023 10:39:04 -0700 Subject: [PATCH 56/62] Stored the returned expanded_ct from compute_CO2_footprint_dynamic(). --- viz_scripts/scaffolding.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/viz_scripts/scaffolding.py b/viz_scripts/scaffolding.py index 675d439..cb2276b 100644 --- a/viz_scripts/scaffolding.py +++ b/viz_scripts/scaffolding.py @@ -400,8 +400,8 @@ def compute_CO2_footprint_dynamic(expanded_ct, dynamic_labels, label_type): def compute_CO2_impact_dynamic(expanded_ct, dynamic_labels): if 'Mode_confirm_kg_CO2' not in expanded_ct.columns: print("Mode confirm footprint not found, computing before impact.") - compute_CO2_footprint_dynamic(expanded_ct, dynamic_labels, "Mode_confirm") - compute_CO2_footprint_dynamic(expanded_ct, dynamic_labels, "Replaced_mode") + expanded_ct = compute_CO2_footprint_dynamic(expanded_ct, dynamic_labels, "Mode_confirm") + expanded_ct = compute_CO2_footprint_dynamic(expanded_ct, dynamic_labels, "Replaced_mode") expanded_ct['CO2_Impact(kg)'] = round ((expanded_ct['Replaced_mode_kg_CO2'] - expanded_ct['Mode_confirm_kg_CO2']), 3) expanded_ct['CO2_Impact(lb)'] = round ((expanded_ct['Replaced_mode_lb_CO2'] - expanded_ct['Mode_confirm_lb_CO2']), 3) From a72ac993ef041a2f7bacaf429f560a3ea4917c86 Mon Sep 17 00:00:00 2001 From: iantei Date: Tue, 7 Nov 2023 12:53:28 -0700 Subject: [PATCH 57/62] Fixed try-catch block to handle case where emissions is not defined. Re-added the removed comments. --- viz_scripts/energy_calculations.ipynb | 61 +++++++++++----------- viz_scripts/generic_timeseries.ipynb | 75 ++++++++++++++------------- 2 files changed, 70 insertions(+), 66 deletions(-) diff --git a/viz_scripts/energy_calculations.ipynb b/viz_scripts/energy_calculations.ipynb index 5e90ee5..84ae325 100644 --- a/viz_scripts/energy_calculations.ipynb +++ b/viz_scripts/energy_calculations.ipynb @@ -222,40 +222,39 @@ " ebco2 = ebco2.sort_values(by=[f'total_{weight_unit}_CO2_emissions'], ascending=False)\n", " return ebco2\n", "\n", - "def display_total_CO2_emissions(data_eb, weight_unit, weight_unit_debug, debug_df, mode_of_interest):\n", - " try:\n", - " plot_title_no_quality=f\"Sketch of Total {weight_unit} of CO2 Emissions of {mode_of_interest} trips\"\n", - " file_name =f'sketch_CO2impact_{mode_of_interest}%s' % file_suffix\n", - " \n", - " ebco2 = ebco2_compute(data_eb, weight_unit)\n", - " net_CO2_emissions = round(sum(ebco2[f'total_{weight_unit}_CO2_emissions']), 2)\n", - " ebco2['boolean'] = ebco2[f'total_{weight_unit}_CO2_emissions'] > 0\n", - "\n", - " x = ebco2[f'total_{weight_unit}_CO2_emissions']\n", - " y = ebco2['Replaced_mode']\n", - " xLabel = f'CO2 Emissions ({weight_unit})'\n", - " yLabel = 'Replaced Mode'\n", - " color = ebco2['boolean']\n", + "def display_total_CO2_emissions(data_eb, weight_unit, weight_unit_debug, mode_of_interest):\n", + " plot_title_no_quality=f\"Sketch of Total {weight_unit} of CO2 Emissions of {mode_of_interest} trips\"\n", + " file_name =f'sketch_CO2impact_{mode_of_interest}%s' % file_suffix\n", + " \n", + " ebco2 = ebco2_compute(data_eb, weight_unit)\n", + " net_CO2_emissions = round(sum(ebco2[f'total_{weight_unit}_CO2_emissions']), 2)\n", + " ebco2['boolean'] = ebco2[f'total_{weight_unit}_CO2_emissions'] > 0\n", + " x = ebco2[f'total_{weight_unit}_CO2_emissions']\n", + " y = ebco2['Replaced_mode']\n", + " xLabel = f'CO2 Emissions ({weight_unit})'\n", + " yLabel = 'Replaced Mode'\n", + " color = ebco2['boolean']\n", "\n", - " plot_title= plot_title_no_quality+f\"\\n Contribution by replaced mode towards a total of {net_CO2_emissions}({weight_unit} CO2 Emissions )\\n\"+quality_text\n", - " CO2_impact(x,y,color,plot_title, xLabel,yLabel,file_name)\n", - " alt_text = store_alt_text_bar(pd.DataFrame(x.values,y), file_name, plot_title)\n", + " plot_title= plot_title_no_quality+f\"\\n Contribution by replaced mode towards a total of {net_CO2_emissions}({weight_unit} CO2 Emissions )\\n\"+quality_text\n", + " CO2_impact(x,y,color,plot_title, xLabel,yLabel,file_name)\n", + " alt_text = store_alt_text_bar(pd.DataFrame(x.values,y), file_name, plot_title)\n", "\n", - " # For debugging: Computing and displaying Total CO2 emissions in both Metric(kg) and Imperial(lb) unit system for easy comparison.\n", - " if (is_debug_mode == True):\n", - " ebco2_debug = ebco2_compute(data_eb, weight_unit_debug)\n", - " if (weight_unit == 'kg'):\n", - " scaffolding.print_CO2_emission_calculations(data_eb, ebco2_debug, ebco2, dynamic_labels)\n", - " else:\n", - " scaffolding.print_CO2_emission_calculations(data_eb, ebco2, ebco2_debug, dynamic_labels)\n", - " except:\n", - " generate_missing_plot(plot_title_no_quality,debug_df,file_name)\n", - " alt_text = store_alt_text_missing(debug_df, file_name, plot_title_no_quality)\n", + " # For debugging: Computing and displaying Total CO2 emissions in both Metric(kg) and Imperial(lb) unit system for easy comparison.\n", + " if (is_debug_mode == True):\n", + " ebco2_debug = ebco2_compute(data_eb, weight_unit_debug)\n", + " if (weight_unit == 'kg'):\n", + " scaffolding.print_CO2_emission_calculations(data_eb, ebco2_debug, ebco2, dynamic_labels)\n", + " else:\n", + " scaffolding.print_CO2_emission_calculations(data_eb, ebco2, ebco2_debug, dynamic_labels)\n", "\n", - "if (len(dynamic_labels) > 0):\n", - " display_total_CO2_emissions(data_eb,'kg','lb', debug_df, mode_of_interest)\n", - "else:\n", - " display_total_CO2_emissions(data_eb,'lb','kg', debug_df, mode_of_interest)" + "try:\n", + " if (len(dynamic_labels) > 0):\n", + " display_total_CO2_emissions(data_eb,'kg','lb', mode_of_interest)\n", + " else:\n", + " display_total_CO2_emissions(data_eb,'lb','kg', mode_of_interest)\n", + "except:\n", + " generate_missing_plot(plot_title_no_quality,debug_df,file_name)\n", + " alt_text = store_alt_text_missing(debug_df, file_name, plot_title_no_quality)" ] } ], diff --git a/viz_scripts/generic_timeseries.ipynb b/viz_scripts/generic_timeseries.ipynb index f311c2c..823653d 100644 --- a/viz_scripts/generic_timeseries.ipynb +++ b/viz_scripts/generic_timeseries.ipynb @@ -148,22 +148,27 @@ "outputs": [], "source": [ "def compute_daily_metrics(data, weight_unit, distance_unit):\n", + " # Sum daily distance traveled for each mode\n", " mode_distance = data.groupby(['user_id','date_time','Mode_confirm'], as_index=False)[[f'{distance_unit}']].sum()\n", " mode_distance.rename(columns={'sum':f'{distance_unit}'}, inplace=True)\n", " mode_distance[f'{distance_unit}'] = mode_distance[f'{distance_unit}'].fillna(0)\n", " \n", + " # Sum daily emissions for each user\n", " emissions = data.groupby(['user_id','date_time'], as_index=False)[[f'Mode_confirm_{weight_unit}_CO2', f'{distance_unit}']].sum()\n", " emissions[f'Mode_confirm_{weight_unit}_CO2'] = emissions[f'Mode_confirm_{weight_unit}_CO2'].fillna(0)\n", " emissions[f'{distance_unit}'] = emissions[f'Mode_confirm_{weight_unit}_CO2'].fillna(0)\n", " \n", + " # Sum daily energy for each user\n", " energy = data.groupby(['user_id','date_time'], as_index=False)[['Mode_confirm_EI(kWH)', f'{distance_unit}']].sum()\n", " energy['Mode_confirm_EI(kWH)'] = energy['Mode_confirm_EI(kWH)'].fillna(0)\n", " energy[f'{distance_unit}'] = energy['Mode_confirm_EI(kWH)'].fillna(0)\n", " \n", + " # Add 7-day rolling avg smoothing to better see trends\n", " mode_counts['trip_count_smooth'] = mode_counts.groupby(['user_id','Mode_confirm'])['trip_count'].apply(lambda x: x.rolling(7,1).mean())\n", " mode_distance[f'{distance_unit}smooth'] = mode_distance.groupby(['user_id','Mode_confirm'])[f'{distance_unit}'].apply(lambda x: x.rolling(7,1).mean())\n", " emissions[f'{distance_unit}_smooth'] = emissions.groupby(['user_id'])[f'{distance_unit}'].apply(lambda x: x.rolling(7,1).mean())\n", " energy[f'{distance_unit}_smooth'] = energy.groupby(['user_id'])[f'{distance_unit}'].apply(lambda x: x.rolling(7,1).mean())\n", + "\n", " return mode_counts, mode_distance, emissions, energy\n", "\n", "if len(expanded_ct) > 0:\n", @@ -206,25 +211,24 @@ "file_name = \"ts_emissions_user%s\"%file_suffix\n", "\n", "def emission_per_week(emissions, unit, debug_df, active_users):\n", - " try:\n", - " # Emissions per week across all users (net impact)\n", - " plot_data = emissions.groupby(['date_time'], as_index=False)[f'Mode_confirm_{unit}_CO2'].agg(['sum'])\n", - " total_sum = plot_data['sum'].sum()\n", - " plot_data = plot_data.merge(active_users, on='date_time')\n", - " plot_data['sum'] = plot_data['sum'] / plot_data['active_users']\n", + " # Emissions per week across all users (net impact)\n", + " plot_data = emissions.groupby(['date_time'], as_index=False)[f'Mode_confirm_{unit}_CO2'].agg(['sum'])\n", + " total_sum = plot_data['sum'].sum()\n", + " plot_data = plot_data.merge(active_users, on='date_time')\n", + " plot_data['sum'] = plot_data['sum'] / plot_data['active_users']\n", "\n", - " plot_title= plot_title_no_quality+\"\\n\"+quality_text\n", - " ylab = f'Emissions ({unit} CO2/day/user)'\n", - " timeseries_plot(plot_data['date_time'], plot_data['sum'], plot_title, ylab, file_name)\n", - " alt_text = store_alt_text_timeseries(plot_data, file_name, plot_title)\n", - " except:\n", - " generate_missing_plot(plot_title_no_quality,debug_df,file_name)\n", - " alt_text = store_alt_text_missing(debug_df, file_name, plot_title_no_quality)\n", + " plot_title= plot_title_no_quality+\"\\n\"+quality_text\n", + " ylab = f'Emissions ({unit} CO2/day/user)'\n", + " timeseries_plot(plot_data['date_time'], plot_data['sum'], plot_title, ylab, file_name)\n", + " alt_text = store_alt_text_timeseries(plot_data, file_name, plot_title)\n", "\n", - "if (len(dynamic_labels)) > 0:\n", - " emission_per_week(emissions, 'kg', debug_df, active_users)\n", - "else:\n", - " emission_per_week(emissions, 'lb', debug_df, active_users)" + "try:\n", + " if (len(dynamic_labels)) > 0:\n", + " emission_per_week(emissions, 'kg', debug_df, active_users)\n", + " else:\n", + " generate_missing_plot(plot_title_no_quality,debug_df,file_name)\n", + "except:\n", + " alt_text = store_alt_text_missing(debug_df, file_name, plot_title_no_quality)" ] }, { @@ -274,26 +278,27 @@ "source": [ "plot_title_no_quality = 'Average Daily Emission Rate (All Users, excluding air)'\n", "file_name = \"ts_emissions_vmt%s\"%file_suffix\n", - "def emissions_per_distance_day(emissions, weight_unit, distance_unit, distance_unit_smooth, debug_df):\n", - " try:\n", - " # Emissions per kilometer per day across all users (travel efficiency)\n", - " # Note that the energy plot will be identical to this one since scale factor is divided out\n", - " emissions[f'CO2_per_{distance_unit}'] = emissions[f'Mode_confirm_{weight_unit}_CO2'] / emissions[f'{distance_unit_smooth}']\n", - " emissions[f'CO2_per_{distance_unit}'] = emissions[f'CO2_per_{distance_unit}'].fillna(0)\n", - " plot_data = emissions.groupby(['date_time'])[f'CO2_per_{distance_unit}'].agg(['mean']).reset_index()\n", "\n", - " plot_title= plot_title_no_quality+\"\\n\"+quality_text\n", - " ylab = f'Emissions ({weight_unit} CO2/{distance_unit}/day)'\n", - " timeseries_plot(plot_data['date_time'], plot_data['mean'], plot_title, ylab, file_name)\n", - " alt_text = store_alt_text_timeseries(plot_data, file_name, plot_title)\n", - " except:\n", - " generate_missing_plot(plot_title_no_quality,debug_df,file_name)\n", - " alt_text = store_alt_text_missing(debug_df, file_name, plot_title_no_quality)\n", + "def emissions_per_distance_day(emissions, weight_unit, distance_unit, distance_unit_smooth,):\n", + " # Emissions per kilometer per day across all users (travel efficiency)\n", + " # Note that the energy plot will be identical to this one since scale factor is divided out\n", + " emissions[f'CO2_per_{distance_unit}'] = emissions[f'Mode_confirm_{weight_unit}_CO2'] / emissions[f'{distance_unit_smooth}']\n", + " emissions[f'CO2_per_{distance_unit}'] = emissions[f'CO2_per_{distance_unit}'].fillna(0)\n", + " plot_data = emissions.groupby(['date_time'])[f'CO2_per_{distance_unit}'].agg(['mean']).reset_index()\n", + " \n", + " plot_title= plot_title_no_quality+\"\\n\"+quality_text\n", + " ylab = f'Emissions ({weight_unit} CO2/{distance_unit}/day)'\n", + " timeseries_plot(plot_data['date_time'], plot_data['mean'], plot_title, ylab, file_name)\n", + " alt_text = store_alt_text_timeseries(plot_data, file_name, plot_title)\n", "\n", - "if (len(dynamic_labels) > 0):\n", - " emissions_per_distance_day(emissions, 'kg', 'km', 'distance_smooth', debug_df)\n", - "else:\n", - " emissions_per_distance_day(emissions, 'lb', 'mile','distance_miles_smooth', debug_df)" + "try:\n", + " if (len(dynamic_labels) > 0):\n", + " emissions_per_distance_day(emissions, 'kg', 'km', 'distance_smooth')\n", + " else:\n", + " emissions_per_distance_day(emissions, 'lb', 'mile','distance_miles_smooth')\n", + "except:\n", + " generate_missing_plot(plot_title_no_quality,debug_df,file_name)\n", + " alt_text = store_alt_text_missing(debug_df, file_name, plot_title_no_quality)" ] }, { From ae3117a52576cdd6efecae00e9242e7f781760d5 Mon Sep 17 00:00:00 2001 From: iantei Date: Tue, 7 Nov 2023 13:26:25 -0700 Subject: [PATCH 58/62] Fixed issue with default mapping - calling the right function. Updated prefix to the new functions to reflect its functionality. --- viz_scripts/generic_timeseries.ipynb | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/viz_scripts/generic_timeseries.ipynb b/viz_scripts/generic_timeseries.ipynb index 823653d..cf0402d 100644 --- a/viz_scripts/generic_timeseries.ipynb +++ b/viz_scripts/generic_timeseries.ipynb @@ -210,7 +210,7 @@ "plot_title_no_quality = 'Net Daily Emissions (All Users, excluding air)'\n", "file_name = \"ts_emissions_user%s\"%file_suffix\n", "\n", - "def emission_per_week(emissions, unit, debug_df, active_users):\n", + "def plot_emission_per_week(emissions, unit, active_users):\n", " # Emissions per week across all users (net impact)\n", " plot_data = emissions.groupby(['date_time'], as_index=False)[f'Mode_confirm_{unit}_CO2'].agg(['sum'])\n", " total_sum = plot_data['sum'].sum()\n", @@ -224,10 +224,11 @@ "\n", "try:\n", " if (len(dynamic_labels)) > 0:\n", - " emission_per_week(emissions, 'kg', debug_df, active_users)\n", + " plot_emission_per_week(emissions, 'kg', active_users)\n", " else:\n", - " generate_missing_plot(plot_title_no_quality,debug_df,file_name)\n", + " plot_emission_per_week(emissions, 'lb', active_users)\n", "except:\n", + " generate_missing_plot(plot_title_no_quality,debug_df,file_name)\n", " alt_text = store_alt_text_missing(debug_df, file_name, plot_title_no_quality)" ] }, @@ -279,7 +280,7 @@ "plot_title_no_quality = 'Average Daily Emission Rate (All Users, excluding air)'\n", "file_name = \"ts_emissions_vmt%s\"%file_suffix\n", "\n", - "def emissions_per_distance_day(emissions, weight_unit, distance_unit, distance_unit_smooth,):\n", + "def plot_emissions_per_distance_day(emissions, weight_unit, distance_unit, distance_unit_smooth,):\n", " # Emissions per kilometer per day across all users (travel efficiency)\n", " # Note that the energy plot will be identical to this one since scale factor is divided out\n", " emissions[f'CO2_per_{distance_unit}'] = emissions[f'Mode_confirm_{weight_unit}_CO2'] / emissions[f'{distance_unit_smooth}']\n", @@ -293,9 +294,9 @@ "\n", "try:\n", " if (len(dynamic_labels) > 0):\n", - " emissions_per_distance_day(emissions, 'kg', 'km', 'distance_smooth')\n", + " plot_emissions_per_distance_day(emissions, 'kg', 'km', 'distance_smooth')\n", " else:\n", - " emissions_per_distance_day(emissions, 'lb', 'mile','distance_miles_smooth')\n", + " plot_emissions_per_distance_day(emissions, 'lb', 'mile','distance_miles_smooth')\n", "except:\n", " generate_missing_plot(plot_title_no_quality,debug_df,file_name)\n", " alt_text = store_alt_text_missing(debug_df, file_name, plot_title_no_quality)" From 8832e797c66cdf22b74e4181d75e03ed8ca73da3 Mon Sep 17 00:00:00 2001 From: iantei Date: Mon, 13 Nov 2023 14:02:54 -0700 Subject: [PATCH 59/62] Remove conversion_kilometer_to_mile and vice versa in scaffolding.py CO2_footprint_default() and compute_CO2_footprint_dynamic(). --- viz_scripts/scaffolding.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/viz_scripts/scaffolding.py b/viz_scripts/scaffolding.py index cb2276b..d727f23 100644 --- a/viz_scripts/scaffolding.py +++ b/viz_scripts/scaffolding.py @@ -357,7 +357,6 @@ def CO2_footprint_default(df, distance, col): """ conversion_lb_to_kilogram = 0.453592 # 1 lb = 0.453592 kg - conversion_mile_to_kilometer = 1.60934 # 1 mile = 1.60934 km conditions_col = [(df[col+'_fuel'] =='gasoline'), (df[col+'_fuel'] == 'diesel'), @@ -385,7 +384,6 @@ def CO2_impact_default(df,distance): def compute_CO2_footprint_dynamic(expanded_ct, dynamic_labels, label_type): conversion_meter_to_kilometer = 0.001 conversion_kilogram_to_lbs = 2.20462 - conversion_kilometer_to_mile = 0.621371 dic_mode_kgCO2PerKm = {mode["value"]: mode["kgCo2PerKm"] for mode in dynamic_labels["MODE"]} From 78e4777a76ce227e7c9b99c915c9a04c0fbba8f1 Mon Sep 17 00:00:00 2001 From: iantei Date: Mon, 13 Nov 2023 14:17:09 -0700 Subject: [PATCH 60/62] Removed f{'distance_unit'} with distance_unit for compute_daily_metrics() in generic_timeseries notebook. --- viz_scripts/generic_timeseries.ipynb | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/viz_scripts/generic_timeseries.ipynb b/viz_scripts/generic_timeseries.ipynb index cf0402d..7a64fcd 100644 --- a/viz_scripts/generic_timeseries.ipynb +++ b/viz_scripts/generic_timeseries.ipynb @@ -149,25 +149,25 @@ "source": [ "def compute_daily_metrics(data, weight_unit, distance_unit):\n", " # Sum daily distance traveled for each mode\n", - " mode_distance = data.groupby(['user_id','date_time','Mode_confirm'], as_index=False)[[f'{distance_unit}']].sum()\n", - " mode_distance.rename(columns={'sum':f'{distance_unit}'}, inplace=True)\n", - " mode_distance[f'{distance_unit}'] = mode_distance[f'{distance_unit}'].fillna(0)\n", + " mode_distance = data.groupby(['user_id','date_time','Mode_confirm'], as_index=False)[[distance_unit]].sum()\n", + " mode_distance.rename(columns={'sum':distance_unit}, inplace=True)\n", + " mode_distance[distance_unit] = mode_distance[distance_unit].fillna(0)\n", " \n", " # Sum daily emissions for each user\n", - " emissions = data.groupby(['user_id','date_time'], as_index=False)[[f'Mode_confirm_{weight_unit}_CO2', f'{distance_unit}']].sum()\n", + " emissions = data.groupby(['user_id','date_time'], as_index=False)[[f'Mode_confirm_{weight_unit}_CO2', distance_unit]].sum()\n", " emissions[f'Mode_confirm_{weight_unit}_CO2'] = emissions[f'Mode_confirm_{weight_unit}_CO2'].fillna(0)\n", - " emissions[f'{distance_unit}'] = emissions[f'Mode_confirm_{weight_unit}_CO2'].fillna(0)\n", + " emissions[distance_unit] = emissions[f'Mode_confirm_{weight_unit}_CO2'].fillna(0)\n", " \n", " # Sum daily energy for each user\n", - " energy = data.groupby(['user_id','date_time'], as_index=False)[['Mode_confirm_EI(kWH)', f'{distance_unit}']].sum()\n", + " energy = data.groupby(['user_id','date_time'], as_index=False)[['Mode_confirm_EI(kWH)', distance_unit]].sum()\n", " energy['Mode_confirm_EI(kWH)'] = energy['Mode_confirm_EI(kWH)'].fillna(0)\n", - " energy[f'{distance_unit}'] = energy['Mode_confirm_EI(kWH)'].fillna(0)\n", + " energy[distance_unit] = energy['Mode_confirm_EI(kWH)'].fillna(0)\n", " \n", " # Add 7-day rolling avg smoothing to better see trends\n", " mode_counts['trip_count_smooth'] = mode_counts.groupby(['user_id','Mode_confirm'])['trip_count'].apply(lambda x: x.rolling(7,1).mean())\n", - " mode_distance[f'{distance_unit}smooth'] = mode_distance.groupby(['user_id','Mode_confirm'])[f'{distance_unit}'].apply(lambda x: x.rolling(7,1).mean())\n", - " emissions[f'{distance_unit}_smooth'] = emissions.groupby(['user_id'])[f'{distance_unit}'].apply(lambda x: x.rolling(7,1).mean())\n", - " energy[f'{distance_unit}_smooth'] = energy.groupby(['user_id'])[f'{distance_unit}'].apply(lambda x: x.rolling(7,1).mean())\n", + " mode_distance[f'{distance_unit}_smooth'] = mode_distance.groupby(['user_id','Mode_confirm'])[distance_unit].apply(lambda x: x.rolling(7,1).mean())\n", + " emissions[f'{distance_unit}_smooth'] = emissions.groupby(['user_id'])[distance_unit].apply(lambda x: x.rolling(7,1).mean())\n", + " energy[f'{distance_unit}_smooth'] = energy.groupby(['user_id'])[distance_unit].apply(lambda x: x.rolling(7,1).mean())\n", "\n", " return mode_counts, mode_distance, emissions, energy\n", "\n", From 932af13439e91028a3cf01a5d016ef537172092e Mon Sep 17 00:00:00 2001 From: iantei Date: Mon, 20 Nov 2023 18:05:54 -0700 Subject: [PATCH 61/62] Update sense keyword in the phrase for dropdown menu for Trip miles by mode (sensed). --- frontend/metrics_program.html | 2 +- frontend/metrics_program_withoutEnergyMetrics.html | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/frontend/metrics_program.html b/frontend/metrics_program.html index 5e7d643..5c1b181 100644 --- a/frontend/metrics_program.html +++ b/frontend/metrics_program.html @@ -6,7 +6,7 @@ - + diff --git a/frontend/metrics_program_withoutEnergyMetrics.html b/frontend/metrics_program_withoutEnergyMetrics.html index a208819..b47e0a5 100644 --- a/frontend/metrics_program_withoutEnergyMetrics.html +++ b/frontend/metrics_program_withoutEnergyMetrics.html @@ -7,7 +7,7 @@ - + From a23ad9f0c7e28648d81ed30535d37d97f3e0b9e9 Mon Sep 17 00:00:00 2001 From: iantei Date: Mon, 20 Nov 2023 18:22:43 -0700 Subject: [PATCH 62/62] Updated the comment to explain this is specically used to test with example-program-label-options for print_CO2_emission_calculations(). --- viz_scripts/scaffolding.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/viz_scripts/scaffolding.py b/viz_scripts/scaffolding.py index d727f23..fa5fd32 100644 --- a/viz_scripts/scaffolding.py +++ b/viz_scripts/scaffolding.py @@ -405,8 +405,11 @@ def compute_CO2_impact_dynamic(expanded_ct, dynamic_labels): expanded_ct['CO2_Impact(lb)'] = round ((expanded_ct['Replaced_mode_lb_CO2'] - expanded_ct['Mode_confirm_lb_CO2']), 3) return expanded_ct -def print_CO2_emission_calculations(data_eb, ebco2_lb, ebco2_kg, dynamic_labels): +# Function to print the emission calculations in both Metric and Imperial System. Helps in debugging for emission calculation. +# Used this function specifically to test with label_options: https://github.com/e-mission/nrel-openpath-deploy-configs/blob/main/label_options/example-program-label-options.json +# Config: https://github.com/e-mission/nrel-openpath-deploy-configs/blob/main/configs/dev-emulator-program.nrel-op.json +def print_CO2_emission_calculations(data_eb, ebco2_lb, ebco2_kg, dynamic_labels): filtered_taxi_data = data_eb[data_eb['Replaced_mode'] == "Taxi/Uber/Lyft"] filtered_bus_data = data_eb[data_eb['Replaced_mode'] == "Bus"] filtered_freeshuttle_data = data_eb[data_eb['Replaced_mode'] == "Free Shuttle"]