From 12e58243b725cbeb26ad1d389630157a60d8a7b5 Mon Sep 17 00:00:00 2001 From: Abby Wheelis Date: Mon, 16 Sep 2024 10:37:37 -0600 Subject: [PATCH 01/32] temporary bump of e-mission-common version for this branch --- viz_scripts/docker/environment36.dashboard.additions.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/viz_scripts/docker/environment36.dashboard.additions.yml b/viz_scripts/docker/environment36.dashboard.additions.yml index 59d26eb..96bf23e 100644 --- a/viz_scripts/docker/environment36.dashboard.additions.yml +++ b/viz_scripts/docker/environment36.dashboard.additions.yml @@ -5,5 +5,6 @@ channels: dependencies: - seaborn=0.11.1 - pip: + - git+https://github.com/JGreenlee/e-mission-common@0.6.0 - nbparameterise==0.6 - devcron==0.4 From bd33afcf86d5ddac0a3dd081fc44e95860012638 Mon Sep 17 00:00:00 2001 From: Abby Wheelis Date: Mon, 16 Sep 2024 10:38:13 -0600 Subject: [PATCH 02/32] refactor trip loading to fetch base mode and footprint --- viz_scripts/scaffolding.py | 34 ++++++++++++++++++++++++++++------ 1 file changed, 28 insertions(+), 6 deletions(-) diff --git a/viz_scripts/scaffolding.py b/viz_scripts/scaffolding.py index 7eec6fd..dfeb6df 100644 --- a/viz_scripts/scaffolding.py +++ b/viz_scripts/scaffolding.py @@ -9,6 +9,8 @@ import emission.storage.timeseries.tcquery as esttc import emission.core.wrapper.localdate as ecwl +from emcommon.util import read_json_resource +import emcommon.metrics.footprint.footprint_calculations as emffc # Module for pretty-printing outputs (e.g. head) to help users # understand what is going on # However, this means that this module can only be used in an ipython notebook @@ -51,16 +53,35 @@ def get_participant_uuids(program, load_test_users): disp.display(participant_list.user_email) return participant_uuid_str -def load_all_confirmed_trips(tq): +async def add_base_mode_footprint(trip_list): + labels = await read_json_resource("label-options.default.json") + value_to_basemode = {mode["value"]: mode.get("base_mode", mode.get("baseMode", "UNKNOWN")) for mode in labels["MODE"]} + + for trip in trip_list: + try: + trip['data']['base_mode'] = value_to_basemode.get(trip['data']['user_input']['mode_confirm'], "UNKNOWN") + footprint = await emffc.calc_footprint_for_trip(trip['data'], {'base_mode' :trip['data']['base_mode']}) + print(footprint) + trip['data']['footprint'] = footprint + except: + trip['data']['base_mode'] = "UNKNOWN" + trip['data']['footprint'] = {} + + return trip_list + +async def load_all_confirmed_trips(tq): agg = esta.TimeSeries.get_aggregate_time_series() - all_ct = agg.get_data_df("analysis/confirmed_trip", tq) + result_it = agg.find_entries(["analysis/confirmed_trip"], tq) + processed_list = await add_base_mode_footprint(result_it) + all_ct = agg.to_data_df("analysis/confirmed_trip", processed_list) + # all_ct = agg.get_data_df("analysis/confirmed_trip", tq) print("Loaded all confirmed trips of length %s" % len(all_ct)) disp.display(all_ct.head()) return all_ct -def load_all_participant_trips(program, tq, load_test_users): +async def load_all_participant_trips(program, tq, load_test_users): participant_list = get_participant_uuids(program, load_test_users) - all_ct = load_all_confirmed_trips(tq) + all_ct = await load_all_confirmed_trips(tq) # CASE 1 of https://github.com/e-mission/em-public-dashboard/issues/69#issuecomment-1256835867 if len(all_ct) == 0: return all_ct @@ -109,7 +130,8 @@ def expand_userinputs(labeled_ct): unique_users = lambda df: len(df.user_id.unique()) if "user_id" in df.columns else 0 trip_label_count = lambda s, df: len(df[s].dropna()) if s in df.columns else 0 -def load_viz_notebook_data(year, month, program, study_type, dynamic_labels, dic_re, dic_pur=None, include_test_users=False): +async def load_viz_notebook_data(year, month, program, study_type, dynamic_labels, dic_re, dic_pur=None, include_test_users=False): + #TODO - see how slow the loading the footprint is compared to just the baseMode, and evaluate if passing param around is needed """ Inputs: year/month/program/study_type = parameters from the visualization notebook dic_* = label mappings; if dic_pur is included it will be used to recode trip purpose @@ -118,7 +140,7 @@ def load_viz_notebook_data(year, month, program, study_type, dynamic_labels, dic """ # Access database tq = get_time_query(year, month) - participant_ct_df = load_all_participant_trips(program, tq, include_test_users) + participant_ct_df = await load_all_participant_trips(program, tq, include_test_users) labeled_ct = filter_labeled_trips(participant_ct_df) expanded_ct = expand_userinputs(labeled_ct) expanded_ct = data_quality_check(expanded_ct) From 2226b296a57cee1196e0b6989a2f107bc3f6e885 Mon Sep 17 00:00:00 2001 From: Abby Wheelis Date: Mon, 16 Sep 2024 10:41:19 -0600 Subject: [PATCH 03/32] await the async load function --- viz_scripts/energy_calculations.ipynb | 2 +- viz_scripts/generic_metrics.ipynb | 2 +- viz_scripts/generic_timeseries.ipynb | 2 +- viz_scripts/mode_specific_metrics.ipynb | 2 +- viz_scripts/mode_specific_timeseries.ipynb | 2 +- 5 files changed, 5 insertions(+), 5 deletions(-) diff --git a/viz_scripts/energy_calculations.ipynb b/viz_scripts/energy_calculations.ipynb index 4095929..f287e8f 100644 --- a/viz_scripts/energy_calculations.ipynb +++ b/viz_scripts/energy_calculations.ipynb @@ -106,7 +106,7 @@ }, "outputs": [], "source": [ - "expanded_ct, file_suffix, quality_text, debug_df = scaffolding.load_viz_notebook_data(year,\n", + "expanded_ct, file_suffix, quality_text, debug_df = await scaffolding.load_viz_notebook_data(year,\n", " month,\n", " program,\n", " study_type,\n", diff --git a/viz_scripts/generic_metrics.ipynb b/viz_scripts/generic_metrics.ipynb index f93de0d..e4673db 100644 --- a/viz_scripts/generic_metrics.ipynb +++ b/viz_scripts/generic_metrics.ipynb @@ -108,7 +108,7 @@ "metadata": {}, "outputs": [], "source": [ - "expanded_ct, file_suffix, quality_text, debug_df = scaffolding.load_viz_notebook_data(year,\n", + "expanded_ct, file_suffix, quality_text, debug_df = await scaffolding.load_viz_notebook_data(year,\n", " month,\n", " program,\n", " study_type,\n", diff --git a/viz_scripts/generic_timeseries.ipynb b/viz_scripts/generic_timeseries.ipynb index 189afdf..63ef911 100644 --- a/viz_scripts/generic_timeseries.ipynb +++ b/viz_scripts/generic_timeseries.ipynb @@ -87,7 +87,7 @@ }, "outputs": [], "source": [ - "expanded_ct, file_suffix, quality_text, debug_df = scaffolding.load_viz_notebook_data(year,\n", + "expanded_ct, file_suffix, quality_text, debug_df = await scaffolding.load_viz_notebook_data(year,\n", " month,\n", " program,\n", " study_type,\n", diff --git a/viz_scripts/mode_specific_metrics.ipynb b/viz_scripts/mode_specific_metrics.ipynb index 0e2e63d..123bf9e 100644 --- a/viz_scripts/mode_specific_metrics.ipynb +++ b/viz_scripts/mode_specific_metrics.ipynb @@ -121,7 +121,7 @@ "metadata": {}, "outputs": [], "source": [ - "expanded_ct, file_suffix, quality_text, debug_df = scaffolding.load_viz_notebook_data(year,\n", + "expanded_ct, file_suffix, quality_text, debug_df = await scaffolding.load_viz_notebook_data(year,\n", " month,\n", " program,\n", " study_type,\n", diff --git a/viz_scripts/mode_specific_timeseries.ipynb b/viz_scripts/mode_specific_timeseries.ipynb index d696794..58ee56d 100644 --- a/viz_scripts/mode_specific_timeseries.ipynb +++ b/viz_scripts/mode_specific_timeseries.ipynb @@ -96,7 +96,7 @@ "metadata": {}, "outputs": [], "source": [ - "expanded_ct, file_suffix, quality_text, debug_df = scaffolding.load_viz_notebook_data(year,\n", + "expanded_ct, file_suffix, quality_text, debug_df = await scaffolding.load_viz_notebook_data(year,\n", " month,\n", " program,\n", " study_type,\n", From 6409725611d7cb37883fb1bffa3c6f79a5191fee Mon Sep 17 00:00:00 2001 From: Abby Wheelis Date: Mon, 16 Sep 2024 13:59:42 -0600 Subject: [PATCH 04/32] create a list from iterator the iterator is consumed on first iteration, must use a list to update items instead --- viz_scripts/scaffolding.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/viz_scripts/scaffolding.py b/viz_scripts/scaffolding.py index dfeb6df..edbcd51 100644 --- a/viz_scripts/scaffolding.py +++ b/viz_scripts/scaffolding.py @@ -72,7 +72,7 @@ async def add_base_mode_footprint(trip_list): async def load_all_confirmed_trips(tq): agg = esta.TimeSeries.get_aggregate_time_series() result_it = agg.find_entries(["analysis/confirmed_trip"], tq) - processed_list = await add_base_mode_footprint(result_it) + processed_list = await add_base_mode_footprint(list(result_it)) all_ct = agg.to_data_df("analysis/confirmed_trip", processed_list) # all_ct = agg.get_data_df("analysis/confirmed_trip", tq) print("Loaded all confirmed trips of length %s" % len(all_ct)) From f2de060ea7751518679fc1b167f3185e4c265dc2 Mon Sep 17 00:00:00 2001 From: Abby Wheelis Date: Mon, 16 Sep 2024 13:59:58 -0600 Subject: [PATCH 05/32] add footprint and replaced mode footprint to the df --- viz_scripts/scaffolding.py | 18 +++++++++++++----- 1 file changed, 13 insertions(+), 5 deletions(-) diff --git a/viz_scripts/scaffolding.py b/viz_scripts/scaffolding.py index edbcd51..b5f31ef 100644 --- a/viz_scripts/scaffolding.py +++ b/viz_scripts/scaffolding.py @@ -60,13 +60,21 @@ async def add_base_mode_footprint(trip_list): for trip in trip_list: try: trip['data']['base_mode'] = value_to_basemode.get(trip['data']['user_input']['mode_confirm'], "UNKNOWN") - footprint = await emffc.calc_footprint_for_trip(trip['data'], {'base_mode' :trip['data']['base_mode']}) - print(footprint) - trip['data']['footprint'] = footprint + trip['data']['mode_confirm_footprint'] = await emffc.calc_footprint_for_trip(trip['data'], {'base_mode' :trip['data']['base_mode']}) + + if 'replaced_mode' in trip['data']['user_input'].keys(): + trip['data']['replaced_base_mode'] = value_to_basemode.get(trip['data']['user_input']['replaced_mode'], "UNKNOWN") + trip['data']['replaced_mode_footprint'] = await emffc.calc_footprint_for_trip(trip['data'], {'base_mode' :trip['data']['replaced_base_mode']}) + else: + trip['data']['replaced_base_mode'] = "UNKNOWN" + trip['data']['replaced_mode_footprint'] = {} + except: trip['data']['base_mode'] = "UNKNOWN" - trip['data']['footprint'] = {} - + trip['data']['replaced_base_mode'] = "UNKNOWN" + trip['data']['mode_confirm_footprint'] = {} + trip['data']['replaced_mode_footprint'] = {} + return trip_list async def load_all_confirmed_trips(tq): From 7ee9d4a033ce8022f79b1d14aae46fc57bf2d0ae Mon Sep 17 00:00:00 2001 From: Abby Wheelis Date: Mon, 16 Sep 2024 16:29:50 -0600 Subject: [PATCH 06/32] separate footprint data from metadata having separate columns makes accessing the data easier, no longer need to break a tuple --- viz_scripts/scaffolding.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/viz_scripts/scaffolding.py b/viz_scripts/scaffolding.py index b5f31ef..985fb28 100644 --- a/viz_scripts/scaffolding.py +++ b/viz_scripts/scaffolding.py @@ -60,11 +60,11 @@ async def add_base_mode_footprint(trip_list): for trip in trip_list: try: trip['data']['base_mode'] = value_to_basemode.get(trip['data']['user_input']['mode_confirm'], "UNKNOWN") - trip['data']['mode_confirm_footprint'] = await emffc.calc_footprint_for_trip(trip['data'], {'base_mode' :trip['data']['base_mode']}) + trip['data']['mode_confirm_footprint'], trip['data']['mode_confirm_footprint_metadata'] = await emffc.calc_footprint_for_trip(trip['data'], {'base_mode' :trip['data']['base_mode']}) if 'replaced_mode' in trip['data']['user_input'].keys(): trip['data']['replaced_base_mode'] = value_to_basemode.get(trip['data']['user_input']['replaced_mode'], "UNKNOWN") - trip['data']['replaced_mode_footprint'] = await emffc.calc_footprint_for_trip(trip['data'], {'base_mode' :trip['data']['replaced_base_mode']}) + trip['data']['replaced_mode_footprint'], trip['data']['replaced_mode_footprint_metadata'] = await emffc.calc_footprint_for_trip(trip['data'], {'base_mode' :trip['data']['replaced_base_mode']}) else: trip['data']['replaced_base_mode'] = "UNKNOWN" trip['data']['replaced_mode_footprint'] = {} From f56d4d2e7806be481ebf0aeb037cff6d627f69b2 Mon Sep 17 00:00:00 2001 From: Abby Wheelis Date: Mon, 16 Sep 2024 16:30:55 -0600 Subject: [PATCH 07/32] replace footprint calculations remove most of the old footprint calculations unpack the footprints into energy and emissions calculate difference between mode and replaced mode --- viz_scripts/energy_calculations.ipynb | 4 +- viz_scripts/scaffolding.py | 149 ++++++-------------------- 2 files changed, 32 insertions(+), 121 deletions(-) diff --git a/viz_scripts/energy_calculations.ipynb b/viz_scripts/energy_calculations.ipynb index f287e8f..e139db4 100644 --- a/viz_scripts/energy_calculations.ipynb +++ b/viz_scripts/energy_calculations.ipynb @@ -114,8 +114,8 @@ " dic_re,\n", " dic_pur=dic_pur,\n", " include_test_users=include_test_users)\n", - "# CASE 1 of https://github.com/e-mission/em-public-dashboard/issues/69#issuecomment-1256835867 \n", - "expanded_ct = scaffolding.add_energy_impact(expanded_ct, df_ei, dic_fuel, dynamic_labels) if 'Replaced_mode' in expanded_ct.columns else expanded_ct" + "\n", + "expanded_ct = scaffolding.unpack_energy_emissions(expanded_ct)" ] }, { diff --git a/viz_scripts/scaffolding.py b/viz_scripts/scaffolding.py index 985fb28..9d79545 100644 --- a/viz_scripts/scaffolding.py +++ b/viz_scripts/scaffolding.py @@ -327,36 +327,6 @@ def load_viz_notebook_survey_data(year, month, program, include_test_users=False return participant_ct_df, labeled_ct, file_suffix -def add_energy_labels(expanded_ct, df_ei, dic_fuel, dynamic_labels): - """ Inputs: - expanded_ct = dataframe of trips that has had Mode_confirm and Replaced_mode added - dic/df_* = label mappings for energy impact and fuel - """ - expanded_ct['Mode_confirm_fuel']= expanded_ct['Mode_confirm'].map(dic_fuel) - expanded_ct = energy_intensity(expanded_ct, df_ei, 'Mode_confirm') - expanded_ct = energy_footprint_kWH(expanded_ct, 'distance_miles', 'Mode_confirm') - - if (len(dynamic_labels) > 0): - expanded_ct = compute_CO2_footprint_dynamic(expanded_ct, dynamic_labels, 'Mode_confirm') - else: - expanded_ct = CO2_footprint_default(expanded_ct, 'distance_miles', 'Mode_confirm') - return expanded_ct - -def add_energy_impact(expanded_ct, df_ei, dic_fuel, dynamic_labels): - # Let's first calculate everything for the mode confirm - # And then calculate everything for the replaced mode - expanded_ct = add_energy_labels(expanded_ct, df_ei, dic_fuel, dynamic_labels) - expanded_ct['Replaced_mode_fuel']= expanded_ct['Replaced_mode'].map(dic_fuel) - expanded_ct = energy_intensity(expanded_ct, df_ei, 'Replaced_mode') - # and then compute the impacts - expanded_ct = energy_impact_kWH(expanded_ct, 'distance_miles') - - if (len(dynamic_labels) > 0): - expanded_ct = compute_CO2_impact_dynamic(expanded_ct, dynamic_labels) - else: - expanded_ct = CO2_impact_default(expanded_ct, 'distance_miles') - return expanded_ct - def get_quality_text(before_df, after_df, mode_of_interest=None, include_test_users=False): """ Inputs: before_df = dataframe prior to filtering (usually participant_ct_df) @@ -416,103 +386,44 @@ def unit_conversions(df): df['distance_miles']= df["distance"]*0.00062 #meters to miles df['distance_kms'] = df["distance"] / 1000 #meters to kms -def energy_intensity(trip_df,mode_intensity_df,col): - """ Inputs: - trip_df = dataframe with data - mode_intensity_df = dataframe with energy/cost/time factors - col = the column for which we want to map the intensity - """ - mode_intensity_df = mode_intensity_df.copy() - mode_intensity_df[col] = mode_intensity_df['mode'] - dic_ei_factor = dict(zip(mode_intensity_df[col],mode_intensity_df['energy_intensity_factor'])) - dic_CO2_factor = dict(zip(mode_intensity_df[col],mode_intensity_df['CO2_factor'])) - dic_ei_trip = dict(zip(mode_intensity_df[col],mode_intensity_df['(kWH)/trip'])) - - trip_df['ei_'+col] = trip_df[col].map(dic_ei_factor) - trip_df['CO2_'+col] = trip_df[col].map(dic_CO2_factor) - trip_df['ei_trip_'+col] = trip_df[col].map(dic_ei_trip) - return trip_df - -def energy_footprint_kWH(df,distance,col): - """ Inputs: - df = dataframe with data - distance = distance in miles - col = Replaced_mode or Mode_confirm - """ - conditions_col = [(df[col+'_fuel'] =='gasoline'), - (df[col+'_fuel'] == 'diesel'), - (df[col+'_fuel'] == 'electric')] - gasoline_col = (df[distance]*df['ei_'+col]*0.000293071) # 1 BTU = 0.000293071 kWH - diesel_col = (df[distance]*df['ei_'+col]*0.000293071) - electric_col = (df[distance]*df['ei_'+col])+ df['ei_trip_'+col] - values_col = [gasoline_col,diesel_col,electric_col] - df[col+'_EI(kWH)'] = np.select(conditions_col, values_col) - return df - -def energy_impact_kWH(df,distance): - if 'Mode_confirm_EI(kWH)' not in df.columns: - print("Mode confirm footprint not found, computing before impact") - df = energy_footprint_kWH(df, distance, "Mode_confirm") - df = energy_footprint_kWH(df, distance, "Replaced_mode") - df['Energy_Impact(kWH)'] = round((df['Replaced_mode_EI(kWH)'] - df['Mode_confirm_EI(kWH)']),3) - return df - -def CO2_footprint_default(df, distance, col): - """ Inputs: - df = dataframe with data - distance = distance in miles - col = Replaced_mode or Mode_confirm - """ - - conversion_lb_to_kilogram = 0.453592 # 1 lb = 0.453592 kg - - conditions_col = [(df[col+'_fuel'] =='gasoline'), - (df[col+'_fuel'] == 'diesel'), - (df[col+'_fuel'] == 'electric')] - gasoline_col = (df[distance]*df['ei_'+col]*0.000001)* df['CO2_'+col] - diesel_col = (df[distance]*df['ei_'+col]*0.000001)* df['CO2_'+col] - electric_col = (((df[distance]*df['ei_'+col])+df['ei_trip_'+col])*0.001)*df['CO2_'+col] - - values_col = [gasoline_col,diesel_col,electric_col] - df[col+'_lb_CO2'] = np.select(conditions_col, values_col) - df[col+'_kg_CO2'] = df[col+'_lb_CO2'] * conversion_lb_to_kilogram - return df - -def CO2_impact_default(df,distance): - if 'Mode_confirm_lb_CO2' not in df.columns: - print("Mode confirm footprint not found, computing before impact") - df = CO2_footprint_default(df, distance, "Mode_confirm") - df = CO2_footprint_default(df, distance, "Replaced_mode") - df['CO2_Impact(lb)'] = round((df['Replaced_mode_lb_CO2'] - df['Mode_confirm_lb_CO2']), 3) - - # Convert the CO2_Impact to be represented in kilogram - df['CO2_Impact(kg)'] = round((df['Replaced_mode_kg_CO2'] - df['Mode_confirm_kg_CO2']), 3) - return df +def extract_kwh(footprint_dict): + if 'kwh' in footprint_dict.keys(): + return footprint_dict['kwh'] + else: + print("missing kwh", footprint_dict) + return np.nan -def compute_CO2_footprint_dynamic(expanded_ct, dynamic_labels, label_type): - conversion_meter_to_kilometer = 0.001 - conversion_kilogram_to_lbs = 2.20462 +def extract_co2(footprint_dict): + if 'kg_co2' in footprint_dict.keys(): + return footprint_dict['kg_co2'] + else: + print("missing co2", footprint_dict) + return np.nan - dic_mode_kgCO2PerKm = {mode["value"]: mode["kgCo2PerKm"] for mode in dynamic_labels["MODE"]} +def unpack_energy_emissions(expanded_ct): + expanded_ct['mode_comfirm_footprint_kg_co2'] = expanded_ct['mode_confirm_footprint'].apply(extract_co2) + expanded_ct['mode_comfirm_footprint_kwh'] = expanded_ct['mode_confirm_footprint'].apply(extract_kwh) + expanded_ct['replaced_mode_footprint_kg_co2'] = expanded_ct['replaced_mode_footprint'].apply(extract_co2) + expanded_ct['replaced_mode_footprint_kwh'] = expanded_ct['replaced_mode_footprint'].apply(extract_kwh) - if label_type.lower() in expanded_ct.columns: - # The expanded_ct['Mode_confirm_kg_CO2'] is CO2 emission in terms of [distance in kms * CO2 emission in kgCO2 per km = kg of CO2] - expanded_ct[label_type+'_kg_CO2'] = ((expanded_ct['distance'] * conversion_meter_to_kilometer )) * (expanded_ct[label_type.lower()].map(dic_mode_kgCO2PerKm)) - expanded_ct[label_type+'_kg_CO2'] = expanded_ct[label_type+'_kg_CO2'].fillna(0) - expanded_ct[label_type+'_lb_CO2'] = expanded_ct[label_type+'_kg_CO2'] * conversion_kilogram_to_lbs + energy_impact(expanded_ct) + CO2_impact(expanded_ct) + expanded_ct['energy_savings'] = expanded_ct['replaced_mode_footprint_kg_co2'] - expanded_ct['mode_comfirm_footprint_kg_co2'] + expanded_ct['emissions_savings'] = expanded_ct['replaced_mode_footprint_kwh'] - expanded_ct['mode_comfirm_footprint_kwh'] return expanded_ct -def compute_CO2_impact_dynamic(expanded_ct, dynamic_labels): - if 'Mode_confirm_kg_CO2' not in expanded_ct.columns: - print("Mode confirm footprint not found, computing before impact.") - expanded_ct = compute_CO2_footprint_dynamic(expanded_ct, dynamic_labels, "Mode_confirm") - expanded_ct = compute_CO2_footprint_dynamic(expanded_ct, dynamic_labels, "Replaced_mode") +def energy_impact(df): + df['Energy_Impact(kWH)'] = round((df['replaced_mode_footprint_kwh'] - df['mode_comfirm_footprint_kwh']),3) - expanded_ct['CO2_Impact(kg)'] = round ((expanded_ct['Replaced_mode_kg_CO2'] - expanded_ct['Mode_confirm_kg_CO2']), 3) - expanded_ct['CO2_Impact(lb)'] = round ((expanded_ct['Replaced_mode_lb_CO2'] - expanded_ct['Mode_confirm_lb_CO2']), 3) - return expanded_ct +def kg_to_lb(kg): + return kg * 2.20462 +def CO2_impact(df): + df['CO2_Impact(kg)'] = round((df['replaced_mode_footprint_kg_co2'] - df['mode_comfirm_footprint_kg_co2']), 3) + df['CO2_Impact(lb)'] = round(kg_to_lb(df['CO2_Impact(kg)']), 3) + + return df # Function to print the emission calculations in both Metric and Imperial System. Helps in debugging for emission calculation. # Used this function specifically to test with label_options: https://github.com/e-mission/nrel-openpath-deploy-configs/blob/main/label_options/example-program-label-options.json From 696417ad403354817d229a8d7e3c2abf6ca96f80 Mon Sep 17 00:00:00 2001 From: Abby Wheelis Date: Mon, 16 Sep 2024 16:42:31 -0600 Subject: [PATCH 08/32] remove old code, add TODOs --- viz_scripts/scaffolding.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/viz_scripts/scaffolding.py b/viz_scripts/scaffolding.py index 9d79545..805d556 100644 --- a/viz_scripts/scaffolding.py +++ b/viz_scripts/scaffolding.py @@ -54,6 +54,7 @@ def get_participant_uuids(program, load_test_users): return participant_uuid_str async def add_base_mode_footprint(trip_list): + #TODO filter ahead of this so only labeled trips get a footprint OR display uncertainties labels = await read_json_resource("label-options.default.json") value_to_basemode = {mode["value"]: mode.get("base_mode", mode.get("baseMode", "UNKNOWN")) for mode in labels["MODE"]} @@ -82,7 +83,6 @@ async def load_all_confirmed_trips(tq): result_it = agg.find_entries(["analysis/confirmed_trip"], tq) processed_list = await add_base_mode_footprint(list(result_it)) all_ct = agg.to_data_df("analysis/confirmed_trip", processed_list) - # all_ct = agg.get_data_df("analysis/confirmed_trip", tq) print("Loaded all confirmed trips of length %s" % len(all_ct)) disp.display(all_ct.head()) return all_ct @@ -429,6 +429,7 @@ def CO2_impact(df): # Used this function specifically to test with label_options: https://github.com/e-mission/nrel-openpath-deploy-configs/blob/main/label_options/example-program-label-options.json # Config: https://github.com/e-mission/nrel-openpath-deploy-configs/blob/main/configs/dev-emulator-program.nrel-op.json def print_CO2_emission_calculations(data_eb, ebco2_lb, ebco2_kg, dynamic_labels): + #TODO update this function with new columns after switching to emcommon emissions filtered_taxi_data = data_eb[data_eb['Replaced_mode'] == "Taxi/Uber/Lyft"] filtered_bus_data = data_eb[data_eb['Replaced_mode'] == "Bus"] filtered_freeshuttle_data = data_eb[data_eb['Replaced_mode'] == "Free Shuttle"] From 391aa173a9b338c536ce2451ab573a289cf014c4 Mon Sep 17 00:00:00 2001 From: Abby Wheelis Date: Tue, 17 Sep 2024 11:53:12 -0600 Subject: [PATCH 09/32] wait for data before proceeding data function is now async because of waiting on returns from e-mission-common --- viz_scripts/generic_metrics_sensed.ipynb | 2 +- viz_scripts/scaffolding.py | 25 ++++++++++++++---------- viz_scripts/survey_metrics.ipynb | 2 +- viz_scripts/survey_responses.ipynb | 2 +- 4 files changed, 18 insertions(+), 13 deletions(-) diff --git a/viz_scripts/generic_metrics_sensed.ipynb b/viz_scripts/generic_metrics_sensed.ipynb index 705c32f..7f465d8 100644 --- a/viz_scripts/generic_metrics_sensed.ipynb +++ b/viz_scripts/generic_metrics_sensed.ipynb @@ -74,7 +74,7 @@ "metadata": {}, "outputs": [], "source": [ - "expanded_ct, file_suffix, quality_text, debug_df = scaffolding.load_viz_notebook_sensor_inference_data(year,\n", + "expanded_ct, file_suffix, quality_text, debug_df = await scaffolding.load_viz_notebook_sensor_inference_data(year,\n", " month,\n", " program,\n", " include_test_users,\n", diff --git a/viz_scripts/scaffolding.py b/viz_scripts/scaffolding.py index 805d556..95a68cd 100644 --- a/viz_scripts/scaffolding.py +++ b/viz_scripts/scaffolding.py @@ -78,18 +78,23 @@ async def add_base_mode_footprint(trip_list): return trip_list -async def load_all_confirmed_trips(tq): +async def load_all_confirmed_trips(tq, add_footprint): agg = esta.TimeSeries.get_aggregate_time_series() result_it = agg.find_entries(["analysis/confirmed_trip"], tq) - processed_list = await add_base_mode_footprint(list(result_it)) - all_ct = agg.to_data_df("analysis/confirmed_trip", processed_list) + print(result_it) + if add_footprint: + processed_list = await add_base_mode_footprint(list(result_it)) + all_ct = agg.to_data_df("analysis/confirmed_trip", processed_list) + else: + all_ct = agg.to_data_df("analysis/confirmed_trip", result_it) + print(all_ct) print("Loaded all confirmed trips of length %s" % len(all_ct)) disp.display(all_ct.head()) return all_ct -async def load_all_participant_trips(program, tq, load_test_users): +async def load_all_participant_trips(program, tq, load_test_users, add_footprint=False): participant_list = get_participant_uuids(program, load_test_users) - all_ct = await load_all_confirmed_trips(tq) + all_ct = await load_all_confirmed_trips(tq, add_footprint) # CASE 1 of https://github.com/e-mission/em-public-dashboard/issues/69#issuecomment-1256835867 if len(all_ct) == 0: return all_ct @@ -148,7 +153,7 @@ async def load_viz_notebook_data(year, month, program, study_type, dynamic_label """ # Access database tq = get_time_query(year, month) - participant_ct_df = await load_all_participant_trips(program, tq, include_test_users) + participant_ct_df = await load_all_participant_trips(program, tq, include_test_users, True) labeled_ct = filter_labeled_trips(participant_ct_df) expanded_ct = expand_userinputs(labeled_ct) expanded_ct = data_quality_check(expanded_ct) @@ -258,14 +263,14 @@ def mapping_color_surveys(dic_options): return colors -def load_viz_notebook_sensor_inference_data(year, month, program, include_test_users=False, sensed_algo_prefix="cleaned"): +async def load_viz_notebook_sensor_inference_data(year, month, program, include_test_users=False, sensed_algo_prefix="cleaned"): """ Inputs: year/month/program = parameters from the visualization notebook Pipeline to load and process the data before use in sensor-based visualization notebooks. """ tq = get_time_query(year, month) - participant_ct_df = load_all_participant_trips(program, tq, include_test_users) + participant_ct_df = await load_all_participant_trips(program, tq, include_test_users, False) expanded_ct = participant_ct_df print(f"Loaded expanded_ct with length {len(expanded_ct)} for {tq}") @@ -312,14 +317,14 @@ def load_viz_notebook_sensor_inference_data(year, month, program, include_test_u return expanded_ct, file_suffix, quality_text, debug_df -def load_viz_notebook_survey_data(year, month, program, include_test_users=False): +async def load_viz_notebook_survey_data(year, month, program, include_test_users=False): """ Inputs: year/month/program/test users = parameters from the visualization notebook Returns: df of all trips taken by participants, df of all trips with user_input """ tq = get_time_query(year, month) - participant_ct_df = load_all_participant_trips(program, tq, include_test_users) + participant_ct_df = await load_all_participant_trips(program, tq, include_test_users, False) labeled_ct = filter_labeled_trips(participant_ct_df) # Document data quality diff --git a/viz_scripts/survey_metrics.ipynb b/viz_scripts/survey_metrics.ipynb index a7d395d..5e46fbe 100644 --- a/viz_scripts/survey_metrics.ipynb +++ b/viz_scripts/survey_metrics.ipynb @@ -93,7 +93,7 @@ "metadata": {}, "outputs": [], "source": [ - "expanded_ct_sensed, file_suffix_sensed, quality_text_sensed, debug_df_sensed = scaffolding.load_viz_notebook_sensor_inference_data(year,\n", + "expanded_ct_sensed, file_suffix_sensed, quality_text_sensed, debug_df_sensed = await scaffolding.load_viz_notebook_sensor_inference_data(year,\n", " month,\n", " program,\n", " include_test_users,\n", diff --git a/viz_scripts/survey_responses.ipynb b/viz_scripts/survey_responses.ipynb index 480ceea..53a633c 100644 --- a/viz_scripts/survey_responses.ipynb +++ b/viz_scripts/survey_responses.ipynb @@ -235,7 +235,7 @@ "outputs": [], "source": [ "#load data - all data and data with labels\n", - "all_confirmed_trips, survey_trips, file_suffix = scaffolding.load_viz_notebook_survey_data(year, month, program, include_test_users)\n", + "all_confirmed_trips, survey_trips, file_suffix = await scaffolding.load_viz_notebook_survey_data(year, month, program, include_test_users)\n", "\n", "try:\n", " #survey counts df\n", From af5ace7d1560afa6b4a1652464ac7956483a88fe Mon Sep 17 00:00:00 2001 From: Abby Wheelis Date: Tue, 17 Sep 2024 11:53:55 -0600 Subject: [PATCH 10/32] rename energy and emissions columns using older names allows for old code to run --- viz_scripts/generic_timeseries.ipynb | 4 ++-- viz_scripts/scaffolding.py | 20 ++++++++++---------- 2 files changed, 12 insertions(+), 12 deletions(-) diff --git a/viz_scripts/generic_timeseries.ipynb b/viz_scripts/generic_timeseries.ipynb index 63ef911..40b1fd6 100644 --- a/viz_scripts/generic_timeseries.ipynb +++ b/viz_scripts/generic_timeseries.ipynb @@ -94,9 +94,9 @@ " dynamic_labels,\n", " dic_re,\n", " include_test_users=include_test_users)\n", - "expanded_ct = scaffolding.add_energy_labels(expanded_ct, df_ei, dic_fuel, dynamic_labels) if \"mode_confirm\" in expanded_ct.columns else expanded_ct\n", + "expanded_ct = scaffolding.unpack_energy_emissions(expanded_ct) if \"mode_confirm\" in expanded_ct.columns else expanded_ct\n", "\n", - "expanded_ct_sensed, file_suffix_sensed, quality_text_sensed, debug_df_sensed = scaffolding.load_viz_notebook_sensor_inference_data(year,\n", + "expanded_ct_sensed, file_suffix_sensed, quality_text_sensed, debug_df_sensed = await scaffolding.load_viz_notebook_sensor_inference_data(year,\n", " month,\n", " program,\n", " include_test_users,\n", diff --git a/viz_scripts/scaffolding.py b/viz_scripts/scaffolding.py index 95a68cd..0cb6e9a 100644 --- a/viz_scripts/scaffolding.py +++ b/viz_scripts/scaffolding.py @@ -406,26 +406,26 @@ def extract_co2(footprint_dict): return np.nan def unpack_energy_emissions(expanded_ct): - expanded_ct['mode_comfirm_footprint_kg_co2'] = expanded_ct['mode_confirm_footprint'].apply(extract_co2) - expanded_ct['mode_comfirm_footprint_kwh'] = expanded_ct['mode_confirm_footprint'].apply(extract_kwh) - expanded_ct['replaced_mode_footprint_kg_co2'] = expanded_ct['replaced_mode_footprint'].apply(extract_co2) - expanded_ct['replaced_mode_footprint_kwh'] = expanded_ct['replaced_mode_footprint'].apply(extract_kwh) + expanded_ct['Mode_confirm_kg_CO2'] = expanded_ct['mode_confirm_footprint'].apply(extract_co2) + expanded_ct['Mode_confirm_lb_CO2'] = kg_to_lb(expanded_ct['Mode_confirm_kg_CO2']) + expanded_ct['Replaced_mode_kg_CO2'] = expanded_ct['replaced_mode_footprint'].apply(extract_co2) + expanded_ct['Replaced_mode_lb_CO2'] = kg_to_lb(expanded_ct['Replaced_mode_kg_CO2']) + CO2_impact(expanded_ct) + expanded_ct['Replaced_mode_EI(kWH)'] = expanded_ct['replaced_mode_footprint'].apply(extract_kwh) + expanded_ct['Mode_confirm_EI(kWH)'] = expanded_ct['mode_confirm_footprint'].apply(extract_kwh) energy_impact(expanded_ct) - CO2_impact(expanded_ct) - expanded_ct['energy_savings'] = expanded_ct['replaced_mode_footprint_kg_co2'] - expanded_ct['mode_comfirm_footprint_kg_co2'] - expanded_ct['emissions_savings'] = expanded_ct['replaced_mode_footprint_kwh'] - expanded_ct['mode_comfirm_footprint_kwh'] return expanded_ct def energy_impact(df): - df['Energy_Impact(kWH)'] = round((df['replaced_mode_footprint_kwh'] - df['mode_comfirm_footprint_kwh']),3) + df['Energy_Impact(kWH)'] = round((df['Replaced_mode_EI(kWH)'] - df['Mode_confirm_EI(kWH)']),3) def kg_to_lb(kg): return kg * 2.20462 def CO2_impact(df): - df['CO2_Impact(kg)'] = round((df['replaced_mode_footprint_kg_co2'] - df['mode_comfirm_footprint_kg_co2']), 3) + df['CO2_Impact(kg)'] = round((df['Replaced_mode_kg_CO2'] - df['Mode_confirm_kg_CO2']), 3) df['CO2_Impact(lb)'] = round(kg_to_lb(df['CO2_Impact(kg)']), 3) return df @@ -452,7 +452,7 @@ def print_CO2_emission_calculations(data_eb, ebco2_lb, ebco2_kg, dynamic_labels) print("With Default mapping:") print("\n") - selected_columns = ['distance','distance_miles', 'Replaced_mode_kg_CO2', 'Replaced_mode_lb_CO2', 'Mode_confirm_kg_CO2','Mode_confirm_lb_CO2', "replaced_mode", "mode_confirm"] + selected_columns = ['distance','distance_miles', 'replaced_mode_footprint_kg_co2', 'mode_comfirm_footprint_kg_co2', "replaced_mode", "mode_confirm"] print("Walk Data:") print(str(filtered_walk_data[selected_columns].head())) From ff0a6979c23f085e66e7486db808eae893201de8 Mon Sep 17 00:00:00 2001 From: Abby Wheelis Date: Tue, 17 Sep 2024 12:16:54 -0600 Subject: [PATCH 11/32] missing await --- viz_scripts/generic_metrics.ipynb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/viz_scripts/generic_metrics.ipynb b/viz_scripts/generic_metrics.ipynb index e4673db..39ade55 100644 --- a/viz_scripts/generic_metrics.ipynb +++ b/viz_scripts/generic_metrics.ipynb @@ -133,7 +133,7 @@ "metadata": {}, "outputs": [], "source": [ - "expanded_ct_sensed, file_suffix_sensed, quality_text_sensed, debug_df_sensed = scaffolding.load_viz_notebook_sensor_inference_data(year,\n", + "expanded_ct_sensed, file_suffix_sensed, quality_text_sensed, debug_df_sensed = await scaffolding.load_viz_notebook_sensor_inference_data(year,\n", " month,\n", " program,\n", " include_test_users,\n", From 814b3c27b8d5049ab96c2edeb511eba8713ad3d9 Mon Sep 17 00:00:00 2001 From: Abby Wheelis Date: Tue, 17 Sep 2024 12:18:10 -0600 Subject: [PATCH 12/32] parameterize footprint calculations only calculating the footprint when needed in a given notebook saves time --- viz_scripts/energy_calculations.ipynb | 3 ++- viz_scripts/generic_timeseries.ipynb | 3 ++- viz_scripts/scaffolding.py | 4 ++-- 3 files changed, 6 insertions(+), 4 deletions(-) diff --git a/viz_scripts/energy_calculations.ipynb b/viz_scripts/energy_calculations.ipynb index e139db4..c1ecc4f 100644 --- a/viz_scripts/energy_calculations.ipynb +++ b/viz_scripts/energy_calculations.ipynb @@ -113,7 +113,8 @@ " dynamic_labels,\n", " dic_re,\n", " dic_pur=dic_pur,\n", - " include_test_users=include_test_users)\n", + " include_test_users=include_test_users,\n", + " add_footprint=True)\n", "\n", "expanded_ct = scaffolding.unpack_energy_emissions(expanded_ct)" ] diff --git a/viz_scripts/generic_timeseries.ipynb b/viz_scripts/generic_timeseries.ipynb index 40b1fd6..e10f769 100644 --- a/viz_scripts/generic_timeseries.ipynb +++ b/viz_scripts/generic_timeseries.ipynb @@ -93,7 +93,8 @@ " study_type,\n", " dynamic_labels,\n", " dic_re,\n", - " include_test_users=include_test_users)\n", + " include_test_users=include_test_users,\n", + " add_footprint=True)\n", "expanded_ct = scaffolding.unpack_energy_emissions(expanded_ct) if \"mode_confirm\" in expanded_ct.columns else expanded_ct\n", "\n", "expanded_ct_sensed, file_suffix_sensed, quality_text_sensed, debug_df_sensed = await scaffolding.load_viz_notebook_sensor_inference_data(year,\n", diff --git a/viz_scripts/scaffolding.py b/viz_scripts/scaffolding.py index 0cb6e9a..d147a70 100644 --- a/viz_scripts/scaffolding.py +++ b/viz_scripts/scaffolding.py @@ -143,7 +143,7 @@ def expand_userinputs(labeled_ct): unique_users = lambda df: len(df.user_id.unique()) if "user_id" in df.columns else 0 trip_label_count = lambda s, df: len(df[s].dropna()) if s in df.columns else 0 -async def load_viz_notebook_data(year, month, program, study_type, dynamic_labels, dic_re, dic_pur=None, include_test_users=False): +async def load_viz_notebook_data(year, month, program, study_type, dynamic_labels, dic_re, dic_pur=None, include_test_users=False, add_footprint=False): #TODO - see how slow the loading the footprint is compared to just the baseMode, and evaluate if passing param around is needed """ Inputs: year/month/program/study_type = parameters from the visualization notebook @@ -153,7 +153,7 @@ async def load_viz_notebook_data(year, month, program, study_type, dynamic_label """ # Access database tq = get_time_query(year, month) - participant_ct_df = await load_all_participant_trips(program, tq, include_test_users, True) + participant_ct_df = await load_all_participant_trips(program, tq, include_test_users, add_footprint) labeled_ct = filter_labeled_trips(participant_ct_df) expanded_ct = expand_userinputs(labeled_ct) expanded_ct = data_quality_check(expanded_ct) From 579b3499db4d36876b7a07033e49deb7053c69f4 Mon Sep 17 00:00:00 2001 From: Abby Wheelis Date: Wed, 18 Sep 2024 10:22:26 -0600 Subject: [PATCH 13/32] use the newest version of e-mission-common updated the version in the build script and updated the calls in scaffolding.py to use the new call pattern some slight data formatting required --- .../environment36.dashboard.additions.yml | 2 +- viz_scripts/scaffolding.py | 37 ++++++++++--------- 2 files changed, 21 insertions(+), 18 deletions(-) diff --git a/viz_scripts/docker/environment36.dashboard.additions.yml b/viz_scripts/docker/environment36.dashboard.additions.yml index 96bf23e..c4298c0 100644 --- a/viz_scripts/docker/environment36.dashboard.additions.yml +++ b/viz_scripts/docker/environment36.dashboard.additions.yml @@ -5,6 +5,6 @@ channels: dependencies: - seaborn=0.11.1 - pip: - - git+https://github.com/JGreenlee/e-mission-common@0.6.0 + - git+https://github.com/JGreenlee/e-mission-common@0.6.1 - nbparameterise==0.6 - devcron==0.4 diff --git a/viz_scripts/scaffolding.py b/viz_scripts/scaffolding.py index d147a70..ff9dc95 100644 --- a/viz_scripts/scaffolding.py +++ b/viz_scripts/scaffolding.py @@ -57,37 +57,40 @@ async def add_base_mode_footprint(trip_list): #TODO filter ahead of this so only labeled trips get a footprint OR display uncertainties labels = await read_json_resource("label-options.default.json") value_to_basemode = {mode["value"]: mode.get("base_mode", mode.get("baseMode", "UNKNOWN")) for mode in labels["MODE"]} - + for trip in trip_list: - try: - trip['data']['base_mode'] = value_to_basemode.get(trip['data']['user_input']['mode_confirm'], "UNKNOWN") - trip['data']['mode_confirm_footprint'], trip['data']['mode_confirm_footprint_metadata'] = await emffc.calc_footprint_for_trip(trip['data'], {'base_mode' :trip['data']['base_mode']}) - - if 'replaced_mode' in trip['data']['user_input'].keys(): - trip['data']['replaced_base_mode'] = value_to_basemode.get(trip['data']['user_input']['replaced_mode'], "UNKNOWN") - trip['data']['replaced_mode_footprint'], trip['data']['replaced_mode_footprint_metadata'] = await emffc.calc_footprint_for_trip(trip['data'], {'base_mode' :trip['data']['replaced_base_mode']}) - else: + #format so emffc can get id for metadata + trip['data']['_id'] = trip['_id'] + if trip['data']['user_input'] != {}: + try: + trip['data']['base_mode'] = value_to_basemode.get(trip['data']['user_input']['mode_confirm'], "UNKNOWN") + trip['data']['mode_confirm_footprint'], trip['data']['mode_confirm_footprint_metadata'] = await emffc.calc_footprint_for_trip(trip['data'], labels, mode_key='mode') + + if 'replaced_mode' in trip['data']['user_input'].keys(): + trip['data']['user_input']['replaced_mode_confirm'] = trip['data']['user_input']['replaced_mode'] + trip['data']['replaced_base_mode'] = value_to_basemode.get(trip['data']['user_input']['replaced_mode'], "UNKNOWN") + trip['data']['replaced_mode_footprint'], trip['data']['replaced_mode_footprint_metadata'] = await emffc.calc_footprint_for_trip(trip['data'], labels, mode_key='replaced_mode') + else: + trip['data']['replaced_base_mode'] = "UNKNOWN" + trip['data']['replaced_mode_footprint'] = {} + + except: + print("hit exception") + trip['data']['base_mode'] = "UNKNOWN" trip['data']['replaced_base_mode'] = "UNKNOWN" + trip['data']['mode_confirm_footprint'] = {} trip['data']['replaced_mode_footprint'] = {} - - except: - trip['data']['base_mode'] = "UNKNOWN" - trip['data']['replaced_base_mode'] = "UNKNOWN" - trip['data']['mode_confirm_footprint'] = {} - trip['data']['replaced_mode_footprint'] = {} return trip_list async def load_all_confirmed_trips(tq, add_footprint): agg = esta.TimeSeries.get_aggregate_time_series() result_it = agg.find_entries(["analysis/confirmed_trip"], tq) - print(result_it) if add_footprint: processed_list = await add_base_mode_footprint(list(result_it)) all_ct = agg.to_data_df("analysis/confirmed_trip", processed_list) else: all_ct = agg.to_data_df("analysis/confirmed_trip", result_it) - print(all_ct) print("Loaded all confirmed trips of length %s" % len(all_ct)) disp.display(all_ct.head()) return all_ct From e70d6937bc5c9bfa1fc74311c7ca1b57c36d7dbb Mon Sep 17 00:00:00 2001 From: Abby Wheelis Date: Sun, 22 Sep 2024 16:44:00 -0600 Subject: [PATCH 14/32] remove label dictionaries Now relying on the default labels from emcommon --- viz_scripts/energy_calculations.ipynb | 21 ------- viz_scripts/generic_metrics.ipynb | 12 ---- viz_scripts/generic_timeseries.ipynb | 30 ++++------ viz_scripts/mapping_dictionaries.ipynb | 65 ---------------------- viz_scripts/mode_specific_metrics.ipynb | 12 ---- viz_scripts/mode_specific_timeseries.ipynb | 23 ++++---- viz_scripts/scaffolding.py | 35 +++++------- 7 files changed, 34 insertions(+), 164 deletions(-) delete mode 100644 viz_scripts/mapping_dictionaries.ipynb diff --git a/viz_scripts/energy_calculations.ipynb b/viz_scripts/energy_calculations.ipynb index c1ecc4f..6bb2cec 100644 --- a/viz_scripts/energy_calculations.ipynb +++ b/viz_scripts/energy_calculations.ipynb @@ -70,25 +70,6 @@ " raise Exception(\"The plots in this notebook are only relevant to programs\")" ] }, - { - "cell_type": "code", - "execution_count": null, - "id": "nearby-fruit", - "metadata": {}, - "outputs": [], - "source": [ - "# Loading mapping dictionaries from mapping_dictionaries notebook\n", - "%store -r df_ei\n", - "%store -r dic_re\n", - "%store -r dic_pur\n", - "%store -r dic_fuel\n", - "\n", - "# convert a dictionary to a defaultdict\n", - "dic_re = defaultdict(lambda: 'Other',dic_re)\n", - "dic_pur = defaultdict(lambda: 'Other',dic_pur)\n", - "dic_fuel = defaultdict(lambda: 'Other',dic_fuel)" - ] - }, { "cell_type": "markdown", "id": "parallel-patch", @@ -111,8 +92,6 @@ " program,\n", " study_type,\n", " dynamic_labels,\n", - " dic_re,\n", - " dic_pur=dic_pur,\n", " include_test_users=include_test_users,\n", " add_footprint=True)\n", "\n", diff --git a/viz_scripts/generic_metrics.ipynb b/viz_scripts/generic_metrics.ipynb index 00fc252..e7cd15b 100644 --- a/viz_scripts/generic_metrics.ipynb +++ b/viz_scripts/generic_metrics.ipynb @@ -63,14 +63,6 @@ "metadata": {}, "outputs": [], "source": [ - "# Loading mapping dictionaries from mapping_dictionaries notebook\n", - "%store -r dic_re\n", - "%store -r dic_pur\n", - "\n", - "# convert a dictionary to a defaultdict\n", - "dic_re = defaultdict(lambda: 'Other',dic_re)\n", - "dic_pur = defaultdict(lambda: 'Other',dic_pur)\n", - "\n", "# get metric vs imperial vars\n", "label_units, short_label, label_units_lower, distance_col, weight_unit = scaffolding.get_units(use_imperial)" ] @@ -114,8 +106,6 @@ " program,\n", " study_type,\n", " dynamic_labels,\n", - " dic_re,\n", - " dic_pur=dic_pur,\n", " include_test_users=include_test_users)" ] }, @@ -161,8 +151,6 @@ " program,\n", " study_type,\n", " dynamic_labels,\n", - " dic_re,\n", - " dic_pur=dic_pur,\n", " include_test_users=include_test_users)" ] }, diff --git a/viz_scripts/generic_timeseries.ipynb b/viz_scripts/generic_timeseries.ipynb index e10f769..43c2ad1 100644 --- a/viz_scripts/generic_timeseries.ipynb +++ b/viz_scripts/generic_timeseries.ipynb @@ -48,6 +48,8 @@ "from plots import *\n", "import scaffolding\n", "\n", + "import emcommon.util as emcu\n", + "\n", "sns.set_style(\"whitegrid\")\n", "sns.set()\n", "%matplotlib inline" @@ -59,15 +61,6 @@ "metadata": {}, "outputs": [], "source": [ - "# Loading mapping dictionaries from mapping_dictionaries notebook\n", - "%store -r df_ei\n", - "%store -r dic_re\n", - "%store -r dic_fuel\n", - "\n", - "# convert a dictionary to a defaultdict\n", - "dic_re = defaultdict(lambda: 'Other',dic_re)\n", - "dic_fuel = defaultdict(lambda: 'Other',dic_fuel)\n", - "\n", "# get metric vs imperial vars\n", "label_units, dist_unit, label_units_lower, distance_col, weight_unit = scaffolding.get_units(use_imperial)" ] @@ -151,12 +144,14 @@ " expanded_ct_sensed.user_id = pd.Categorical(expanded_ct_sensed.user_id)\n", " expanded_ct_sensed.date_time = pd.Categorical(expanded_ct_sensed.date_time)\n", "\n", + "if len(dynamic_labels) > 0:\n", + " labels=dynamic_labels\n", + "else:\n", + " labels = await emcu.read_json_resource(\"label-options.default.json\")\n", + "dic_mode_mapping = scaffolding.mapping_labels(labels, \"MODE\")\n", + "\n", "if \"Mode_confirm\" in expanded_ct.columns:\n", - " if (len(dynamic_labels) > 0):\n", - " dic_mode_mapping = scaffolding.mapping_labels(dynamic_labels, \"MODE\")\n", - " data.Mode_confirm = pd.Categorical(data.Mode_confirm, ordered=True, categories=np.unique(list(dic_mode_mapping.values())))\n", - " else:\n", - " data.Mode_confirm = pd.Categorical(data.Mode_confirm, ordered=True, categories=np.unique(list(dic_re.values())))" + " data.Mode_confirm = pd.Categorical(data.Mode_confirm, ordered=True, categories=np.unique(list(dic_mode_mapping.values())))" ] }, { @@ -379,11 +374,8 @@ " # Re-establish categorical variable to not include Other and Non-trips\n", " plot_data = plot_data[~plot_data['Mode_confirm'].isin(['Not a Trip','Other'])]\n", "\n", - " if (len(dynamic_labels) > 0):\n", - " dic_mode_mapping = scaffolding.mapping_labels(dynamic_labels, label_type = \"MODE\")\n", - " plot_data.Mode_confirm = pd.Categorical(plot_data.Mode_confirm, ordered=True, categories=np.unique(list(dic_mode_mapping.values()))) \n", - " else:\n", - " plot_data.Mode_confirm = pd.Categorical(plot_data.Mode_confirm, ordered=True, categories=np.unique(list(dic_re.values())))\n", + " plot_data.Mode_confirm = pd.Categorical(plot_data.Mode_confirm, ordered=True, categories=np.unique(list(dic_mode_mapping.values()))) \n", + "\n", " plot_title= plot_title_no_quality+\"\\n\"+quality_text\n", " ylab = 'Proportion of All Trips'\n", " legend_title = 'Confirmed Mode'\n", diff --git a/viz_scripts/mapping_dictionaries.ipynb b/viz_scripts/mapping_dictionaries.ipynb deleted file mode 100644 index aa6046d..0000000 --- a/viz_scripts/mapping_dictionaries.ipynb +++ /dev/null @@ -1,65 +0,0 @@ -{ - "cells": [ - { - "cell_type": "code", - "execution_count": null, - "id": "available-fusion", - "metadata": {}, - "outputs": [], - "source": [ - "import pandas as pd" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "obvious-chapter", - "metadata": {}, - "outputs": [], - "source": [ - "df_pur = pd.read_csv(r'auxiliary_files/purpose_labels.csv')\n", - "df_re = pd.read_csv(r'auxiliary_files/mode_labels.csv')\n", - "df_ei = pd.read_csv(r'auxiliary_files/energy_intensity.csv')\n", - "\n", - "#dictionaries:\n", - "dic_pur = dict(zip(df_pur['purpose_confirm'],df_pur['bin_purpose'])) # bin purpose\n", - "dic_re = dict(zip(df_re['replaced_mode'],df_re['mode_clean'])) # bin modes\n", - "dic_fuel = dict(zip(df_ei['mode'],df_ei['fuel']))" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "younger-indication", - "metadata": {}, - "outputs": [], - "source": [ - "%store df_ei\n", - "%store dic_re\n", - "%store dic_pur\n", - "%store dic_fuel" - ] - } - ], - "metadata": { - "kernelspec": { - "display_name": "Python 3 (ipykernel)", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.7.12" - } - }, - "nbformat": 4, - "nbformat_minor": 5 -} diff --git a/viz_scripts/mode_specific_metrics.ipynb b/viz_scripts/mode_specific_metrics.ipynb index a1d362e..a165630 100644 --- a/viz_scripts/mode_specific_metrics.ipynb +++ b/viz_scripts/mode_specific_metrics.ipynb @@ -76,14 +76,6 @@ "metadata": {}, "outputs": [], "source": [ - "# Loading mapping dictionaries from mapping_dictionaries notebook\n", - "%store -r dic_re\n", - "%store -r dic_pur\n", - "\n", - "# convert a dictionary to a defaultdict\n", - "dic_re = defaultdict(lambda: 'Other',dic_re)\n", - "dic_pur = defaultdict(lambda: 'Other',dic_pur)\n", - "\n", "# get metric vs imperial vars\n", "label_units, short_label, label_units_lower, distance_col, weight_unit = scaffolding.get_units(use_imperial)" ] @@ -127,8 +119,6 @@ " program,\n", " study_type,\n", " dynamic_labels,\n", - " dic_re,\n", - " dic_pur=dic_pur,\n", " include_test_users=include_test_users)" ] }, @@ -152,8 +142,6 @@ " program,\n", " study_type,\n", " dynamic_labels,\n", - " dic_re,\n", - " dic_pur=dic_pur,\n", " include_test_users=include_test_users)" ] }, diff --git a/viz_scripts/mode_specific_timeseries.ipynb b/viz_scripts/mode_specific_timeseries.ipynb index 58ee56d..2342951 100644 --- a/viz_scripts/mode_specific_timeseries.ipynb +++ b/viz_scripts/mode_specific_timeseries.ipynb @@ -47,6 +47,8 @@ "from plots import *\n", "import scaffolding\n", "\n", + "import emcommon.util as emcu\n", + "\n", "sns.set_style(\"whitegrid\")\n", "sns.set()\n", "%matplotlib inline" @@ -71,14 +73,6 @@ "metadata": {}, "outputs": [], "source": [ - "# Loading mapping dictionaries from mapping_dictionaries notebook\n", - "%store -r dic_re\n", - "%store -r dic_pur\n", - "\n", - "# convert a dictionary to a defaultdict\n", - "dic_re = defaultdict(lambda: 'Other',dic_re)\n", - "dic_pur = defaultdict(lambda: 'Other',dic_pur)\n", - "\n", "# get metric vs imperial vars\n", "label_units, short_label, label_units_lower, distance_col, weight_unit = scaffolding.get_units(use_imperial)" ] @@ -101,8 +95,6 @@ " program,\n", " study_type,\n", " dynamic_labels,\n", - " dic_re,\n", - " dic_pur=dic_pur,\n", " include_test_users=include_test_users)" ] }, @@ -185,9 +177,14 @@ " mode_distance_interest = mode_distance[mode_distance['mode_confirm']==mode_of_interest].copy()\n", "\n", " # Mapping new mode labels with dictionaries\n", - " mode_counts['Mode_confirm'] = mode_counts['mode_confirm'].map(dic_re)\n", - " mode_counts_interest['Mode_confirm'] = mode_counts_interest['mode_confirm'].map(dic_re)\n", - " mode_distance_interest['Mode_confirm'] = mode_distance_interest['mode_confirm'].map(dic_re)\n", + " if len(dynamic_labels) > 0:\n", + " labels=dynamic_labels\n", + " else:\n", + " labels = await emcu.read_json_resource(\"label-options.default.json\")\n", + " dic_mode_mapping = scaffolding.mapping_labels(labels, \"MODE\")\n", + " mode_counts['Mode_confirm'] = mode_counts['mode_confirm'].map(dic_mode_mapping)\n", + " mode_counts_interest['Mode_confirm'] = mode_counts_interest['mode_confirm'].map(dic_mode_mapping)\n", + " mode_distance_interest['Mode_confirm'] = mode_distance_interest['mode_confirm'].map(dic_mode_mapping)\n", " quality_text = scaffolding.get_quality_text(expanded_ct, expanded_ct[expanded_ct['mode_confirm'] == mode_of_interest], mode_of_interest, include_test_users)" ] }, diff --git a/viz_scripts/scaffolding.py b/viz_scripts/scaffolding.py index a9aaafe..3d57e96 100644 --- a/viz_scripts/scaffolding.py +++ b/viz_scripts/scaffolding.py @@ -181,7 +181,7 @@ def _select_max_label(row): unique_users = lambda df: len(df.user_id.unique()) if "user_id" in df.columns else 0 trip_label_count = lambda s, df: len(df[s].dropna()) if s in df.columns else 0 -async def load_viz_notebook_data(year, month, program, study_type, dynamic_labels, dic_re, dic_pur=None, include_test_users=False, add_footprint=False): +async def load_viz_notebook_data(year, month, program, study_type, dynamic_labels, include_test_users=False, add_footprint=False): #TODO - see how slow the loading the footprint is compared to just the baseMode, and evaluate if passing param around is needed """ Inputs: year/month/program/study_type = parameters from the visualization notebook @@ -195,7 +195,7 @@ async def load_viz_notebook_data(year, month, program, study_type, dynamic_label labeled_ct = filter_labeled_trips(participant_ct_df) expanded_ct = expand_userinputs(labeled_ct) expanded_ct = data_quality_check(expanded_ct) - expanded_ct = await map_trip_data(expanded_ct, study_type, dynamic_labels, dic_re, dic_pur) + expanded_ct = await map_trip_data(expanded_ct, study_type, dynamic_labels) # Document data quality file_suffix = get_file_suffix(year, month, program) @@ -215,7 +215,7 @@ async def load_viz_notebook_data(year, month, program, study_type, dynamic_label return expanded_ct, file_suffix, quality_text, debug_df -async def map_trip_data(expanded_trip_df, study_type, dynamic_labels, dic_re, dic_pur): +async def map_trip_data(expanded_trip_df, study_type, dynamic_labels): # Change meters to miles # CASE 2 of https://github.com/e-mission/em-public-dashboard/issues/69#issuecomment-1256835867 if "distance" in expanded_trip_df.columns: @@ -231,22 +231,16 @@ async def map_trip_data(expanded_trip_df, study_type, dynamic_labels, dic_re, di # Map new mode labels with translations dictionary from dynamic_labels # CASE 2 of https://github.com/e-mission/em-public-dashboard/issues/69#issuecomment-1256835867 if "mode_confirm" in expanded_trip_df.columns: - if (len(dynamic_labels)): - dic_mode_mapping = mapping_labels(dynamic_labels, "MODE") - expanded_trip_df['Mode_confirm'] = expanded_trip_df['mode_confirm'].map(dic_mode_mapping) - else: - expanded_trip_df['Mode_confirm'] = expanded_trip_df['mode_confirm'].map(dic_re) + dic_mode_mapping = mapping_labels(labels, "MODE") + expanded_trip_df['Mode_confirm'] = expanded_trip_df['mode_confirm'].map(dic_mode_mapping) # If the 'mode_confirm' is not available as the list of keys in the dynamic_labels or label_options.default.json, then, we should transform it as 'other' mode_values = [item['value'] for item in labels['MODE']] expanded_trip_df['mode_confirm_w_other'] = expanded_trip_df['mode_confirm'].apply(lambda mode: 'other' if mode not in mode_values else mode) if study_type == 'program': # CASE 2 of https://github.com/e-mission/em-public-dashboard/issues/69#issuecomment-1256835867 if 'replaced_mode' in expanded_trip_df.columns: - if (len(dynamic_labels)): - dic_replaced_mapping = mapping_labels(dynamic_labels, "REPLACED_MODE") - expanded_trip_df['Replaced_mode'] = expanded_trip_df['replaced_mode'].map(dic_replaced_mapping) - else: - expanded_trip_df['Replaced_mode'] = expanded_trip_df['replaced_mode'].map(dic_re) + dic_replaced_mapping = mapping_labels(labels, "REPLACED_MODE") + expanded_trip_df['Replaced_mode'] = expanded_trip_df['replaced_mode'].map(dic_replaced_mapping) replaced_modes = [item['value'] for item in labels['REPLACED_MODE']] expanded_trip_df['replaced_mode_w_other'] = expanded_trip_df['replaced_mode'].apply(lambda mode: 'other' if mode not in replaced_modes else mode) else: @@ -256,18 +250,15 @@ async def map_trip_data(expanded_trip_df, study_type, dynamic_labels, dic_re, di # Trip purpose mapping # CASE 2 of https://github.com/e-mission/em-public-dashboard/issues/69#issuecomment-1256835867 - if dic_pur is not None and "purpose_confirm" in expanded_trip_df.columns: - if (len(dynamic_labels)): - dic_purpose_mapping = mapping_labels(dynamic_labels, "PURPOSE") - expanded_trip_df['Trip_purpose'] = expanded_trip_df['purpose_confirm'].map(dic_purpose_mapping) - else: - expanded_trip_df['Trip_purpose'] = expanded_trip_df['purpose_confirm'].map(dic_pur) + if "purpose_confirm" in expanded_trip_df.columns: + dic_purpose_mapping = mapping_labels(labels, "PURPOSE") + expanded_trip_df['Trip_purpose'] = expanded_trip_df['purpose_confirm'].map(dic_purpose_mapping) purpose_values = [item['value'] for item in labels['PURPOSE']] expanded_trip_df['purpose_confirm_w_other'] = expanded_trip_df['purpose_confirm'].apply(lambda value: 'other' if value not in purpose_values else value) return expanded_trip_df -async def load_viz_notebook_inferred_data(year, month, program, study_type, dynamic_labels, dic_re, dic_pur=None, include_test_users=False): +async def load_viz_notebook_inferred_data(year, month, program, study_type, dynamic_labels, include_test_users=False): """ Inputs: year/month/program/study_type = parameters from the visualization notebook dic_* = label mappings; if dic_pur is included it will be used to recode trip purpose @@ -276,10 +267,10 @@ async def load_viz_notebook_inferred_data(year, month, program, study_type, dyna """ # Access database tq = get_time_query(year, month) - participant_ct_df = load_all_participant_trips(program, tq, include_test_users) + participant_ct_df = await load_all_participant_trips(program, tq, include_test_users) inferred_ct = filter_inferred_trips(participant_ct_df) expanded_it = expand_inferredlabels(inferred_ct) - expanded_it = await map_trip_data(expanded_it, study_type, dynamic_labels, dic_re, dic_pur) + expanded_it = await map_trip_data(expanded_it, study_type, dynamic_labels) # Document data quality file_suffix = get_file_suffix(year, month, program) From d44e3c637c5a8e43c4d7fd93a59e82c4ca0f70b4 Mon Sep 17 00:00:00 2001 From: Abby Wheelis Date: Sun, 22 Sep 2024 16:45:01 -0600 Subject: [PATCH 15/32] remove csvs --- .../auxiliary_files/energy_intensity.csv | 17 ------- viz_scripts/auxiliary_files/mode_labels.csv | 33 ------------- .../auxiliary_files/purpose_labels.csv | 47 ------------------- 3 files changed, 97 deletions(-) delete mode 100644 viz_scripts/auxiliary_files/energy_intensity.csv delete mode 100644 viz_scripts/auxiliary_files/mode_labels.csv delete mode 100644 viz_scripts/auxiliary_files/purpose_labels.csv diff --git a/viz_scripts/auxiliary_files/energy_intensity.csv b/viz_scripts/auxiliary_files/energy_intensity.csv deleted file mode 100644 index c203241..0000000 --- a/viz_scripts/auxiliary_files/energy_intensity.csv +++ /dev/null @@ -1,17 +0,0 @@ -mode,fuel,(kWH)/trip,EI(kWH/PMT),energy_intensity_factor,energy_intensity_units,CO2_factor,CO2_factor_units -"Gas Car, drove alone",gasoline,0,,5170,BTU/PMT,157.2,lb_CO2/MMBTU -"Gas Car, with others",gasoline,0,,2585,BTU/PMT,157.2,lb_CO2/MMBTU -"E-car, drove alone",electric,0,0.25,0.25,kWH/PMT,1166,lb_CO2/MWH -"E-car, with others",electric,0,0.125,0.125,kWH/PMT,1166,lb_CO2/MWH -Taxi/Uber/Lyft,gasoline,0,,7214,BTU/PMT,157.2,lb_CO2/MMBTU -Bus,diesel,0,,4560,BTU/PMT,161.3,lb_CO2/MMBTU -Free Shuttle,diesel,0,,4560,BTU/PMT,161.3,lb_CO2/MMBTU -Train,electric,0,0.37,0.37,kWH/PMT,1166,lb_CO2/MWH -Scooter share,electric,0.0041,0.027,0.027,kWH/PMT,1166,lb_CO2/MWH -E-bike,electric,0,0.022,0.022,kWH/PMT,1166,lb_CO2/MWH -Bikeshare,human_powered,0.09,0,0,kWH/PMT,1166,lb_CO2/MWH -Walk,human_powered,0,,0,,0,0 -Skate board,human_powered,0,,0,,0,0 -Regular Bike,human_powered,0,,0,,0,0 -Not a Trip,none,0,,0,,0,0 -No Travel,none,0,,0,,0,0 diff --git a/viz_scripts/auxiliary_files/mode_labels.csv b/viz_scripts/auxiliary_files/mode_labels.csv deleted file mode 100644 index 551d4ee..0000000 --- a/viz_scripts/auxiliary_files/mode_labels.csv +++ /dev/null @@ -1,33 +0,0 @@ -replaced_mode,mode_confirm,mode_clean -drove_alone,drove_alone,"Gas Car, drove alone" -e_car_drove_alone,e_car_drove_alone,"E-car, drove alone" -work_vehicle,work_vehicle,"Gas Car, drove alone" -bus,bus,Bus -train,train,Train -free_shuttle,free_shuttle,Free Shuttle -"train,_bus and walk","train,_bus and walk",Train -train_and pilot e-bike,train_and pilot e-bike,Train -taxi,taxi,Taxi/Uber/Lyft -friend_picked me up,friend_picked me up,"Gas Car, with others" -carpool_w/ friend to work,carpool_w/ friend to work,"Gas Car, with others" -friend_carpool to work,friend_carpool to work,"Gas Car, with others" -carpool_to work,carpool_to work,"Gas Car, with others" -friend/co_worker carpool,friend/co_worker carpool,"Gas Car, with others" -carpool_to lunch,carpool_to lunch,"Gas Car, with others" -carpool,carpool,"Gas Car, with others" -carpool_for lunch,carpool_for lunch,"Gas Car, with others" -carpool_lunch,carpool_lunch,"Gas Car, with others" -shared_ride,shared_ride,"Gas Car, with others" -e_car_shared_ride,e_car_shared_ride,"E-car, with others" -bikeshare,bikeshare,Bikeshare -scootershare,scootershare,Scooter share -pilot_ebike,pilot_ebike,E-bike -e-bike,e-bike,E-bike -walk,walk,Walk -skateboard,skateboard,Skate board -bike,bike,Regular Bike -the_friend who drives us to work was running errands after the shift before dropping me off. not a trip of mine.,the_friend who drives us to work was running errands after the shift before dropping me off. not a trip of mine.,Not a Trip -not_a_trip,not_a_trip,Not a Trip -no_travel,,No Travel -same_mode,,Same Mode -air,,Airplane diff --git a/viz_scripts/auxiliary_files/purpose_labels.csv b/viz_scripts/auxiliary_files/purpose_labels.csv deleted file mode 100644 index 298e8aa..0000000 --- a/viz_scripts/auxiliary_files/purpose_labels.csv +++ /dev/null @@ -1,47 +0,0 @@ -purpose_confirm,bin_purpose -work_travel,Work -work,Work -home,Home -meal,Meal -shopping,Shopping -personal_med,Personal/Medical -exercise,Recreation/Exercise -transit_transfer,Transit transfer -pick_drop,Pick-up/Drop off -entertainment,Entertainment/Social -car_mechanic,Other -school,School -revisado_bike,Other -placas_de carro,Other -community_walk,Entertainment/Social -gardening,Entertainment/Social -visiting,Entertainment/Social -church,Religious -community_garden,Entertainment/Social -community_meeting,Entertainment/Social -visit_a friend,Entertainment/Social -aseguranza,Other -meeting_bike,Entertainment/Social -gas_station,Other -iglesia,Religious -curso,School -mi_hija recién aliviada,Entertainment/Social -servicio_comunitario,Entertainment/Social -pago_de aseguranza,Other -grupo_comunitario,Entertainment/Social -caminata_comunitaria,Entertainment/Social -bank,Other -religious,Religious -no_travel,No travel -work_break - short walk,Entertainment/Social -work_- lunch break,Meal -friend_was running errands before dropping me off after work,Other -"multiple_errands, etc.",Other -lunch_break,Meal -break,Entertainment/Social -pet,Entertainment/Social -recording_performance at park,Entertainment/Social -not_a trip,not_a_trip -on_the way home,Home -other,Other -nan,nan \ No newline at end of file From 6f2335ac4e1633805550e91abb86c9ac32303fd4 Mon Sep 17 00:00:00 2001 From: iantei Date: Mon, 23 Sep 2024 11:07:25 -0700 Subject: [PATCH 16/32] Remove COPY auxillary_files in Dockerfile since we do not have auxillary_files dir anymore. --- viz_scripts/Dockerfile | 1 - 1 file changed, 1 deletion(-) diff --git a/viz_scripts/Dockerfile b/viz_scripts/Dockerfile index 52c5aa5..195a6f2 100644 --- a/viz_scripts/Dockerfile +++ b/viz_scripts/Dockerfile @@ -16,7 +16,6 @@ RUN /bin/bash -c "source setup/activate.sh && conda env update --name emission - RUN mkdir -p /usr/src/app/saved-notebooks WORKDIR /usr/src/app/saved-notebooks -COPY auxiliary_files ./auxiliary_files COPY bin ./bin COPY *.ipynb . COPY *.py . From 768c38bcf1dc46e789279c6624261ea737b2a42c Mon Sep 17 00:00:00 2001 From: iantei Date: Mon, 23 Sep 2024 11:09:03 -0700 Subject: [PATCH 17/32] Remove mapping_dictionaries from crontab. We don't have mapping_dictionaries anymore. --- viz_scripts/docker/crontab | 1 - 1 file changed, 1 deletion(-) diff --git a/viz_scripts/docker/crontab b/viz_scripts/docker/crontab index ca37fe5..95b588c 100644 --- a/viz_scripts/docker/crontab +++ b/viz_scripts/docker/crontab @@ -1,4 +1,3 @@ -0 7 * * * python bin/update_mappings.py mapping_dictionaries.ipynb >> /var/log/intake.stdinout 2>&1 0 8 * * * python bin/generate_plots.py generic_metrics.ipynb default >> /var/log/intake.stdinout 2>&1 0 8 * * * python bin/generate_plots.py generic_metrics_sensed.ipynb default >> /var/log/intake.stdinout 2>&1 0 8 * * * python bin/generate_plots.py generic_timeseries.ipynb default >> /var/log/intake.stdinout 2>&1 From 5255e1dc64442a3aefbb15a4aa3b9951e458a575 Mon Sep 17 00:00:00 2001 From: iantei Date: Mon, 23 Sep 2024 11:10:00 -0700 Subject: [PATCH 18/32] Remove dic_re as the param from load_viz_notebook_data(). --- viz_scripts/generic_timeseries.ipynb | 1 - 1 file changed, 1 deletion(-) diff --git a/viz_scripts/generic_timeseries.ipynb b/viz_scripts/generic_timeseries.ipynb index 43c2ad1..61a152b 100644 --- a/viz_scripts/generic_timeseries.ipynb +++ b/viz_scripts/generic_timeseries.ipynb @@ -85,7 +85,6 @@ " program,\n", " study_type,\n", " dynamic_labels,\n", - " dic_re,\n", " include_test_users=include_test_users,\n", " add_footprint=True)\n", "expanded_ct = scaffolding.unpack_energy_emissions(expanded_ct) if \"mode_confirm\" in expanded_ct.columns else expanded_ct\n", From e15de8ebe1d30214150f3b86f5deabce3ec4ef1b Mon Sep 17 00:00:00 2001 From: iantei Date: Mon, 23 Sep 2024 11:22:37 -0700 Subject: [PATCH 19/32] Add check to see if there is mode_confirm_footprint in expanded_ct.columns before calling unpack_energy_emissions(). --- viz_scripts/energy_calculations.ipynb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/viz_scripts/energy_calculations.ipynb b/viz_scripts/energy_calculations.ipynb index 6bb2cec..79eccf1 100644 --- a/viz_scripts/energy_calculations.ipynb +++ b/viz_scripts/energy_calculations.ipynb @@ -95,7 +95,7 @@ " include_test_users=include_test_users,\n", " add_footprint=True)\n", "\n", - "expanded_ct = scaffolding.unpack_energy_emissions(expanded_ct)" + "expanded_ct = scaffolding.unpack_energy_emissions(expanded_ct) if \"mode_confirm_footprint\" in expanded_ct.columns else expanded_ct" ] }, { From a804f22c64f8a86bd53408d871de061ac26caa05 Mon Sep 17 00:00:00 2001 From: iantei Date: Mon, 23 Sep 2024 11:25:24 -0700 Subject: [PATCH 20/32] Update check for mode_confirm_footprint instead of mode_confirm, as unpack_energy_emissions look up for mode_confirm_footprint over mode_confirm. --- viz_scripts/generic_timeseries.ipynb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/viz_scripts/generic_timeseries.ipynb b/viz_scripts/generic_timeseries.ipynb index 61a152b..b237322 100644 --- a/viz_scripts/generic_timeseries.ipynb +++ b/viz_scripts/generic_timeseries.ipynb @@ -87,7 +87,7 @@ " dynamic_labels,\n", " include_test_users=include_test_users,\n", " add_footprint=True)\n", - "expanded_ct = scaffolding.unpack_energy_emissions(expanded_ct) if \"mode_confirm\" in expanded_ct.columns else expanded_ct\n", + "expanded_ct = scaffolding.unpack_energy_emissions(expanded_ct) if \"mode_confirm_footprint\" in expanded_ct.columns else expanded_ct\n", "\n", "expanded_ct_sensed, file_suffix_sensed, quality_text_sensed, debug_df_sensed = await scaffolding.load_viz_notebook_sensor_inference_data(year,\n", " month,\n", From 1ca1e560f14ae1c87cac4d673919d8cf7121f364 Mon Sep 17 00:00:00 2001 From: iantei Date: Mon, 23 Sep 2024 12:06:53 -0700 Subject: [PATCH 21/32] Use internal label over Display label for trip_purpose_query. --- viz_scripts/generic_metrics.ipynb | 9 +-------- 1 file changed, 1 insertion(+), 8 deletions(-) diff --git a/viz_scripts/generic_metrics.ipynb b/viz_scripts/generic_metrics.ipynb index e7cd15b..792507f 100644 --- a/viz_scripts/generic_metrics.ipynb +++ b/viz_scripts/generic_metrics.ipynb @@ -263,14 +263,7 @@ "file_name = f\"ntrips_commute_mode_confirm{file_suffix}\"\n", "\n", "try:\n", - " # Preprocess to find commute trips\n", - " if (len(dynamic_labels)):\n", - " purpose_map_label = scaffolding.mapping_labels(dynamic_labels, \"PURPOSE\")\n", - " translation_work = purpose_map_label['work']\n", - " trip_purpose_query = f\"Trip_purpose == '{translation_work}'\"\n", - " else:\n", - " trip_purpose_query = \"Trip_purpose == 'Work'\"\n", - "\n", + " trip_purpose_query = \"purpose_confirm == 'work'\"\n", " expanded_ct_commute = expanded_ct.query(trip_purpose_query)\n", " commute_quality_text = scaffolding.get_quality_text(expanded_ct, expanded_ct_commute, \"commute\", include_test_users) if not expanded_ct.empty else \"\"\n", " expanded_ct_inferred_commute = expanded_ct_inferred.query(trip_purpose_query)\n", From 1fbe879debe308acb686e7e920cf4554c4ec50d0 Mon Sep 17 00:00:00 2001 From: iantei Date: Mon, 23 Sep 2024 12:17:43 -0700 Subject: [PATCH 22/32] Update the bar label for commute trips. --- viz_scripts/generic_metrics.ipynb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/viz_scripts/generic_metrics.ipynb b/viz_scripts/generic_metrics.ipynb index 792507f..b0a7ce6 100644 --- a/viz_scripts/generic_metrics.ipynb +++ b/viz_scripts/generic_metrics.ipynb @@ -282,7 +282,7 @@ " plot_and_text_stacked_bar_chart(expanded_ct_commute, lambda df: df.groupby(\"mode_confirm_w_other\").agg({distance_col: 'count'}).sort_values(by=distance_col, ascending=False), \n", " \"Labeled by user\\n (Confirmed trips)\", ax[0], text_results[0], colors_mode, debug_df, values_to_translations)\n", " plot_and_text_stacked_bar_chart(expanded_ct_inferred_commute, lambda df: df.groupby(\"mode_confirm_w_other\").agg({distance_col: 'count'}).sort_values(by=distance_col, ascending=False), \n", - " \"Labeled by user\\n (Confirmed trips)\", ax[1], text_results[1], colors_mode, debug_df_inferred, values_to_translations)\n", + " \"Labeled and Inferred by OpenPATH\\n (Confirmed trips)\", ax[1], text_results[1], colors_mode, debug_df_inferred, values_to_translations)\n", " set_title_and_save(fig, text_results, plot_title, file_name)\n", "except (AttributeError, KeyError, pd.errors.UndefinedVariableError) as e:\n", " plt.clf()\n", From aaa07ac4ae566284585483dc20a1be8859bd0be3 Mon Sep 17 00:00:00 2001 From: iantei Date: Mon, 23 Sep 2024 13:09:02 -0700 Subject: [PATCH 23/32] Update stacked_bar_quality_text_commute_labeled/inferred for commute trip charts. --- viz_scripts/generic_metrics.ipynb | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/viz_scripts/generic_metrics.ipynb b/viz_scripts/generic_metrics.ipynb index b0a7ce6..2ecc1ab 100644 --- a/viz_scripts/generic_metrics.ipynb +++ b/viz_scripts/generic_metrics.ipynb @@ -280,9 +280,9 @@ " fig, ax = plt.subplots(nrows=2, ncols=1, figsize=(15,2*2), sharex=True) \n", " text_results = [[\"Unmodified Alt Text\", \"Unmodified HTML\"], [\"Unmodified Alt Text\", \"Unmodified HTML\"]]\n", " plot_and_text_stacked_bar_chart(expanded_ct_commute, lambda df: df.groupby(\"mode_confirm_w_other\").agg({distance_col: 'count'}).sort_values(by=distance_col, ascending=False), \n", - " \"Labeled by user\\n (Confirmed trips)\", ax[0], text_results[0], colors_mode, debug_df, values_to_translations)\n", + " \"Labeled by user\\n\"+stacked_bar_quality_text_commute_labeled, ax[0], text_results[0], colors_mode, debug_df, values_to_translations)\n", " plot_and_text_stacked_bar_chart(expanded_ct_inferred_commute, lambda df: df.groupby(\"mode_confirm_w_other\").agg({distance_col: 'count'}).sort_values(by=distance_col, ascending=False), \n", - " \"Labeled and Inferred by OpenPATH\\n (Confirmed trips)\", ax[1], text_results[1], colors_mode, debug_df_inferred, values_to_translations)\n", + " \"Labeled and Inferred by OpenPATH\\n\"+stacked_bar_quality_text_commute_inferred, ax[1], text_results[1], colors_mode, debug_df_inferred, values_to_translations)\n", " set_title_and_save(fig, text_results, plot_title, file_name)\n", "except (AttributeError, KeyError, pd.errors.UndefinedVariableError) as e:\n", " plt.clf()\n", From ea3cd89506df47b7e10cb80c56a21cc4a490aaf5 Mon Sep 17 00:00:00 2001 From: iantei Date: Mon, 23 Sep 2024 14:54:20 -0700 Subject: [PATCH 24/32] Remove check for replaced_mode since replaced_mode are only available for program, not study. --- viz_scripts/scaffolding.py | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/viz_scripts/scaffolding.py b/viz_scripts/scaffolding.py index 3d57e96..3463d16 100644 --- a/viz_scripts/scaffolding.py +++ b/viz_scripts/scaffolding.py @@ -164,16 +164,14 @@ def _select_max_label(row): return row['user_input'] max_entry = max(row['inferred_labels'], key=lambda x: x['p']) return max_entry['labels'] if max_entry['p'] > row['confidence_threshold'] else { - 'mode_confirm': 'uncertain', - 'purpose_confirm': 'uncertain', - 'replaced_mode': 'uncertain' + 'mode_confirm': 'uncertain' } labeled_inferred_labels = labeled_inferred_ct.apply(_select_max_label, axis=1).apply(pd.Series) disp.display(labeled_inferred_labels.head()) expanded_labeled_inferred_ct = pd.concat([labeled_inferred_ct, labeled_inferred_labels], axis=1) - # Filter out the dataframe in which mode_confirm, purpose_confirm and replaced_mode is uncertain - expanded_labeled_inferred_ct = expanded_labeled_inferred_ct[(expanded_labeled_inferred_ct['mode_confirm'] != 'uncertain') & (expanded_labeled_inferred_ct['purpose_confirm'] != 'uncertain') & (expanded_labeled_inferred_ct['replaced_mode'] != 'uncertain')] + # Filter out the dataframe in which mode_confirm is uncertain + expanded_labeled_inferred_ct = expanded_labeled_inferred_ct[(expanded_labeled_inferred_ct['mode_confirm'] != 'uncertain')] disp.display(expanded_labeled_inferred_ct.head()) return expanded_labeled_inferred_ct From 4ccd1593bba1efc64e89c979d447265b1229bfa1 Mon Sep 17 00:00:00 2001 From: iantei Date: Mon, 23 Sep 2024 16:17:58 -0700 Subject: [PATCH 25/32] Add a check to not execute generic_metrics notebook for survey metrics data. --- viz_scripts/generic_metrics.ipynb | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/viz_scripts/generic_metrics.ipynb b/viz_scripts/generic_metrics.ipynb index 2ecc1ab..ab0a581 100644 --- a/viz_scripts/generic_metrics.ipynb +++ b/viz_scripts/generic_metrics.ipynb @@ -32,7 +32,8 @@ "include_test_users = False\n", "dynamic_labels = {}\n", "use_imperial = True\n", - "sensed_algo_prefix = \"cleaned\"" + "sensed_algo_prefix = \"cleaned\"\n", + "survey_info = {}" ] }, { @@ -56,6 +57,20 @@ "%matplotlib inline" ] }, + { + "cell_type": "code", + "execution_count": null, + "id": "eaf90438", + "metadata": {}, + "outputs": [], + "source": [ + "# Do not run this notebook if it has survey_info; nbclient will run up through this cell\n", + "if survey_info != {}:\n", + " ipython = get_ipython()\n", + " ipython._showtraceback = scaffolding.no_traceback_handler\n", + " raise Exception(\"The plots in this notebook are relevant to survey metrics\")" + ] + }, { "cell_type": "code", "execution_count": null, From 22f720cc245d6d5522a1904ddebd7f92d7d6a9ae Mon Sep 17 00:00:00 2001 From: iantei Date: Mon, 23 Sep 2024 16:22:07 -0700 Subject: [PATCH 26/32] Update the exception message in stopping execution of survey info related data for generic_metrics noteboook. --- viz_scripts/generic_metrics.ipynb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/viz_scripts/generic_metrics.ipynb b/viz_scripts/generic_metrics.ipynb index ab0a581..f5b27e7 100644 --- a/viz_scripts/generic_metrics.ipynb +++ b/viz_scripts/generic_metrics.ipynb @@ -68,7 +68,7 @@ "if survey_info != {}:\n", " ipython = get_ipython()\n", " ipython._showtraceback = scaffolding.no_traceback_handler\n", - " raise Exception(\"The plots in this notebook are relevant to survey metrics\")" + " raise Exception(\"The plots in this notebook are irrelevant to survey info\")" ] }, { From 468d40ed6279c4acca1d24641366869a9e29594e Mon Sep 17 00:00:00 2001 From: iantei Date: Tue, 24 Sep 2024 09:56:23 -0700 Subject: [PATCH 27/32] Change the notebook execution check to look up for trip-labels as MULTILABEL for generic_metrics notebook --- viz_scripts/generic_metrics.ipynb | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/viz_scripts/generic_metrics.ipynb b/viz_scripts/generic_metrics.ipynb index f5b27e7..2f173df 100644 --- a/viz_scripts/generic_metrics.ipynb +++ b/viz_scripts/generic_metrics.ipynb @@ -64,11 +64,11 @@ "metadata": {}, "outputs": [], "source": [ - "# Do not run this notebook if it has survey_info; nbclient will run up through this cell\n", - "if survey_info != {}:\n", + "# Do not run this notebook if it does not have trip-labels as MULTILABEL; nbclient will run up through this cell\n", + "if not survey_info.get('trip-labels', None) == 'MULTILABEL':\n", " ipython = get_ipython()\n", " ipython._showtraceback = scaffolding.no_traceback_handler\n", - " raise Exception(\"The plots in this notebook are irrelevant to survey info\")" + " raise Exception(\"The plots in this notebook are only relevant for MULTILABEL trip-labels\")" ] }, { From 2ebb1cf8716081ad64c7e5be900653023fd5e106 Mon Sep 17 00:00:00 2001 From: Abby Wheelis Date: Wed, 25 Sep 2024 11:01:14 -0600 Subject: [PATCH 28/32] revert addition of emcommon import --- viz_scripts/docker/environment36.dashboard.additions.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/viz_scripts/docker/environment36.dashboard.additions.yml b/viz_scripts/docker/environment36.dashboard.additions.yml index c4298c0..59d26eb 100644 --- a/viz_scripts/docker/environment36.dashboard.additions.yml +++ b/viz_scripts/docker/environment36.dashboard.additions.yml @@ -5,6 +5,5 @@ channels: dependencies: - seaborn=0.11.1 - pip: - - git+https://github.com/JGreenlee/e-mission-common@0.6.1 - nbparameterise==0.6 - devcron==0.4 From 3f43e00c156cb0776e29a7e5a24a70808a2fe182 Mon Sep 17 00:00:00 2001 From: Abby Wheelis Date: Wed, 25 Sep 2024 11:11:37 -0600 Subject: [PATCH 29/32] update condition for generic metrics rather than halt on survey info that does not have multilabel halt on survey info that does have enketo blank survey info means default, which means multilabel, and we should still proceed --- viz_scripts/generic_metrics.ipynb | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/viz_scripts/generic_metrics.ipynb b/viz_scripts/generic_metrics.ipynb index 2f173df..b0992f1 100644 --- a/viz_scripts/generic_metrics.ipynb +++ b/viz_scripts/generic_metrics.ipynb @@ -64,11 +64,11 @@ "metadata": {}, "outputs": [], "source": [ - "# Do not run this notebook if it does not have trip-labels as MULTILABEL; nbclient will run up through this cell\n", - "if not survey_info.get('trip-labels', None) == 'MULTILABEL':\n", + "# Do not run this notebook if it has trip-labels as ENKETO nbclient will run up through this cell\n", + "if survey_info.get('trip-labels', None) == 'ENKETO':\n", " ipython = get_ipython()\n", " ipython._showtraceback = scaffolding.no_traceback_handler\n", - " raise Exception(\"The plots in this notebook are only relevant for MULTILABEL trip-labels\")" + " raise Exception(\"The plots in this notebook are not relecant for ENKETO trip-labels\")" ] }, { From 1a8629dcb46f0803f2982d0bfbb190aa47aa277c Mon Sep 17 00:00:00 2001 From: Abby Wheelis Date: Wed, 25 Sep 2024 11:15:47 -0600 Subject: [PATCH 30/32] correct import --- viz_scripts/scaffolding.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/viz_scripts/scaffolding.py b/viz_scripts/scaffolding.py index 3463d16..3e46eba 100644 --- a/viz_scripts/scaffolding.py +++ b/viz_scripts/scaffolding.py @@ -11,8 +11,6 @@ import emission.core.wrapper.localdate as ecwl import emcommon.diary.base_modes as emcdb import emcommon.util as emcu - -from emcommon.util import read_json_resource import emcommon.metrics.footprint.footprint_calculations as emffc # Module for pretty-printing outputs (e.g. head) to help users # understand what is going on @@ -58,7 +56,7 @@ def get_participant_uuids(program, load_test_users): async def add_base_mode_footprint(trip_list): #TODO filter ahead of this so only labeled trips get a footprint OR display uncertainties - labels = await read_json_resource("label-options.default.json") + labels = await emcu.read_json_resource("label-options.default.json") value_to_basemode = {mode["value"]: mode.get("base_mode", mode.get("baseMode", "UNKNOWN")) for mode in labels["MODE"]} for trip in trip_list: From 29f99b1442b7f64309ceda9f11e57253ab2f458a Mon Sep 17 00:00:00 2001 From: iantei Date: Wed, 25 Sep 2024 15:25:30 -0700 Subject: [PATCH 31/32] Remove metrics_program/study_withoutEnergyMetrics.html, and its use is index.html. --- frontend/index.html | 15 +++----- .../metrics_program_withoutEnergyMetrics.html | 34 ------------------- .../metrics_study_withoutEnergyMetrics.html | 19 ----------- 3 files changed, 5 insertions(+), 63 deletions(-) delete mode 100644 frontend/metrics_program_withoutEnergyMetrics.html delete mode 100644 frontend/metrics_study_withoutEnergyMetrics.html diff --git a/frontend/index.html b/frontend/index.html index 1997a0d..309c3ff 100644 --- a/frontend/index.html +++ b/frontend/index.html @@ -426,16 +426,13 @@ }); } else if (data.intro.program_or_study == 'program') { //CASE: PROGRAM - // Note: We're disabling energy metrics on public dashboard when dynamic labels are available. - // TODO: Remove the if (data.label_options) in future when energy computation is handled properly. if (dynamic_labels) { - load_file = "metrics_program_withoutEnergyMetrics.html" console.log("Dynamic Labels are available for: " + STUDY_CONFIG) } else { - load_file = "metrics_program.html" console.log("Dynamic Labels are unavailable for: " + STUDY_CONFIG) } + load_file = "metrics_program.html" $.get(load_file, function (response) { const configuredResponse = response.replaceAll("${data.intro.mode_studied}", mode_studied); console.log("configuring units"); @@ -454,15 +451,13 @@ else // CASE: STUDY { if (dynamic_labels) { - console.log("Dynamic Labels are available for: " + STUDY_CONFIG) - load_file = 'metrics_study_withoutEnergyMetrics.html' - // $('#metric').load('metrics_study_withoutEnergyMetrics.html'); + console.log("Dynamic Labels are available for: " + STUDY_CONFIG) } else { console.log("Dynamic Labels are unavailable for: " + STUDY_CONFIG) - load_file = 'metrics_study.html' - // $('#metric').load('metrics_study.html'); - }; + } + load_file = 'metrics_study.html' + // $('#metric').load('metrics_study.html'); console.log("loading study, replacing units") $.get(load_file, function (response) { console.log("configuring units"); diff --git a/frontend/metrics_program_withoutEnergyMetrics.html b/frontend/metrics_program_withoutEnergyMetrics.html deleted file mode 100644 index ab42bcb..0000000 --- a/frontend/metrics_program_withoutEnergyMetrics.html +++ /dev/null @@ -1,34 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - diff --git a/frontend/metrics_study_withoutEnergyMetrics.html b/frontend/metrics_study_withoutEnergyMetrics.html deleted file mode 100644 index 7f336bf..0000000 --- a/frontend/metrics_study_withoutEnergyMetrics.html +++ /dev/null @@ -1,19 +0,0 @@ - - - - - - - - - - - - - - - - - - - From edfc6ecede00d9f803bd63409b2b847cca4fd772 Mon Sep 17 00:00:00 2001 From: iantei Date: Wed, 25 Sep 2024 15:28:41 -0700 Subject: [PATCH 32/32] Remove check to build ts_energy_user only when dynamic labels is available. --- viz_scripts/generic_timeseries.ipynb | 29 ++++++++++++++-------------- 1 file changed, 14 insertions(+), 15 deletions(-) diff --git a/viz_scripts/generic_timeseries.ipynb b/viz_scripts/generic_timeseries.ipynb index b237322..54ab23a 100644 --- a/viz_scripts/generic_timeseries.ipynb +++ b/viz_scripts/generic_timeseries.ipynb @@ -252,23 +252,22 @@ "metadata": {}, "outputs": [], "source": [ - "if (len(dynamic_labels) == 0):\n", - " plot_title_no_quality = 'Net Daily Energy (All Users, excluding air)'\n", - " file_name = \"ts_energy_user%s\"%file_suffix\n", + "plot_title_no_quality = 'Net Daily Energy (All Users, excluding air)'\n", + "file_name = \"ts_energy_user%s\"%file_suffix\n", "\n", - " try:\n", - " # Energy per week across all users (net impact)\n", - " plot_data = energy.groupby(['date_time'], as_index=False)['Mode_confirm_EI(kWH)'].agg(['sum'])\n", - " plot_data = plot_data.merge(active_users, on='date_time')\n", - " plot_data['sum'] = plot_data['sum'] / plot_data['active_users']\n", + "try:\n", + " # Energy per week across all users (net impact)\n", + " plot_data = energy.groupby(['date_time'], as_index=False)['Mode_confirm_EI(kWH)'].agg(['sum'])\n", + " plot_data = plot_data.merge(active_users, on='date_time')\n", + " plot_data['sum'] = plot_data['sum'] / plot_data['active_users']\n", "\n", - " plot_title= plot_title_no_quality+\"\\n\"+quality_text\n", - " ylab = 'Energy (kWH/day/user)'\n", - " timeseries_plot(plot_data['date_time'], plot_data['sum'], plot_title, ylab, file_name)\n", - " alt_text = store_alt_text_timeseries(plot_data, file_name, plot_title)\n", - " except:\n", - " generate_missing_plot(plot_title_no_quality,debug_df,file_name)\n", - " alt_text = store_alt_text_missing(debug_df, file_name, plot_title_no_quality)" + " plot_title= plot_title_no_quality+\"\\n\"+quality_text\n", + " ylab = 'Energy (kWH/day/user)'\n", + " timeseries_plot(plot_data['date_time'], plot_data['sum'], plot_title, ylab, file_name)\n", + " alt_text = store_alt_text_timeseries(plot_data, file_name, plot_title)\n", + "except:\n", + " generate_missing_plot(plot_title_no_quality,debug_df,file_name)\n", + " alt_text = store_alt_text_missing(debug_df, file_name, plot_title_no_quality)" ] }, {