Changes on generic_metric and mode_specific_metrics in relation to Mo…

docker-compose.dev.yml

@@ -26,7 +26,7 @@ services:
       - DB_HOST=db
       - WEB_SERVER_HOST=0.0.0.0
       - CRON_MODE=
-      - STUDY_CONFIG=stage-program
+      - STUDY_CONFIG=dev-emulator-study
     ports:
       # ipynb in numbers
       - "47962:8888"

viz_scripts/bin/generate_plots.py

@@ -34,13 +34,34 @@
     dynamic_config = json.loads(r.text)
     print(f"Successfully downloaded config with version {dynamic_config['version']} "\
         f"for {dynamic_config['intro']['translated_text']['en']['deployment_name']} "\
-        f"and data collection URL {dynamic_config['server']['connectUrl']}")
+        f"and data collection URL {dynamic_config['server']['connectUrl'] if 'server' in dynamic_config else 'default'}")
 
 if dynamic_config['intro']['program_or_study'] == 'program':
     mode_studied = dynamic_config['intro']['mode_studied']
 else:
     mode_studied = None
 
+# dynamic_labels can  be referenced from 
+# https://github.com/e-mission/nrel-openpath-deploy-configs/blob/main/label_options/example-study-label-options.json
+dynamic_labels = { }
+
+# Check if the dynamic config contains dynamic labels 'label_options'
+# Parse through the dynamic_labels_url:
+if 'label_options' in dynamic_config:
+    dynamic_labels_url = dynamic_config['label_options']
+
+    if  dynamic_labels_url:
+        req = requests.get(dynamic_labels_url)
+        if req.status_code != 200:
+            print(f"Unable to download dynamic_labels, status code: {req.status_code}")
+        else:
+            dynamic_labels = json.loads(req.text)
+            print("Dynamic labels download was successful.")
+    else:
+        print("Dynamic labels URL is unavailable.")
+else:
+    print("Dynamic labels are not available.")
+
 if args.date is None:
     start_date = arrow.get(int(dynamic_config['intro']['start_year']),
         int(dynamic_config['intro']['start_month']), 1)
@@ -70,7 +91,9 @@ def compute_for_date(month, year):
         study_type=dynamic_config['intro']['program_or_study'],
         mode_of_interest=mode_studied,
         include_test_users=dynamic_config.get('metrics', {}).get('include_test_users', False),
+        dynamic_labels = dynamic_labels,
         sensed_algo_prefix=dynamic_config.get('metrics', {}).get('sensed_algo_prefix', "cleaned"))
+
 
     print(f"Running at {arrow.get()} with params {params}")
 

viz_scripts/energy_calculations.ipynb

@@ -30,7 +30,8 @@
     "program = \"default\"\n",
     "study_type = \"program\"\n",
     "mode_of_interest = \"e-bike\"\n",
-    "include_test_users = False"
+    "include_test_users = False\n",
+    "dynamic_labels = { }"
    ]
   },
   {
@@ -107,6 +108,7 @@
     "                                                                            month,\n",
     "                                                                            program,\n",
     "                                                                            study_type,\n",
+    "                                                                            dynamic_labels,\n",
     "                                                                            dic_re,\n",
     "                                                                            dic_pur=dic_pur,\n",
     "                                                                            include_test_users=include_test_users)\n",

viz_scripts/generic_metrics.ipynb

@@ -30,7 +30,8 @@
     "program = \"default\"\n",
     "study_type = \"study\"\n",
     "mode_of_interest = None\n",
-    "include_test_users = False"
+    "include_test_users = False\n",
+    "dynamic_labels = { }"
    ]
   },
   {
@@ -88,6 +89,7 @@
     "                                                                            month,\n",
     "                                                                            program,\n",
     "                                                                            study_type,\n",
+    "                                                                            dynamic_labels,\n",
     "                                                                            dic_re,\n",
     "                                                                            dic_pur=dic_pur,\n",
     "                                                                            include_test_users=include_test_users)"
@@ -295,7 +297,7 @@
     "    \n",
     "    plot_title= plot_title_no_quality+\"\\n\"+quality_text\n",
     "\n",
-    "    barplot_mode(data,x,y,plot_title,file_name)\n",
+    "    barplot_mode(data,x,y,plot_title, expanded_ct['Mode_confirm'].dropna().unique().tolist(), file_name)\n",
     "    alt_text = store_alt_text_bar(pd.DataFrame(data['Average (miles)'].values, data['Mode_confirm']), file_name, plot_title)\n",
     "except:\n",
     "    generate_missing_plot(plot_title_no_quality,debug_df,file_name)\n",

viz_scripts/generic_timeseries.ipynb

@@ -27,7 +27,8 @@
     "program = \"default\"\n",
     "study_type = \"study\"\n",
     "mode_of_interest = None\n",
-    "include_test_users = False"
+    "include_test_users = False\n",
+    "dynamic_labels = {}"
    ]
   },
   {
@@ -85,6 +86,7 @@
     "                                                                            month,\n",
     "                                                                            program,\n",
     "                                                                            study_type,\n",
+    "                                                                            dynamic_labels,\n",
     "                                                                            dic_re,\n",
     "                                                                            include_test_users=include_test_users)\n",
     "expanded_ct = scaffolding.add_energy_labels(expanded_ct, df_ei, dic_fuel) if \"mode_confirm\" in expanded_ct.columns else expanded_ct"

viz_scripts/mode_specific_metrics.ipynb

@@ -30,7 +30,8 @@
     "program = \"default\"\n",
     "study_type = \"program\"\n",
     "mode_of_interest = \"e-bike\"\n",
-    "include_test_users = False"
+    "include_test_users = False\n",
+    "dynamic_labels = { }"
    ]
   },
   {
@@ -102,6 +103,7 @@
     "                                                                            month,\n",
     "                                                                            program,\n",
     "                                                                            study_type,\n",
+    "                                                                            dynamic_labels,\n",
     "                                                                            dic_re,\n",
     "                                                                            dic_pur=dic_pur,\n",
     "                                                                            include_test_users=include_test_users)"
@@ -266,7 +268,7 @@
     "    y2 = \"Count\"\n",
     "\n",
     "    plot_title= plot_title_no_quality+\"\\n\"+quality_text\n",
-    "    barplot_mode(data,x,y,plot_title,file_name)\n",
+    "    barplot_mode(data,x,y,plot_title, expanded_ct['Mode_confirm'].dropna().unique().tolist(), file_name)\n",
     "    alt_text = store_alt_text_bar(pd.DataFrame(data['Average (miles)'].values, data.Replaced_mode), file_name, plot_title)\n",
     "except:\n",
     "    generate_missing_plot(plot_title_no_quality,debug_df,file_name)\n",

viz_scripts/mode_specific_timeseries.ipynb

@@ -27,7 +27,8 @@
     "program = \"default\"\n",
     "study_type = \"program\"\n",
     "mode_of_interest = \"e-bike\"\n",
-    "include_test_users = False"
+    "include_test_users = False\n",
+    "dynamic_labels = { }"
    ]
   },
   {
@@ -95,6 +96,7 @@
     "                                                                            month,\n",
     "                                                                            program,\n",
     "                                                                            study_type,\n",
+    "                                                                            dynamic_labels,\n",
     "                                                                            dic_re,\n",
     "                                                                            dic_pur=dic_pur,\n",
     "                                                                            include_test_users=include_test_users)"

viz_scripts/plots.py

@@ -58,36 +58,15 @@ def format_pct(pct, values):
 
 
 def pie_chart_mode(plot_title,labels,values,file_name):
-    all_labels= ['Gas Car, drove alone',
-                 'Bus', 
-                 'Train', 
-                 'Free Shuttle',
-                 'Taxi/Uber/Lyft', 
-                 'Gas Car, with others', 
-                 'Bikeshare',
-                 'Scooter share',
-                 'E-bike', 
-                 'Walk', 
-                 'Skate board', 
-                 'Regular Bike', 
-                 'Not a Trip',
-                 'No Travel', 
-                 'Same Mode', 
-                 'E-car, drove alone',
-                 'E-car, with others',
-                 'Air',
-                 'Other']
-
-    val2labeldf = pd.DataFrame({"labels": labels, "values": values})
 
-    colours = dict(zip(all_labels, plt.cm.tab20.colors[:len(all_labels)]))
+    colors = dict(zip(labels, plt.cm.tab20.colors[:len(labels)]))
     fig, ax = plt.subplots(figsize=(10, 10), subplot_kw=dict(aspect="equal"))
 
     m_labels, m_values = merge_small_entries(labels, values)
 
     wedges, texts, autotexts = ax.pie(m_values,
                                       labels = m_labels,
-                                      colors=[colours[key] for key in labels],
+                                      colors=[colors[key] for key in labels],
                                       pctdistance=0.75,
                                       autopct= lambda pct: format_pct(pct, values),
                                       textprops={'size': 23})
@@ -127,22 +106,8 @@ def pie_chart_sensed_mode(plot_title,labels,values,file_name):
     plt.show()
 
 def pie_chart_purpose(plot_title,labels,values,file_name):
-    labels_trip= ['Work', 
-                  'Home',
-                  'Meal',
-                  'Shopping',
-                  'Personal/Medical',
-                  'Recreation/Exercise', 
-                  'Transit transfer', 
-                  'Pick-up/Drop off',
-                  'Entertainment/Social',
-                  'Other',
-                  'School',
-                  'Religious',
-                  'No travel', 
-                  'not_a_trip']
 
-    colours = dict(zip(labels_trip, plt.cm.tab20.colors[:len(labels_trip)]))
+    colors = dict(zip(labels, plt.cm.tab20.colors[:len(labels)]))
     fig, ax = plt.subplots(figsize=(10, 10), subplot_kw=dict(aspect="equal"))
 
     m_labels, m_values = merge_small_entries(labels, values)
@@ -154,7 +119,7 @@ def func(pct, values):
 
     wedges, texts, autotexts = ax.pie(m_values,
                                       labels = m_labels,
-                                      colors=[colours[key] for key in labels],
+                                      colors=[colors[key] for key in labels],
                                       pctdistance=0.85,
                                       autopct=lambda pct: func(pct, values),
                                       textprops={'size': 23})
@@ -211,32 +176,12 @@ def energy_impact(x,y,color,plot_title,file_name):
     plt.legend(labels=objects, handles=patches, loc='upper right', borderaxespad=0, fontsize=15, frameon=True)
     plt.savefig(SAVE_DIR+file_name+".png", bbox_inches='tight')
 
-def barplot_mode(data,x,y,plot_title,file_name):
-    all_labels= ['Gas Car, drove alone',
-                 'Bus', 
-                 'Train', 
-                 'Free Shuttle',
-                 'Taxi/Uber/Lyft', 
-                 'Gas Car, with others', 
-                 'Bikeshare',
-                 'Scooter share',
-                 'E-bike', 
-                 'Walk', 
-                 'Skate board', 
-                 'Regular Bike', 
-                 'Not a Trip',
-                 'No Travel', 
-                 'Same Mode', 
-                 'E-car, drove alone',
-                 'E-car, with others',
-                 'Air',
-                 'Other']
-
-    colours = dict(zip(all_labels, plt.cm.tab20.colors[:len(all_labels)]))
+def barplot_mode(data,x,y,plot_title, labels, file_name):
+    colors = dict(zip(labels, plt.cm.tab20.colors[:len(labels)]))
     sns.set(font_scale=1.5)
     f = plt.subplots(figsize=(15, 6))
     sns.set(style='whitegrid')
-    ax = sns.barplot(x=x, y=y, palette=colours,data=data, ci=None)
+    ax = sns.barplot(x=x, y=y, palette=colors,data=data, ci=None)
     plt.xlabel(x, fontsize=23)
     plt.ylabel(y, fontsize=23)
     plt.title(plot_title, fontsize=25)

viz_scripts/scaffolding.py

@@ -1,6 +1,8 @@
 import pandas as pd
 import numpy as np
 import sys
+import requests
+import json
 
 import emission.storage.timeseries.abstract_timeseries as esta
 import emission.storage.timeseries.tcquery as esttc
@@ -106,7 +108,7 @@ def expand_userinputs(labeled_ct):
 unique_users = lambda df: len(df.user_id.unique()) if "user_id" in df.columns else 0
 trip_label_count = lambda s, df: len(df[s].dropna()) if s in df.columns else 0
 
-def load_viz_notebook_data(year, month, program, study_type, dic_re, dic_pur=None, include_test_users=False):
+def load_viz_notebook_data(year, month, program, study_type, dynamic_labels, dic_re, dic_pur=None, include_test_users=False):
     """ Inputs:
     year/month/program/study_type = parameters from the visualization notebook
     dic_* = label mappings; if dic_pur is included it will be used to recode trip purpose
@@ -124,15 +126,26 @@ def load_viz_notebook_data(year, month, program, study_type, dic_re, dic_pur=Non
     # CASE 2 of https://github.com/e-mission/em-public-dashboard/issues/69#issuecomment-1256835867
     if "distance" in expanded_ct.columns:
         unit_conversions(expanded_ct)
-
-    # Mapping new mode labels with dictionaries
+
+    # Extract translations key
+    dic_translations = dict()
+    if "translations" in dynamic_labels and "en" in dynamic_labels["translations"]:
+        dic_translations = dynamic_labels["translations"]["en"]
+
+    # Select the mapping based on availability of dynamic_labels
+    if dic_translations:
+        dic_mapping = dic_translations
+    else:
+        dic_mapping = dic_re
+
+    # Map new mode labels with translations dictionary from dynamic_labels
     # CASE 2 of https://github.com/e-mission/em-public-dashboard/issues/69#issuecomment-1256835867
     if "mode_confirm" in expanded_ct.columns:
-        expanded_ct['Mode_confirm']= expanded_ct['mode_confirm'].map(dic_re)
+        expanded_ct['Mode_confirm'] = expanded_ct['mode_confirm'].map(dic_mapping)
     if study_type == 'program':
         # CASE 2 of https://github.com/e-mission/em-public-dashboard/issues/69#issuecomment-1256835867
         if 'replaced_mode' in expanded_ct.columns:
-            expanded_ct['Replaced_mode']= expanded_ct['replaced_mode'].map(dic_re)
+            expanded_ct['Replaced_mode'] = expanded_ct['replaced_mode'].map(dic_mapping)
         else:
             print("This is a program, but no replaced modes found. Likely cold start case. Ignoring replaced mode mapping")
     else:
@@ -141,7 +154,7 @@ def load_viz_notebook_data(year, month, program, study_type, dic_re, dic_pur=Non
     # Trip purpose mapping
     # CASE 2 of https://github.com/e-mission/em-public-dashboard/issues/69#issuecomment-1256835867
     if dic_pur is not None and "purpose_confirm" in expanded_ct.columns:
-        expanded_ct['Trip_purpose']= expanded_ct['purpose_confirm'].map(dic_pur)
+        expanded_ct['Trip_purpose'] = expanded_ct['purpose_confirm'].map(dic_pur)
 
     # Document data quality
     file_suffix = get_file_suffix(year, month, program)