diff --git a/viz_scripts/bin/generate_plots.py b/viz_scripts/bin/generate_plots.py index 966e5fd..4175edb 100644 --- a/viz_scripts/bin/generate_plots.py +++ b/viz_scripts/bin/generate_plots.py @@ -33,14 +33,21 @@ else: dynamic_config = json.loads(r.text) print(f"Successfully downloaded config with version {dynamic_config['version']} "\ - f"for {dynamic_config['intro']['translated_text']['en']['deployment_name']} "\ - f"and data collection URL {dynamic_config['server']['connectUrl']}") + f"for {dynamic_config['intro']['translated_text']['en']['deployment_name']} ") if dynamic_config['intro']['program_or_study'] == 'program': mode_studied = dynamic_config['intro']['mode_studied'] else: mode_studied = None +# Check if the dynamic config contains dynamic labels 'label_options' +# Passing the boolean flag is not enough, bcos we need to trace through the dynamic_config +if 'label_options' in dynamic_config: + has_dynamic_labels = True + dynamic_labels_url = dynamic_config['label_options'] +else: + has_dynamic_labels = False + if args.date is None: start_date = arrow.get(int(dynamic_config['intro']['start_year']), int(dynamic_config['intro']['start_month']), 1) @@ -69,8 +76,9 @@ def compute_for_date(month, year): program=args.program, study_type=dynamic_config['intro']['program_or_study'], mode_of_interest=mode_studied, - include_test_users=dynamic_config.get('metrics', {}).get('include_test_users', False)) - + include_test_users=dynamic_config.get('metrics', {}).get('include_test_users', False), + has_dynamic_labels = has_dynamic_labels, + dynamic_labels_url = dynamic_labels_url) print(f"Running at {arrow.get()} with params {params}") # Make a notebook object with these definitions diff --git a/viz_scripts/docker/load_mongodump.sh b/viz_scripts/docker/load_mongodump.sh index fb17cd7..d69854e 100644 --- a/viz_scripts/docker/load_mongodump.sh +++ b/viz_scripts/docker/load_mongodump.sh @@ -1,9 +1,9 @@ MONGODUMP_FILE=$1 echo "Copying file to docker container" -docker cp $MONGODUMP_FILE em-public-dashboard_db_1:/tmp +docker cp $MONGODUMP_FILE em-public-dashboard-db-1:/tmp FILE_NAME=`basename $MONGODUMP_FILE` echo "Restoring the dump from $FILE_NAME" -docker exec -e MONGODUMP_FILE=$FILE_NAME em-public-dashboard_db_1 bash -c 'cd /tmp && tar xvf $MONGODUMP_FILE && mongorestore' +docker exec -e MONGODUMP_FILE=$FILE_NAME em-public-dashboard-db-1 bash -c 'cd /tmp && tar xvf $MONGODUMP_FILE && mongorestore' diff --git a/viz_scripts/generic_metrics.ipynb b/viz_scripts/generic_metrics.ipynb index b59a6e4..c457c54 100644 --- a/viz_scripts/generic_metrics.ipynb +++ b/viz_scripts/generic_metrics.ipynb @@ -30,7 +30,9 @@ "program = \"default\"\n", "study_type = \"study\"\n", "mode_of_interest = None\n", - "include_test_users = False" + "include_test_users = False\n", + "has_dynamic_labels = True\n", + "dynamic_labels_url = \"https://raw.githubusercontent.com/e-mission/nrel-openpath-deploy-configs/main/label_options/example-program-label-options.json\"\n" ] }, { @@ -43,7 +45,7 @@ "from collections import defaultdict\n", "\n", "import numpy as np\n", - "import pandas as pd\n", + "import pandas as pd4\n", "\n", "from plots import *\n", "import scaffolding\n", @@ -88,6 +90,8 @@ " month,\n", " program,\n", " study_type,\n", + " has_dynamic_labels,\n", + " dynamic_labels_url,\n", " dic_re,\n", " dic_pur=dic_pur,\n", " include_test_users=include_test_users)" @@ -392,7 +396,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.7.12" + "version": "3.9.16" } }, "nbformat": 4, diff --git a/viz_scripts/scaffolding.py b/viz_scripts/scaffolding.py index 01b4158..91925fa 100644 --- a/viz_scripts/scaffolding.py +++ b/viz_scripts/scaffolding.py @@ -1,5 +1,7 @@ import pandas as pd import numpy as np +import requests +import json import sys import emission.storage.timeseries.abstract_timeseries as esta @@ -106,7 +108,7 @@ def expand_userinputs(labeled_ct): unique_users = lambda df: len(df.user_id.unique()) if "user_id" in df.columns else 0 trip_label_count = lambda s, df: len(df[s].dropna()) if s in df.columns else 0 -def load_viz_notebook_data(year, month, program, study_type, dic_re, dic_pur=None, include_test_users=False): +def load_viz_notebook_data(year, month, program, study_type, has_dynamic_labels, dynamic_labels_url, dic_re, dic_pur=None, include_test_users=False): """ Inputs: year/month/program/study_type = parameters from the visualization notebook dic_* = label mappings; if dic_pur is included it will be used to recode trip purpose @@ -126,17 +128,35 @@ def load_viz_notebook_data(year, month, program, study_type, dic_re, dic_pur=Non unit_conversions(expanded_ct) # Mapping new mode labels with dictionaries - # CASE 2 of https://github.com/e-mission/em-public-dashboard/issues/69#issuecomment-1256835867 - if "mode_confirm" in expanded_ct.columns: - expanded_ct['Mode_confirm']= expanded_ct['mode_confirm'].map(dic_re) - if study_type == 'program': - # CASE 2 of https://github.com/e-mission/em-public-dashboard/issues/69#issuecomment-1256835867 - if 'replaced_mode' in expanded_ct.columns: - expanded_ct['Replaced_mode']= expanded_ct['replaced_mode'].map(dic_re) + + if has_dynamic_labels: + print("This json has dynamic label") + req = requests.get(dynamic_labels_url) + if req.status_code != 200: + print("Unable to download dynamic_labels") else: - print("This is a program, but no replaced modes found. Likely cold start case. Ignoring replaced mode mapping") + print("dynamic labels download was successful.") + dynamic_labels = json.loads(req.text) + + # Extract translations section + translations_data = dynamic_labels["translations"]["en"] + + # Map the "key" to the "en" translations + expanded_ct["Mode_confirm"] = expanded_ct["mode_confirm"].map(translations_data) + else: - print("This is a study, not expecting any replaced modes.") + print("This json doesn't have dynamic label") + # CASE 2 of https://github.com/e-mission/em-public-dashboard/issues/69#issuecomment-1256835867 + if "mode_confirm" in expanded_ct.columns: + expanded_ct['Mode_confirm']= expanded_ct['mode_confirm'].map(dic_re) + if study_type == 'program': + # CASE 2 of https://github.com/e-mission/em-public-dashboard/issues/69#issuecomment-1256835867 + if 'replaced_mode' in expanded_ct.columns: + expanded_ct['Replaced_mode']= expanded_ct['replaced_mode'].map(dic_re) + else: + print("This is a program, but no replaced modes found. Likely cold start case. Ignoring replaced mode mapping") + else: + print("This is a study, not expecting any replaced modes.") # Trip purpose mapping # CASE 2 of https://github.com/e-mission/em-public-dashboard/issues/69#issuecomment-1256835867