[WIP] Feat/read mitigations

.eslintignore

@@ -5,6 +5,7 @@
 \.ignore
 \.reports
 cypress/integration/language.spec.ts
+data
 node_modules
 static
 tsconfig.json

.gitignore

@@ -2,16 +2,15 @@
 *.swp
 *log
 \.build
-\.idea
 \.cache
 \.env
+\.generated
+\.idea
 \.output
 \.reports
 \.vscode
 cypress/screenshots
 cypress/videos
+data/parsers.log
 node_modules
 .DS_Store
-
-## data
-data/parsers.log

config/nodemon/analyze.json

@@ -15,6 +15,7 @@
     "config",
     "package.json",
     "postcss.config.js",
+    "schemas",
     "src",
     "stylelint.config.js",
     "tsconfig.json",

config/nodemon/dev.json

@@ -15,6 +15,7 @@
     "config",
     "package.json",
     "postcss.config.js",
+    "schemas",
     "stylelint.config.js",
     "tsconfig.json",
     "tslint.json",

config/nodemon/prod.json

@@ -13,9 +13,10 @@
     ".prettierrc",
     "babel.config.js",
     "config",
+    "infra/lambda-at-edge/modifyOutgoingHeaders.lambda.js",
     "package.json",
     "postcss.config.js",
-    "infra/lambda-at-edge/modifyOutgoingHeaders.lambda.js",
+    "schemas",
     "src",
     "static",
     "stylelint.config.js",

config/webpack/lib/webpackLoadJavascript.ts

@@ -115,5 +115,10 @@ export default function webpackLoadJavaScript({
         },
       ],
     },
+    {
+      test: /\.ya?ml$/,
+      type: 'json', // Required by Webpack v4
+      use: 'yaml-loader',
+    },
   ].filter(Boolean)
 }

data/.gitignore

@@ -1,3 +1,3 @@
-raw_data/*
-
-*pyc
+*.pyc
+generated_types/
+raw_data/

data/Makefile

@@ -1,4 +1,16 @@
 
+.PHONY: all clean data-for-app
+
+all: clean types data-for-app
+
+clean: clean-types
+
+clean-types:
+	rm -rf .generated/types/**
+
+types: clean-types
+	cd .. && yarn install && yarn schema:totypes
+
 data-for-app:
 	python3 generate_data.py \
 		--output-scenarios ../src/assets/data/scenarios/scenarios.json \

data/mitigationMeasures.tsv

@@ -0,0 +1,9 @@
+Timestamp	Country/State	Mitigation measure	Start date	End date	Source of information (e.g. official url)	Contributor (your name, country)
+4/5/2020 23:41:45	USA-California	Soft population lockdown	2020-04-19		https://www.gov.ca.gov/2020/03/19/governor-gavin-newsom-issues-stay-at-home-order/	Sreekanth, USA
+4/5/2020 23:42:16	ITA-Emilia Romagna	Nurseries/schools/universities closure	2020-02-23	2020-04-30	http://www.salute.gov.it/portale/news/p3_2_1_1_1.jsp?lingua=italiano&menu=notizie&p=dalministero&id=4089	Andrea, Italy
+4/5/2020 23:48:20	ITA-Emilia Romagna	Ban on mass gatherings	2020-02-24	2020-04-30	https://www.regione.emilia-romagna.it/notizie/2020/febbraio/coronavirus-manifestazioni-pubbliche-servizi-e-attivita-quelle-da-sospendere-e-quelle-che-possono-proseguire	Andrea, Italy
+4/6/2020 0:43:36	USA-Utah	Soft population lockdown	2020-03-17	2020-05-15
+4/6/2020 1:56:35	Netherlands	Ban on mass gatherings	2020-04-01	2020-04-28	https://twitter.com/MinVWS/status/1245350203638468608?s=20	CT Netherlands
+4/6/2020 8:37:41	USA-California	Soft population lockdown	2020-03-19	2020-05-03	https://www.fmcsa.dot.gov/emergency/california-governor-newsom-shelter-place-executive-order-n33-20	Debbie Lefkowitz, United States
+4/6/2020 8:44:44	USA-Florida	Restaurants/entertainment/shops closure	2020-04-02		https://www.flgov.com/2020-executive-orders/	Matthew W Magnuson
+4/6/2020 10:16:22	CHE-Bern	Ban on mass gatherings	2020-02-28		https://www.admin.ch/gov/de/start/dokumentation/medienmitteilungen.msg-id-78289.html

data/parsers/utils.py

@@ -4,11 +4,14 @@
 import os
 import re
 import sys
+import yaml
+
 sys.path.append('..')
-from paths import TMP_CASES, BASE_PATH, JSON_DIR, SOURCES_FILE, TSV_DIR
+from paths import TMP_CASES, BASE_PATH, JSON_DIR, SOURCES_FILE, TSV_DIR, SCHEMA_CASECOUNTS
 
 from datetime import datetime
 from collections import defaultdict
+from jsonschema import validate
 
 # ------------------------------------------------------------------------
 # Globals
@@ -189,7 +192,21 @@ def add_country_code(regions, exceptions, code):
     return res
 
 
-def store_json(newdata, json_file):
+def store_json(case_counts, json_file):
+    """ Validate and store data to .json file
+    Arguments:
+    - case_counts: a dict of lists of dicts for case counts
+    """
+
+    #convert dict of lists of dicts to list of dicts of lists of dicts
+    newdata = []
+    for k in case_counts:
+        newdata.append({'country': k, 'empiricalData': case_counts[k]})
+
+    with open(os.path.join(BASE_PATH, SCHEMA_CASECOUNTS), "r") as f:
+        schema = yaml.load(f, Loader=yaml.FullLoader)
+        validate(newdata, schema)
+
     with open(json_file, 'w') as fh:
         json.dump(newdata, fh)
 

data/paths.py

@@ -9,3 +9,6 @@
 TMP_SCENARIOS = "scenario_defaults.json"
 PARSERS_LOG_FILE = 'parsers.log'
 FIT_PARAMETERS = 'fit_parameters.json'
+SCHEMA_CASECOUNTS = '../schemas/CaseCounts.yml'
+SCHEMA_SCENARIOS = '../schemas/Scenarios.yml'
+MITIGATION_TABLE = 'mitigationMeasures.tsv'

data/scripts/mitigationMeasures.py

@@ -0,0 +1,47 @@
+import csv
+import sys, os
+sys.path.append('..')
+from collections import defaultdict
+from paths import MITIGATION_TABLE, BASE_PATH
+
+##
+mitigationMeasures = {
+"Ban on mass gatherings":{"color": "#7fc97f", "value": 0.1},
+"Nurseries/schools/universities closure": {"color": "#beaed4", "value":0.1},
+"Restaurants/entertainment/shops closure": {"color": "#fdc086", "value":0.1},
+"Soft population lockdown": {"color": "#ffff99", "value":0.1},
+"Hard population lockdown": {"color": "#386cb0", "value":0.1},
+"Contact Tracing": {"color": "#f0027f", "value":0.1},
+"Intervention #1": {"color": "#bf5b17", "value":0.1},
+"Intervention #2": {"color": "#666666", "value":0.1},
+}
+
+# we probably have to find a way to deal with supersets: national measures that
+# supplant regional ones. Another case to deal with are bans of escalating strictness:
+# mass gatherings with 1000, 500, 100, 50, 5 people
+def read_table():
+    measures = defaultdict(list)
+    with open(os.path.join(BASE_PATH, MITIGATION_TABLE), 'r') as fh:
+        rdr = csv.reader(fh, delimiter='\t')
+        header = rdr.__next__()
+        tMin_idx = header.index("Start date")
+        tMax_idx = header.index("End date")
+        key_idx = header.index("Country/State")
+        measure_idx = header.index("Mitigation measure")
+
+        for row in rdr:
+            if row[measure_idx] not in mitigationMeasures:
+                print("unknown mitigation measure, skipping")
+                continue
+            tmp = {}
+            tmp["tMin"] = row[tMin_idx]
+            tmp["tMax"] = row[tMax_idx]
+            tmp["name"] = row[measure_idx]
+            measures[row[key_idx]].append(tmp)
+
+    return measures
+
+
+
+
+

data/scripts/mitigations_JHU.py

@@ -0,0 +1,109 @@
+import os, sys, io
+import csv
+import json
+import requests
+from datetime import date
+
+from collections import defaultdict
+from .utils import store_data
+
+# ------------------------------------------------------------------------
+# Globals
+
+measure_categories = { # for use in covid-19 scenarios
+  '1':['Ban on mass gatherings'],
+  '2':['Nurseries/schools/universities closure'],
+  '3':['Restaurants/entertainment/shops closure'],
+  '4':['Household confinement'],
+  '5':['Contact Tracing'],
+  '6':['Borders closure/limitation of movements/symptoms screening'],
+  '7':['Isolation/quarantine of cases'],
+  '8':['Office closures'],
+  '9':['Universal mask'],
+  '10':['NA']}
+
+# --> TO DO: assign a reduction range to each measure in measure_categories
+
+# Mapping of HIT mitigation measures (JHU) to covid-19 categories
+mapping = {
+  'Universal facemask policies':[measure_categories['9']],
+  'Testing of symptomatic individuals':[measure_categories['5']],
+  'Testing of asymptomatic individuals':[measure_categories['5']],
+  'Symptom screening when entering by sea':[measure_categories['6']],
+  'Symptom screening when entering by land':[measure_categories['6']],
+  'Symptom screening when entering by air':[measure_categories['6']],
+  'Symptom screening at checkpoints within borders':[measure_categories['6']],
+  'State of emergency':[measure_categories['10']],
+  'Secondary school closures':[measure_categories['2']],
+  'School closures of unknown type':[measure_categories['2']],
+  'Retail store closures (excluding essentials)':[measure_categories['3']],
+  'Restaurant closures (excluding takeout/delivery)':[measure_categories['3']],
+  'Quarantine of travelers':[measure_categories['7']],
+  'Quarantine of other asymptomatic individuals':[measure_categories['7']],
+  'Quarantine of contacts suspected (symptomatic) cases':[measure_categories['7']],
+  'Quarantine of contacts confirmed cases':[measure_categories['7']],
+  'Public transportation closures':[measure_categories['10']],
+  'Public space closures':[measure_categories['3']],
+  'Primary school closures':[measure_categories['2']],
+  'Post-secondary school closures':[measure_categories['2']],
+  'Office closures':[measure_categories['8']],
+  'Nursing home/long-term care closures':[measure_categories['2']],
+  'Nursery school closures':[measure_categories['2']],
+  'Military and police deployment':[measure_categories['10']],
+  'Limiting size of gatherings':[measure_categories['1']],
+  'Limiting number of patrons in restaurants':[measure_categories['1']],
+  'Limiting movement within borders':[measure_categories['6']],
+  'Leisure, entertainment, and religious venue closures':[measure_categories['3']],
+  'Household confinement':[measure_categories['4']],
+  'Home isolation of suspected (symptomatic) cases':[measure_categories['7']],
+  'Home isolation of non-hospitalized confirmed cases':[measure_categories['7']],
+  'Home isolation of confirmed cases discharged from the hospital':[measure_categories['7']],
+  'Contact tracing':[measure_categories['5']],
+  'Border closures for leaving by sea':[measure_categories['6']],
+  'Border closures for leaving by land':[measure_categories['6']],
+  'Border closures for leaving by air':[measure_categories['6']],
+  'Border closures for entering by sea':[measure_categories['6']],
+  'Border closures for entering by land':[measure_categories['6']],
+  'Border closures for entering by air':[measure_categories['6']]}
+
+# --> TO DO: URL changes dynamically, need to fix this !
+URL = "https://iddynamics.jhsph.edu/apps/connect/content/25/_w_f79c18ee/session/089681125bfd645ece0c677db0997ecb/download/download_data?w=f79c18ee"
+r  = requests.get(URL)
+cols = ['timestamp','state','mitigation_measure','mitigation_category','status','update_date','end_date']
+
+# ------------------------------------------------------------------------
+# Main point of entry
+
+def parse():
+    r  = requests.get(URL)
+
+    if not r.ok:
+        print(f"Failed to fetch {URL}", file=sys.stderr)
+        exit(1)
+        r.close()
+
+    mitigations = r.text
+    r.close()
+
+    mitigations = defaultdict(list)
+    mtg  = io.StringIO(r.text)
+    mtg = csv.reader(mtg)
+    next(mtg, None)
+
+    for row in mtg:
+        timestamp = row[1]
+        country = row[3]
+        state = row[6]
+        measure = row[8]
+        if measure in mapping.keys():
+          measure_category = mapping[measure]
+        else:
+          measure_category = 'NA'
+        status = row[11] # status can be "strongly/partially/not implemented"... this may hide end_dates...!
+        if row[9]!="NA": # skip measure if not date was given
+            tmp_date = list(map(int,row[9].split('-')))
+            update_date = date(tmp_date[0],tmp_date[1],tmp_date[2])
+            mitigations[country].append([timestamp,state,measure,measure_category,status,update_date,None])
+
+    # --> TO DO: store_data: agree how/where to save it.
+    #store_data(mitigations, 'mitigationMeasures',  cols)