diff --git a/gen_publication_count.py b/gen_publication_count.py deleted file mode 100644 index c8ed70e..0000000 --- a/gen_publication_count.py +++ /dev/null @@ -1,35 +0,0 @@ -import json - -import pandas -import requests - -# get data -res = requests.get("https://publications-covid19.scilifelab.se/publications.json") -txt = res.json() -df = pandas.json_normalize(txt["publications"]) -df = df[["type", "published"]] - -# for now, assume dates with day as 00 are 1st day -df.replace("-00", "-01", regex=True, inplace=True) - -# make the cumulative sum of papers published -df.published = pandas.to_datetime(df["published"], format="%Y-%m-%d").dt.date -currdate = pandas.to_datetime("today").date() -df = df[df["published"] < currdate] -df.sort_values(by="published", inplace=True) -df1 = df["published"].value_counts().sort_index().reset_index() -df1["cumulativecount"] = df1["published"].cumsum() - -# find number of papers published in each month -df["year"] = pandas.DatetimeIndex(df["published"]).year -df["month"] = pandas.DatetimeIndex(df["published"]).month -df["index"] = pandas.to_datetime(df[["year", "month"]].assign(day=1)) -df2 = df.groupby(["index"]).size().reset_index(name="Number Added") - -outdata = {} -outdata["dates"] = list(str(val) for val in df1["index"]) -outdata["cumsums"] = list(df1["cumulativecount"]) -outdata["months"] = list(str(val).replace(" ", "T") for val in df2["index"]) -outdata["per_month"] = list(df2["Number Added"]) - -print(json.dumps(outdata)) diff --git a/gen_symptoms_data.py b/gen_symptoms_data.py deleted file mode 100644 index 97946e2..0000000 --- a/gen_symptoms_data.py +++ /dev/null @@ -1,28 +0,0 @@ -import csv -import json - -import pandas -import requests - -# data -req = requests.get("https://blobserver.dc.scilifelab.se/blob/CSSS_estimates.csv") -reader = csv.reader(req.text.splitlines()) -data = list(reader)[-21:] -df1 = pandas.DataFrame( - data[0:], columns=["Lan", "Datum", "Uppskattning", "Low_CI", "High_CI"] -) - -# format data -df1["Datum"] = pandas.to_datetime(df1["Datum"]) -df1.sort_values(by="Datum", ascending=False, inplace=True) -df1.drop_duplicates("Lan", keep="first", inplace=True) -df1["Uppskattning"] = pandas.to_numeric(df1["Uppskattning"], errors="coerce") -df1["Uppskattning"] = df1["Uppskattning"].fillna(-0.1) - -outdata = {} - -outdata["z"] = list(df1["Uppskattning"]) -outdata["custom"] = [[val] for val in outdata["z"]] -outdata["Lan"] = list(df1["Lan"]) - -print(json.dumps(outdata)) diff --git a/requirements.txt b/requirements.txt index d6b71fc..5791573 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,28 +1,13 @@ +# DC-DYNAMIC DEPENDENCIES beautifulsoup4==4.11.1 -certifi==2022.9.24 -chardet==5.0.0 -charset-normalizer==2.1.0 -cycler==0.11.0 -DateTime==4.5 -et-xmlfile==1.1.0 -fonttools==4.35.0 -idna==3.3 -kiwisolver==1.4.4 +requests==2.32.3 lxml==4.9.1 -matplotlib==3.5.3 -numpy==1.23.2 -openpyxl==3.0.10 -packaging==21.3 -pandas==1.4.3 -Pillow==9.2.0 -plotly==5.10.0 -pyparsing==3.0.9 -python-dateutil==2.8.2 -pytz==2022.2.1 -requests==2.28.1 -six==1.16.0 -soupsieve==2.3.2.post1 -tenacity==8.0.1 -urllib3==1.26.13 -wordcloud==1.8.2.2 -zope.interface==5.4.0 + +# VISUALISATIONS DEPENDENCIES +matplotlib==3.9.1.post1 +numpy==2.0.1 +openpyxl==3.1.5 +pandas==2.2.2 +pillow==10.4.0 +plotly==5.23.0 +wordcloud==1.9.3 diff --git a/runner_daily.sh b/runner_daily.sh index bf473ce..3a3acf6 100755 --- a/runner_daily.sh +++ b/runner_daily.sh @@ -1,5 +1,7 @@ export CODE_PATH=/code +# Feed SciLifeLab figshare data to Slack python "$CODE_PATH"/slack_figshare.py +# Update EBI index file PYTHONPATH="$CODE_PATH"/pathogens-portal-scripts/EBI_indexing python "$CODE_PATH"/pathogens-portal-scripts/EBI_indexing/update_index_json.py diff --git a/runner_every10mins.sh b/runner_every10mins.sh index 32448f6..0519259 100644 --- a/runner_every10mins.sh +++ b/runner_every10mins.sh @@ -1,20 +1,13 @@ export CODE_PATH=/code -# Covid quatification SLU - +# Wastewater SARS-CoV-2 quantification SLU PYTHONPATH="$CODE_PATH"/pathogens-portal-visualisations/wastewater python "$CODE_PATH"/pathogens-portal-visualisations/wastewater/combined_slu_regular.py > "$CODE_PATH"/output/wastewater_combined_slu_regular.json -# Covid quatification KTH -#PYTHONPATH="$CODE_PATH"/pathogens-portal-visualisations/wastewater python "$CODE_PATH"/pathogens-portal-visualisations/wastewater/combined_stockholm_regular.py > "$CODE_PATH"/output/wastewater_combined_stockholm.json -#PYTHONPATH="$CODE_PATH"/pathogens-portal-visualisations/wastewater python "$CODE_PATH"/pathogens-portal-visualisations/wastewater/quant_malmo_kthplot.py > "$CODE_PATH"/output/wastewater_kthmalmo.json -# Covid quatification GU -PYTHONPATH="$CODE_PATH"/pathogens-portal-visualisations/wastewater python "$CODE_PATH"/pathogens-portal-visualisations/wastewater/gothenburg_covid.py > "$CODE_PATH"/output/wastewater_gothenburg.json -# Enteric virus GU -PYTHONPATH="$CODE_PATH"/pathogens-portal-visualisations/wastewater python "$CODE_PATH"/pathogens-portal-visualisations/wastewater/enteric_viruses_gu.py > "$CODE_PATH"/output/enteric_graph_gu.json -# Influenza virus SLU + +# Wastewater influenza quantification SLU PYTHONPATH="$CODE_PATH"/pathogens-portal-visualisations/wastewater python "$CODE_PATH"/pathogens-portal-visualisations/wastewater/combined_slu_influenza_a.py > "$CODE_PATH"/output/wastewater_slu_infA.json PYTHONPATH="$CODE_PATH"/pathogens-portal-visualisations/wastewater python "$CODE_PATH"/pathogens-portal-visualisations/wastewater/combined_slu_influenza_b.py > "$CODE_PATH"/output/wastewater_slu_infB.json -# Serology +# Serology statistics SARS-CoV-2 PYTHONPATH="$CODE_PATH"/pathogens-portal-visualisations/serology python "$CODE_PATH"/pathogens-portal-visualisations/serology/weekly-serology-tests.py > "$CODE_PATH"/output/weekly_serology_tests.json PYTHONPATH="$CODE_PATH"/pathogens-portal-visualisations/serology python "$CODE_PATH"/pathogens-portal-visualisations/serology/cumulative-serology-tests.py > "$CODE_PATH"/output/cumulative_serology_tests.json diff --git a/runner_hourly.sh b/runner_hourly.sh index 9e55beb..c62a498 100755 --- a/runner_hourly.sh +++ b/runner_hourly.sh @@ -1,7 +1,9 @@ export CODE_PATH=/code +# Post items from SciLifeLab feed to Slack channel python "$CODE_PATH"/slack_feeds.py $CODE_PATH/output + # Upload generated files for filename in $(ls $CODE_PATH/output) ; do curl "https://blobserver.dc.scilifelab.se/blob/$filename" -H "x-accesskey: $ACCESS_KEY" --upload-file "output/$filename" diff --git a/runner_weekly.sh b/runner_weekly.sh index d03753a..4069405 100755 --- a/runner_weekly.sh +++ b/runner_weekly.sh @@ -1,12 +1,13 @@ export CODE_PATH=/code +# Wordclouds generator PYTHONPATH="$CODE_PATH"/pathogens-portal-visualisations/Wordcloud python "$CODE_PATH"/gen_clouds.py # Publication related updates PYTHONPATH="$CODE_PATH"/pathogens-portal-visualisations/Count_publications python "$CODE_PATH"/pathogens-portal-visualisations/Count_publications/count_publications.py > "$CODE_PATH"/output/COVID_publication_count.json -#python "$CODE_PATH"/gen_publication_count.py > "$CODE_PATH"/output/covid-portal-publication-counts.json python "$CODE_PATH"/gen_recent_pub.py > "$CODE_PATH"/output/covid-portal-recent10.json + # Upload generated files for filename in $(ls $CODE_PATH/output) ; do curl "https://blobserver.dc.scilifelab.se/blob/$filename" -H "x-accesskey: $ACCESS_KEY" --upload-file "output/$filename"