Merge pull request #33 from ScilifelabDataCentre/Paul/FREYA-886/dc-dy…

…namic-cleanup

FREYA-886 :: dc-dynamic cleanup

requirements.txt

@@ -1,28 +1,13 @@
+# DC-DYNAMIC DEPENDENCIES
 beautifulsoup4==4.11.1
-certifi==2022.9.24
-chardet==5.0.0
-charset-normalizer==2.1.0
-cycler==0.11.0
-DateTime==4.5
-et-xmlfile==1.1.0
-fonttools==4.35.0
-idna==3.3
-kiwisolver==1.4.4
+requests==2.32.3
 lxml==4.9.1
-matplotlib==3.5.3
-numpy==1.23.2
-openpyxl==3.0.10
-packaging==21.3
-pandas==1.4.3
-Pillow==9.2.0
-plotly==5.10.0
-pyparsing==3.0.9
-python-dateutil==2.8.2
-pytz==2022.2.1
-requests==2.28.1
-six==1.16.0
-soupsieve==2.3.2.post1
-tenacity==8.0.1
-urllib3==1.26.13
-wordcloud==1.8.2.2
-zope.interface==5.4.0
+
+# VISUALISATIONS DEPENDENCIES
+matplotlib==3.9.1.post1
+numpy==2.0.1
+openpyxl==3.1.5
+pandas==2.2.2
+pillow==10.4.0
+plotly==5.23.0
+wordcloud==1.9.3

runner_daily.sh

@@ -1,5 +1,7 @@
 export CODE_PATH=/code
 
+# Feed SciLifeLab figshare data to Slack
 python "$CODE_PATH"/slack_figshare.py
 
+# Update EBI index file
 PYTHONPATH="$CODE_PATH"/pathogens-portal-scripts/EBI_indexing python "$CODE_PATH"/pathogens-portal-scripts/EBI_indexing/update_index_json.py

runner_every10mins.sh

@@ -1,20 +1,13 @@
 export CODE_PATH=/code
 
-# Covid quatification SLU
-
+# Wastewater SARS-CoV-2 quantification SLU
 PYTHONPATH="$CODE_PATH"/pathogens-portal-visualisations/wastewater python "$CODE_PATH"/pathogens-portal-visualisations/wastewater/combined_slu_regular.py > "$CODE_PATH"/output/wastewater_combined_slu_regular.json
-# Covid quatification KTH
-#PYTHONPATH="$CODE_PATH"/pathogens-portal-visualisations/wastewater python "$CODE_PATH"/pathogens-portal-visualisations/wastewater/combined_stockholm_regular.py > "$CODE_PATH"/output/wastewater_combined_stockholm.json
-#PYTHONPATH="$CODE_PATH"/pathogens-portal-visualisations/wastewater python "$CODE_PATH"/pathogens-portal-visualisations/wastewater/quant_malmo_kthplot.py > "$CODE_PATH"/output/wastewater_kthmalmo.json
-# Covid quatification GU
-PYTHONPATH="$CODE_PATH"/pathogens-portal-visualisations/wastewater python "$CODE_PATH"/pathogens-portal-visualisations/wastewater/gothenburg_covid.py > "$CODE_PATH"/output/wastewater_gothenburg.json
-# Enteric virus GU
-PYTHONPATH="$CODE_PATH"/pathogens-portal-visualisations/wastewater python "$CODE_PATH"/pathogens-portal-visualisations/wastewater/enteric_viruses_gu.py > "$CODE_PATH"/output/enteric_graph_gu.json
-# Influenza virus SLU
+
+# Wastewater influenza quantification SLU
 PYTHONPATH="$CODE_PATH"/pathogens-portal-visualisations/wastewater python "$CODE_PATH"/pathogens-portal-visualisations/wastewater/combined_slu_influenza_a.py > "$CODE_PATH"/output/wastewater_slu_infA.json
 PYTHONPATH="$CODE_PATH"/pathogens-portal-visualisations/wastewater python "$CODE_PATH"/pathogens-portal-visualisations/wastewater/combined_slu_influenza_b.py > "$CODE_PATH"/output/wastewater_slu_infB.json
 
-# Serology
+# Serology statistics SARS-CoV-2
 PYTHONPATH="$CODE_PATH"/pathogens-portal-visualisations/serology python "$CODE_PATH"/pathogens-portal-visualisations/serology/weekly-serology-tests.py > "$CODE_PATH"/output/weekly_serology_tests.json
 PYTHONPATH="$CODE_PATH"/pathogens-portal-visualisations/serology python "$CODE_PATH"/pathogens-portal-visualisations/serology/cumulative-serology-tests.py > "$CODE_PATH"/output/cumulative_serology_tests.json
 

runner_hourly.sh

@@ -1,7 +1,9 @@
 export CODE_PATH=/code
 
+# Post items from SciLifeLab feed to Slack channel
 python "$CODE_PATH"/slack_feeds.py $CODE_PATH/output
 
+
 # Upload generated files
 for filename in $(ls $CODE_PATH/output) ; do
     curl "https://blobserver.dc.scilifelab.se/blob/$filename" -H "x-accesskey: $ACCESS_KEY" --upload-file "output/$filename"

runner_weekly.sh

@@ -1,12 +1,13 @@
 export CODE_PATH=/code
 
+# Wordclouds generator
 PYTHONPATH="$CODE_PATH"/pathogens-portal-visualisations/Wordcloud python "$CODE_PATH"/gen_clouds.py
 
 # Publication related updates
 PYTHONPATH="$CODE_PATH"/pathogens-portal-visualisations/Count_publications python "$CODE_PATH"/pathogens-portal-visualisations/Count_publications/count_publications.py > "$CODE_PATH"/output/COVID_publication_count.json
-#python "$CODE_PATH"/gen_publication_count.py > "$CODE_PATH"/output/covid-portal-publication-counts.json
 python "$CODE_PATH"/gen_recent_pub.py > "$CODE_PATH"/output/covid-portal-recent10.json
 
+
 # Upload generated files
 for filename in $(ls $CODE_PATH/output) ; do
     curl "https://blobserver.dc.scilifelab.se/blob/$filename" -H "x-accesskey: $ACCESS_KEY" --upload-file "output/$filename"