fix for weekly dash getting NaT date values #9426

Summary
Jobs
- build
Run details
- Usage
- Workflow file

Workflow file for this run

	name: Update Thailand Covid Data
	on:
	push:
	branches:
	- main
	pull_request:
	branches-ignore:
	- "releases/**"
	schedule: # can be up to 20min late. So better to have more updates
	# - cron: "/19 23 * *" # Quick checks for briefing after 7am
	# - cron: "55 23 * * SUN" # after midnight is delayed
	- cron: "00 06 * * MON,WED" # 13:00 Dashboard
	# - cron: "/15 00-02 * *" # Quick checks for briefing after 7am (but often after 8:30am)
	# - cron: "45 23 * * *" # 07:15 Dashboard
	# - cron: "02 00 * * *" # 08:02 Dashboard
	# - cron: "25 00 * * *" # 08:45 Dashboard
	# - cron: "25 05 * * *" # 12:25 daily briefing - official tweet
	# - cron: "/59 06-09 * MON" # Quick checks for briefing after 1pm
	# - cron: "00 06 * * *" # 13:00 daily briefing - briefing pdf
	# - cron: "40 06 * * *" # 13:40 daily briefing - briefing pdf
	# - cron: "50 07 * * *" # 14:50 in case briefing was uploaded late - #TODO: currently even later
	# - cron: "20 10 * * *" # 17:20 new PUI number
	# - cron: "20 14 * * *" # 23:20 test data/situation report/vaccination report
	jobs:
	build:
	runs-on: ubuntu-latest
	strategy:
	fail-fast: false
	# permissions:
	# statuses: write
	steps:
	- uses: actions/checkout@v2
	with:
	submodules: recursive
	# - name: Get Cached docs
	# uses: actions/checkout@v2
	# with:
	# repository: djay/covid_thailand_inputs
	# path: inputs
	# lfs: true
	- uses: szenius/set-timezone@v1.0
	with:
	timezoneLinux: "Asia/Bangkok"
	- name: Get current date
	id: date
	run: \|
	echo "::set-output name=date::$(TZ=Asia/Bangkok date +'%Y-%m-%d')"
	echo "::set-output name=yesterday::$(TZ=Asia/Bangkok date --date=yesterday +'%Y-%m-%d')"
	echo "::set-output name=month::$(TZ=Asia/Bangkok date +'%Y-%m')"
	echo "::set-output name=year::$(TZ=Asia/Bangkok date +'%Y')"
	echo "::set-output name=hour::$(TZ=Asia/Bangkok date +'%H')"
	echo "::set-output name=time::$(TZ=Asia/Bangkok date +'%H:%M')"
	echo "::set-output name=briefingurl::http://media.thaigov.go.th/uploads/public_img/source/$(TZ=Asia/Bangkok date +'%d%m')$(($(TZ=Asia/Bangkok date +'%Y')-1957)).pdf"
	echo "::set-output name=briefingfile::inputs/briefings/$(TZ=Asia/Bangkok date +'%d%m')$(($(TZ=Asia/Bangkok date +'%Y')-1957)).pdf"
	- name: Check briefing doc available
	id: briefing
	continue-on-error: true
	uses: jtalk/url-health-check-action@v1.5
	with:
	url: ${{ steps.date.outputs.briefingurl }}
	follow-redirect: yes
	- name: Check api available
	id: api
	continue-on-error: true
	run: \|
	(curl -sL https://covid19.ddc.moph.go.th/api/Cases/today-cases-by-provinces \| grep ${{ steps.date.outputs.date }})
	- name: api available or it's a push
	id: goready
	if: ${{ (github.event_name != 'schedule' \|\| steps.api.outcome == 'success') }}
	run: \|
	echo "'${{ github.event_name }}'(github.event_name)' != 'schedule' OR"
	echo "${{ steps.date.outputs.hour }}(steps.date.outputs.hour) < 13 OR"
	echo "'${{ steps.briefing.outcome }}'(steps.briefing.outcome) == 'success' AND"
	- name: Get release datasets
	uses: dsaltares/fetch-gh-release-asset@master
	with:
	file: "datasets.tar.gz"
	token: ${{ secrets.GITHUB_TOKEN }}
	- name: Extract datasets
	run: \|
	tar -xzvf datasets.tar.gz
	rm datasets.tar.gz
	- name: Api not done yet
	id: goapi
	if: ${{ steps.api.outcome == 'success' }}
	continue-on-error: true
	run: (! grep ${{ steps.date.outputs.date }} api/cases_by_province.csv )
	- name: Dash not done yet
	id: dash
	# if: ${{ steps.api.outcome == 'success' && steps.goapi.outcome != 'success' }}
	continue-on-error: true
	run: (! grep ${{ steps.date.outputs.date }} api/moph_dashboard_prov.csv )
	- name: Cache multiple paths
	id: cache
	if: ${{ steps.goready.outcome == 'success' \|\| steps.dash.outcome == 'success' \|\| steps.briefing.outcome == 'success'}}
	uses: pat-s/always-upload-cache@v2.1.5
	with:
	path: \|
	~/.cache/pip
	${{ env.pythonLocation }}
	/tmp/tika-server.jar
	/tmp/tika-server.jar.md5
	# key is evaluated only at the start so no way we can based it on files downloaded
	# without checking them all first. Time is hack that will cause caching on every run
	# see https://github.com/actions/cache/issues/135
	key: ${{ runner.os }}-${{ steps.date.outputs.date }}-${{ steps.date.outputs.time }}
	restore-keys: \|
	${{ runner.os }}-${{ steps.date.outputs.date }}
	${{ runner.os }}-${{ steps.date.outputs.yesterday }}
	${{ runner.os }}-${{ steps.date.outputs.month }}
	${{ runner.os }}-${{ steps.date.outputs.year }}
	${{ runner.os }}
	- name: Get release with cached docs
	id: docsrelease
	if: ${{ steps.goready.outcome == 'success' \|\| steps.dash.outcome == 'success' \|\| steps.briefing.outcome == 'success'}}
	uses: dsaltares/fetch-gh-release-asset@master
	with:
	file: "inputs.tar.gz"
	token: ${{ secrets.GITHUB_TOKEN }}
	- name: Get release with cached briefings
	id: briefingssrelease
	continue-on-error: true
	if: ${{ steps.goready.outcome == 'success' \|\| steps.dash.outcome == 'success' \|\| steps.briefing.outcome == 'success'}}
	uses: dsaltares/fetch-gh-release-asset@master
	with:
	file: "briefings.tar.gz"
	token: ${{ secrets.GITHUB_TOKEN }}
	- name: Get release with cached situation
	continue-on-error: true
	if: ${{ steps.goready.outcome == 'success' \|\| steps.dash.outcome == 'success' \|\| steps.briefing.outcome == 'success'}}
	uses: dsaltares/fetch-gh-release-asset@master
	with:
	file: "situation.tar.gz"
	token: ${{ secrets.GITHUB_TOKEN }}
	- name: Get release with cached vaccinations
	continue-on-error: true
	if: ${{ steps.goready.outcome == 'success' \|\| steps.dash.outcome == 'success' \|\| steps.briefing.outcome == 'success'}}
	uses: dsaltares/fetch-gh-release-asset@master
	with:
	file: "vaccinations.tar.gz"
	token: ${{ secrets.GITHUB_TOKEN }}
	- name: Get release with cached testing
	continue-on-error: true
	if: ${{ steps.goready.outcome == 'success' \|\| steps.dash.outcome == 'success' \|\| steps.briefing.outcome == 'success'}}
	uses: dsaltares/fetch-gh-release-asset@master
	with:
	file: "testing.tar.gz"
	token: ${{ secrets.GITHUB_TOKEN }}
	- name: Extract cached docs # TODO; should be before or after proper cache? or conditional on cache being empty?
	if: ${{ steps.docsrelease.outcome == 'success' }}
	run: \|
	tar -xzvf inputs.tar.gz
	tar -xzvf briefings.tar.gz \|\| true
	tar -xzvf situation.tar.gz \|\| true
	tar -xzvf vaccinations.tar.gz \|\| true
	tar -xzvf testing.tar.gz \|\| true
	rm inputs.tar.gz briefings.tar.gz situation.tar.gz vaccinations.tar.gz testing.tar.gz \|\| true
	- name: du
	if: ${{ steps.docsrelease.outcome == 'success' }}
	run: \|
	du -ha * \| sort -h
	- name: go if briefing ready
	id: gofull
	if: ${{ github.event_name != 'schedule' \|\| (steps.briefing.outcome == 'success' && hashFiles(steps.date.outputs.briefingfile) == '') }}
	run: \|
	echo "'${{ github.event_name }}'(github.event_name)' != 'schedule' OR"
	echo "${{ steps.date.outputs.hour }}(steps.date.outputs.hour) < 13 OR"
	echo "('${{ steps.briefing.outcome }}'(steps.briefing.outcome) == 'success' AND"
	echo " '${{ hashFiles(steps.date.outputs.briefingfile) }}'(hashFiles(${{ steps.date.outputs.briefingfile }}) == '')"
	- name: Apt update
	if: ${{ steps.gofull.outcome == 'success' \|\| steps.goapi.outcome == 'success' \|\| steps.dash.outcome == 'success' }}
	id: dep_apt
	run: \|
	sudo apt-get update
	sudo apt-get install -y openvpn unzip ghostscript python3-tk
	# sudo apt-get update -y --ignore-missing \|\| true
	# uses: awalsh128/cache-apt-pkgs-action@latest
	# with:
	# packages: openvpn unzip ghostscript python3-tk
	# version: 1.7
	# execute_install_scripts: true
	- name: Set up Python
	id: goscrape
	if: ${{ steps.gofull.outcome == 'success' \|\| steps.goapi.outcome == 'success' \|\| steps.dash.outcome == 'success' }}
	uses: actions/setup-python@v2
	with:
	python-version: 3.9.15
	cache: 'pip'
	- uses: actions/setup-java@v1
	if: ${{ steps.goscrape.outcome == 'success' }}
	with:
	java-version: 15
	- name: Install dependencies
	if: ${{ steps.goscrape.outcome == 'success' }}
	id: dep
	env:
	TIKA_VERSION: 2.5.0
	run: \|
	python3 -m pip install --upgrade pip
	pip install wheel
	pip install -r requirements.txt
	python -c 'from ctypes.util import find_library; print(find_library("gs"))'
	# Prep tika so gets a new version outside of VPN
	tika-python config mime-types \|\| true
	# - name: Setup VPN
	# id: setup-vpn
	# run: \|
	# sudo wget https://my.surfshark.com/vpn/api/v1/server/configurations
	# sudo unzip configurations
	# (echo -e "${{ secrets.OVPN_USERNAME }}\n${{ secrets.OVPN_PASSWORD }}") > userpass.txt
	# sudo openvpn --config th-bkk.prod.surfshark.com_tcp.ovpn --daemon --auth-user-pass userpass.txt
	# rm userpass.txt
	# # Wait for OpenVPN to connect
	# sleep 5
	# tail -30 /var/log/syslog
	# while ! sudo pgrep openvpn > /dev/null; do
	# sleep 1
	# done

	# while ! sudo grep -q "Initialization Sequence Completed" /var/log/syslog; do
	# sleep 1
	# done

	# # Do something after OpenVPN connects
	# echo "OpenVPN connected"
	- name: is dash ready
	id: godash
	if: ${{ steps.dep.outcome == 'success' && steps.dep.dash != 'success' && github.event_name == 'schedule'}}
	continue-on-error: true
	# run: python -c "import covid_data_dash; covid_data_dash.check_dash_ready()"
	run: true
	- name: Scrape Covid Data API
	id: updateapi
	if: ${{ steps.gofull.outcome != 'success' && steps.godash.outcome != 'success' && steps.goapi.outcome == 'success' }}
	run: \|
	# sudo docker run -t -p 8118:8118 -p 9050:9050 -l "TH" -d dperson/torproxy
	python covid_data_api.py
	env:
	# TIKA_VERSION: 1.24 # Prevent delays in upgrades
	DRIVE_API_KEY: ${{ secrets.DRIVE_API_KEY }}
	- name: Scrape Covid Data
	id: update
	if: ${{ steps.gofull.outcome == 'success' \|\| steps.godash.outcome == 'success' \|\| github.event_name != 'schedule' }}
	run: \|
	python covid_plot.py
	env:
	# TIKA_VERSION: 1.24 # Prevent delays in upgrades
	DRIVE_API_KEY: ${{ secrets.DRIVE_API_KEY }}
	- name: Run tests # Do after because we don't download new files to make quicker
	if: ${{ github.event_name != 'schedule'}}
	run: \|
	# sudo docker run -t -p 8118:8118 -p 9050:9050 -l "TH" -d dperson/torproxy
	pytest
	env:
	# TIKA_VERSION: 1.24 # Prevent delays in upgrades
	DRIVE_API_KEY: ${{ secrets.DRIVE_API_KEY }}
	- name: outputfiles.txt
	continue-on-error: true
	run:
	du inputs \| tee outputfiles.txt
	# - name: Stop VPN
	# id: stop-vpn
	# run: \|
	# sudo pkill -f "openvpn"
	- id: can-deploy
	env:
	MY_KEY: ${{ secrets.NETLIFY_AUTH_TOKEN }}
	if: "${{ env.MY_KEY != '' }}"
	run: echo "::set-output name=key::true"
	# - name: Push cached docs to repo
	# run: \|
	# cd inputs
	# git config user.name github-actions
	# git config user.email github-actions@github.com
	# git add .
	# git commit -m "generated"
	# git push

	- uses: ruby/setup-ruby@v1
	id: gonetlify
	if: ${{ (steps.update.outcome == 'success' \|\| steps.updateapi.outcome == 'success') && steps.can-deploy.outputs.key == 'true' }}
	with:
	ruby-version: 2.7.2 # Not needed with a .ruby-version file
	bundler-cache: true # runs 'bundle install' and caches installed gems automatically
	- name: Make links relative so PRs still work
	if: ${{ steps.gonetlify.outcome == 'success' && github.ref != 'refs/heads/main'}}
	run: \|
	sed -i 's#https://practical-ritchie-cca141\.netlify\.app#.#g' README.md
	- name: Jekyll Build
	if: ${{ steps.gonetlify.outcome == 'success' }}
	run: \|
	bundle config set --local path vendor/bundle
	bundler install --jobs 4 --retry 3
	sed -i 's#!\[.\]($._.[^$]*\)\.png)#<a href="\1.svg" target="_black"><object width="100%" type="image/svg+xml" data="\1.svg"><img src="\1.png" /></object></a>#g' README.md
	sed -Ei 's#\[(.)\]($._.[^$]*\)\.png)#<a href="\2.svg" target="_black">\1</a>#g' README.md

	cp *.md docs
	cp -R assets docs/assets
	cp _config.yml docs/_config.yml
	bundle exec jekyll build --destination=./_site
	- name: Inject slug/short variables
	uses: rlespinasse/github-slug-action@v3.x
	- name: Deploy gh-pages
	if: ${{ steps.gonetlify.outcome == 'success' && github.ref == 'refs/heads/main' }}
	uses: peaceiris/actions-gh-pages@v3
	with:
	github_token: ${{ secrets.GITHUB_TOKEN }}
	publish_dir: ./docs
	publish_branch: gh-pages
	enable_jekyll: true
	force_orphan: true
	- name: path urls for netlify
	if: ${{ steps.gonetlify.outcome == 'success' }}
	run: \|
	sed -i 's/https:\/\/practical-ritchie-cca141.netlify.app//g' README.md
	sed -i 's/https:\/\/practical-ritchie-cca141.netlify.app//g' downloads.md
	- name: Deploy to Netlify
	if: ${{ steps.gonetlify.outcome == 'success' }}
	id: netlify
	uses: nwtgck/actions-netlify@v1.2.2
	with:
	publish-dir: './_site'
	production-branch: main
	github-token: ${{ secrets.GITHUB_TOKEN }}
	deploy-message: "Deploy from GitHub Actions"
	enable-pull-request-comment: false
	enable-commit-comment: true
	overwrites-pull-request-comment: true
	env:
	NETLIFY_AUTH_TOKEN: ${{ secrets.NETLIFY_AUTH_TOKEN }}
	NETLIFY_SITE_ID: ${{ secrets.NETLIFY_SITE_ID }}
	timeout-minutes: 3
	# - name: Upload png to wiki
	# if: ${{ steps.netlify.outcome == 'success' && github.ref == 'refs/heads/main' }}
	# uses: kai-tub/external-repo-sync-action@v1
	# with:
	# source-directory: "outputs"
	# include-patterns: "*.png"
	# env:
	# GH_ACCESS_TOKEN: ${{ secrets.GH_ACCESS_TOKEN }}
	# - name: Upload json to wiki
	# if: ${{ steps.netlify.outcome == 'success' && github.ref == 'refs/heads/main' }}
	# uses: kai-tub/external-repo-sync-action@v1
	# with:
	# source-directory: "api"
	# include-patterns: "*"
	# env:
	# GH_ACCESS_TOKEN: ${{ secrets.GH_ACCESS_TOKEN }}
	- name: Run tests # Do after because we don't download new files to make quicker
	if: ${{ github.event_name != 'schedule'}}
	run: \|
	# sudo docker run -t -p 8118:8118 -p 9050:9050 -l "TH" -d dperson/torproxy
	pytest
	env:
	TIKA_VERSION: 1.24 # Prevent delays in upgrades
	DRIVE_API_KEY: ${{ secrets.DRIVE_API_KEY }}
	- name: Make cached docs bundle
	id: makerelease
	if: ${{ steps.netlify.outcome == 'success' && github.ref == 'refs/heads/main' }}
	run: \|
	tar -czvf briefings.tar.gz -T <(\ls -1 inputs/briefings/*)
	tar -czvf situation.tar.gz -T <(\ls -1 inputs/situation_/)
	tar -czvf vaccinations.tar.gz -T <(\ls -1 inputs/vaccinations/*)
	tar -czvf testing.tar.gz -T <(\ls -1 inputs/testing_moph/* inputs/variants/*)
	tar --exclude="inputs/.git" --exclude="_site" --exclude="._" --exclude="inputs/json/.csv" --exclude="inputs/briefings/" --exclude="inputs/situation_/" --exclude="inputs/vaccinations/" --exclude="inputs/variants/" --exclude="inputs/testing_moph/" -czvf inputs.tar.gz inputs
	find api inputs/json -path "api/" -o -name ".csv" \| tar -czvf datasets.tar.gz -T -
	- uses: "marvinpinto/action-automatic-releases@latest"
	if: ${{ steps.makerelease.outcome == 'success' }}
	with:
	repo_token: "${{ secrets.GITHUB_TOKEN }}"
	prerelease: false
	automatic_release_tag: ${{ steps.date.outputs.date }}
	title: Dataset and input files for covidthailand for ${{ steps.date.outputs.date }}
	files: \|
	briefings.tar.gz
	situation.tar.gz
	vaccinations.tar.gz
	testing.tar.gz
	datasets.tar.gz
	inputs.tar.gz
	- name: Upload inputs.tar.gz to existing release
	if: ${{ steps.makerelease.outcome == 'success' }}
	uses: svenstaro/upload-release-action@v2
	with:
	repo_token: ${{ secrets.GITHUB_TOKEN }}
	file: ./inputs.tar.gz
	asset_name: inputs.tar.gz
	tag: 1
	overwrite: true
	body: "Updated docs"
	- name: Upload datasets.tar.gz to existing release
	if: ${{ steps.makerelease.outcome == 'success' }}
	uses: svenstaro/upload-release-action@v2
	with:
	repo_token: ${{ secrets.GITHUB_TOKEN }}
	file: ./datasets.tar.gz
	asset_name: datasets.tar.gz
	tag: 1
	overwrite: true
	body: "Updated docs"
	- name: Upload vaccinations.tar.gz to existing release
	if: ${{ steps.makerelease.outcome == 'success' }}
	uses: svenstaro/upload-release-action@v2
	with:
	repo_token: ${{ secrets.GITHUB_TOKEN }}
	file: ./vaccinations.tar.gz
	asset_name: vaccinations.tar.gz
	tag: 1
	overwrite: true
	body: "Updated vaccinations"
	- name: Upload testing.tar.gz to existing release
	if: ${{ steps.makerelease.outcome == 'success' }}
	uses: svenstaro/upload-release-action@v2
	with:
	repo_token: ${{ secrets.GITHUB_TOKEN }}
	file: ./testing.tar.gz
	asset_name: testing.tar.gz
	tag: 1
	overwrite: true
	body: "Updated testing"
	# - name: Upload briefings.tar.gz to existing release
	# if: ${{ steps.makerelease.outcome == 'success' }}
	# uses: svenstaro/upload-release-action@v2
	# with:
	# repo_token: ${{ secrets.GITHUB_TOKEN }}
	# file: ./briefings.tar.gz
	# asset_name: briefings.tar.gz
	# tag: 1
	# overwrite: true
	# body: "Updated briefings"
	# - name: Upload situation.tar.gz to existing release
	# if: ${{ steps.makerelease.outcome == 'success' }}
	# uses: svenstaro/upload-release-action@v2
	# with:
	# repo_token: ${{ secrets.GITHUB_TOKEN }}
	# file: ./situation.tar.gz
	# asset_name: situation.tar.gz
	# tag: 1
	# overwrite: true
	# body: "Updated situation"

Provide feedback

Saved searches

Use saved searches to filter your results more quickly

fix for weekly dash getting NaT date values #9426

Workflow file

fix for weekly dash getting NaT date values #9426

Jobs

Run details

Workflow file for this run