update requirements and python3 instructions (#92)

* update requirements and python3 instructions

* update GitHub actions

* update ES8 settings

* fix ref

* use ES keyword parameters

* remove setup step from README

* f-strings housekeeping

* fixes

* Update search.py

* ES: remove option for index basename

* Order query results of get-bps-metadata.sh
- Add pypandoc as requirements-dev.txt

* Update default.env for ES8 setup

* Use WDR_SEARCH_INDEX_BASENAME config in search.py

* Bump version to 0.2.dev0

* Remove'_ type': '_doc'c'; removed in ES8

* Minor README.md fixes

* Improve registry.py logging and revised save function for consistent error handling

* Added missing commands in data-registry-ingest.sh
- remove double print(count)

* Syntax fix in UVIndex.get_waf_path()

* Typo fix for UVIndex.station.gaw_id

* Uncomment commands in data-registry-ingest.sh meant for testing

* Flake8 fixes and formatting adjustments

* flake8 fix

* Update debian/control to reflect dependency changes

* Update requirements-dev.txt

---------

Co-authored-by: Kevin Ngai <ngaik@woudc-px-dev3.cmc.ec.gc.ca>

.github/workflows/main.yml

@@ -26,7 +26,7 @@ jobs:
       GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}"
     strategy:
       matrix:
-        python-version: [3.7]
+        python-version: [3.8]
     steps:
     - uses: actions/checkout@v2
     - uses: actions/setup-python@v2

Makefile

@@ -54,7 +54,7 @@ help:
 	@echo " createdb: create PostgreSQL database"
 	@echo " dropdb: drop PostgreSQL database"
 	@echo " setup: create models and search index"
-	@echo " setup_data: download core metadata
+	@echo " setup_data: download core metadata"
 	@echo " teardown: delete models and search index"
 	@echo " test: run tests"
 	@echo " coverage: run code coverage"
@@ -92,7 +92,7 @@ init:
 	woudc-data-registry admin init -d $(DATA)
 
 package:
-	python setup.py sdist bdist_wheel
+	python3 setup.py sdist bdist_wheel
 
 setup:
 	woudc-data-registry admin registry setup
@@ -107,6 +107,6 @@ teardown:
 	woudc-data-registry admin search teardown
 
 test:
-	python setup.py test
+	python3 setup.py test
 
 .PHONY: clean coverage createdb dropdb flake8 help init package setup setup_data teardown test

README.md

@@ -31,19 +31,20 @@ are automatically installed during installation.
 python3 -m venv --system-site-packages woudc-data-registry
 cd woudc-data-registry
 source bin/activate
+cd ..
 
 # clone woudc-extcsv and install
 git clone https://github.com/woudc/woudc-extcsv.git
 cd woudc-extcsv
 pip install -r requirements.txt
-python setup.py install
+python3 setup.py install
 cd ..
 
 # clone codebase and install
 git clone https://github.com/woudc/woudc-data-registry.git
 cd woudc-data-registry
-python setup.py build
-python setup.py install
+python3 setup.py build
+python3 setup.py install
 # for PostgreSQL backends
 pip install -r requirements-pg.txt
 
@@ -69,10 +70,6 @@ woudc-data-registry admin registry setup
 woudc-data-registry admin search setup
 
 # load core metadata
-
-# fetch WMO country list
-mkdir data
-curl -o data/wmo-countries.json https://www.wmo.int/cpdb/data/membersandterritories.json
 woudc-data-registry admin init -d data/
 
 # cleanups
@@ -104,35 +101,35 @@ eval "$(_WOUDC_DATA_REGISTRY_COMPLETE=source woudc-data-registry)"
 # list all instances of foo (where foo is one of:
 #  project|dataset|contributor|country|station|instrument|deployment)
 woudc-data-registry <foo> list
- e.g.
+# e.g.
 woudc-data-registry contributor list
 
 # show a specific instance of foo with a given registry identifier
 woudc-data-registry <foo> show <identifier>
- e.g.
+# e.g.
 woudc-data-registry station show 023
 woudc-data-registry instrument show ECC:2Z:4052:002:OzoneSonde
 
 # add a new instance of foo (contributor|country|station|instrument|deployment)
 woudc-data-registry <foo> add <options>
- e.g.
+# e.g.
 woudc-data-registry deployment add -s 001 -c MSC:WOUDC
 woudc-data-registry contributor add -id foo -n "Contributor name" -c Canada -w IV -u https://example.org -e you@example.org -f foouser -g -75,45
 
 # update an existing instance of foo with a given registry identifier
 woudc-data-registry <foo> update -id <identifier> <options>
- e.g.
+# e.g.
 woudc-data-registry station update -n "New station name"
 woudc-data-registry deployment update --end-date 'Deployment end date'
 
 # delete an instance of foo with a given registry identifier
 woudc-data-registry <foo> delete <identifier>
- e.g.
+# e.g.
 woudc-data-registry deployment delete 018:MSC:WOUDC
 
 # for more information about options on operation (add|update):
 woudc-data-registry <foo> <operation> --help
- e.g.
+# e.g.
 woudc-data-registry instrument update --help
 ```
 
@@ -223,18 +220,18 @@ pip install -r requirements-dev.txt
 # build local copy of https://woudc.github.io/woudc-data-registry
 cd docs
 make html
-python -m http.server  # view on http://localhost:8000/
+python3 -m http.server  # view on http://localhost:8000/
 ```
 
 #### Running Tests
 
 ```bash
 # run tests like this:
 cd woudc_data_registry/tests
-python test_data_registry.py
+python3 test_data_registry.py
 
 # or this:
-python setup.py test
+python3 setup.py test
 
 # measure code coverage
 coverage run --source=woudc_data_registry -m unittest woudc_data_registry.tests.test_data_registry

data-registry-ingest.sh

@@ -1,54 +1,60 @@
 # The purpose of this script is to ingest all of the operational WOUDC Archive datasets and rename the generated ingest reports to avoid overwriting
 # Inital setup and table initialization of WOUDC Data Registry is required to run this script
+# Replace YYYY-MM-DD with today's date before proceeding
 
-woudc-data-registry data ingest /path/to/TotalOzone_1.0_1 -y -r /path/to/operator/report
-mv /path/to/operator/report/operator-report.csv /path/to/operator/report/TotalOzone_1.0_1.csv
-mv /path/to/operator/report/run_report /path/to/operator/report/TotalOzone_1.0_1_run_report
+woudc-data-registry data ingest /apps/data/web/woudc-archive/Archive-NewFormat/TotalOzone_1.0_1 -y -r /apps/data/wdr-ingest/YYYY-MM-DD
+mv /apps/data/wdr-ingest/YYYY-MM-DD/operator-report.csv /apps/data/wdr-ingest/YYYY-MM-DD/TotalOzone_1.0_1.csv
+mv /apps/data/wdr-ingest/YYYY-MM-DD/run_report /apps/data/wdr-ingest/YYYY-MM-DD/TotalOzone_1.0_1_run_report
 
-woudc-data-registry data ingest /path/to/TotalOzone_2.0_1 -y -r /path/to/operator/report
-mv /path/to/operator/report/operator-report.csv /path/to/operator/report/TotalOzone_2.0_1.csv
-mv /path/to/operator/report/run_report /path/to/operator/report/TotalOzone_2.0_1_run_report
+woudc-data-registry data ingest /apps/data/web/woudc-archive/Archive-NewFormat/TotalOzone_2.0_1 -y -r /apps/data/wdr-ingest/YYYY-MM-DD
+mv /apps/data/wdr-ingest/YYYY-MM-DD/operator-report.csv /apps/data/wdr-ingest/YYYY-MM-DD/TotalOzone_2.0_1.csv
+mv /apps/data/wdr-ingest/YYYY-MM-DD/run_report /apps/data/wdr-ingest/YYYY-MM-DD/TotalOzone_2.0_1_run_report
 
-woudc-data-registry data ingest /path/to/Broad-band_1.0_1 -y -r /path/to/operator/report
-mv /path/to/operator/report/operator-report.csv /path/to/operator/report/Broad-band_1.0_1.csv
-mv /path/to/operator/report/run_report /path/to/operator/report/Broad-band_1.0_1_run_report
+woudc-data-registry data ingest /apps/data/web/woudc-archive/Archive-NewFormat/Broad-band_1.0_1 -y -r /apps/data/wdr-ingest/YYYY-MM-DD
+mv /apps/data/wdr-ingest/YYYY-MM-DD/operator-report.csv /apps/data/wdr-ingest/YYYY-MM-DD/Broad-band_1.0_1.csv
+mv /apps/data/wdr-ingest/YYYY-MM-DD/run_report /apps/data/wdr-ingest/YYYY-MM-DD/Broad-band_1.0_1_run_report
 
-woudc-data-registry data ingest /path/to/Broad-band_2.0_1 -y -r /path/to/operator/report
-mv /path/to/operator/report/operator-report.csv /path/to/operator/report/Broad-band_2.0_1.csv
-mv /path/to/operator/report/run_report /path/to/operator/report/Broad-band_2.0_1_run_report
+woudc-data-registry data ingest /apps/data/web/woudc-archive/Archive-NewFormat/Broad-band_2.0_1 -y -r /apps/data/wdr-ingest/YYYY-MM-DD
+mv /apps/data/wdr-ingest/YYYY-MM-DD/operator-report.csv /apps/data/wdr-ingest/YYYY-MM-DD/Broad-band_2.0_1.csv
+mv /apps/data/wdr-ingest/YYYY-MM-DD/run_report /apps/data/wdr-ingest/YYYY-MM-DD/Broad-band_2.0_1_run_report
 
-woudc-data-registry data ingest /path/to/Lidar_1.0_1 -y -r /path/to/operator/report
-mv /path/to/operator/report/operator-report.csv /path/to/operator/report/Lidar_1.0_1.csv
-mv /path/to/operator/report/run_report /path/to/operator/report/Lidar_1.0_1_run_report
+woudc-data-registry data ingest /apps/data/web/woudc-archive/Archive-NewFormat/Lidar_1.0_1 -y -r /apps/data/wdr-ingest/YYYY-MM-DD
+mv /apps/data/wdr-ingest/YYYY-MM-DD/operator-report.csv /apps/data/wdr-ingest/YYYY-MM-DD/Lidar_1.0_1.csv
+mv /apps/data/wdr-ingest/YYYY-MM-DD/run_report /apps/data/wdr-ingest/YYYY-MM-DD/Lidar_1.0_1_run_report
 
-woudc-data-registry data ingest /path/to/Multi-band_1.0_1 -y -r /path/to/operator/report
-mv /path/to/operator/report/operator-report.csv /path/to/operator/report/Multi-band_1.0_1.csv
-mv /path/to/operator/report/run_report /path/to/operator/report/Multi-band_1.0_1_run_report
+woudc-data-registry data ingest /apps/data/web/woudc-archive/Archive-NewFormat/Multi-band_1.0_1 -y -r /apps/data/wdr-ingest/YYYY-MM-DD
+mv /apps/data/wdr-ingest/YYYY-MM-DD/operator-report.csv /apps/data/wdr-ingest/YYYY-MM-DD/Multi-band_1.0_1.csv
+mv /apps/data/wdr-ingest/YYYY-MM-DD/run_report /apps/data/wdr-ingest/YYYY-MM-DD/Multi-band_1.0_1_run_report
 
-woudc-data-registry data ingest /path/to/RocketSonde_1.0_1 -y -r /path/to/operator/report
-mv /path/to/operator/report/operator-report.csv /path/to/operator/report/RocketSonde_1.0_1.csv
-mv /path/to/operator/report/run_report /path/to/operator/report/RocketSonde_1.0_1_run_report
+woudc-data-registry data ingest /apps/data/web/woudc-archive/Archive-NewFormat/RocketSonde_1.0_1 -y -r /apps/data/wdr-ingest/YYYY-MM-DD
+mv /apps/data/wdr-ingest/YYYY-MM-DD/operator-report.csv /apps/data/wdr-ingest/YYYY-MM-DD/RocketSonde_1.0_1.csv
+mv /apps/data/wdr-ingest/YYYY-MM-DD/run_report /apps/data/wdr-ingest/YYYY-MM-DD/RocketSonde_1.0_1_run_report
 
-woudc-data-registry data ingest /path/to/OzoneSonde_1.0_1 -y -r /path/to/operator/report
-mv /path/to/operator/report/operator-report.csv /path/to/operator/report/OzoneSonde_1.0_1.csv
-mv /path/to/operator/report/run_report /path/to/operator/report/OzoneSonde_1.0_1_run_report
+woudc-data-registry data ingest /apps/data/web/woudc-archive/Archive-NewFormat/OzoneSonde_1.0_1 -y -r /apps/data/wdr-ingest/YYYY-MM-DD
+mv /apps/data/wdr-ingest/YYYY-MM-DD/operator-report.csv /apps/data/wdr-ingest/YYYY-MM-DD/OzoneSonde_1.0_1.csv
+mv /apps/data/wdr-ingest/YYYY-MM-DD/run_report /apps/data/wdr-ingest/YYYY-MM-DD/OzoneSonde_1.0_1_run_report
 
-woudc-data-registry data ingest /path/to/TotalOzoneObs_1.0_1 -y -r /path/to/operator/report
-mv /path/to/operator/report/operator-report.csv /path/to/operator/report/TotalOzoneObs_1.0_1.csv
-mv /path/to/operator/report/run_report /path/to/operator/report/TotalOzoneObs_1.0_1_run_report
+woudc-data-registry data ingest /apps/data/web/woudc-archive/Archive-NewFormat/TotalOzoneObs_1.0_1 -y -r /apps/data/wdr-ingest/YYYY-MM-DD
+mv /apps/data/wdr-ingest/YYYY-MM-DD/operator-report.csv /apps/data/wdr-ingest/YYYY-MM-DD/TotalOzoneObs_1.0_1.csv
+mv /apps/data/wdr-ingest/YYYY-MM-DD/run_report /apps/data/wdr-ingest/YYYY-MM-DD/TotalOzoneObs_1.0_1_run_report
 
-woudc-data-registry data ingest /path/to/Spectral_1.0_1 -y -r /path/to/operator/report
-mv /path/to/operator/report/operator-report.csv /path/to/operator/report/Spectral_1.0_1.csv
-mv /path/to/operator/report/run_report /path/to/operator/report/Spectral_1.0_1_run_report
+woudc-data-registry data ingest /apps/data/web/woudc-archive/Archive-NewFormat/Spectral_1.0_1 -y -r /apps/data/wdr-ingest/YYYY-MM-DD
+mv /apps/data/wdr-ingest/YYYY-MM-DD/operator-report.csv /apps/data/wdr-ingest/YYYY-MM-DD/Spectral_1.0_1.csv
+mv /apps/data/wdr-ingest/YYYY-MM-DD/run_report /apps/data/wdr-ingest/YYYY-MM-DD/Spectral_1.0_1_run_report
 
-woudc-data-registry data ingest /path/to/Spectral_2.0_1 -y -r /path/to/operator/report
-mv /path/to/operator/report/operator-report.csv /path/to/operator/report/Spectral_2.0_1.csv
-mv /path/to/operator/report/run_report /path/to/operator/report/Spectral_2.0_1_run_report
+woudc-data-registry data ingest /apps/data/web/woudc-archive/Archive-NewFormat/Spectral_2.0_1 -y -r /apps/data/wdr-ingest/YYYY-MM-DD
+mv /apps/data/wdr-ingest/YYYY-MM-DD/operator-report.csv /apps/data/wdr-ingest/YYYY-MM-DD/Spectral_2.0_1.csv
+mv /apps/data/wdr-ingest/YYYY-MM-DD/run_report /apps/data/wdr-ingest/YYYY-MM-DD/Spectral_2.0_1_run_report
 
-woudc-data-registry data ingest /path/to/UmkehrN14_1.0_1 -y -r /path/to/operator/report
-mv /path/to/operator/report/operator-report.csv /path/to/operator/report/UmkehrN14_1.0_1.csv
-mv /path/to/operator/report/run_report /path/to/operator/report/UmkehrN14_1.0_1_run_report
+woudc-data-registry data ingest /apps/data/web/woudc-archive/Archive-NewFormat/UmkehrN14_1.0_1 -y -r /apps/data/wdr-ingest/YYYY-MM-DD
+mv /apps/data/wdr-ingest/YYYY-MM-DD/operator-report.csv /apps/data/wdr-ingest/YYYY-MM-DD/UmkehrN14_1.0_1.csv
+mv /apps/data/wdr-ingest/YYYY-MM-DD/run_report /apps/data/wdr-ingest/YYYY-MM-DD/UmkehrN14_1.0_1_run_report
 
-woudc-data-registry data ingest /path/to/UmkehrN14_2.0_1 -y -r /path/to/operator/report
-mv /path/to/operator/report/operator-report.csv /path/to/operator/report/UmkehrN14_2.0_1.csv
-mv /path/to/operator/report/run_report /path/to/operator/report/UmkehrN14_2.0_1_run_report
+woudc-data-registry data ingest /apps/data/web/woudc-archive/Archive-NewFormat/UmkehrN14_2.0_1 -y -r /apps/data/wdr-ingest/YYYY-MM-DD
+mv /apps/data/wdr-ingest/YYYY-MM-DD/operator-report.csv /apps/data/wdr-ingest/YYYY-MM-DD/UmkehrN14_2.0_1.csv
+mv /apps/data/wdr-ingest/YYYY-MM-DD/run_report /apps/data/wdr-ingest/YYYY-MM-DD/UmkehrN14_2.0_1_run_report
+
+## Table generation process into registry for totalozone, uv-index and ozonesonde
+woudc-data-registry product totalozone generate -y /apps/data/web/woudc-archive/Archive-NewFormat/
+woudc-data-registry product uv-index generate -y /apps/data/web/woudc-archive/Archive-NewFormat/
+woudc-data-registry product ozonesonde generate -y /apps/data/web/woudc-archive/Archive-NewFormat/

debian/control

@@ -2,14 +2,14 @@ Source: woudc-data-registry
 Section: python
 Priority: optional
 Maintainer: WOUDC <ec.woudc.ec@canada.ca>
-Build-Depends: debhelper (>= 9), python3, python3-setuptools
+Build-Depends: debhelper (>= 9), python3 (>= 3.8.10), python3-setuptools
 Standards-Version: 3.9.5
-X-Python-Version: >= 3.4
+X-Python-Version: >= 3.8
 Vcs-Git: https://github.com/woudc/woudc-data-registry.git
 
 Package: woudc-data-registry
 Architecture: all
-Depends: elasticsearch (>=5.5.0), postgresql, python3-click, python-elasticsearch, python3-psycopg2, python3-requests, python3-sqlalchemy, woudc-extcsv
+Depends: elasticsearch (>= 8.12.0), postgresql, python3-click, python3-elasticsearch, python3-psycopg2, python3-requests, python3-sqlalchemy, woudc-extcsv
 Homepage: https://woudc.org
 Description: WOUDC Data Registry is a platform that manages Ozone and
  Ultraviolet Radiation data in support of the World Ozone and Ultraviolet

default.env

@@ -14,28 +14,31 @@ export WDR_DB_USERNAME=postgres
 export WDR_DB_PASSWORD=postgres
 export PGPASSWORD=$WDR_DB_PASSWORD
 
-# SQLite
+# alternative SQLite
 export WDR_DB_TYPE=sqlite
 export WDR_DB_NAME=test.db
 
 ## search index configuration
 export WDR_SEARCH_TYPE=elasticsearch
-export WDR_SEARCH_INDEX_BASENAME=woudc-data-registry
-export WDR_SEARCH_URL=http://localhost:9200/
-export WDR_SEARCH_USERNAME=elasticsearch
-export WDR_SEARCH_PASSWORD=elasticsearch
+export WDR_SEARCH_INDEX_BASENAME=woudc_data_registry
+export WDR_SEARCH_USERNAME=elastic
+export WDR_SEARCH_PASSWORD=<secret>
+export WDR_SEARCH_URL=https://${WDR_SEARCH_USERNAME}:${WDR_SEARCH_PASSWORD}@woudc-geo-dev3.cmc.ec.gc.ca/elasticsearch/
+# export WDR_SEARCH_URL=http://username:password@localhost:9200/
 
 ## waf configuration
 export WDR_WAF_BASEURL=https://woudc.org/archive/
 export WDR_WAF_BASEDIR=/tmp
 
-# table configurations
+# table configurations; optional: WDR_TABLE_SCHEMA, WDR_TABLE_CONFIG
+export WDR_TABLE_SCHEMA=/path/to/data/tables-schema.json
+export WDR_TABLE_CONFIG=/path/to/data/migrate/tables-backfilling.yml
 export WDR_ERROR_CONFIG=/path/to/data/errors.csv
 export WDR_ALIAS_CONFIG=/path/to/data/aliases.yml
 export WDR_EXTRA_CONFIG=/path/to/data/extra-options.yml
 
 # UV Index formula table configuration
 export WDR_UV_INDEX_FORMULA_LOOKUP=/path/to/data/uv-index-formula-lookup.csv
 
-# enable shell autocompletion
-eval "$(_WOUDC_DATA_REGISTRY_COMPLETE=source woudc-data-registry)"
+# enable shell autocompletion - optional
+# eval "$(_WOUDC_DATA_REGISTRY_COMPLETE=source woudc-data-registry)"

docs/installation.rst

@@ -33,8 +33,8 @@ Instructions
      | pip install -r requirements.txt     # Core dependencies
      | pip install -r requirements-pg.txt  # For PostgreSQL backends
      |
-     | python setup.py build
-     | python setup.py install
+     | python3 setup.py build
+     | python3 setup.py install
 
 * Set up the project::
      | . /path/to/environment/config.env  # Set environment variables