add option to publish specific stations to a certain topic (#720) #3864
Workflow file for this run
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: test docker deployment ⚙️ | |
| on: [ push, pull_request ] | |
| jobs: | |
| main: | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@v2 | |
| - uses: actions/setup-python@v2 | |
| name: setup Python | |
| with: | |
| python-version: 3.8 | |
| - name: install testing requirements 📦 | |
| working-directory: tests | |
| run: | | |
| pip3 install -r requirements.txt | |
| pip3 install check-jsonschema | |
| - name: cache schemas 📦 | |
| run: | | |
| pywis-pubsub schema sync | |
| curl -s https://raw.githubusercontent.com/wmo-im/wcmp2/main/schemas/wcmp2-bundled.json --output /tmp/wcmp2-bundled.json | |
| - name: display Docker and Python versions 📦 | |
| run: | | |
| docker version | |
| docker compose version | |
| python3 -V | |
| - name: setup wis2box configuration, replace localhost with IP on host 📦 | |
| run: | | |
| export IP=$(hostname -I | awk '{print $1}') | |
| cp tests/test.env wis2box.env | |
| sed -i "s/localhost/$IP/g" wis2box.env | |
| cat wis2box.env | |
| python3 wis2box-ctl.py config | |
| - name: build wis2box | |
| run: | | |
| python3 wis2box-ctl.py build | |
| python3 wis2box-ctl.py update | |
| - name: start containers ⚙️ | |
| run: | | |
| python3 wis2box-ctl.py start | |
| python3 wis2box-ctl.py status -a | |
| - name: show environment and check collections exist ⚙️ | |
| run: | | |
| sleep 30 | |
| python3 wis2box-ctl.py execute wis2box environment show | |
| curl http://localhost/oapi/collections/stations | |
| curl http://localhost/oapi/collections/discovery-metadata | |
| curl http://localhost/oapi/collections/messages | |
| - name: downloader subscribe to topics | |
| run: | | |
| python3 wis2box-ctl.py execute wis2box auth add-token -p wis2downloader github123 -y | |
| echo '{"topic": "origin/a/wis2/+/data/core/#"}' > test.json | |
| curl -X POST http://localhost/wis2downloader/subscriptions -H "Content-Type: application/json" -H "Authorization: Bearer github123" -d @test.json | |
| - name: populate stations from CSV 📡 | |
| run: | | |
| python3 wis2box-ctl.py execute wis2box metadata station publish-collection | |
| - name: add Malawi synop data (csv2bufr synop_bufr template) 🇲🇼 | |
| env: | |
| TOPIC_HIERARCHY: mw-mw_met_centre.data.core.weather.surface-based-observations.synop | |
| CHANNEL: origin/a/wis2/mw-mw_met_centre/data/core/weather/surface-based-observations/synop | |
| TERRITORY: MWI | |
| DISCOVERY_METADATA: /data/wis2box/metadata/discovery/mw-surface-weather-observations.yml | |
| DISCOVERY_METADATA_ID: urn:wmo:md:mw-mw_met_centre:surface-weather-observations | |
| TEST_DATA: /data/wis2box/observations/malawi | |
| TEST_DATA_UPDATE: /data/wis2box/observations/malawi_update | |
| run: | | |
| python3 wis2box-ctl.py execute wis2box dataset publish $DISCOVERY_METADATA | |
| python3 wis2box-ctl.py execute wis2box metadata station add-topic --territory-name $TERRITORY $CHANNEL | |
| curl -s http://localhost/oapi/collections/discovery-metadata/items/$DISCOVERY_METADATA_ID --output /tmp/$DISCOVERY_METADATA_ID | |
| check-jsonschema --schemafile /tmp/wcmp2-bundled.json /tmp/$DISCOVERY_METADATA_ID | |
| python3 wis2box-ctl.py execute wis2box data ingest -mdi $DISCOVERY_METADATA_ID -p $TEST_DATA | |
| python3 wis2box-ctl.py execute wis2box data ingest -mdi $DISCOVERY_METADATA_ID -p $TEST_DATA_UPDATE | |
| - name: add Italy synop data (bufr2bufr) 🇮🇹 | |
| env: | |
| TOPIC_HIERARCHY: it-roma_met_centre.data.core.weather.surface-based-observations.synop | |
| CHANNEL: origin/a/wis2/it-roma_met_centre/data/core/weather/surface-based-observations/synop | |
| TERRITORY: ITA | |
| DISCOVERY_METADATA: /data/wis2box/metadata/discovery/it-surface-weather-observations.yml | |
| DISCOVERY_METADATA_ID: urn:wmo:md:it-roma_met_centre:surface-weather-observations | |
| TEST_DATA: /data/wis2box/observations/italy | |
| run: | | |
| python3 wis2box-ctl.py execute wis2box dataset publish $DISCOVERY_METADATA | |
| python3 wis2box-ctl.py execute wis2box metadata station add-topic --territory-name $TERRITORY $CHANNEL | |
| curl -s http://localhost/oapi/collections/discovery-metadata/items/$DISCOVERY_METADATA_ID --output /tmp/$DISCOVERY_METADATA_ID | |
| check-jsonschema --schemafile /tmp/wcmp2-bundled.json /tmp/$DISCOVERY_METADATA_ID | |
| python3 wis2box-ctl.py execute wis2box data ingest -mdi $DISCOVERY_METADATA_ID -p $TEST_DATA | |
| - name: add Algeria synop data (bufr2bufr) 🇩🇿 | |
| env: | |
| TOPIC_HIERARCHY: dz-alger_met_centre.data.core.weather.surface-based-observations.synop | |
| CHANNEL: origin/a/wis2/dz-alger_met_centre/data/core/weather/surface-based-observations/synop | |
| TERRITORY: DZA | |
| DISCOVERY_METADATA: /data/wis2box/metadata/discovery/dz-surface-weather-observations.yml | |
| DISCOVERY_METADATA_ID: urn:wmo:md:dz-alger_met_centre:surface-weather-observations | |
| TEST_DATA: /data/wis2box/observations/algeria | |
| run: | | |
| python3 wis2box-ctl.py execute wis2box dataset publish $DISCOVERY_METADATA | |
| python3 wis2box-ctl.py execute wis2box metadata station add-topic --territory-name $TERRITORY $CHANNEL | |
| curl -s http://localhost/oapi/collections/discovery-metadata/items/$DISCOVERY_METADATA_ID --output /tmp/$DISCOVERY_METADATA_ID | |
| check-jsonschema --schemafile /tmp/wcmp2-bundled.json /tmp/$DISCOVERY_METADATA_ID | |
| python3 wis2box-ctl.py execute wis2box data ingest -mdi $DISCOVERY_METADATA_ID -p $TEST_DATA | |
| - name: add Romania synop data (synop2bufr and csv2bufr aws-template) 🇷🇴 | |
| env: | |
| TOPIC_HIERARCHY: ro-rnimh.data.core.weather.surface-based-observations.synop | |
| CHANNEL: origin/a/wis2/ro-rnimh/data/core/weather/surface-based-observations/synop | |
| TERRITORY: ROU | |
| DISCOVERY_METADATA: /data/wis2box/metadata/discovery/ro-synoptic-weather-observations.yml | |
| DISCOVERY_METADATA_ID: urn:wmo:md:ro-rnimh:synoptic-weather-observations | |
| TEST_DATA: /data/wis2box/observations/romania | |
| run: | | |
| python3 wis2box-ctl.py execute wis2box dataset publish $DISCOVERY_METADATA | |
| python3 wis2box-ctl.py execute wis2box metadata station add-topic --territory-name $TERRITORY $CHANNEL | |
| curl -s http://localhost/oapi/collections/discovery-metadata/items/$DISCOVERY_METADATA_ID --output /tmp/$DISCOVERY_METADATA_ID | |
| check-jsonschema --schemafile /tmp/wcmp2-bundled.json /tmp/$DISCOVERY_METADATA_ID | |
| python3 wis2box-ctl.py execute wis2box data ingest -mdi $DISCOVERY_METADATA_ID -p $TEST_DATA | |
| - name: add Congo synop data (synop2bufr) 🇨🇩 | |
| env: | |
| TOPIC_HIERARCHY: cd-brazza_met_centre.data.core.weather.surface-based-observations.synop | |
| CHANNEL: origin/a/wis2/cd-brazza_met_centre/data/core/weather/surface-based-observations/synop | |
| TERRITORY: COD | |
| DISCOVERY_METADATA: /data/wis2box/metadata/discovery/cd-surface-weather-observations.yml | |
| DISCOVERY_METADATA_ID: urn:wmo:md:cd-brazza_met_centre:surface-weather-observations | |
| TEST_DATA: /data/wis2box/observations/congo | |
| run: | | |
| python3 wis2box-ctl.py execute wis2box dataset publish $DISCOVERY_METADATA | |
| python3 wis2box-ctl.py execute wis2box metadata station add-topic --territory-name $TERRITORY $CHANNEL | |
| curl -s http://localhost/oapi/collections/discovery-metadata/items/$DISCOVERY_METADATA_ID --output /tmp/$DISCOVERY_METADATA_ID | |
| check-jsonschema --schemafile /tmp/wcmp2-bundled.json /tmp/$DISCOVERY_METADATA_ID | |
| python3 wis2box-ctl.py execute wis2box data ingest -mdi $DISCOVERY_METADATA_ID -p $TEST_DATA | |
| - name: add example ship data (bufr2bufr) WMO | |
| env: | |
| TOPIC_HIERARCHY: int-wmo-test.data.core.weather.surface-based-observations.ship | |
| CHANNEL: origin/a/wis2/int-wmo-test/data/core/weather/surface-based-observations/ship | |
| DISCOVERY_METADATA: /data/wis2box/metadata/discovery/int-wmo-test-ship.yml | |
| DISCOVERY_METADATA_ID: urn:wmo:md:int-wmo-test:surface-weather-observations:ship | |
| TEST_DATA: /data/wis2box/observations/wmo/ship | |
| run: | | |
| python3 wis2box-ctl.py execute wis2box dataset publish $DISCOVERY_METADATA | |
| python3 wis2box-ctl.py execute wis2box metadata station add-topic --wsi 0-22000-0-3WL23XC $CHANNEL | |
| python3 wis2box-ctl.py execute wis2box metadata station add-topic --wsi 0-22000-0-4EEWQMW $CHANNEL | |
| python3 wis2box-ctl.py execute wis2box metadata station add-topic --wsi 0-22000-0-4SGNZUJ $CHANNEL | |
| python3 wis2box-ctl.py execute wis2box metadata station add-topic --wsi 0-22000-0-EUCDE09 $CHANNEL | |
| python3 wis2box-ctl.py execute wis2box metadata station add-topic --wsi 0-22000-0-EUCDE34 $CHANNEL | |
| curl -s http://localhost/oapi/collections/discovery-metadata/items/$DISCOVERY_METADATA_ID --output /tmp/$DISCOVERY_METADATA_ID | |
| check-jsonschema --schemafile /tmp/wcmp2-bundled.json /tmp/$DISCOVERY_METADATA_ID | |
| python3 wis2box-ctl.py execute wis2box data ingest -mdi $DISCOVERY_METADATA_ID -p $TEST_DATA | |
| - name: add example buoy data (bufr2bufr) WMO | |
| env: | |
| TOPIC_HIERARCHY: int-wmo-test.data.core.weather.surface-based-observations.buoy | |
| CHANNEL: origin/a/wis2/int-wmo-test/data/core/weather/surface-based-observations/buoy | |
| DISCOVERY_METADATA: /data/wis2box/metadata/discovery/int-wmo-test-buoy.yml | |
| DISCOVERY_METADATA_ID: urn:wmo:md:int-wmo-test:surface-weather-observations:buoy | |
| TEST_DATA: /data/wis2box/observations/wmo/buoy | |
| run: | | |
| python3 wis2box-ctl.py execute wis2box dataset publish $DISCOVERY_METADATA | |
| python3 wis2box-ctl.py execute wis2box metadata station add-topic --wsi 0-22000-0-1400011 $CHANNEL | |
| curl -s http://localhost/oapi/collections/discovery-metadata/items/$DISCOVERY_METADATA_ID --output /tmp/$DISCOVERY_METADATA_ID | |
| check-jsonschema --schemafile /tmp/wcmp2-bundled.json /tmp/$DISCOVERY_METADATA_ID | |
| python3 wis2box-ctl.py execute wis2box data ingest -mdi $DISCOVERY_METADATA_ID -p $TEST_DATA | |
| - name: add example wind profiler data (bufr2bufr) WMO | |
| env: | |
| TOPIC_HIERARCHY: int-wmo-test.data.core.weather.surface-based-observations.wind_profiler | |
| CHANNEL: origin/a/wis2/int-wmo-test/data/core/weather/surface-based-observations/wind_profiler | |
| DISCOVERY_METADATA: /data/wis2box/metadata/discovery/int-wmo-test-wind_profiler.yml | |
| DISCOVERY_METADATA_ID: urn:wmo:md:int-wmo-test:surface-weather-observations:wind_profiler | |
| TEST_DATA: /data/wis2box/observations/wmo/wind_profiler | |
| run: | | |
| python3 wis2box-ctl.py execute wis2box dataset publish $DISCOVERY_METADATA | |
| python wis2box-ctl.py execute wis2box metadata station add-topic --wsi 0-702-0-48698 $CHANNEL | |
| curl -s http://localhost/oapi/collections/discovery-metadata/items/$DISCOVERY_METADATA_ID --output /tmp/$DISCOVERY_METADATA_ID | |
| check-jsonschema --schemafile /tmp/wcmp2-bundled.json /tmp/$DISCOVERY_METADATA_ID | |
| python3 wis2box-ctl.py execute wis2box data ingest -mdi $DISCOVERY_METADATA_ID -p $TEST_DATA | |
| - name: add China GRIB2 data (universal pipeline) 🇨🇳 | |
| env: | |
| TOPIC_HIERARCHY: cn-cma.data.core.weather.prediction.forecast.medium-range.probabilistic.global | |
| DISCOVERY_METADATA: /data/wis2box/metadata/discovery/cn-grapes-geps-global.yml | |
| DISCOVERY_METADATA_ID: urn:wmo:md:cn-cma:grapes-geps-global | |
| TEST_DATA: /data/wis2box/observations/china | |
| run: | | |
| python3 wis2box-ctl.py execute wis2box dataset publish $DISCOVERY_METADATA | |
| curl -s http://localhost/oapi/collections/discovery-metadata/items/$DISCOVERY_METADATA_ID --output /tmp/$DISCOVERY_METADATA_ID | |
| check-jsonschema --schemafile /tmp/wcmp2-bundled.json /tmp/$DISCOVERY_METADATA_ID | |
| python3 wis2box-ctl.py execute wis2box data ingest -mdi $DISCOVERY_METADATA_ID -p $TEST_DATA | |
| - name: sleep 30 seconds then run integration tests ⚙️ | |
| run: | | |
| sleep 30 | |
| docker logs wis2box-management | |
| pytest -s tests/integration | |
| - name: run flake8 ⚙️ | |
| run: | | |
| find . -type f -name "*.py" | xargs flake8 | |
| - name: failed tests 🚩 | |
| if: ${{ failure() }} | |
| run: | | |
| docker logs -n1000 wis2box-management | |
| docker logs -n1000 wis2box-api |