From 15d1a75bb51fefa7a072f5f93ddeae3bbc6f117e Mon Sep 17 00:00:00 2001 From: Maaike Date: Thu, 21 Nov 2024 19:46:52 +0100 Subject: [PATCH] add mappings-info process and test (#78) * add mappings-info process and test --- docker/entrypoint.sh | 3 + docker/pygeoapi-config.yml | 5 + docker_compose_test/docker-compose.yml | 2 + docker_compose_test/test.env | 7 +- .../data/mappings/my_csv2bufr_mappings.json | 93 +++++++++++++ .../tests/integration/test_wis2box.py | 47 +++++++ wis2box_api/plugins/process/mappings_info.py | 131 ++++++++++++++++++ 7 files changed, 286 insertions(+), 2 deletions(-) create mode 100644 docker_compose_test/tests/data/mappings/my_csv2bufr_mappings.json create mode 100644 wis2box_api/plugins/process/mappings_info.py diff --git a/docker/entrypoint.sh b/docker/entrypoint.sh index 01cf7a2..ca7e549 100755 --- a/docker/entrypoint.sh +++ b/docker/entrypoint.sh @@ -47,6 +47,9 @@ function error() { exit -1 } +# create mappings directory /data/wi2box/mappings if it does not exist +mkdir -p /data/wis2box/mappings + # Workdir cd /pygeoapi diff --git a/docker/pygeoapi-config.yml b/docker/pygeoapi-config.yml index 02ff9c9..98a3b72 100644 --- a/docker/pygeoapi-config.yml +++ b/docker/pygeoapi-config.yml @@ -67,6 +67,11 @@ resources: type: process processor: name: wis2box_api.plugins.process.dataset_info.DatasetInfoProcessor + + mappings-info: + type: process + processor: + name: wis2box_api.plugins.process.mappings_info.MappingsInfoProcessor pygeometa-metadata-generate: type: process diff --git a/docker_compose_test/docker-compose.yml b/docker_compose_test/docker-compose.yml index bd10e0f..190483e 100644 --- a/docker_compose_test/docker-compose.yml +++ b/docker_compose_test/docker-compose.yml @@ -14,6 +14,8 @@ services: retries: 100 ports: - "4343:80" + volumes: + - ./tests/data/mappings:/data/wis2box/mappings minio: image: minio/minio:RELEASE.2022-12-02T19-19-22Z.fips diff --git a/docker_compose_test/test.env b/docker_compose_test/test.env index a575086..0f25448 100644 --- a/docker_compose_test/test.env +++ b/docker_compose_test/test.env @@ -1,4 +1,4 @@ -WIS2BOX_URL=http://localhost +WIS2BOX_URL=http://localhost:4343 WIS2BOX_API_URL=http://localhost:4343/oapi WIS2BOX_LOGGING_LOGLEVEL=INFO @@ -46,4 +46,7 @@ MINIO_NOTIFY_MQTT_TOPIC_WIS2BOX=wis2box/storage MINIO_NOTIFY_MQTT_QOS_WIS2BOX=1 BUFR_ORIGINATING_CENTRE=0 -BUFR_ORIGINATING_SUBCENTRE=0 \ No newline at end of file +BUFR_ORIGINATING_SUBCENTRE=0 + +# CSV2BUFR_TEMPLATES +CSV2BUFR_TEMPLATES=${WIS2BOX_DATADIR}/mappings \ No newline at end of file diff --git a/docker_compose_test/tests/data/mappings/my_csv2bufr_mappings.json b/docker_compose_test/tests/data/mappings/my_csv2bufr_mappings.json new file mode 100644 index 0000000..3a71c60 --- /dev/null +++ b/docker_compose_test/tests/data/mappings/my_csv2bufr_mappings.json @@ -0,0 +1,93 @@ +{ + "conformsTo": "csv2bufr-template-v2.json", + "metadata": { + "label": "my-custom-template", + "description": "This is a custom template to test the mapping_info process", + "version": "3", + "author": "TEST USER", + "editor": "TEST USER", + "dateCreated": "2024-11-06", + "dateModified": "2024-11-06", + "id": "my-custom-template" + }, + "inputShortDelayedDescriptorReplicationFactor": [], + "inputDelayedDescriptorReplicationFactor": [], + "inputExtendedDelayedDescriptorReplicationFactor": [], + "number_header_rows": 1, + "column_names_row": 1, + "quoting": "QUOTE_NONE", + "header": [ + {"eccodes_key": "edition", "value": "const:4"}, + {"eccodes_key": "masterTableNumber", "value": "const:0"}, + {"eccodes_key": "updateSequenceNumber", "value": "const:0"}, + {"eccodes_key": "dataCategory", "value": "const:0"}, + {"eccodes_key": "internationalDataSubCategory", "value": "const:6"}, + {"eccodes_key": "masterTablesVersionNumber", "value": "const:39"}, + {"eccodes_key": "typicalYear", "value": "data:year"}, + {"eccodes_key": "typicalMonth", "value": "data:month"}, + {"eccodes_key": "typicalDay", "value": "data:day"}, + {"eccodes_key": "typicalHour", "value": "const:23"}, + {"eccodes_key": "typicalMinute", "value": "const:59"}, + {"eccodes_key": "numberOfSubsets", "value": "const:1"}, + {"eccodes_key": "observedData", "value": "const:1"}, + {"eccodes_key": "compressedData", "value": "const:0"}, + {"eccodes_key": "unexpandedDescriptors", "value": "array:307075"} + ], + "data": [ + {"eccodes_key": "#1#wigosIdentifierSeries", "value":"data:wsi_series", "valid_min": "const:0", "valid_max": "const:0"}, + {"eccodes_key": "#1#wigosIssuerOfIdentifier", "value":"data:wsi_issuer", "valid_min": "const:0", "valid_max": "const:65534"}, + {"eccodes_key": "#1#wigosIssueNumber", "value":"data:wsi_issue_number", "valid_min": "const:0", "valid_max": "const:65534"}, + {"eccodes_key": "#1#wigosLocalIdentifierCharacter", "value":"data:wsi_local"}, + {"eccodes_key": "#1#blockNumber", "value": "data:wmo_block_number", "valid_min": "const:0", "valid_max": "const:99"}, + {"eccodes_key": "#1#stationNumber", "value": "data:wmo_station_number", "valid_min": "const:0", "valid_max": "const:999"}, + {"eccodes_key": "#1#latitude", "value": "data:latitude", "valid_min": "const:-90.0", "valid_max": "const:90.0"}, + {"eccodes_key": "#1#longitude", "value": "data:longitude", "valid_min": "const:-180.0", "valid_max": "const:180.0"}, + {"eccodes_key": "#1#heightOfStationGroundAboveMeanSeaLevel", "value":"data:station_height_above_msl", "valid_min": "const:-400", "valid_max": "const:9000"}, + {"eccodes_key": "#1#methodUsedToCalculateTheAverageDailyTemperature", "value": "data:averaging_method", "valid_min": "const:0", "valid_max": "const:255"}, + {"eccodes_key": "#1#sitingAndMeasurementQualityClassificationForTemperature", "value": "data:temperature_siting_classification", "valid_min": "const:0", "valid_max": "const:255"}, + {"eccodes_key": "#1#sitingAndMeasurementQualityClassificationForPrecipitation", "value": "data:precipitation_siting_classification", "valid_min": "const:0", "valid_max": "const:255"}, + {"eccodes_key": "#1#year", "value": "data:year", "valid_min": "const:1800", "valid_max": "const:2100"}, + {"eccodes_key": "#1#month", "value": "data:month", "valid_min": "const:1", "valid_max": "const:12"}, + {"eccodes_key": "#1#day", "value": "data:day", "valid_min": "const:1", "valid_max": "const:31"}, + {"eccodes_key": "#1#timePeriod", "value": "data:precipitation_day_offset", "valid_min": "const:-1", "valid_max": "const:0"}, + {"eccodes_key": "#1#hour", "value": "data:precipitation_hour", "valid_min": "const:0", "valid_max": "const:23"}, + {"eccodes_key": "#1#minute", "value": "data:precipitation_minute", "valid_min": "const:0", "valid_max": "const:59"}, + {"eccodes_key": "#1#second", "value": "data:precipitation_second", "valid_min": "const:0", "valid_max": "const:59"}, + {"eccodes_key": "#1#totalAccumulatedPrecipitation", "value": "data:precipitation", "valid_min": "const:0", "valid_max": "const:2000"}, + {"eccodes_key": "#1#totalAccumulatedPrecipitation->associatedField", "value": "data:precipitation_flag", "valid_min": "const:0", "valid_max": "const:7"}, + {"eccodes_key": "#1#totalAccumulatedPrecipitation->associatedField->associatedFieldSignificance", "value": "const:5"}, + {"eccodes_key": "#2#timePeriod", "value": "data:fresh_snow_day_offset", "valid_min": "const:-1", "valid_max": "const:0"}, + {"eccodes_key": "#2#hour", "value": "data:fresh_snow_hour", "valid_min": "const:0", "valid_max": "const:23"}, + {"eccodes_key": "#2#minute", "value": "data:fresh_snow_minute", "valid_min": "const:0", "valid_max": "const:59"}, + {"eccodes_key": "#2#second", "value": "data:fresh_snow_second", "valid_min": "const:0", "valid_max": "const:59"}, + {"eccodes_key": "#3#timePeriod", "value": "data:total_snow_day_offset", "valid_min": "const:-1", "valid_max": "const:0"}, + {"eccodes_key": "#3#hour", "value": "data:total_snow_hour", "valid_min": "const:0", "valid_max": "const:23"}, + {"eccodes_key": "#3#minute", "value": "data:total_snow_minute", "valid_min": "const:0", "valid_max": "const:59"}, + {"eccodes_key": "#3#second", "value": "data:total_snow_second", "valid_min": "const:0", "valid_max": "const:59"}, + {"eccodes_key": "#1#heightOfSensorAboveLocalGroundOrDeckOfMarinePlatform", "value": "data:thermometer_height", "valid_min": "const:0", "valid_max": "const:655.35"}, + {"eccodes_key": "#4#timePeriod", "value": "data:maximum_temperature_day_offset", "valid_min": "const:-1", "valid_max": "const:0"}, + {"eccodes_key": "#4#hour", "value": "data:maximum_temperature_hour", "valid_min": "const:0", "valid_max": "const:23"}, + {"eccodes_key": "#4#minute", "value": "data:maximum_temperature_minute", "valid_min": "const:0", "valid_max": "const:59"}, + {"eccodes_key": "#4#second", "value": "data:maximum_temperature_second", "valid_min": "const:0", "valid_max": "const:59"}, + {"eccodes_key": "#1#firstOrderStatistics", "value": "const:2"}, + {"eccodes_key": "#1#airTemperature", "value": "data:maximum_temperature", "valid_min": "const:183.15", "valid_max": "const:343.15"}, + {"eccodes_key": "#1#airTemperature->associatedField", "value": "data:maximum_temperature_flag", "valid_min": "const:0", "valid_max": "const:7"}, + {"eccodes_key": "#1#airTemperature->associatedField->associatedFieldSignificance", "value": "const:5"}, + {"eccodes_key": "#5#timePeriod", "value": "data:minimum_temperature_day_offset", "valid_min": "const:-1", "valid_max": "const:0"}, + {"eccodes_key": "#5#hour", "value": "data:minimum_temperature_hour", "valid_min": "const:0", "valid_max": "const:23"}, + {"eccodes_key": "#5#minute", "value": "data:minimum_temperature_minute", "valid_min": "const:0", "valid_max": "const:59"}, + {"eccodes_key": "#5#second", "value": "data:minimum_temperature_second", "valid_min": "const:0", "valid_max": "const:59"}, + {"eccodes_key": "#2#firstOrderStatistics", "value": "const:3"}, + {"eccodes_key": "#2#airTemperature", "value": "data:minimum_temperature", "valid_min": "const:183.15", "valid_max": "const:343.15"}, + {"eccodes_key": "#2#airTemperature->associatedField", "value": "data:minimum_temperature_flag", "valid_min": "const:0", "valid_max": "const:7"}, + {"eccodes_key": "#2#airTemperature->associatedField->associatedFieldSignificance", "value": "const:5"}, + {"eccodes_key": "#6#timePeriod", "value": "data:average_temperature_day_offset", "valid_min": "const:-1", "valid_max": "const:0"}, + {"eccodes_key": "#6#hour", "value": "data:average_temperature_hour", "valid_min": "const:0", "valid_max": "const:23"}, + {"eccodes_key": "#6#minute", "value": "data:average_temperature_minute", "valid_min": "const:0", "valid_max": "const:59"}, + {"eccodes_key": "#6#second", "value": "data:average_temperature_second", "valid_min": "const:0", "valid_max": "const:59"}, + {"eccodes_key": "#3#firstOrderStatistics", "value": "const:4"}, + {"eccodes_key": "#3#airTemperature", "value": "data:average_temperature", "valid_min": "const:183.15", "valid_max": "const:343.15"}, + {"eccodes_key": "#3#airTemperature->associatedField", "value": "data:average_temperature_flag", "valid_min": "const:0", "valid_max": "const:7"}, + {"eccodes_key": "#3#airTemperature->associatedField->associatedFieldSignificance", "value": "const:5"} + ] +} \ No newline at end of file diff --git a/docker_compose_test/tests/integration/test_wis2box.py b/docker_compose_test/tests/integration/test_wis2box.py index ee242fa..c17cb01 100644 --- a/docker_compose_test/tests/integration/test_wis2box.py +++ b/docker_compose_test/tests/integration/test_wis2box.py @@ -356,3 +356,50 @@ def test_cap2geojson(): assert 'items' in output assert len(output['items']) == 1 assert output['items'][0]['features'][0]['properties'] == cap_geojson['features'][0]['properties'] # noqa + + +def test_mappings_info(): + """Test mappings_info process""" + + process_name = 'mappings-info' + data = { + 'inputs': { + 'plugin': 'wis2box.data.csv2bufr.ObservationDataCSV2BUFR' + } + } + + # execute the process and get the result + result = requests.post(f'{API_URL}/processes/{process_name}/execution', json=data) # noqa + + # check the status code + assert result.status_code == 200 + + response = result.json() + + expected_response = { + "templates": [ + { + "id": "/data/wis2box/mappings/my_csv2bufr_mappings.json", + "title": "my_csv2bufr_mappings" + }, + { + "id": "daycli-template", + "title": "DayCLI" + }, + { + "id": "CampbellAfrica-v1-template", + "title": "WIS2-pilot-template-2021" + }, + { + "id": "aws-template", + "title": "AWS" + } + ] + } + + # compare that all expected templates are present, regardless of the order + assert len(response['templates']) == len(expected_response['templates']) + templates = response['templates'] + for template in expected_response['templates']: + assert template['id'] in [t['id'] for t in templates] + assert template['title'] in [t['title'] for t in templates] diff --git a/wis2box_api/plugins/process/mappings_info.py b/wis2box_api/plugins/process/mappings_info.py new file mode 100644 index 0000000..ad447b1 --- /dev/null +++ b/wis2box_api/plugins/process/mappings_info.py @@ -0,0 +1,131 @@ +############################################################################### +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +############################################################################### + +import logging + +import csv2bufr.templates as c2bt + +from pygeoapi.process.base import BaseProcessor, ProcessorExecuteError + +LOGGER = logging.getLogger(__name__) + +PROCESS_DEF = { + 'version': '0.1.0', + 'id': 'wmo-get-ra', + 'title': { + 'en': 'Get available mapping templates in the system' + }, + 'description': { + 'en': 'Get available mapping templates in the system' + }, + 'keywords': [], + 'links': [], + 'inputs': { + 'plugin': { + 'title': 'Plugin', + 'description': 'Plugin ID', + 'schema': { + 'type': 'string' + } + } + }, + 'outputs': { + 'result': { + 'title': 'Mapping templates', + 'description': 'Mapping templates', + 'schema': { + 'type': 'object', + 'contentMediaType': 'application/json' + } + } + }, + 'example': { + 'inputs': { + 'plugin': 'wis2box.data.csv2bufr.ObservationDataCSV2BUFR' + } + } +} + + +class MappingsInfoProcessor(BaseProcessor): + """WMO RA Processor""" + + def __init__(self, processor_def): + """ + Initialize object + + :param processor_def: provider definition + + :returns: wis2box_api.plugins.process.mappings_info.MappingsInfoProcessor # noqa + """ + + super().__init__(processor_def, PROCESS_DEF) + + def execute(self, data): + """ + Execute Process + + :param data: processor arguments + + :returns: 'application/json' + """ + + mimetype = 'application/json' + + valid_plugins = [ + 'wis2box.data.csv2bufr.ObservationDataCSV2BUFR' + ] + + plugin_id = data.get('plugin') + + if plugin_id not in valid_plugins: + raise ProcessorExecuteError('Invalid plugin ID') + + templates = [] + if plugin_id == 'wis2box.data.csv2bufr.ObservationDataCSV2BUFR': + for template in c2bt.list_templates().values(): + LOGGER.info(template) + id_ = template['path'] + title = template['path'] + if '/opt/csv2bufr/templates/' in id_: + id_ = id_.replace('/opt/csv2bufr/templates/', '').replace('.json', '') # noqa + # to ensure backward compatibility with existing titles + if title.find('aws-template') != -1: + title = 'AWS' + elif title.find('daycli-template') != -1: + title = 'DayCLI' + elif title.find('CampbellAfrica-v1-template') != -1: + title = 'WIS2-pilot-template-2021' + else: + # extract title from filename + title = id_.split('/')[-1].replace('.json', '') + templates.append({ + 'id': id_, + 'title': title + }) + outputs = { + 'templates': templates + } + + return mimetype, outputs + + def __repr__(self): + return ' {}'.format(self.name)