Skip to content

Commit

Permalink
add mappings-info process and test (#78)
Browse files Browse the repository at this point in the history 
* add mappings-info process and test
  • Loading branch information
@maaikelimper
maaikelimper authored Nov 21, 2024
1 parent 679d7f6 commit 15d1a75
Show file tree
Hide file tree
Showing 7 changed files with 286 additions and 2 deletions.
3 changes: 3 additions & 0 deletions docker/entrypoint.sh
View file
Original file line number Diff line number Diff line change
Expand Up @@ -47,6 +47,9 @@ function error() {
exit -1
}

# create mappings directory /data/wi2box/mappings if it does not exist
mkdir -p /data/wis2box/mappings

# Workdir
cd /pygeoapi

Expand Down
5 changes: 5 additions & 0 deletions docker/pygeoapi-config.yml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -67,6 +67,11 @@ resources:
type: process
processor:
name: wis2box_api.plugins.process.dataset_info.DatasetInfoProcessor

mappings-info:
type: process
processor:
name: wis2box_api.plugins.process.mappings_info.MappingsInfoProcessor

pygeometa-metadata-generate:
type: process
Expand Down
2 changes: 2 additions & 0 deletions docker_compose_test/docker-compose.yml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,8 @@ services:
retries: 100
ports:
- "4343:80"
volumes:
- ./tests/data/mappings:/data/wis2box/mappings

minio:
image: minio/minio:RELEASE.2022-12-02T19-19-22Z.fips
Expand Down
7 changes: 5 additions & 2 deletions docker_compose_test/test.env
View file
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
WIS2BOX_URL=http://localhost
WIS2BOX_URL=http://localhost:4343
WIS2BOX_API_URL=http://localhost:4343/oapi

WIS2BOX_LOGGING_LOGLEVEL=INFO
Expand Down Expand Up @@ -46,4 +46,7 @@ MINIO_NOTIFY_MQTT_TOPIC_WIS2BOX=wis2box/storage
MINIO_NOTIFY_MQTT_QOS_WIS2BOX=1

BUFR_ORIGINATING_CENTRE=0
BUFR_ORIGINATING_SUBCENTRE=0
BUFR_ORIGINATING_SUBCENTRE=0

# CSV2BUFR_TEMPLATES
CSV2BUFR_TEMPLATES=${WIS2BOX_DATADIR}/mappings
93 changes: 93 additions & 0 deletions docker_compose_test/tests/data/mappings/my_csv2bufr_mappings.json
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,93 @@
{
"conformsTo": "csv2bufr-template-v2.json",
"metadata": {
"label": "my-custom-template",
"description": "This is a custom template to test the mapping_info process",
"version": "3",
"author": "TEST USER",
"editor": "TEST USER",
"dateCreated": "2024-11-06",
"dateModified": "2024-11-06",
"id": "my-custom-template"
},
"inputShortDelayedDescriptorReplicationFactor": [],
"inputDelayedDescriptorReplicationFactor": [],
"inputExtendedDelayedDescriptorReplicationFactor": [],
"number_header_rows": 1,
"column_names_row": 1,
"quoting": "QUOTE_NONE",
"header": [
{"eccodes_key": "edition", "value": "const:4"},
{"eccodes_key": "masterTableNumber", "value": "const:0"},
{"eccodes_key": "updateSequenceNumber", "value": "const:0"},
{"eccodes_key": "dataCategory", "value": "const:0"},
{"eccodes_key": "internationalDataSubCategory", "value": "const:6"},
{"eccodes_key": "masterTablesVersionNumber", "value": "const:39"},
{"eccodes_key": "typicalYear", "value": "data:year"},
{"eccodes_key": "typicalMonth", "value": "data:month"},
{"eccodes_key": "typicalDay", "value": "data:day"},
{"eccodes_key": "typicalHour", "value": "const:23"},
{"eccodes_key": "typicalMinute", "value": "const:59"},
{"eccodes_key": "numberOfSubsets", "value": "const:1"},
{"eccodes_key": "observedData", "value": "const:1"},
{"eccodes_key": "compressedData", "value": "const:0"},
{"eccodes_key": "unexpandedDescriptors", "value": "array:307075"}
],
"data": [
{"eccodes_key": "#1#wigosIdentifierSeries", "value":"data:wsi_series", "valid_min": "const:0", "valid_max": "const:0"},
{"eccodes_key": "#1#wigosIssuerOfIdentifier", "value":"data:wsi_issuer", "valid_min": "const:0", "valid_max": "const:65534"},
{"eccodes_key": "#1#wigosIssueNumber", "value":"data:wsi_issue_number", "valid_min": "const:0", "valid_max": "const:65534"},
{"eccodes_key": "#1#wigosLocalIdentifierCharacter", "value":"data:wsi_local"},
{"eccodes_key": "#1#blockNumber", "value": "data:wmo_block_number", "valid_min": "const:0", "valid_max": "const:99"},
{"eccodes_key": "#1#stationNumber", "value": "data:wmo_station_number", "valid_min": "const:0", "valid_max": "const:999"},
{"eccodes_key": "#1#latitude", "value": "data:latitude", "valid_min": "const:-90.0", "valid_max": "const:90.0"},
{"eccodes_key": "#1#longitude", "value": "data:longitude", "valid_min": "const:-180.0", "valid_max": "const:180.0"},
{"eccodes_key": "#1#heightOfStationGroundAboveMeanSeaLevel", "value":"data:station_height_above_msl", "valid_min": "const:-400", "valid_max": "const:9000"},
{"eccodes_key": "#1#methodUsedToCalculateTheAverageDailyTemperature", "value": "data:averaging_method", "valid_min": "const:0", "valid_max": "const:255"},
{"eccodes_key": "#1#sitingAndMeasurementQualityClassificationForTemperature", "value": "data:temperature_siting_classification", "valid_min": "const:0", "valid_max": "const:255"},
{"eccodes_key": "#1#sitingAndMeasurementQualityClassificationForPrecipitation", "value": "data:precipitation_siting_classification", "valid_min": "const:0", "valid_max": "const:255"},
{"eccodes_key": "#1#year", "value": "data:year", "valid_min": "const:1800", "valid_max": "const:2100"},
{"eccodes_key": "#1#month", "value": "data:month", "valid_min": "const:1", "valid_max": "const:12"},
{"eccodes_key": "#1#day", "value": "data:day", "valid_min": "const:1", "valid_max": "const:31"},
{"eccodes_key": "#1#timePeriod", "value": "data:precipitation_day_offset", "valid_min": "const:-1", "valid_max": "const:0"},
{"eccodes_key": "#1#hour", "value": "data:precipitation_hour", "valid_min": "const:0", "valid_max": "const:23"},
{"eccodes_key": "#1#minute", "value": "data:precipitation_minute", "valid_min": "const:0", "valid_max": "const:59"},
{"eccodes_key": "#1#second", "value": "data:precipitation_second", "valid_min": "const:0", "valid_max": "const:59"},
{"eccodes_key": "#1#totalAccumulatedPrecipitation", "value": "data:precipitation", "valid_min": "const:0", "valid_max": "const:2000"},
{"eccodes_key": "#1#totalAccumulatedPrecipitation->associatedField", "value": "data:precipitation_flag", "valid_min": "const:0", "valid_max": "const:7"},
{"eccodes_key": "#1#totalAccumulatedPrecipitation->associatedField->associatedFieldSignificance", "value": "const:5"},
{"eccodes_key": "#2#timePeriod", "value": "data:fresh_snow_day_offset", "valid_min": "const:-1", "valid_max": "const:0"},
{"eccodes_key": "#2#hour", "value": "data:fresh_snow_hour", "valid_min": "const:0", "valid_max": "const:23"},
{"eccodes_key": "#2#minute", "value": "data:fresh_snow_minute", "valid_min": "const:0", "valid_max": "const:59"},
{"eccodes_key": "#2#second", "value": "data:fresh_snow_second", "valid_min": "const:0", "valid_max": "const:59"},
{"eccodes_key": "#3#timePeriod", "value": "data:total_snow_day_offset", "valid_min": "const:-1", "valid_max": "const:0"},
{"eccodes_key": "#3#hour", "value": "data:total_snow_hour", "valid_min": "const:0", "valid_max": "const:23"},
{"eccodes_key": "#3#minute", "value": "data:total_snow_minute", "valid_min": "const:0", "valid_max": "const:59"},
{"eccodes_key": "#3#second", "value": "data:total_snow_second", "valid_min": "const:0", "valid_max": "const:59"},
{"eccodes_key": "#1#heightOfSensorAboveLocalGroundOrDeckOfMarinePlatform", "value": "data:thermometer_height", "valid_min": "const:0", "valid_max": "const:655.35"},
{"eccodes_key": "#4#timePeriod", "value": "data:maximum_temperature_day_offset", "valid_min": "const:-1", "valid_max": "const:0"},
{"eccodes_key": "#4#hour", "value": "data:maximum_temperature_hour", "valid_min": "const:0", "valid_max": "const:23"},
{"eccodes_key": "#4#minute", "value": "data:maximum_temperature_minute", "valid_min": "const:0", "valid_max": "const:59"},
{"eccodes_key": "#4#second", "value": "data:maximum_temperature_second", "valid_min": "const:0", "valid_max": "const:59"},
{"eccodes_key": "#1#firstOrderStatistics", "value": "const:2"},
{"eccodes_key": "#1#airTemperature", "value": "data:maximum_temperature", "valid_min": "const:183.15", "valid_max": "const:343.15"},
{"eccodes_key": "#1#airTemperature->associatedField", "value": "data:maximum_temperature_flag", "valid_min": "const:0", "valid_max": "const:7"},
{"eccodes_key": "#1#airTemperature->associatedField->associatedFieldSignificance", "value": "const:5"},
{"eccodes_key": "#5#timePeriod", "value": "data:minimum_temperature_day_offset", "valid_min": "const:-1", "valid_max": "const:0"},
{"eccodes_key": "#5#hour", "value": "data:minimum_temperature_hour", "valid_min": "const:0", "valid_max": "const:23"},
{"eccodes_key": "#5#minute", "value": "data:minimum_temperature_minute", "valid_min": "const:0", "valid_max": "const:59"},
{"eccodes_key": "#5#second", "value": "data:minimum_temperature_second", "valid_min": "const:0", "valid_max": "const:59"},
{"eccodes_key": "#2#firstOrderStatistics", "value": "const:3"},
{"eccodes_key": "#2#airTemperature", "value": "data:minimum_temperature", "valid_min": "const:183.15", "valid_max": "const:343.15"},
{"eccodes_key": "#2#airTemperature->associatedField", "value": "data:minimum_temperature_flag", "valid_min": "const:0", "valid_max": "const:7"},
{"eccodes_key": "#2#airTemperature->associatedField->associatedFieldSignificance", "value": "const:5"},
{"eccodes_key": "#6#timePeriod", "value": "data:average_temperature_day_offset", "valid_min": "const:-1", "valid_max": "const:0"},
{"eccodes_key": "#6#hour", "value": "data:average_temperature_hour", "valid_min": "const:0", "valid_max": "const:23"},
{"eccodes_key": "#6#minute", "value": "data:average_temperature_minute", "valid_min": "const:0", "valid_max": "const:59"},
{"eccodes_key": "#6#second", "value": "data:average_temperature_second", "valid_min": "const:0", "valid_max": "const:59"},
{"eccodes_key": "#3#firstOrderStatistics", "value": "const:4"},
{"eccodes_key": "#3#airTemperature", "value": "data:average_temperature", "valid_min": "const:183.15", "valid_max": "const:343.15"},
{"eccodes_key": "#3#airTemperature->associatedField", "value": "data:average_temperature_flag", "valid_min": "const:0", "valid_max": "const:7"},
{"eccodes_key": "#3#airTemperature->associatedField->associatedFieldSignificance", "value": "const:5"}
]
}
47 changes: 47 additions & 0 deletions docker_compose_test/tests/integration/test_wis2box.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -356,3 +356,50 @@ def test_cap2geojson():
assert 'items' in output
assert len(output['items']) == 1
assert output['items'][0]['features'][0]['properties'] == cap_geojson['features'][0]['properties'] # noqa


def test_mappings_info():
"""Test mappings_info process"""

process_name = 'mappings-info'
data = {
'inputs': {
'plugin': 'wis2box.data.csv2bufr.ObservationDataCSV2BUFR'
}
}

# execute the process and get the result
result = requests.post(f'{API_URL}/processes/{process_name}/execution', json=data) # noqa

# check the status code
assert result.status_code == 200

response = result.json()

expected_response = {
"templates": [
{
"id": "/data/wis2box/mappings/my_csv2bufr_mappings.json",
"title": "my_csv2bufr_mappings"
},
{
"id": "daycli-template",
"title": "DayCLI"
},
{
"id": "CampbellAfrica-v1-template",
"title": "WIS2-pilot-template-2021"
},
{
"id": "aws-template",
"title": "AWS"
}
]
}

# compare that all expected templates are present, regardless of the order
assert len(response['templates']) == len(expected_response['templates'])
templates = response['templates']
for template in expected_response['templates']:
assert template['id'] in [t['id'] for t in templates]
assert template['title'] in [t['title'] for t in templates]
131 changes: 131 additions & 0 deletions wis2box_api/plugins/process/mappings_info.py
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,131 @@
###############################################################################
#
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
#
###############################################################################

import logging

import csv2bufr.templates as c2bt

from pygeoapi.process.base import BaseProcessor, ProcessorExecuteError

LOGGER = logging.getLogger(__name__)

PROCESS_DEF = {
'version': '0.1.0',
'id': 'wmo-get-ra',
'title': {
'en': 'Get available mapping templates in the system'
},
'description': {
'en': 'Get available mapping templates in the system'
},
'keywords': [],
'links': [],
'inputs': {
'plugin': {
'title': 'Plugin',
'description': 'Plugin ID',
'schema': {
'type': 'string'
}
}
},
'outputs': {
'result': {
'title': 'Mapping templates',
'description': 'Mapping templates',
'schema': {
'type': 'object',
'contentMediaType': 'application/json'
}
}
},
'example': {
'inputs': {
'plugin': 'wis2box.data.csv2bufr.ObservationDataCSV2BUFR'
}
}
}


class MappingsInfoProcessor(BaseProcessor):
"""WMO RA Processor"""

def __init__(self, processor_def):
"""
Initialize object
:param processor_def: provider definition
:returns: wis2box_api.plugins.process.mappings_info.MappingsInfoProcessor # noqa
"""

super().__init__(processor_def, PROCESS_DEF)

def execute(self, data):
"""
Execute Process
:param data: processor arguments
:returns: 'application/json'
"""

mimetype = 'application/json'

valid_plugins = [
'wis2box.data.csv2bufr.ObservationDataCSV2BUFR'
]

plugin_id = data.get('plugin')

if plugin_id not in valid_plugins:
raise ProcessorExecuteError('Invalid plugin ID')

templates = []
if plugin_id == 'wis2box.data.csv2bufr.ObservationDataCSV2BUFR':
for template in c2bt.list_templates().values():
LOGGER.info(template)
id_ = template['path']
title = template['path']
if '/opt/csv2bufr/templates/' in id_:
id_ = id_.replace('/opt/csv2bufr/templates/', '').replace('.json', '') # noqa
# to ensure backward compatibility with existing titles
if title.find('aws-template') != -1:
title = 'AWS'
elif title.find('daycli-template') != -1:
title = 'DayCLI'
elif title.find('CampbellAfrica-v1-template') != -1:
title = 'WIS2-pilot-template-2021'
else:
# extract title from filename
title = id_.split('/')[-1].replace('.json', '')
templates.append({
'id': id_,
'title': title
})
outputs = {
'templates': templates
}

return mimetype, outputs

def __repr__(self):
return '<TemplatesInfoProcessor> {}'.format(self.name)

0 comments on commit 15d1a75

Please sign in to comment.