From 3b2e88213ea15c845b5a3aad9b0868215b10559f Mon Sep 17 00:00:00 2001 From: Jules Dejaeghere Date: Sun, 30 Jun 2024 17:57:24 +0200 Subject: [PATCH 1/2] Add RIO IFDM client --- src/open_irceline/__init__.py | 2 +- src/open_irceline/api.py | 40 +- src/open_irceline/data.py | 25 +- src/open_irceline/forecast.py | 46 +- src/open_irceline/rio.py | 50 +- .../ifdm_interpolation_feature_info.json | 17 + ...{capabilities.xml => rio_capabilities.xml} | 0 tests/fixtures/rio_ifdm_capabilities.xml | 694 ++++++++++++++++++ tests/test_api_forecast.py | 5 +- tests/test_api_rio.py | 4 +- tests/test_api_rio_ifdm.py | 102 +++ 11 files changed, 921 insertions(+), 64 deletions(-) create mode 100644 tests/fixtures/ifdm_interpolation_feature_info.json rename tests/fixtures/{capabilities.xml => rio_capabilities.xml} (100%) create mode 100644 tests/fixtures/rio_ifdm_capabilities.xml create mode 100644 tests/test_api_rio_ifdm.py diff --git a/src/open_irceline/__init__.py b/src/open_irceline/__init__.py index 00e8314..d830345 100644 --- a/src/open_irceline/__init__.py +++ b/src/open_irceline/__init__.py @@ -1,6 +1,6 @@ from .api import IrcelineApiError from .rio import IrcelineRioClient from .forecast import IrcelineForecastClient -from .data import RioFeature, ForecastFeature, FeatureValue, BelAqiIndex +from .data import RioFeature, ForecastFeature, FeatureValue __version__ = '2.0.0' diff --git a/src/open_irceline/api.py b/src/open_irceline/api.py index bb374a2..a89c165 100644 --- a/src/open_irceline/api.py +++ b/src/open_irceline/api.py @@ -2,17 +2,18 @@ import socket from abc import ABC, abstractmethod from typing import Tuple, List, Set +from xml.etree import ElementTree import aiohttp import async_timeout +from aiohttp import ClientResponse from .data import IrcelineFeature from .utils import SizedDict _rio_wfs_base_url = 'https://geo.irceline.be/wfs' _forecast_wms_base_url = 'https://geo.irceline.be/forecast/wms' -# noinspection HttpUrlsUsage -# There is not HTTPS version of this endpoint +_rio_ifdm_wms_base_url = 'https://geobelair.irceline.be/rioifdm/wms' _user_agent = 'github.com/jdejaegh/python-irceline' @@ -65,3 +66,38 @@ async def _api_wrapper(self, url: str, querystring: dict = None, headers: dict = raise IrcelineApiError(f"Something really wrong happened! {exception}") from exception +class IrcelineBaseWmsClient(IrcelineBaseClient, ABC): + _default_querystring = {"service": "WMS", + "version": "1.1.1", + "request": "GetFeatureInfo", + "info_format": "application/json", + "width": "1", + "height": "1", + "srs": "EPSG:4326", + "X": "1", + "Y": "1"} + _epsilon = 0.00001 + _base_url = None + + @staticmethod + def _parse_capabilities(xml_string: str) -> Set[str]: + try: + root = ElementTree.fromstring(xml_string) + except ElementTree.ParseError: + return set() + + path = './/Capability/Layer/Layer/Name' + feature_type_names = {t.text for t in root.findall(path)} + return feature_type_names + + async def get_capabilities(self) -> Set[str]: + """ + Fetch the list of possible features from the WMS server + :return: set of features available on the WMS server + """ + querystring = {"service": "WMS", + "version": "1.1.1", + "request": "GetCapabilities"} + r: ClientResponse = await self._api_wrapper(self._base_url, querystring) + + return self._parse_capabilities(await r.text()) diff --git a/src/open_irceline/data.py b/src/open_irceline/data.py index fd4cb41..7f8c4c7 100644 --- a/src/open_irceline/data.py +++ b/src/open_irceline/data.py @@ -1,5 +1,5 @@ from datetime import datetime, date -from enum import StrEnum, Enum +from enum import StrEnum from typing import TypedDict @@ -30,6 +30,14 @@ class RioFeature(IrcelineFeature): SO2_HMEAN = 'rio:so2_hmean' +class RioIfdmFeature(IrcelineFeature): + PM25_HMEAN = 'rioifdm:pm25_hmean' + NO2_HMEAN = 'rioifdm:no2_hmean' + PM10_HMEAN = 'rioifdm:pm10_hmean' + O3_HMEAN = 'rioifdm:o3_hmean' + BELAQI = 'rioifdm:belaqi' + + class ForecastFeature(IrcelineFeature): NO2_MAXHMEAN = 'forecast:no2_maxhmean' NO2_DMEAN = 'forecast:no2_dmean' @@ -40,20 +48,7 @@ class ForecastFeature(IrcelineFeature): BELAQI = 'forecast:belaqi' -class BelAqiIndex(Enum): - EXCELLENT = 1 - VERY_GOOD = 2 - GOOD = 3 - FAIRLY_GOOD = 4 - MODERATE = 5 - POOR = 6 - VERY_POOR = 7 - BAD = 8 - VERY_BAD = 9 - HORRIBLE = 10 - - class FeatureValue(TypedDict): # Timestamp at which the value was computed timestamp: datetime | date | None - value: int | float | BelAqiIndex | None + value: int | float | None diff --git a/src/open_irceline/forecast.py b/src/open_irceline/forecast.py index d4ba038..c2b5efd 100644 --- a/src/open_irceline/forecast.py +++ b/src/open_irceline/forecast.py @@ -1,16 +1,15 @@ from datetime import date, timedelta, datetime from itertools import product -from typing import List, Tuple, Dict, Set -from xml.etree import ElementTree +from typing import List, Tuple, Dict from aiohttp import ClientResponse, ClientResponseError -from .api import IrcelineBaseClient, _forecast_wms_base_url, IrcelineApiError +from .api import IrcelineApiError, IrcelineBaseWmsClient, _forecast_wms_base_url from .data import ForecastFeature, FeatureValue -class IrcelineForecastClient(IrcelineBaseClient): - _epsilon = 0.00001 +class IrcelineForecastClient(IrcelineBaseWmsClient): + _base_url = _forecast_wms_base_url async def get_data(self, features: List[ForecastFeature], @@ -26,22 +25,14 @@ async def get_data(self, timestamp = date.today() result = dict() lat, lon = position - base_querystring = {"service": "WMS", - "version": "1.1.1", - "request": "GetFeatureInfo", - "info_format": "application/json", - "width": "1", - "height": "1", - "srs": "EPSG:4326", - "bbox": f"{lon},{lat},{lon + self._epsilon},{lat + self._epsilon}", - "X": "1", - "Y": "1"} + base_querystring = (self._default_querystring | + {"bbox": f"{lon},{lat},{lon + self._epsilon},{lat + self._epsilon}"}) for feature, d in product(features, range(4)): querystring = base_querystring | {"layers": f"{feature}_d{d}", "query_layers": f"{feature}_d{d}"} try: - r: ClientResponse = await self._api_wrapper(_forecast_wms_base_url, querystring) + r: ClientResponse = await self._api_wrapper(self._base_url, querystring) r: dict = await r.json() result[(feature, timestamp + timedelta(days=d))] = FeatureValue( value=r.get('features', [{}])[0].get('properties', {}).get('GRAY_INDEX'), @@ -50,26 +41,3 @@ async def get_data(self, result[(feature, timestamp + timedelta(days=d))] = FeatureValue(value=None, timestamp=None) return result - - async def get_capabilities(self) -> Set[str]: - """ - Fetch the list of possible features from the WMS server - :return: set of features available on the WMS server - """ - querystring = {"service": "WMS", - "version": "1.1.1", - "request": "GetCapabilities"} - r: ClientResponse = await self._api_wrapper(_forecast_wms_base_url, querystring) - - return self._parse_capabilities(await r.text()) - - @staticmethod - def _parse_capabilities(xml_string: str) -> Set[str]: - try: - root = ElementTree.fromstring(xml_string) - except ElementTree.ParseError: - return set() - - path = './/Capability/Layer/Layer/Name' - feature_type_names = {t.text for t in root.findall(path)} - return feature_type_names diff --git a/src/open_irceline/rio.py b/src/open_irceline/rio.py index 93a839a..3cea0b5 100644 --- a/src/open_irceline/rio.py +++ b/src/open_irceline/rio.py @@ -2,15 +2,19 @@ from typing import List, Tuple, Dict, Set from xml.etree import ElementTree -from aiohttp import ClientResponse +from aiohttp import ClientResponse, ClientResponseError -from .api import IrcelineBaseClient, _rio_wfs_base_url, IrcelineApiError -from .data import RioFeature, FeatureValue +from .api import IrcelineBaseClient, _rio_wfs_base_url, IrcelineApiError, _rio_ifdm_wms_base_url, IrcelineBaseWmsClient +from .data import RioFeature, FeatureValue, RioIfdmFeature from .utils import epsg_transform class IrcelineRioClient(IrcelineBaseClient): - """API client for RIO interpolated IRCEL - CELINE open data""" + """ + API client for RIO interpolated IRCEL - CELINE open data + RIO is more coarse grained for interpolation than RIO IFDM and allows to request multiple features in the same + request, which may be faster. + """ async def get_data(self, features: List[RioFeature], @@ -124,3 +128,41 @@ def _format_result(prefix: str, data: dict, features: List[RioFeature]) -> dict: result[name] = FeatureValue(timestamp=timestamp, value=value) return result + + +class IrcelineRioIfdmClient(IrcelineBaseWmsClient): + """ + API client for RIO IFDM interpolated IRCEL - CELINE open data + RIO IFDM is more fine-grained for interpolation than RIO but only allows one feature to be request at a time, which + may be slower + """ + _base_url = _rio_ifdm_wms_base_url + + async def get_data(self, + features: List[RioIfdmFeature], + position: Tuple[float, float] + ) -> Dict[RioIfdmFeature, FeatureValue]: + """ + Get interpolated concentrations for the given features at the given position. + :param features: pollutants to get the forecasts for + :param position: (lat, long) + :return: dict where key is RioIfdmFeature and value is a FeatureValue + """ + result = dict() + lat, lon = position + base_querystring = (self._default_querystring | + {"bbox": f"{lon},{lat},{lon + self._epsilon},{lat + self._epsilon}"}) + print({"bbox": f"{lon},{lat},{lon + self._epsilon},{lat + self._epsilon}"}) + + for feature in features: + querystring = base_querystring | {"layers": f"{feature}", "query_layers": f"{feature}"} + try: + r: ClientResponse = await self._api_wrapper(self._base_url, querystring) + r: dict = await r.json() + result[feature] = FeatureValue( + value=r.get('features', [{}])[0].get('properties', {}).get('GRAY_INDEX'), + timestamp=datetime.fromisoformat(r.get('timeStamp')) if 'timeStamp' in r else None) + except (IrcelineApiError, ClientResponseError, IndexError): + result[feature] = FeatureValue(value=None, timestamp=None) + + return result diff --git a/tests/fixtures/ifdm_interpolation_feature_info.json b/tests/fixtures/ifdm_interpolation_feature_info.json new file mode 100644 index 0000000..42b1fba --- /dev/null +++ b/tests/fixtures/ifdm_interpolation_feature_info.json @@ -0,0 +1,17 @@ +{ + "type": "FeatureCollection", + "features": [ + { + "type": "Feature", + "id": "", + "geometry": null, + "properties": { + "GRAY_INDEX": 84.33950805664062 + } + } + ], + "totalFeatures": "unknown", + "numberReturned": 1, + "timeStamp": "2024-06-30T15:43:07.222Z", + "crs": null +} \ No newline at end of file diff --git a/tests/fixtures/capabilities.xml b/tests/fixtures/rio_capabilities.xml similarity index 100% rename from tests/fixtures/capabilities.xml rename to tests/fixtures/rio_capabilities.xml diff --git a/tests/fixtures/rio_ifdm_capabilities.xml b/tests/fixtures/rio_ifdm_capabilities.xml new file mode 100644 index 0000000..4bffb10 --- /dev/null +++ b/tests/fixtures/rio_ifdm_capabilities.xml @@ -0,0 +1,694 @@ + + + + + OGC:WMS + IRCEL - CELINE - Web Map Service + A compliant implementation of WMS plus most of the SLD extension (dynamic styling). Can also generate + PDF, SVG, KML, GeoRSS + + + WFS + WMS + GEOSERVER + + + + + IRCEL - CELINE + Belgian Interregional Environment Agency + + + + +
Gaucheretstraat 92-94 Rue Gaucheret
+ Brussels + + 1030 + Belgium + + +(32)(0)2 227 57 01 + + info@irceline.be + + NONE + NONE + + + + + application/vnd.ogc.wms_xml + text/xml + + + + + + + + + + + + + image/png + application/atom xml + application/atom+xml + application/json;type=utfgrid + application/openlayers + application/openlayers2 + application/openlayers3 + application/pdf + application/rss xml + application/rss+xml + application/vnd.google-earth.kml + application/vnd.google-earth.kml xml + application/vnd.google-earth.kml+xml + application/vnd.google-earth.kml+xml;mode=networklink + application/vnd.google-earth.kmz + application/vnd.google-earth.kmz xml + application/vnd.google-earth.kmz+xml + application/vnd.google-earth.kmz;mode=networklink + atom + image/geotiff + image/geotiff8 + image/gif + image/gif;subtype=animated + image/jpeg + image/png8 + image/png; mode=8bit + image/svg + image/svg xml + image/svg+xml + image/tiff + image/tiff8 + image/vnd.jpeg-png + image/vnd.jpeg-png8 + kml + kmz + openlayers + rss + text/html; subtype=openlayers + text/html; subtype=openlayers2 + text/html; subtype=openlayers3 + utfgrid + + + + + + + + + + text/plain + application/vnd.ogc.gml + text/xml + application/vnd.ogc.gml/3.1.1 + text/xml; subtype=gml/3.1.1 + text/html + application/json + + + + + + + + + + + + + application/vnd.ogc.wms_xml + + + + + + + + + + image/png + image/jpeg + application/json + image/gif + + + + + + + + + + application/vnd.ogc.sld+xml + + + + + + + + + + + application/vnd.ogc.se_xml + application/vnd.ogc.se_inimage + application/vnd.ogc.se_blank + application/json + + + + IRCEL - CELINE - Web Map Service + A compliant implementation of WMS plus most of the SLD extension (dynamic styling). Can also + generate PDF, SVG, KML, GeoRSS + + + EPSG:3857 + EPSG:4258 + EPSG:4326 + EPSG:31370 + EPSG:900913 + + + + + http://geo.irceline.be + + belaqi + belaqi + + + belaqi + WCS + GeoTIFF + + EPSG:31370 + + + + + + belaqi_dm1 + belaqi_dm1 + + + belaqi_dmean_dm1 + WCS + GeoTIFF + + EPSG:31370 + + + + + + belaqi_dm2 + belaqi_dm2 + + + belaqi_dmean_dm2 + WCS + GeoTIFF + + EPSG:31370 + + + + + + belaqi_dm3 + belaqi_dm3 + + + belaqi_dmean_dm3 + WCS + GeoTIFF + + EPSG:31370 + + + + + + no2_dm1 + no2_dm1 + + + no2_dmean_dm1 + WCS + GeoTIFF + + EPSG:31370 + + + + + + no2_dm2 + no2_dm2 + + + no2_dmean_dm2 + WCS + GeoTIFF + + EPSG:31370 + + + + + + no2_dm3 + no2_dm3 + + + no2_dmean_dm3 + WCS + GeoTIFF + + EPSG:31370 + + + + + + no2_hmean + no2_hmean + + + no2_hmean + WCS + GeoTIFF + + EPSG:31370 + + + + + + o3_dm1 + o3_dm1 + + + o3_max8hmean_dm1 + WCS + GeoTIFF + + EPSG:31370 + + + + + + o3_dm2 + o3_dm2 + + + o3_max8hmean_dm2 + WCS + GeoTIFF + + EPSG:31370 + + + + + + o3_dm3 + o3_dm3 + + + o3_max8hmean_dm3 + WCS + GeoTIFF + + EPSG:31370 + + + + + + o3_hmean + o3_hmean + + + o3_hmean + WCS + GeoTIFF + + EPSG:31370 + + + + + + pm10_dm1 + pm10_dm1 + + + pm10_dmean_dm1 + WCS + GeoTIFF + + EPSG:31370 + + + + + + pm10_dm2 + pm10_dm2 + + + pm10_dmean_dm2 + WCS + GeoTIFF + + EPSG:31370 + + + + + + pm10_dm3 + pm10_dm3 + + + pm10_dmean_dm3 + WCS + GeoTIFF + + EPSG:31370 + + + + + + pm10_hmean + pm10_hmean + + + pm10_hmean + WCS + GeoTIFF + + EPSG:31370 + + + + + + pm25_dm1 + pm25_dm1 + + + pm25_dmean_dm1 + WCS + GeoTIFF + + EPSG:31370 + + + + + + pm25_dm2 + pm25_dm2 + + + pm25_dmean_dm2 + WCS + GeoTIFF + + EPSG:31370 + + + + + + pm25_dm3 + pm25_dm3 + + + pm25_dmean_dm3 + WCS + GeoTIFF + + EPSG:31370 + + + + + + pm25_hmean + pm25_hmean + + + pm25_hmean + WCS + GeoTIFF + + EPSG:31370 + + + + + + + diff --git a/tests/test_api_forecast.py b/tests/test_api_forecast.py index 628636a..4a5a703 100644 --- a/tests/test_api_forecast.py +++ b/tests/test_api_forecast.py @@ -95,7 +95,7 @@ async def test_api_forecast(): client = IrcelineForecastClient(session) features = [ForecastFeature.NO2_DMEAN, ForecastFeature.O3_MAXHMEAN] - _ = await client.get_data(features, pos) + result = await client.get_data(features, pos) base = {"service": "WMS", "version": "1.1.1", @@ -119,6 +119,9 @@ async def test_api_forecast(): session.request.assert_has_calls(calls, any_order=True) + for k, v in result.items(): + assert v['value'] == 10.853286743164062 + def test_parse_capabilities_with_error(): result = IrcelineForecastClient._parse_capabilities("wow there no valid XML") diff --git a/tests/test_api_rio.py b/tests/test_api_rio.py index ec41631..f7dc1f5 100644 --- a/tests/test_api_rio.py +++ b/tests/test_api_rio.py @@ -64,7 +64,7 @@ async def test_format_result_dmean(): def test_parse_capabilities(): - data = get_api_data('capabilities.xml', plain=True) + data = get_api_data('rio_capabilities.xml', plain=True) result = IrcelineRioClient._parse_capabilities(data) expected = {'rio:so2_anmean_be', 'rio:o3_hmean', 'rio:bc_anmean_vl', 'rio:o3_anmean_be', 'rio:pm10_hmean_vl', @@ -117,7 +117,7 @@ async def test_api_rio(): async def test_api_rio_get_capabilities(): - session = get_mock_session(text_file='capabilities.xml') + session = get_mock_session(text_file='rio_capabilities.xml') client = IrcelineRioClient(session) _ = await client.get_capabilities() diff --git a/tests/test_api_rio_ifdm.py b/tests/test_api_rio_ifdm.py new file mode 100644 index 0000000..7b92fdc --- /dev/null +++ b/tests/test_api_rio_ifdm.py @@ -0,0 +1,102 @@ +from datetime import datetime +from unittest.mock import call + +from freezegun import freeze_time + +from src.open_irceline.api import _rio_ifdm_wms_base_url, _user_agent +from src.open_irceline.data import RioIfdmFeature, FeatureValue +from src.open_irceline.rio import IrcelineRioIfdmClient +from tests.conftest import get_api_data, get_mock_session + + +def test_parse_capabilities(): + data = get_api_data('rio_ifdm_capabilities.xml', plain=True) + result = IrcelineRioIfdmClient._parse_capabilities(data) + + expected = {'no2_dm3', 'belaqi_dm2', 'pm10_hmean', 'belaqi_dm1', 'pm25_dm3', 'pm25_dm2', 'pm10_dm1', 'o3_dm3', + 'no2_dm1', 'pm10_dm3', 'pm25_dm1', 'belaqi', 'belaqi_dm3', 'pm10_dm2', 'o3_dm2', 'pm25_hmean', 'o3_dm1', + 'o3_hmean', 'no2_dm2', 'no2_hmean'} + + assert result == expected + + for f in RioIfdmFeature: + assert f"{f.split(':')[1]}" in result + + +async def test_aget_capabilities(): + session = get_mock_session(text_file='rio_ifdm_capabilities.xml') + + client = IrcelineRioIfdmClient(session) + _ = await client.get_capabilities() + + session.request.assert_called_once_with( + method='GET', + url=_rio_ifdm_wms_base_url, + params={"service": "WMS", + "version": "1.1.1", + "request": "GetCapabilities"}, + headers={'User-Agent': _user_agent} + ) + + +@freeze_time(datetime.fromisoformat("2024-06-30T13:00:21.520Z")) +async def test_api_forecast_error(): + pos = (50.4657, 4.8647) + session = get_mock_session('forecast_wms_feature_info_invalid.json') + + client = IrcelineRioIfdmClient(session) + + features = [RioIfdmFeature.NO2_HMEAN, RioIfdmFeature.O3_HMEAN] + result = await client.get_data(features, pos) + + for k, v in result.items(): + assert v == FeatureValue(timestamp=datetime.fromisoformat("2024-06-30T13:00:21.520Z"), value=None) + + +async def test_api_forecast(): + pos = (50.4657, 4.8647) + lat, lon = pos + session = get_mock_session('forecast_wms_feature_info.json') + + client = IrcelineRioIfdmClient(session) + + features = [RioIfdmFeature.NO2_HMEAN, RioIfdmFeature.O3_HMEAN] + result = await client.get_data(features, pos) + + base = {"service": "WMS", + "version": "1.1.1", + "request": "GetFeatureInfo", + "info_format": "application/json", + "width": "1", + "height": "1", + "srs": "EPSG:4326", + "bbox": f"{lon},{lat},{lon + 0.00001},{lat + 0.00001}", + "X": "1", + "Y": "1"} + + calls = [call( + method='GET', + url=_rio_ifdm_wms_base_url, + params=base | {"layers": f"{feature}", + "query_layers": f"{feature}"}, + headers={'User-Agent': _user_agent}, + ) + for feature in features] + + session.request.assert_has_calls(calls, any_order=True) + + for k, v in result.items(): + assert v['value'] == 10.853286743164062 + + +async def test_api_forecast_no_field(): + pos = (50.4657, 4.8647) + session = get_mock_session('forecast_wms_feature_info_no_field.json') + + client = IrcelineRioIfdmClient(session) + + features = [RioIfdmFeature.NO2_HMEAN, RioIfdmFeature.O3_HMEAN] + result = await client.get_data(features, pos) + + for k, v in result.items(): + assert v == FeatureValue(timestamp=None, value=None) From 5b899da28ef48002021245a41e6389b08e431e71 Mon Sep 17 00:00:00 2001 From: Jules Dejaeghere Date: Sun, 30 Jun 2024 18:16:23 +0200 Subject: [PATCH 2/2] Use concurrent requests for WMS clients --- src/open_irceline/__init__.py | 4 ++-- src/open_irceline/api.py | 4 +--- src/open_irceline/forecast.py | 34 +++++++++++++++++++--------- src/open_irceline/rio.py | 30 ++++++++++++++++--------- src/open_irceline/utils.py | 38 ------------------------------- tests/test_utils.py | 42 +---------------------------------- 6 files changed, 46 insertions(+), 106 deletions(-) diff --git a/src/open_irceline/__init__.py b/src/open_irceline/__init__.py index d830345..5188fe4 100644 --- a/src/open_irceline/__init__.py +++ b/src/open_irceline/__init__.py @@ -1,6 +1,6 @@ from .api import IrcelineApiError -from .rio import IrcelineRioClient +from .data import RioFeature, ForecastFeature, FeatureValue, RioIfdmFeature from .forecast import IrcelineForecastClient -from .data import RioFeature, ForecastFeature, FeatureValue +from .rio import IrcelineRioClient, IrcelineRioIfdmClient __version__ = '2.0.0' diff --git a/src/open_irceline/api.py b/src/open_irceline/api.py index a89c165..110ce46 100644 --- a/src/open_irceline/api.py +++ b/src/open_irceline/api.py @@ -9,7 +9,6 @@ from aiohttp import ClientResponse from .data import IrcelineFeature -from .utils import SizedDict _rio_wfs_base_url = 'https://geo.irceline.be/wfs' _forecast_wms_base_url = 'https://geo.irceline.be/forecast/wms' @@ -22,9 +21,8 @@ class IrcelineApiError(Exception): class IrcelineBaseClient(ABC): - def __init__(self, session: aiohttp.ClientSession, cache_size: int = 20) -> None: + def __init__(self, session: aiohttp.ClientSession) -> None: self._session = session - self._cache = SizedDict(cache_size) @abstractmethod async def get_data(self, diff --git a/src/open_irceline/forecast.py b/src/open_irceline/forecast.py index c2b5efd..c29168e 100644 --- a/src/open_irceline/forecast.py +++ b/src/open_irceline/forecast.py @@ -1,3 +1,4 @@ +import asyncio from datetime import date, timedelta, datetime from itertools import product from typing import List, Tuple, Dict @@ -28,16 +29,27 @@ async def get_data(self, base_querystring = (self._default_querystring | {"bbox": f"{lon},{lat},{lon + self._epsilon},{lat + self._epsilon}"}) - for feature, d in product(features, range(4)): - querystring = base_querystring | {"layers": f"{feature}_d{d}", - "query_layers": f"{feature}_d{d}"} - try: - r: ClientResponse = await self._api_wrapper(self._base_url, querystring) - r: dict = await r.json() - result[(feature, timestamp + timedelta(days=d))] = FeatureValue( - value=r.get('features', [{}])[0].get('properties', {}).get('GRAY_INDEX'), - timestamp=datetime.fromisoformat(r.get('timeStamp')) if 'timeStamp' in r else None) - except (IrcelineApiError, ClientResponseError, IndexError): - result[(feature, timestamp + timedelta(days=d))] = FeatureValue(value=None, timestamp=None) + tasks = [asyncio.create_task(self._get_single_feature(base_querystring, d, feature, timestamp)) + for feature, d in product(features, range(4))] + results = await asyncio.gather(*tasks) + for r in results: + result |= r + + return result + + async def _get_single_feature(self, base_querystring: dict, d: int, feature: ForecastFeature, + timestamp: date) -> dict: + result = dict() + + querystring = base_querystring | {"layers": f"{feature}_d{d}", + "query_layers": f"{feature}_d{d}"} + try: + r: ClientResponse = await self._api_wrapper(self._base_url, querystring) + r: dict = await r.json() + result[(feature, timestamp + timedelta(days=d))] = FeatureValue( + value=r.get('features', [{}])[0].get('properties', {}).get('GRAY_INDEX'), + timestamp=datetime.fromisoformat(r.get('timeStamp')) if 'timeStamp' in r else None) + except (IrcelineApiError, ClientResponseError, IndexError): + result[(feature, timestamp + timedelta(days=d))] = FeatureValue(value=None, timestamp=None) return result diff --git a/src/open_irceline/rio.py b/src/open_irceline/rio.py index 3cea0b5..42dfd7e 100644 --- a/src/open_irceline/rio.py +++ b/src/open_irceline/rio.py @@ -1,3 +1,4 @@ +import asyncio from datetime import datetime, date, UTC, timedelta from typing import List, Tuple, Dict, Set from xml.etree import ElementTree @@ -152,17 +153,24 @@ async def get_data(self, lat, lon = position base_querystring = (self._default_querystring | {"bbox": f"{lon},{lat},{lon + self._epsilon},{lat + self._epsilon}"}) - print({"bbox": f"{lon},{lat},{lon + self._epsilon},{lat + self._epsilon}"}) - for feature in features: - querystring = base_querystring | {"layers": f"{feature}", "query_layers": f"{feature}"} - try: - r: ClientResponse = await self._api_wrapper(self._base_url, querystring) - r: dict = await r.json() - result[feature] = FeatureValue( - value=r.get('features', [{}])[0].get('properties', {}).get('GRAY_INDEX'), - timestamp=datetime.fromisoformat(r.get('timeStamp')) if 'timeStamp' in r else None) - except (IrcelineApiError, ClientResponseError, IndexError): - result[feature] = FeatureValue(value=None, timestamp=None) + tasks = [asyncio.create_task(self._get_single_feature(base_querystring, feature)) for feature in features] + results = await asyncio.gather(*tasks) + + for r in results: + result |= r + + return result + async def _get_single_feature(self, base_querystring: dict, feature: RioIfdmFeature) -> dict: + result = dict() + querystring = base_querystring | {"layers": f"{feature}", "query_layers": f"{feature}"} + try: + r: ClientResponse = await self._api_wrapper(self._base_url, querystring) + r: dict = await r.json() + result[feature] = FeatureValue( + value=r.get('features', [{}])[0].get('properties', {}).get('GRAY_INDEX'), + timestamp=datetime.fromisoformat(r.get('timeStamp')) if 'timeStamp' in r else None) + except (IrcelineApiError, ClientResponseError, IndexError): + result[feature] = FeatureValue(value=None, timestamp=None) return result diff --git a/src/open_irceline/utils.py b/src/open_irceline/utils.py index 65b573a..952bbf9 100644 --- a/src/open_irceline/utils.py +++ b/src/open_irceline/utils.py @@ -1,4 +1,3 @@ -from collections import OrderedDict from typing import Tuple from pyproj import Transformer @@ -6,31 +5,6 @@ _project_transform = Transformer.from_crs('EPSG:4326', 'EPSG:31370', always_xy=False) -class SizedDict(OrderedDict): - """Dictionary with a maximum size. When more items are added, the least recently accessed element is evicted""" - - def __init__(self, size: int): - super().__init__() - self._size = size - - def __setitem__(self, key, value): - super().__setitem__(key, value) - self.move_to_end(key) - if len(self) > self._size: - self.popitem(False) - - def __getitem__(self, key): - self.move_to_end(key) - return super().__getitem__(key) - - def get(self, __key, __default=None): - self.move_to_end(__key) - return super().get(__key, __default) - - def update(self, __m, **kwargs): - raise NotImplementedError() - - def epsg_transform(position: Tuple[float, float]) -> Tuple[int, int]: """ Convert 'EPSG:4326' coordinates to 'EPSG:31370' coordinates @@ -39,15 +13,3 @@ def epsg_transform(position: Tuple[float, float]) -> Tuple[int, int]: """ result = _project_transform.transform(position[0], position[1]) return round(result[0]), round(result[1]) - - -def round_coordinates(x: float, y: float, step=.05) -> Tuple[float, float]: - """ - Round the coordinate to the precision given by step - :param x: latitude - :param y: longitude - :param step: precision of the rounding - :return: x and y round to the closest step increment - """ - n = 1 / step - return round(x * n) / n, round(y * n) / n diff --git a/tests/test_utils.py b/tests/test_utils.py index 1423bc2..49dff05 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -1,44 +1,4 @@ -import pytest - -from src.open_irceline.utils import SizedDict, round_coordinates, epsg_transform - - -def test_sized_dict(): - s_dict = SizedDict(5) - assert len(s_dict) == 0 - - s_dict['a'] = 1 - s_dict['b'] = 2 - s_dict['c'] = 3 - s_dict['d'] = 4 - s_dict['e'] = 5 - assert len(s_dict) == 5 - - s_dict['f'] = 6 - assert 'a' not in s_dict - assert s_dict['f'] == 6 - assert len(s_dict) == 5 - - s_dict['b'] = 42 - s_dict['g'] = 7 - assert s_dict.get('f') == 6 - assert s_dict['g'] == 7 - assert s_dict['b'] == 42 - assert 'c' not in s_dict - assert len(s_dict) == 5 - - del s_dict['b'] - assert len(s_dict) == 4 - assert 'b' not in s_dict - - with pytest.raises(NotImplementedError): - s_dict.update({'a': 1}) - - -def test_round_coord(): - x, y = round_coordinates(50.4657, 4.8647) - assert x == 50.45 - assert y == 4.85 +from src.open_irceline.utils import epsg_transform def test_epsg_transform():