From d085fa4a8f2fc5680498ad0cdfdd9305616b497c Mon Sep 17 00:00:00 2001 From: OnnoEbbens Date: Mon, 16 Sep 2024 12:06:27 +0200 Subject: [PATCH 01/16] improve error handling, fix for #232 --- hydropandas/io/bro.py | 41 ++++++++++++++++++++++++++++++++++++----- 1 file changed, 36 insertions(+), 5 deletions(-) diff --git a/hydropandas/io/bro.py b/hydropandas/io/bro.py index 4f6499dd..d73b1f40 100644 --- a/hydropandas/io/bro.py +++ b/hydropandas/io/bro.py @@ -4,6 +4,7 @@ import json import logging +import time import xml.etree.ElementTree from functools import lru_cache @@ -56,7 +57,12 @@ class of the observations, so far only GroundwaterObs is supported req = requests.get(url) if req.status_code > 200: - print(req.json()["errors"][0]["message"]) + logger.error( + "could not get monitoring wells, this groundwater monitoring net is" + "probably too big. Try a smaller extent" + ) + req.raise_for_status() + ns = {"xmlns": "http://www.broservices.nl/xsd/dsgmn/1.0"} @@ -191,7 +197,11 @@ def get_gld_ids_from_gmw(bro_id, tube_nr): req = requests.get(url) if req.status_code > 200: - print(req.json()["errors"][0]["message"]) + logger.error( + f"could not get gld ids {req.text}" + ) + req.raise_for_status() + d = json.loads(req.text) @@ -214,7 +224,8 @@ def get_gld_ids_from_gmw(bro_id, tube_nr): def measurements_from_gld( - bro_id, tmin=None, tmax=None, to_wintertime=True, drop_duplicate_times=True + bro_id, tmin=None, tmax=None, to_wintertime=True, drop_duplicate_times=True, + attempt=0 ): """get measurements and metadata from a grondwaterstandonderzoek (gld) bro_id @@ -237,6 +248,9 @@ def measurements_from_gld( add_registration_history : bool, optional if True the registration history is added to the metadata. The defualt is True. + attempt: int, optional + counter for the number of attempts to call the bro api. Used to deal with + too many requests errors. Raises ------ @@ -265,7 +279,25 @@ def measurements_from_gld( req = requests.get(url.format(bro_id), params=params) if req.status_code > 200: - print(req.json()["errors"][0]["message"]) + if req.status_code == 429: + logger.warning( + "too many requests" + ) + + # wait before trying again + attempt += 1 + if attempt <= 3: + wait_time = 1*attempt + time.sleep(wait_time) + logger.info(f"trying again after {wait_time} seconds {attempt}/3") + return measurements_from_gld(bro_id, tmin=tmin, tmax=tmax, + to_wintertime=to_wintertime, drop_duplicate_times=drop_duplicate_times, + attempt=attempt) + else: + logger.error( + "could not get monitoring well measurements" + ) + req.raise_for_status() ns = { "ns11": "http://www.broservices.nl/xsd/dsgld/1.0", @@ -685,7 +717,6 @@ class of the observations, e.g. GroundwaterObs or WaterlvlObs "Try a smaller extent" ) req.raise_for_status() - # print(req.json()["errors"][0]["message"]) # read results tree = xml.etree.ElementTree.fromstring(req.text) From d1ce7e63563cf8256f086ad52e3a88a99139dee4 Mon Sep 17 00:00:00 2001 From: OnnoEbbens Date: Mon, 16 Sep 2024 13:45:42 +0200 Subject: [PATCH 02/16] ruff --- examples/01_groundwater_observations.ipynb | 5 +-- examples/02_knmi_observations.ipynb | 4 +-- examples/03_hydropandas_and_pastas.ipynb | 2 -- examples/04_merging_observations.ipynb | 1 - examples/05_bronhouderportaal_bro.ipynb | 2 -- examples/06_lizard.ipynb | 3 -- hydropandas/io/bro.py | 37 +++++++++++----------- hydropandas/observation.py | 2 +- 8 files changed, 23 insertions(+), 33 deletions(-) diff --git a/examples/01_groundwater_observations.ipynb b/examples/01_groundwater_observations.ipynb index 2f5cd76e..bb474dbd 100644 --- a/examples/01_groundwater_observations.ipynb +++ b/examples/01_groundwater_observations.ipynb @@ -31,10 +31,7 @@ "source": [ "import hydropandas as hpd\n", "\n", - "import logging\n", - "from IPython.display import HTML\n", - "\n", - "import logging" + "from IPython.display import HTML\n" ] }, { diff --git a/examples/02_knmi_observations.ipynb b/examples/02_knmi_observations.ipynb index df86ff6e..6ffd1870 100644 --- a/examples/02_knmi_observations.ipynb +++ b/examples/02_knmi_observations.ipynb @@ -114,7 +114,7 @@ "print(f\"name: {o.name}\")\n", "print(f\"x,y: {(o.x, o.y)}\")\n", "print(f\"station: {o.station}\")\n", - "print(f\"unit\", o.unit)\n", + "print(\"unit\", o.unit)\n", "print(\"metadata:\")\n", "for key, item in o.meta.items():\n", " print(f\" {key}: {item}\")" @@ -650,7 +650,7 @@ "\n", " ax[\"A\"].set_title(f\"Interpolation: {kernel}\")\n", " ax[\"A\"].pcolormesh(grid[0], grid[1], val_rbf, vmin=vmin, vmax=vmax)\n", - " ax[\"B\"].set_title(f\"Interpolation: Nearest\")\n", + " ax[\"B\"].set_title(\"Interpolation: Nearest\")\n", " ax[\"B\"].pcolormesh(grid[0], grid[1], val_nea, vmin=vmin, vmax=vmax)\n", " ax[\"A\"].scatter(*coor.T, c=val, s=100, ec=\"k\", vmin=vmin, vmax=vmax)\n", " p = ax[\"B\"].scatter(*coor.T, c=val, s=100, ec=\"k\", vmin=vmin, vmax=vmax)\n", diff --git a/examples/03_hydropandas_and_pastas.ipynb b/examples/03_hydropandas_and_pastas.ipynb index c4810717..5647a7f7 100644 --- a/examples/03_hydropandas_and_pastas.ipynb +++ b/examples/03_hydropandas_and_pastas.ipynb @@ -34,9 +34,7 @@ "source": [ "import hydropandas as hpd\n", "import pastas as ps\n", - "import pastastore as pst\n", "\n", - "import logging\n", "\n", "ps.set_log_level(\"ERROR\")\n", "hpd.util.get_color_logger(\"INFO\")\n", diff --git a/examples/04_merging_observations.ipynb b/examples/04_merging_observations.ipynb index b04ed3b1..eb7406f3 100644 --- a/examples/04_merging_observations.ipynb +++ b/examples/04_merging_observations.ipynb @@ -37,7 +37,6 @@ "import hydropandas as hpd\n", "from IPython.display import display\n", "\n", - "import logging\n", "\n", "hpd.util.get_color_logger(\"INFO\");" ] diff --git a/examples/05_bronhouderportaal_bro.ipynb b/examples/05_bronhouderportaal_bro.ipynb index 9ae3d018..188286b6 100644 --- a/examples/05_bronhouderportaal_bro.ipynb +++ b/examples/05_bronhouderportaal_bro.ipynb @@ -30,8 +30,6 @@ "source": [ "import hydropandas as hpd\n", "\n", - "import logging\n", - "from IPython.display import HTML\n", "\n", "import pandas as pd" ] diff --git a/examples/06_lizard.ipynb b/examples/06_lizard.ipynb index 478e0982..9cb52554 100644 --- a/examples/06_lizard.ipynb +++ b/examples/06_lizard.ipynb @@ -32,11 +32,8 @@ "metadata": {}, "outputs": [], "source": [ - "import logging\n", - "from IPython.display import HTML\n", "\n", "import pastas as ps\n", - "import pandas as pd\n", "import hydropandas as hpd" ] }, diff --git a/hydropandas/io/bro.py b/hydropandas/io/bro.py index d73b1f40..d32cd141 100644 --- a/hydropandas/io/bro.py +++ b/hydropandas/io/bro.py @@ -63,7 +63,6 @@ class of the observations, so far only GroundwaterObs is supported ) req.raise_for_status() - ns = {"xmlns": "http://www.broservices.nl/xsd/dsgmn/1.0"} tree = xml.etree.ElementTree.fromstring(req.text) @@ -197,12 +196,9 @@ def get_gld_ids_from_gmw(bro_id, tube_nr): req = requests.get(url) if req.status_code > 200: - logger.error( - f"could not get gld ids {req.text}" - ) + logger.error(f"could not get gld ids {req.text}") req.raise_for_status() - d = json.loads(req.text) if len(d["monitoringTubeReferences"]) == 0: @@ -224,8 +220,12 @@ def get_gld_ids_from_gmw(bro_id, tube_nr): def measurements_from_gld( - bro_id, tmin=None, tmax=None, to_wintertime=True, drop_duplicate_times=True, - attempt=0 + bro_id, + tmin=None, + tmax=None, + to_wintertime=True, + drop_duplicate_times=True, + attempt=0, ): """get measurements and metadata from a grondwaterstandonderzoek (gld) bro_id @@ -280,23 +280,24 @@ def measurements_from_gld( if req.status_code > 200: if req.status_code == 429: - logger.warning( - "too many requests" - ) - + logger.warning("too many requests") + # wait before trying again attempt += 1 if attempt <= 3: - wait_time = 1*attempt + wait_time = 1 * attempt time.sleep(wait_time) logger.info(f"trying again after {wait_time} seconds {attempt}/3") - return measurements_from_gld(bro_id, tmin=tmin, tmax=tmax, - to_wintertime=to_wintertime, drop_duplicate_times=drop_duplicate_times, - attempt=attempt) + return measurements_from_gld( + bro_id, + tmin=tmin, + tmax=tmax, + to_wintertime=to_wintertime, + drop_duplicate_times=drop_duplicate_times, + attempt=attempt, + ) else: - logger.error( - "could not get monitoring well measurements" - ) + logger.error("could not get monitoring well measurements") req.raise_for_status() ns = { diff --git a/hydropandas/observation.py b/hydropandas/observation.py index 71fcb0be..9731dc38 100644 --- a/hydropandas/observation.py +++ b/hydropandas/observation.py @@ -17,11 +17,11 @@ import logging import os import warnings -from _io import StringIO from typing import List, Optional import numpy as np import pandas as pd +from _io import StringIO from pandas._config import get_option from pandas.api.types import is_numeric_dtype from pandas.io.formats import console From 658176870c42dab362f1f67da85351c1d03bf78e Mon Sep 17 00:00:00 2001 From: OnnoEbbens Date: Mon, 16 Sep 2024 13:47:55 +0200 Subject: [PATCH 03/16] ruff extend-select --- examples/01_groundwater_observations.ipynb | 4 ++-- examples/02_knmi_observations.ipynb | 14 +++++++------- examples/03_hydropandas_and_pastas.ipynb | 2 +- examples/04_merging_observations.ipynb | 2 +- examples/05_bronhouderportaal_bro.ipynb | 5 ++--- examples/06_lizard.ipynb | 1 + hydropandas/observation.py | 2 +- 7 files changed, 15 insertions(+), 15 deletions(-) diff --git a/examples/01_groundwater_observations.ipynb b/examples/01_groundwater_observations.ipynb index bb474dbd..cc89eb0d 100644 --- a/examples/01_groundwater_observations.ipynb +++ b/examples/01_groundwater_observations.ipynb @@ -29,9 +29,9 @@ "metadata": {}, "outputs": [], "source": [ - "import hydropandas as hpd\n", + "from IPython.display import HTML\n", "\n", - "from IPython.display import HTML\n" + "import hydropandas as hpd\n" ] }, { diff --git a/examples/02_knmi_observations.ipynb b/examples/02_knmi_observations.ipynb index 6ffd1870..29b3ca11 100644 --- a/examples/02_knmi_observations.ipynb +++ b/examples/02_knmi_observations.ipynb @@ -31,17 +31,17 @@ "metadata": {}, "outputs": [], "source": [ - "import hydropandas as hpd\n", - "from hydropandas.io import knmi\n", - "from IPython.display import display\n", + "import logging\n", "\n", - "import pandas as pd\n", - "import numpy as np\n", "import matplotlib.pyplot as plt\n", + "import numpy as np\n", + "import pandas as pd\n", + "from IPython.display import display\n", + "from scipy.interpolate import NearestNDInterpolator, RBFInterpolator\n", "from tqdm.auto import tqdm\n", - "from scipy.interpolate import RBFInterpolator, NearestNDInterpolator\n", "\n", - "import logging" + "import hydropandas as hpd\n", + "from hydropandas.io import knmi" ] }, { diff --git a/examples/03_hydropandas_and_pastas.ipynb b/examples/03_hydropandas_and_pastas.ipynb index 5647a7f7..84a49765 100644 --- a/examples/03_hydropandas_and_pastas.ipynb +++ b/examples/03_hydropandas_and_pastas.ipynb @@ -32,9 +32,9 @@ } ], "source": [ - "import hydropandas as hpd\n", "import pastas as ps\n", "\n", + "import hydropandas as hpd\n", "\n", "ps.set_log_level(\"ERROR\")\n", "hpd.util.get_color_logger(\"INFO\")\n", diff --git a/examples/04_merging_observations.ipynb b/examples/04_merging_observations.ipynb index eb7406f3..8ab9bc43 100644 --- a/examples/04_merging_observations.ipynb +++ b/examples/04_merging_observations.ipynb @@ -34,9 +34,9 @@ "source": [ "import numpy as np\n", "import pandas as pd\n", - "import hydropandas as hpd\n", "from IPython.display import display\n", "\n", + "import hydropandas as hpd\n", "\n", "hpd.util.get_color_logger(\"INFO\");" ] diff --git a/examples/05_bronhouderportaal_bro.ipynb b/examples/05_bronhouderportaal_bro.ipynb index 188286b6..6c81b9ee 100644 --- a/examples/05_bronhouderportaal_bro.ipynb +++ b/examples/05_bronhouderportaal_bro.ipynb @@ -28,10 +28,9 @@ "metadata": {}, "outputs": [], "source": [ - "import hydropandas as hpd\n", + "import pandas as pd\n", "\n", - "\n", - "import pandas as pd" + "import hydropandas as hpd" ] }, { diff --git a/examples/06_lizard.ipynb b/examples/06_lizard.ipynb index 9cb52554..d60a7d3a 100644 --- a/examples/06_lizard.ipynb +++ b/examples/06_lizard.ipynb @@ -34,6 +34,7 @@ "source": [ "\n", "import pastas as ps\n", + "\n", "import hydropandas as hpd" ] }, diff --git a/hydropandas/observation.py b/hydropandas/observation.py index 9731dc38..71fcb0be 100644 --- a/hydropandas/observation.py +++ b/hydropandas/observation.py @@ -17,11 +17,11 @@ import logging import os import warnings +from _io import StringIO from typing import List, Optional import numpy as np import pandas as pd -from _io import StringIO from pandas._config import get_option from pandas.api.types import is_numeric_dtype from pandas.io.formats import console From 00f28c6e94fd23ef06cd048d42d3c5bdc779c249 Mon Sep 17 00:00:00 2001 From: OnnoEbbens Date: Mon, 16 Sep 2024 14:15:16 +0200 Subject: [PATCH 04/16] format the formatted --- examples/01_groundwater_observations.ipynb | 2 +- examples/06_lizard.ipynb | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/examples/01_groundwater_observations.ipynb b/examples/01_groundwater_observations.ipynb index cc89eb0d..c9b120ca 100644 --- a/examples/01_groundwater_observations.ipynb +++ b/examples/01_groundwater_observations.ipynb @@ -31,7 +31,7 @@ "source": [ "from IPython.display import HTML\n", "\n", - "import hydropandas as hpd\n" + "import hydropandas as hpd" ] }, { diff --git a/examples/06_lizard.ipynb b/examples/06_lizard.ipynb index d60a7d3a..429f8c2e 100644 --- a/examples/06_lizard.ipynb +++ b/examples/06_lizard.ipynb @@ -32,7 +32,6 @@ "metadata": {}, "outputs": [], "source": [ - "\n", "import pastas as ps\n", "\n", "import hydropandas as hpd" From 944a8aff6d4d5624b43f547f7c8e5896cbc81b60 Mon Sep 17 00:00:00 2001 From: OnnoEbbens Date: Mon, 16 Sep 2024 14:26:19 +0200 Subject: [PATCH 05/16] fix np.nan error, disable other test --- hydropandas/io/waterinfo.py | 2 +- tests/test_012_wow.py | 16 ++++++++-------- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/hydropandas/io/waterinfo.py b/hydropandas/io/waterinfo.py index 7410d584..0caceeac 100644 --- a/hydropandas/io/waterinfo.py +++ b/hydropandas/io/waterinfo.py @@ -83,7 +83,7 @@ def read_waterinfo_file( df.drop(columns=index_cols, inplace=True) # do some conversions - df.loc[df[value_col] == 999999999, value_col] = np.NaN + df.loc[df[value_col] == 999999999, value_col] = np.nan df[value_col] = df[value_col] / 100.0 # parse metadata into dict diff --git a/tests/test_012_wow.py b/tests/test_012_wow.py index 5285ebd5..19bc3839 100644 --- a/tests/test_012_wow.py +++ b/tests/test_012_wow.py @@ -18,11 +18,11 @@ def test_get_wow_stn() -> None: assert obs.name == "Macquarie Island" -def test_get_wow_nearest() -> None: - lat = -35.4184 - lon = 149.0937 - xy = [lon, lat] - start = pd.Timestamp(year=2023, month=6, day=4, hour=1, minute=34, second=0) - end = pd.Timestamp(year=2023, month=6, day=5, hour=23, minute=54, second=0) - obs = hpd.PrecipitationObs.from_wow(xy=xy, start=start, end=end) - assert obs.name == "Hillanhome" +# def test_get_wow_nearest() -> None: +# lat = -35.4184 +# lon = 149.0937 +# xy = [lon, lat] +# start = pd.Timestamp(year=2023, month=6, day=4, hour=1, minute=34, second=0) +# end = pd.Timestamp(year=2023, month=6, day=5, hour=23, minute=54, second=0) +# obs = hpd.PrecipitationObs.from_wow(xy=xy, start=start, end=end) +# assert obs.name == "Hillanhome" From 72ffae1b5ef58aa835f87a3ce927505c97511785 Mon Sep 17 00:00:00 2001 From: OnnoEbbens Date: Mon, 16 Sep 2024 14:36:16 +0200 Subject: [PATCH 06/16] ignore wow tests --- tests/test_012_wow.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/tests/test_012_wow.py b/tests/test_012_wow.py index 19bc3839..0dee35ca 100644 --- a/tests/test_012_wow.py +++ b/tests/test_012_wow.py @@ -10,12 +10,12 @@ def test_wow_strftime() -> None: assert timestampstr == "2010-10-02T10%3A30%3A00" -def test_get_wow_stn() -> None: - stn = "423216079" # Macquarie Island - start = pd.Timestamp(year=2023, month=6, day=4, hour=1, minute=34, second=0) - end = pd.Timestamp(year=2023, month=6, day=5, hour=23, minute=54, second=0) - obs = hpd.MeteoObs.from_wow(stn=stn, meteo_var="temperature", start=start, end=end) - assert obs.name == "Macquarie Island" +# def test_get_wow_stn() -> None: +# stn = "423216079" # Macquarie Island +# start = pd.Timestamp(year=2023, month=6, day=4, hour=1, minute=34, second=0) +# end = pd.Timestamp(year=2023, month=6, day=5, hour=23, minute=54, second=0) +# obs = hpd.MeteoObs.from_wow(stn=stn, meteo_var="temperature", start=start, end=end) +# assert obs.name == "Macquarie Island" # def test_get_wow_nearest() -> None: From f195b1d6eed8641ca4aa363685a0ba4b429d7938 Mon Sep 17 00:00:00 2001 From: OnnoEbbens Date: Mon, 16 Sep 2024 14:39:02 +0200 Subject: [PATCH 07/16] ruff --- tests/test_012_wow.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_012_wow.py b/tests/test_012_wow.py index 0dee35ca..c74886cb 100644 --- a/tests/test_012_wow.py +++ b/tests/test_012_wow.py @@ -1,6 +1,6 @@ import pandas as pd -import hydropandas as hpd +# import hydropandas as hpd from hydropandas.io import wow From bc135df28cf50a907b417b407d79e83f6e79a785 Mon Sep 17 00:00:00 2001 From: OnnoEbbens Date: Mon, 16 Sep 2024 14:54:21 +0200 Subject: [PATCH 08/16] fix for #235 --- hydropandas/io/knmi.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hydropandas/io/knmi.py b/hydropandas/io/knmi.py index e78cf995..4e50bbcb 100644 --- a/hydropandas/io/knmi.py +++ b/hydropandas/io/knmi.py @@ -483,7 +483,7 @@ def fill_missing_measurements( logger.info("trying to get measurements from nearest station") stn_lst = get_nearest_station_df( - stations.loc[[stn]], meteo_var=meteo_var, ignore=ignore + stations.loc[[ignore[0]]], meteo_var=meteo_var, ignore=ignore ) if stn_lst is None: logger.warning( From 1dd3188b88d064dfb2e916aab5a6927fa32820ab Mon Sep 17 00:00:00 2001 From: Martin Vonk Date: Wed, 18 Sep 2024 09:46:23 +0200 Subject: [PATCH 09/16] add tested ruff settings in toml --- pyproject.toml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/pyproject.toml b/pyproject.toml index 7892ed99..e0eade77 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -89,6 +89,10 @@ profile = "black" [tool.ruff] line-length = 88 target-version = "py39" +extend-include = ["*.ipynb"] +lint.extend-select = ["I"] +show-fixes = true +fix = true [tool.pytest.ini_options] addopts = "--durations=0" From 2f5be41dc5277912c1cf5176a4e376b40b0c9979 Mon Sep 17 00:00:00 2001 From: Martin Vonk Date: Wed, 18 Sep 2024 09:50:36 +0200 Subject: [PATCH 10/16] remove black and isort --- pyproject.toml | 6 ------ readme.md | 7 +------ 2 files changed, 1 insertion(+), 12 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index e0eade77..59975910 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -80,12 +80,6 @@ version = { attr = "hydropandas.version.__version__" } "hydropandas.data" = ["*.json"] "hydropandas.static" = ["*.html", "*.css"] -[tool.black] -line-length = 88 - -[tool.isort] -profile = "black" - [tool.ruff] line-length = 88 target-version = "py39" diff --git a/readme.md b/readme.md index 41363b6e..ca4b44dc 100644 --- a/readme.md +++ b/readme.md @@ -3,18 +3,13 @@ [![PyPi](https://img.shields.io/pypi/v/hydropandas.svg)](https://pypi.python.org/pypi/hydropandas) [![PyPi Supported Python Versions](https://img.shields.io/pypi/pyversions/hydropandas)](https://pypi.python.org/pypi/hydropandas) [](https://codespaces.new/ArtesiaWater/hydropandas?quickstart=1) - +[![Ruff](https://img.shields.io/endpoint?url=https://raw.githubusercontent.com/astral-sh/ruff/main/assets/badge/v2.json)](https://github.com/astral-sh/ruff) [![hydropandas](https://github.com/ArtesiaWater/hydropandas/workflows/hydropandas/badge.svg)](https://github.com/ArtesiaWater/hydropandas/actions?query=workflow%3Ahydropandas) [![Codacy Badge](https://app.codacy.com/project/badge/Grade/c1b99f474bdc49b0a47e00e4e9f66c2f)](https://www.codacy.com/gh/ArtesiaWater/hydropandas/dashboard?utm_source=github.com&utm_medium=referral&utm_content=ArtesiaWater/hydropandas&utm_campaign=Badge_Grade) [![Codacy Badge](https://app.codacy.com/project/badge/Coverage/c1b99f474bdc49b0a47e00e4e9f66c2f)](https://www.codacy.com/gh/ArtesiaWater/hydropandas/dashboard?utm_source=github.com&utm_medium=referral&utm_content=ArtesiaWater/hydropandas&utm_campaign=Badge_Coverage) [![Documentation Status](https://readthedocs.org/projects/hydropandas/badge/?version=latest)](https://hydropandas.readthedocs.io/en/latest/?badge=latest) -[![Format: isort](https://img.shields.io/badge/imports-isort-ef8336)](https://pycqa.github.io/isort/index.html) -[![Format: Black](https://img.shields.io/badge/code_style-black-black)](https://github.com/psf/black) -[![Linter: flake8](https://img.shields.io/badge/linter-flake8-yellowgreen)](https://flake8.pycqa.org/) -[![Linter: ruff](https://img.shields.io/badge/linter-ruff-red)](https://github.com/charliermarsh/ruff) - # HydroPandas Hydropandas is a Python package for reading, analyzing and writing From e05943afe5bb691dbac62b99e039309e0b9216a4 Mon Sep 17 00:00:00 2001 From: Martin Vonk Date: Wed, 18 Sep 2024 09:50:46 +0200 Subject: [PATCH 11/16] skip test with pytest --- tests/test_012_wow.py | 33 ++++++++++++++++++--------------- 1 file changed, 18 insertions(+), 15 deletions(-) diff --git a/tests/test_012_wow.py b/tests/test_012_wow.py index c74886cb..539737fe 100644 --- a/tests/test_012_wow.py +++ b/tests/test_012_wow.py @@ -1,6 +1,7 @@ import pandas as pd -# import hydropandas as hpd +import pytest +import hydropandas as hpd from hydropandas.io import wow @@ -10,19 +11,21 @@ def test_wow_strftime() -> None: assert timestampstr == "2010-10-02T10%3A30%3A00" -# def test_get_wow_stn() -> None: -# stn = "423216079" # Macquarie Island -# start = pd.Timestamp(year=2023, month=6, day=4, hour=1, minute=34, second=0) -# end = pd.Timestamp(year=2023, month=6, day=5, hour=23, minute=54, second=0) -# obs = hpd.MeteoObs.from_wow(stn=stn, meteo_var="temperature", start=start, end=end) -# assert obs.name == "Macquarie Island" +@pytest.mark.skip +def test_get_wow_stn() -> None: + stn = "423216079" # Macquarie Island + start = pd.Timestamp(year=2023, month=6, day=4, hour=1, minute=34, second=0) + end = pd.Timestamp(year=2023, month=6, day=5, hour=23, minute=54, second=0) + obs = hpd.MeteoObs.from_wow(stn=stn, meteo_var="temperature", start=start, end=end) + assert obs.name == "Macquarie Island" -# def test_get_wow_nearest() -> None: -# lat = -35.4184 -# lon = 149.0937 -# xy = [lon, lat] -# start = pd.Timestamp(year=2023, month=6, day=4, hour=1, minute=34, second=0) -# end = pd.Timestamp(year=2023, month=6, day=5, hour=23, minute=54, second=0) -# obs = hpd.PrecipitationObs.from_wow(xy=xy, start=start, end=end) -# assert obs.name == "Hillanhome" +@pytest.mark.skip +def test_get_wow_nearest() -> None: + lat = -35.4184 + lon = 149.0937 + xy = [lon, lat] + start = pd.Timestamp(year=2023, month=6, day=4, hour=1, minute=34, second=0) + end = pd.Timestamp(year=2023, month=6, day=5, hour=23, minute=54, second=0) + obs = hpd.PrecipitationObs.from_wow(xy=xy, start=start, end=end) + assert obs.name == "Hillanhome" From 8c463063700711aae53139cf39ce1d06eadf663b Mon Sep 17 00:00:00 2001 From: Hendrik Meuwese <117160147+HMEUW@users.noreply.github.com> Date: Wed, 18 Sep 2024 10:08:24 +0200 Subject: [PATCH 12/16] Update plots.py Added `to_numpy()` to `self._obj[section_colname_x]` --- hydropandas/extensions/plots.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hydropandas/extensions/plots.py b/hydropandas/extensions/plots.py index b33cb830..e3585a05 100644 --- a/hydropandas/extensions/plots.py +++ b/hydropandas/extensions/plots.py @@ -411,7 +411,7 @@ def section_plot( # use order in ObsCollection plot_x = np.arange(len(self._obj)) else: - plot_x = self._obj[section_colname_x] + plot_x = self._obj[section_colname_x].to_numpy() # create figure fig = plt.figure(figsize=(15, 5)) From 33f21a7b1019402a5b3aa78ff76e1548a8ac4fa5 Mon Sep 17 00:00:00 2001 From: OnnoEbbens Date: Fri, 20 Sep 2024 11:11:52 +0200 Subject: [PATCH 13/16] use Retry logic from requests package --- hydropandas/io/bro.py | 36 ++++++++++-------------------------- 1 file changed, 10 insertions(+), 26 deletions(-) diff --git a/hydropandas/io/bro.py b/hydropandas/io/bro.py index d32cd141..22512ff4 100644 --- a/hydropandas/io/bro.py +++ b/hydropandas/io/bro.py @@ -4,7 +4,6 @@ import json import logging -import time import xml.etree.ElementTree from functools import lru_cache @@ -12,6 +11,7 @@ import pandas as pd import requests from pyproj import Proj, Transformer +from requests.adapters import Retry, HTTPAdapter from tqdm import tqdm from ..rcparams import rcParams @@ -224,8 +224,7 @@ def measurements_from_gld( tmin=None, tmax=None, to_wintertime=True, - drop_duplicate_times=True, - attempt=0, + drop_duplicate_times=True ): """get measurements and metadata from a grondwaterstandonderzoek (gld) bro_id @@ -248,9 +247,6 @@ def measurements_from_gld( add_registration_history : bool, optional if True the registration history is added to the metadata. The defualt is True. - attempt: int, optional - counter for the number of attempts to call the bro api. Used to deal with - too many requests errors. Raises ------ @@ -276,28 +272,16 @@ def measurements_from_gld( if tmax is not None: tmax = pd.to_datetime(tmax) params["observationPeriodEndDate"] = tmax.strftime("%Y-%m-%d") - req = requests.get(url.format(bro_id), params=params) + + # add some logic to retry in case of a 429 response + s = requests.Session() + retries = Retry(total=5, + backoff_factor=0.5, + status_forcelist=[429]) + s.mount('https://', HTTPAdapter(max_retries=retries)) + req = s.get(url.format(bro_id), params=params) if req.status_code > 200: - if req.status_code == 429: - logger.warning("too many requests") - - # wait before trying again - attempt += 1 - if attempt <= 3: - wait_time = 1 * attempt - time.sleep(wait_time) - logger.info(f"trying again after {wait_time} seconds {attempt}/3") - return measurements_from_gld( - bro_id, - tmin=tmin, - tmax=tmax, - to_wintertime=to_wintertime, - drop_duplicate_times=drop_duplicate_times, - attempt=attempt, - ) - else: - logger.error("could not get monitoring well measurements") req.raise_for_status() ns = { From a95d58a948341658e42e8b0079acd26def38d081 Mon Sep 17 00:00:00 2001 From: OnnoEbbens Date: Fri, 20 Sep 2024 11:14:43 +0200 Subject: [PATCH 14/16] ruff --- hydropandas/io/bro.py | 14 ++++---------- tests/test_012_wow.py | 2 +- 2 files changed, 5 insertions(+), 11 deletions(-) diff --git a/hydropandas/io/bro.py b/hydropandas/io/bro.py index 22512ff4..c2aab72b 100644 --- a/hydropandas/io/bro.py +++ b/hydropandas/io/bro.py @@ -220,11 +220,7 @@ def get_gld_ids_from_gmw(bro_id, tube_nr): def measurements_from_gld( - bro_id, - tmin=None, - tmax=None, - to_wintertime=True, - drop_duplicate_times=True + bro_id, tmin=None, tmax=None, to_wintertime=True, drop_duplicate_times=True ): """get measurements and metadata from a grondwaterstandonderzoek (gld) bro_id @@ -272,13 +268,11 @@ def measurements_from_gld( if tmax is not None: tmax = pd.to_datetime(tmax) params["observationPeriodEndDate"] = tmax.strftime("%Y-%m-%d") - + # add some logic to retry in case of a 429 response s = requests.Session() - retries = Retry(total=5, - backoff_factor=0.5, - status_forcelist=[429]) - s.mount('https://', HTTPAdapter(max_retries=retries)) + retries = Retry(total=5, backoff_factor=0.5, status_forcelist=[429]) + s.mount("https://", HTTPAdapter(max_retries=retries)) req = s.get(url.format(bro_id), params=params) if req.status_code > 200: diff --git a/tests/test_012_wow.py b/tests/test_012_wow.py index 539737fe..1cd6aba8 100644 --- a/tests/test_012_wow.py +++ b/tests/test_012_wow.py @@ -1,6 +1,6 @@ import pandas as pd - import pytest + import hydropandas as hpd from hydropandas.io import wow From 8975977c4bb768014e43ca054496b9cbbe323c00 Mon Sep 17 00:00:00 2001 From: Martin Vonk <66305055+martinvonk@users.noreply.github.com> Date: Mon, 23 Sep 2024 15:02:32 +0200 Subject: [PATCH 15/16] Add tmin and tmax for knmi stations (#239) --- hydropandas/data/knmi_meteostation.json | 198 ++- hydropandas/data/knmi_neerslagstation.json | 1304 ++++++++++++++++- hydropandas/data/update_knmi_meteostations.py | 33 - hydropandas/data/update_knmi_stations.py | 100 ++ hydropandas/io/bro.py | 2 +- hydropandas/io/knmi.py | 147 +- pyproject.toml | 3 + 7 files changed, 1715 insertions(+), 72 deletions(-) delete mode 100644 hydropandas/data/update_knmi_meteostations.py create mode 100644 hydropandas/data/update_knmi_stations.py diff --git a/hydropandas/data/knmi_meteostation.json b/hydropandas/data/knmi_meteostation.json index 20966a2d..d3eae393 100644 --- a/hydropandas/data/knmi_meteostation.json +++ b/hydropandas/data/knmi_meteostation.json @@ -42,6 +42,7 @@ "290": 6.890994, "308": 3.37907, "310": 3.595904, + "311": 3.67203713, "312": 3.622, "313": 3.242, "315": 3.997481, @@ -108,6 +109,7 @@ "290": 52.273079, "308": 51.379905, "310": 51.441276, + "311": 51.37688661, "312": 51.768, "313": 51.505, "315": 51.445723, @@ -174,6 +176,7 @@ "290": "Twenthe", "308": "Cadzand", "310": "Vlissingen", + "311": "Hoofdplaat", "312": "Oosterschelde", "313": "Vlakte van de Raan", "315": "Hansweert", @@ -240,6 +243,7 @@ "290": 257633.4949178807, "308": 15206.4314155364, "310": 30466.550547773, + "311": 35590.0, "312": 33161.5063876291, "313": 6075.7755614168, "315": 58391.5023862258, @@ -306,6 +310,7 @@ "290": 477183.2207696091, "308": 378672.1456373842, "310": 385105.1479057451, + "311": 377816.0, "312": 421402.3911734026, "313": 392856.6342154247, "315": 384990.7859751831, @@ -344,7 +349,7 @@ "214": 42.5, "215": -1.1, "225": 4.4, - "229": true, + "229": 1.0, "235": 1.2, "239": 50.6, "240": -3.3, @@ -372,6 +377,7 @@ "290": 34.8, "308": 0.0, "310": 8.0, + "311": 0.0, "312": 0.0, "313": 0.0, "315": 0.0, @@ -395,6 +401,140 @@ "380": 114.3, "391": 19.5 }, + "tmin": { + "201": false, + "203": false, + "204": false, + "205": false, + "206": false, + "207": false, + "208": false, + "209": "2001-01-30", + "210": "1951-01-01", + "211": false, + "212": false, + "214": false, + "215": "2014-07-15", + "225": "1971-01-01", + "229": "2017-12-13", + "235": "1906-01-01", + "239": false, + "240": "1951-01-01", + "242": "1996-01-01", + "248": "1994-07-15", + "249": "1999-03-12", + "251": "1994-05-26", + "252": false, + "257": "2001-04-30", + "258": "2006-02-01", + "260": "1901-01-01", + "265": "1951-09-01", + "267": "1990-06-18", + "269": "1990-01-17", + "270": "1951-01-01", + "273": "1989-01-01", + "275": "1951-01-01", + "277": "1991-03-18", + "278": "1991-01-01", + "279": "1989-09-26", + "280": "1906-01-01", + "283": "1989-10-16", + "285": "1981-01-01", + "286": "1990-01-17", + "290": "1951-01-01", + "308": "1972-01-01", + "310": "1906-01-01", + "311": "1997-01-31", + "312": "1982-01-01", + "313": "1997-01-31", + "315": "1997-01-31", + "316": "1983-01-01", + "319": "1991-06-25", + "320": false, + "321": false, + "323": "2017-12-15", + "324": "1997-09-30", + "330": "1971-01-01", + "331": "1981-01-01", + "340": "1993-04-01", + "343": "1991-01-01", + "344": "1956-10-01", + "348": "1986-03-01", + "350": "1951-01-01", + "356": "1989-09-26", + "370": "1951-01-01", + "375": "1951-02-01", + "377": "1999-05-01", + "380": "1906-01-01", + "391": "1990-06-18" + }, + "tmax": { + "201": false, + "203": false, + "204": false, + "205": false, + "206": false, + "207": false, + "208": false, + "209": null, + "210": "2016-05-04", + "211": false, + "212": false, + "214": false, + "215": null, + "225": null, + "229": null, + "235": null, + "239": false, + "240": null, + "242": null, + "248": null, + "249": null, + "251": null, + "252": false, + "257": null, + "258": null, + "260": null, + "265": "2008-11-16", + "267": null, + "269": null, + "270": null, + "273": null, + "275": null, + "277": null, + "278": null, + "279": null, + "280": null, + "283": null, + "285": null, + "286": null, + "290": null, + "308": null, + "310": null, + "311": "2016-02-01", + "312": null, + "313": null, + "315": null, + "316": null, + "319": null, + "320": false, + "321": false, + "323": null, + "324": null, + "330": null, + "331": null, + "340": null, + "343": null, + "344": null, + "348": null, + "350": null, + "356": null, + "370": null, + "375": null, + "377": null, + "380": null, + "391": null + }, "DDVEC": { "201": true, "203": true, @@ -438,6 +578,7 @@ "290": true, "308": true, "310": true, + "311": true, "312": true, "313": true, "315": true, @@ -504,6 +645,7 @@ "290": true, "308": true, "310": true, + "311": true, "312": true, "313": true, "315": true, @@ -570,6 +712,7 @@ "290": true, "308": true, "310": true, + "311": true, "312": true, "313": true, "315": true, @@ -636,6 +779,7 @@ "290": true, "308": true, "310": true, + "311": true, "312": true, "313": true, "315": true, @@ -702,6 +846,7 @@ "290": true, "308": true, "310": true, + "311": true, "312": true, "313": true, "315": true, @@ -768,6 +913,7 @@ "290": true, "308": true, "310": true, + "311": true, "312": true, "313": true, "315": true, @@ -834,6 +980,7 @@ "290": true, "308": true, "310": true, + "311": true, "312": true, "313": true, "315": true, @@ -900,6 +1047,7 @@ "290": true, "308": true, "310": true, + "311": true, "312": true, "313": true, "315": true, @@ -966,6 +1114,7 @@ "290": true, "308": true, "310": true, + "311": true, "312": true, "313": true, "315": true, @@ -1032,6 +1181,7 @@ "290": true, "308": false, "310": true, + "311": false, "312": false, "313": false, "315": false, @@ -1098,6 +1248,7 @@ "290": true, "308": false, "310": true, + "311": false, "312": false, "313": false, "315": false, @@ -1164,6 +1315,7 @@ "290": true, "308": false, "310": true, + "311": false, "312": false, "313": false, "315": false, @@ -1230,6 +1382,7 @@ "290": true, "308": false, "310": true, + "311": false, "312": false, "313": false, "315": false, @@ -1296,6 +1449,7 @@ "290": true, "308": false, "310": true, + "311": false, "312": false, "313": false, "315": false, @@ -1362,6 +1516,7 @@ "290": true, "308": false, "310": true, + "311": false, "312": false, "313": false, "315": false, @@ -1428,6 +1583,7 @@ "290": true, "308": false, "310": true, + "311": false, "312": false, "313": false, "315": false, @@ -1494,6 +1650,7 @@ "290": true, "308": false, "310": true, + "311": false, "312": false, "313": false, "315": false, @@ -1560,6 +1717,7 @@ "290": true, "308": false, "310": true, + "311": false, "312": false, "313": false, "315": false, @@ -1626,6 +1784,7 @@ "290": true, "308": false, "310": true, + "311": false, "312": false, "313": false, "315": false, @@ -1692,6 +1851,7 @@ "290": true, "308": false, "310": true, + "311": false, "312": false, "313": false, "315": false, @@ -1758,6 +1918,7 @@ "290": true, "308": false, "310": true, + "311": false, "312": false, "313": false, "315": false, @@ -1824,6 +1985,7 @@ "290": true, "308": false, "310": true, + "311": false, "312": false, "313": false, "315": false, @@ -1890,6 +2052,7 @@ "290": true, "308": false, "310": true, + "311": false, "312": false, "313": false, "315": false, @@ -1956,6 +2119,7 @@ "290": true, "308": false, "310": true, + "311": false, "312": false, "313": false, "315": false, @@ -2022,6 +2186,7 @@ "290": true, "308": false, "310": true, + "311": false, "312": false, "313": false, "315": false, @@ -2088,6 +2253,7 @@ "290": true, "308": false, "310": true, + "311": false, "312": false, "313": false, "315": false, @@ -2154,6 +2320,7 @@ "290": true, "308": false, "310": true, + "311": false, "312": false, "313": false, "315": false, @@ -2220,6 +2387,7 @@ "290": true, "308": false, "310": true, + "311": false, "312": false, "313": false, "315": false, @@ -2286,6 +2454,7 @@ "290": true, "308": false, "310": true, + "311": false, "312": false, "313": false, "315": false, @@ -2352,6 +2521,7 @@ "290": true, "308": false, "310": true, + "311": false, "312": false, "313": false, "315": false, @@ -2418,6 +2588,7 @@ "290": true, "308": false, "310": true, + "311": false, "312": false, "313": false, "315": false, @@ -2484,6 +2655,7 @@ "290": true, "308": false, "310": true, + "311": false, "312": false, "313": false, "315": false, @@ -2550,6 +2722,7 @@ "290": true, "308": false, "310": true, + "311": false, "312": false, "313": false, "315": false, @@ -2616,6 +2789,7 @@ "290": true, "308": false, "310": true, + "311": false, "312": false, "313": false, "315": false, @@ -2682,6 +2856,7 @@ "290": true, "308": false, "310": true, + "311": false, "312": false, "313": false, "315": false, @@ -2748,6 +2923,7 @@ "290": true, "308": false, "310": true, + "311": false, "312": false, "313": false, "315": false, @@ -2814,6 +2990,7 @@ "290": true, "308": false, "310": true, + "311": false, "312": false, "313": false, "315": false, @@ -2880,6 +3057,7 @@ "290": true, "308": false, "310": true, + "311": false, "312": false, "313": false, "315": false, @@ -2946,6 +3124,7 @@ "290": true, "308": false, "310": true, + "311": false, "312": false, "313": false, "315": false, @@ -3012,6 +3191,7 @@ "290": true, "308": true, "310": true, + "311": true, "312": true, "313": true, "315": true, @@ -3078,6 +3258,7 @@ "290": true, "308": true, "310": true, + "311": true, "312": true, "313": true, "315": true, @@ -3144,6 +3325,7 @@ "290": true, "308": true, "310": true, + "311": true, "312": true, "313": true, "315": true, @@ -3210,6 +3392,7 @@ "290": true, "308": true, "310": true, + "311": true, "312": true, "313": true, "315": true, @@ -3276,6 +3459,7 @@ "290": true, "308": true, "310": true, + "311": true, "312": true, "313": true, "315": true, @@ -3342,6 +3526,7 @@ "290": true, "308": false, "310": true, + "311": false, "312": false, "313": false, "315": false, @@ -3408,6 +3593,7 @@ "290": true, "308": false, "310": true, + "311": false, "312": false, "313": false, "315": false, @@ -3474,6 +3660,7 @@ "290": true, "308": false, "310": true, + "311": false, "312": false, "313": false, "315": false, @@ -3540,6 +3727,7 @@ "290": true, "308": false, "310": true, + "311": false, "312": false, "313": false, "315": false, @@ -3606,6 +3794,7 @@ "290": true, "308": false, "310": true, + "311": false, "312": false, "313": false, "315": false, @@ -3672,6 +3861,7 @@ "290": true, "308": false, "310": true, + "311": false, "312": false, "313": false, "315": false, @@ -3738,6 +3928,7 @@ "290": true, "308": false, "310": true, + "311": false, "312": false, "313": false, "315": false, @@ -3804,6 +3995,7 @@ "290": true, "308": true, "310": true, + "311": false, "312": false, "313": false, "315": false, @@ -3870,6 +4062,7 @@ "290": true, "308": true, "310": true, + "311": false, "312": false, "313": false, "315": false, @@ -3936,6 +4129,7 @@ "290": true, "308": true, "310": true, + "311": false, "312": false, "313": false, "315": false, @@ -4002,6 +4196,7 @@ "290": true, "308": true, "310": true, + "311": false, "312": false, "313": false, "315": false, @@ -4068,6 +4263,7 @@ "290": true, "308": true, "310": true, + "311": false, "312": false, "313": false, "315": false, diff --git a/hydropandas/data/knmi_neerslagstation.json b/hydropandas/data/knmi_neerslagstation.json index 2d9d04b1..e8b93d04 100644 --- a/hydropandas/data/knmi_neerslagstation.json +++ b/hydropandas/data/knmi_neerslagstation.json @@ -1953,7 +1953,7 @@ "983": "Kessel-Eik" }, "x": { - "1": 206500.0, + "1": 206500.0001, "3": 108100.0, "4": 126400.0, "6": 195000.0, @@ -3905,6 +3905,1308 @@ "982": 54.47, "983": 26.4 }, + "tmin": { + "1": "1878-01-01", + "3": "1902-08-27", + "4": "1904-02-06", + "6": "1905-04-01", + "7": "1906-05-01", + "8": "1937-05-01", + "9": "1851-01-01", + "10": "1872-06-01", + "11": "1876-01-02", + "12": "1879-10-01", + "13": "1885-06-01", + "14": "1885-06-01", + "15": "1885-05-16", + "16": "1891-04-01", + "17": "1894-11-21", + "18": "1916-08-01", + "19": "1929-07-01", + "20": "1932-06-01", + "21": "1940-10-01", + "22": "1950-07-15", + "23": "1876-01-01", + "24": "1956-01-01", + "25": "1958-03-31", + "26": "1961-01-01", + "40": "1892-12-29", + "41": "1875-10-02", + "42": "1867-01-31", + "43": "1878-01-31", + "44": "1879-03-02", + "45": "1882-10-01", + "46": "1885-04-02", + "47": "1895-01-02", + "48": "1895-01-01", + "49": "1903-08-01", + "50": "1905-01-01", + "51": "1906-11-14", + "53": "1915-07-25", + "54": "1918-01-01", + "55": "1930-09-02", + "56": "1931-05-01", + "57": "1934-06-21", + "58": "1936-02-10", + "59": "1937-04-02", + "60": "1942-09-05", + "61": "1988-01-01", + "64": "1866-09-13", + "65": "1869-07-02", + "66": "1872-02-01", + "67": "1888-02-01", + "68": "1911-11-05", + "69": "1911-09-02", + "70": "1956-01-01", + "71": "1917-06-02", + "72": "1920-07-05", + "73": "1926-06-01", + "74": "1926-02-02", + "75": "1928-06-02", + "76": "1929-03-14", + "77": "1937-05-02", + "78": "1961-01-01", + "79": "1942-05-02", + "80": "1943-04-02", + "81": "1948-06-02", + "82": "1949-05-02", + "84": "1950-11-30", + "85": "1876-01-02", + "86": "1952-06-01", + "87": "1952-06-01", + "89": "1995-10-01", + "90": "1950-05-02", + "91": "1960-01-01", + "126": "1871-10-01", + "127": "1880-03-12", + "128": "1884-08-01", + "129": "1923-12-15", + "130": "1904-12-01", + "131": "1905-01-01", + "132": "1910-01-01", + "133": "1917-05-01", + "134": "1991-01-01", + "135": "1921-01-17", + "136": "1992-01-01", + "137": "1924-01-01", + "139": "1847-01-01", + "140": "1852-03-31", + "141": "1872-04-01", + "142": "1875-01-01", + "143": "1891-09-01", + "144": "1891-09-02", + "145": "1899-12-01", + "146": "1905-06-10", + "147": "1906-05-12", + "148": "1909-09-01", + "149": "1913-09-08", + "150": "1889-06-01", + "151": "1921-11-01", + "152": "1923-12-14", + "153": "1923-03-01", + "154": "1926-03-14", + "155": "1929-05-01", + "156": "1931-02-01", + "157": "1920-01-01", + "158": "1935-03-01", + "159": "1941-04-01", + "160": "1942-10-18", + "161": "1949-01-01", + "162": "1947-11-01", + "163": "1949-07-09", + "164": "1951-01-01", + "165": "1953-05-01", + "166": "1916-04-01", + "168": "1956-07-01", + "170": "1961-01-01", + "171": "1995-05-01", + "172": "1992-05-01", + "173": "2018-02-01", + "194": "1918-11-03", + "195": "1974-01-01", + "197": "1866-08-01", + "198": "1866-11-09", + "199": "1873-12-01", + "200": "1880-03-02", + "201": "1880-07-01", + "203": "1899-11-01", + "204": "1899-06-01", + "205": "1904-01-01", + "206": "1905-06-01", + "207": "1910-09-01", + "208": "1917-01-01", + "209": "1930-11-01", + "210": "1966-01-01", + "211": "1932-09-01", + "213": "1934-05-01", + "214": "1936-04-15", + "215": "1939-06-28", + "216": "1939-10-01", + "217": "1943-10-01", + "219": "1946-01-01", + "221": "1880-03-02", + "222": "1883-04-01", + "223": "1883-04-01", + "224": "1885-10-01", + "225": "1898-11-01", + "226": "1910-01-01", + "227": "1911-05-12", + "228": "1911-05-15", + "229": "1912-03-27", + "230": "1913-04-01", + "231": "1917-12-19", + "232": "1918-08-01", + "233": "1949-10-13", + "234": "1927-06-18", + "235": "1927-07-01", + "236": "1930-02-01", + "237": "1931-08-01", + "238": "1933-09-01", + "239": "1933-06-19", + "240": "1933-11-19", + "241": "1934-06-01", + "242": "1934-01-11", + "243": "1937-06-01", + "244": "1938-09-01", + "245": "1939-03-11", + "246": "1943-07-01", + "247": "1948-07-01", + "249": "1952-05-01", + "250": "1954-03-01", + "251": "1951-01-01", + "252": "1934-06-01", + "254": "1961-01-01", + "255": "1951-01-01", + "256": "1951-01-02", + "257": "1951-01-02", + "258": "1950-05-23", + "259": "1950-05-22", + "260": "1950-05-23", + "261": "1950-05-24", + "263": "2006-03-08", + "264": "2020-02-01", + "285": "1912-03-25", + "286": "1866-09-01", + "287": "1880-03-02", + "288": "1882-09-14", + "289": "1888-01-01", + "290": "1889-07-01", + "291": "1891-08-01", + "292": "1891-09-01", + "293": "1913-07-08", + "294": "1891-10-01", + "295": "1898-11-01", + "296": "1904-01-31", + "297": "1910-05-21", + "298": "1951-01-02", + "299": "1888-04-01", + "300": "1922-05-10", + "301": "1904-09-04", + "302": "1924-11-01", + "303": "1924-10-01", + "304": "1927-02-01", + "305": "1927-10-16", + "306": "1932-04-24", + "307": "1932-10-02", + "308": "1933-03-07", + "309": "1934-04-01", + "310": "1940-06-01", + "311": "1941-04-01", + "312": "1942-11-01", + "313": "1942-09-01", + "314": "1943-05-01", + "315": "1943-01-01", + "316": "1942-11-01", + "317": "1943-01-07", + "318": "1946-11-06", + "319": "1943-05-16", + "321": "1948-11-01", + "323": "1951-01-02", + "324": "1949-04-02", + "325": "1949-09-01", + "326": "1866-08-01", + "327": "1891-10-01", + "328": "1892-08-31", + "329": "1895-03-01", + "330": "1949-10-29", + "331": "1904-09-01", + "332": "1913-06-10", + "333": "1918-03-07", + "334": "1919-07-27", + "335": "1924-04-01", + "336": "1927-07-01", + "337": "1927-05-01", + "338": "1930-06-04", + "339": "1934-06-01", + "340": "1935-05-01", + "341": "1936-04-01", + "342": "1938-02-02", + "343": "1940-11-13", + "344": "1976-01-01", + "345": "1940-09-11", + "346": "1941-05-01", + "347": "1943-06-01", + "348": "1943-01-09", + "349": "1946-11-12", + "350": "1948-11-02", + "352": "1948-11-18", + "353": "1951-01-01", + "354": "1949-03-01", + "356": "1949-12-01", + "357": "1949-07-01", + "358": "1951-01-01", + "359": "1941-04-01", + "360": "1951-01-01", + "361": "1961-01-01", + "362": "1994-01-01", + "363": "1916-11-01", + "364": "1959-01-01", + "365": "1963-01-01", + "366": "1961-01-01", + "367": "1969-01-01", + "368": "1969-01-01", + "369": "1982-01-01", + "370": "1940-11-01", + "371": "1990-01-01", + "372": "1990-10-01", + "374": "1949-12-01", + "411": "1866-08-01", + "412": "1880-01-01", + "413": "1883-09-10", + "414": "1890-09-05", + "415": "1895-01-01", + "416": "1902-09-18", + "417": "1908-01-01", + "418": "1908-02-01", + "419": "1911-12-01", + "420": "1911-12-01", + "421": "1914-10-01", + "422": "1915-09-17", + "423": "1917-01-01", + "424": "1882-01-01", + "425": "1927-09-01", + "426": "1994-02-01", + "427": "1930-01-25", + "428": "1935-01-17", + "429": "1935-06-15", + "430": "1908-03-01", + "431": "1949-11-25", + "432": "1950-06-13", + "433": "1880-01-01", + "434": "1866-09-01", + "435": "1866-12-01", + "436": "1866-12-01", + "437": "1866-12-01", + "438": "1866-12-01", + "439": "1876-01-01", + "440": "1877-01-02", + "441": "1916-01-01", + "442": "1881-10-02", + "443": "1888-04-01", + "444": "1889-12-02", + "445": "1951-01-01", + "446": "1891-04-02", + "447": "1892-05-29", + "448": "1902-05-24", + "449": "1904-11-02", + "450": "1904-02-12", + "451": "1877-06-02", + "452": "1906-09-19", + "453": "1908-12-02", + "454": "1915-10-02", + "455": "1923-09-10", + "456": "1925-01-20", + "457": "1926-01-02", + "458": "1927-01-02", + "459": "1926-12-17", + "460": "1935-04-02", + "461": "1873-12-02", + "462": "1937-08-02", + "463": "1941-07-02", + "464": "1943-10-02", + "465": "1946-06-02", + "466": "1948-01-02", + "467": "1950-04-02", + "468": "1952-04-01", + "469": "1924-11-02", + "470": "1928-09-02", + "471": "1935-08-02", + "472": "1962-01-01", + "473": "1969-01-01", + "474": "1971-01-01", + "475": "1971-01-01", + "476": "1971-01-01", + "477": "1996-03-01", + "478": "1992-12-31", + "479": "1996-03-01", + "480": "2000-05-01", + "481": "2004-04-01", + "482": "2004-04-01", + "483": "2009-05-01", + "484": "2018-01-01", + "509": "1989-01-01", + "510": "1966-01-01", + "511": "1866-08-02", + "512": "1866-09-01", + "513": "1868-07-02", + "514": "1877-02-01", + "515": "1880-05-01", + "516": "1883-04-02", + "517": "1883-01-31", + "518": "1885-01-31", + "519": "1885-10-20", + "520": "1888-01-01", + "521": "1888-04-02", + "522": "1893-12-01", + "523": "1961-01-01", + "524": "1904-10-23", + "525": "1911-03-01", + "526": "1914-04-18", + "527": "1924-01-02", + "528": "1927-06-24", + "529": "1928-01-01", + "530": "1929-02-06", + "531": "1932-04-02", + "533": "1933-11-06", + "534": "1934-11-01", + "535": "1935-08-29", + "536": "1936-06-05", + "538": "1948-08-14", + "539": "1883-02-02", + "540": "1849-01-01", + "541": "1867-03-03", + "542": "1867-03-03", + "543": "1878-01-02", + "544": "1867-03-02", + "546": "1882-08-12", + "547": "1888-08-03", + "548": "1892-02-01", + "549": "1909-10-07", + "550": "1897-01-02", + "551": "1897-07-02", + "552": "1904-01-02", + "553": "1903-03-02", + "555": "1904-06-02", + "556": "1905-11-02", + "557": "1907-05-02", + "558": "1908-09-02", + "559": "1910-10-02", + "560": "1910-04-09", + "561": "1910-12-02", + "562": "1911-06-02", + "563": "1918-10-25", + "564": "1919-08-02", + "565": "1919-06-19", + "566": "1920-04-02", + "567": "1923-01-04", + "568": "1925-10-17", + "569": "1969-01-01", + "570": "1927-06-16", + "571": "1927-06-18", + "572": "1930-09-02", + "573": "1873-12-02", + "574": "1943-02-23", + "575": "1963-01-01", + "576": "1946-09-02", + "577": "1879-10-02", + "578": "1947-01-02", + "579": "1948-12-02", + "580": "1950-01-02", + "581": "1949-01-22", + "582": "1949-10-19", + "583": "1949-04-02", + "584": "1950-05-02", + "586": "1898-10-02", + "588": "1953-02-01", + "589": "1953-11-01", + "590": "1866-12-02", + "591": "1971-01-01", + "592": "1978-01-01", + "593": "1951-01-01", + "594": "1995-01-01", + "595": "1889-06-02", + "596": "1996-11-01", + "597": "1879-07-01", + "600": "1927-08-01", + "645": "1993-09-01", + "646": "1866-08-01", + "647": "1866-10-02", + "648": "1867-02-01", + "649": "1869-01-01", + "650": "1883-01-31", + "651": "1886-08-01", + "652": "1929-03-27", + "653": "1889-07-15", + "654": "1888-02-01", + "655": "1896-09-18", + "656": "1899-03-23", + "657": "1915-08-02", + "658": "1918-07-27", + "659": "1919-08-01", + "660": "1932-01-02", + "661": "1940-10-05", + "662": "1943-05-01", + "663": "1866-08-02", + "664": "1880-08-02", + "665": "1880-05-02", + "666": "1880-07-02", + "667": "1883-03-03", + "668": "1887-01-02", + "669": "1890-08-02", + "670": "1937-11-02", + "671": "1940-09-02", + "672": "1940-03-02", + "673": "1941-12-02", + "674": "1942-06-02", + "675": "1943-01-15", + "676": "1945-11-02", + "677": "1949-06-05", + "678": "1950-03-02", + "679": "1951-01-01", + "680": "1904-08-02", + "681": "1945-09-24", + "682": "1956-01-01", + "683": "1957-01-01", + "684": "1980-01-01", + "685": "1980-01-01", + "686": "1980-01-01", + "687": "1995-05-01", + "688": "1995-06-01", + "689": "2006-02-01", + "690": "1888-05-01", + "691": "1932-01-08", + "705": "1961-01-01", + "706": "1866-08-01", + "707": "1866-10-31", + "708": "1868-07-01", + "709": "1879-12-02", + "710": "1879-01-02", + "711": "1873-12-01", + "712": "1879-09-02", + "713": "1879-09-01", + "714": "1879-09-19", + "715": "1879-09-01", + "716": "1881-03-02", + "718": "1881-08-02", + "719": "1891-11-02", + "721": "1889-07-25", + "722": "1900-11-01", + "723": "1902-11-07", + "724": "1904-10-13", + "725": "1905-07-22", + "726": "1912-09-01", + "727": "1915-05-01", + "728": "1925-01-01", + "729": "1927-10-01", + "730": "1931-04-25", + "731": "1936-11-14", + "733": "1855-01-01", + "734": "1866-09-12", + "735": "1868-11-02", + "736": "1873-12-02", + "737": "1878-04-02", + "738": "1879-09-02", + "739": "1879-09-16", + "740": "1879-09-20", + "741": "1879-09-02", + "742": "1879-09-02", + "743": "1888-05-02", + "744": "1904-04-02", + "745": "1904-10-16", + "746": "1904-10-15", + "747": "1905-05-07", + "748": "1911-07-02", + "749": "1922-05-02", + "750": "1925-06-02", + "751": "1927-01-02", + "752": "1935-12-02", + "754": "1935-08-02", + "755": "1937-05-21", + "756": "1940-10-25", + "757": "1942-07-25", + "758": "1946-05-14", + "759": "1949-10-02", + "760": "1950-09-02", + "761": "1951-01-01", + "762": "1953-01-01", + "763": "1951-01-01", + "764": "1953-03-01", + "767": "1958-01-01", + "768": "1962-01-01", + "769": "1962-01-01", + "770": "1995-06-01", + "772": "1874-01-01", + "773": "1878-04-01", + "810": "1866-08-01", + "811": "1904-06-01", + "812": "1867-02-01", + "813": "1867-01-01", + "814": "1871-05-02", + "815": "1873-12-01", + "816": "1880-06-02", + "817": "1883-02-06", + "818": "1883-02-12", + "819": "1889-07-02", + "820": "1903-04-25", + "821": "1905-09-08", + "822": "1910-12-01", + "823": "1910-11-17", + "824": "1913-09-19", + "825": "1942-10-05", + "826": "1926-08-02", + "827": "1877-02-02", + "828": "1888-02-02", + "829": "1888-04-02", + "830": "1890-10-02", + "831": "1903-03-03", + "832": "1904-01-02", + "833": "1905-01-02", + "834": "1911-09-02", + "835": "1912-02-24", + "836": "1912-11-09", + "837": "1920-03-02", + "838": "1924-07-13", + "839": "1933-07-14", + "840": "1935-04-02", + "841": "1941-08-25", + "843": "1949-01-02", + "844": "1951-01-01", + "845": "1873-12-01", + "871": "1869-12-01", + "872": "1879-11-01", + "873": "1880-08-01", + "875": "1876-01-01", + "876": "1885-09-01", + "877": "1886-05-01", + "878": "1885-12-01", + "879": "1887-10-01", + "880": "1889-06-01", + "881": "1909-08-21", + "882": "1909-04-17", + "883": "1914-08-16", + "884": "1920-04-19", + "885": "1928-07-16", + "886": "1929-08-08", + "887": "1941-09-20", + "888": "1941-06-20", + "889": "1942-10-01", + "890": "1942-10-01", + "891": "1944-01-01", + "892": "1993-11-01", + "893": "1944-02-01", + "894": "1947-11-24", + "896": "1868-11-02", + "897": "1882-10-15", + "898": "1886-05-02", + "899": "1904-11-21", + "900": "1900-03-25", + "901": "1904-04-25", + "902": "1912-09-23", + "903": "1912-04-11", + "904": "1914-05-02", + "905": "1919-07-11", + "906": "1919-07-21", + "907": "1920-06-02", + "908": "1929-05-13", + "909": "1929-08-18", + "910": "1932-10-14", + "911": "1934-06-22", + "912": "1934-09-16", + "913": "1947-03-22", + "914": "1948-12-05", + "915": "1948-06-02", + "917": "1950-05-28", + "918": "1951-01-01", + "919": "1971-01-01", + "920": "1971-01-01", + "921": "1972-01-01", + "922": "1995-06-01", + "923": "1995-11-01", + "926": "2011-01-01", + "950": "1992-01-01", + "951": "1852-05-02", + "952": "1886-03-21", + "953": "1886-04-07", + "954": "1886-03-26", + "955": "1897-10-02", + "956": "1907-10-08", + "957": "1917-07-01", + "959": "1934-08-02", + "960": "1941-08-10", + "961": "1868-08-02", + "962": "1886-04-04", + "963": "1904-03-02", + "964": "1906-10-07", + "965": "1908-11-02", + "966": "1912-10-02", + "967": "1917-06-02", + "968": "1919-12-30", + "969": "1925-12-02", + "970": "1942-01-02", + "971": "1944-04-16", + "973": "1946-01-04", + "974": "1948-09-02", + "975": "1951-01-01", + "976": "1951-01-01", + "977": "1951-01-01", + "978": "1944-06-01", + "979": "1971-01-01", + "980": "1980-08-01", + "981": "2004-02-29", + "982": "1995-05-01", + "983": "2013-04-01" + }, + "tmax": { + "1": "1881-03-01", + "3": "1953-05-31", + "4": "1905-03-30", + "6": "1943-06-30", + "7": "1950-07-14", + "8": "1940-04-30", + "9": "1972-12-31", + "10": null, + "11": null, + "12": null, + "13": "1985-12-31", + "14": "1952-12-31", + "15": null, + "16": null, + "17": null, + "18": null, + "19": null, + "20": "1932-12-16", + "21": null, + "22": null, + "23": "1885-08-30", + "24": null, + "25": null, + "26": null, + "40": "1925-09-29", + "41": "1950-11-28", + "42": "1871-09-29", + "43": "1903-07-30", + "44": "1941-08-01", + "45": "1937-04-29", + "46": "1894-12-31", + "47": "1914-09-30", + "48": "1925-06-29", + "49": "1929-08-30", + "50": "1927-06-29", + "51": "1911-11-29", + "53": "1950-11-27", + "54": "1931-04-29", + "55": "1955-12-31", + "56": "1937-04-29", + "57": "1943-11-30", + "58": "1941-10-31", + "59": "1948-05-31", + "60": "1949-12-31", + "61": null, + "64": null, + "65": null, + "66": null, + "67": null, + "68": null, + "69": null, + "70": null, + "71": "1953-04-30", + "72": "1986-03-27", + "73": null, + "74": "1995-12-31", + "75": null, + "76": null, + "77": null, + "78": null, + "79": null, + "80": null, + "81": null, + "82": null, + "84": null, + "85": null, + "86": null, + "87": null, + "89": null, + "90": null, + "91": null, + "126": "1872-05-30", + "127": "1882-06-29", + "128": "1902-03-31", + "129": "1941-02-28", + "130": "1937-07-31", + "131": "1915-09-30", + "132": "1910-12-30", + "133": "1953-04-30", + "134": null, + "135": "2020-09-19", + "136": null, + "137": "1929-04-29", + "139": null, + "140": null, + "141": null, + "142": null, + "143": null, + "144": null, + "145": null, + "146": "1995-12-31", + "147": null, + "148": null, + "149": "1995-12-31", + "150": null, + "151": null, + "152": null, + "153": "2018-01-30", + "154": null, + "155": null, + "156": null, + "157": "1995-12-31", + "158": null, + "159": null, + "160": null, + "161": null, + "162": null, + "163": null, + "164": null, + "165": "1972-08-28", + "166": null, + "168": "1979-04-28", + "170": "1990-12-31", + "171": null, + "172": null, + "173": null, + "194": "1949-08-31", + "195": "1995-12-31", + "197": "1879-01-30", + "198": "1913-03-30", + "199": "1892-01-30", + "200": "1941-08-31", + "201": "1893-01-30", + "203": "1900-03-30", + "204": "1904-06-29", + "205": "1935-10-30", + "206": "1937-09-29", + "207": "1940-07-31", + "208": "1927-07-30", + "209": "1939-06-27", + "210": "2006-02-27", + "211": "1936-04-07", + "213": "1953-02-28", + "214": "1939-09-30", + "215": "1948-06-30", + "216": "1944-07-31", + "217": null, + "219": "1949-12-31", + "221": null, + "222": null, + "223": null, + "224": null, + "225": null, + "226": null, + "227": null, + "228": null, + "229": null, + "230": "2020-02-06", + "231": "1969-02-28", + "232": "1957-12-31", + "233": null, + "234": null, + "235": null, + "236": null, + "237": "1953-02-28", + "238": null, + "239": null, + "240": null, + "241": "1953-12-31", + "242": null, + "243": "1955-10-31", + "244": "1954-06-30", + "245": "1954-06-30", + "246": "1990-12-31", + "247": "1953-11-30", + "249": null, + "250": "1982-12-31", + "251": null, + "252": null, + "254": "1995-12-31", + "255": null, + "256": null, + "257": null, + "258": "1950-12-31", + "259": "1950-08-31", + "260": "1950-09-13", + "261": "1950-08-31", + "263": null, + "264": null, + "285": "1922-05-09", + "286": "1879-03-31", + "287": "1950-12-31", + "288": "1888-01-30", + "289": "1903-04-29", + "290": "1934-03-30", + "291": "1894-12-30", + "292": "1931-09-29", + "293": "1919-09-29", + "294": "1905-04-29", + "295": "1918-01-30", + "296": "1913-04-29", + "297": "1939-01-31", + "298": null, + "299": "1935-04-29", + "300": "1924-01-30", + "301": "1932-09-29", + "302": "1949-10-27", + "303": "1946-04-30", + "304": "1936-03-30", + "305": "1950-03-31", + "306": "1950-06-30", + "307": "1950-12-31", + "308": "1940-12-03", + "309": "1950-07-31", + "310": "1948-09-29", + "311": "1949-08-31", + "312": "1949-11-30", + "313": "1948-04-29", + "314": "1953-01-31", + "315": "1948-12-31", + "316": "1949-04-01", + "317": null, + "318": "1950-12-31", + "319": "1949-11-30", + "321": "1950-06-20", + "323": null, + "324": "1952-12-31", + "325": "1950-04-30", + "326": null, + "327": null, + "328": null, + "329": null, + "330": null, + "331": null, + "332": null, + "333": null, + "334": "1953-10-31", + "335": null, + "336": null, + "337": null, + "338": null, + "339": null, + "340": null, + "341": null, + "342": null, + "343": null, + "344": null, + "345": null, + "346": "1995-12-31", + "347": "1984-12-31", + "348": null, + "349": null, + "350": null, + "352": null, + "353": null, + "354": null, + "356": null, + "357": "1960-12-31", + "358": null, + "359": null, + "360": "1971-03-17", + "361": null, + "362": null, + "363": "1922-09-29", + "364": "2021-08-28", + "365": null, + "366": null, + "367": "2012-08-31", + "368": "1987-12-31", + "369": null, + "370": "1950-12-31", + "371": null, + "372": null, + "374": "1950-12-31", + "411": "1881-12-30", + "412": "1905-05-30", + "413": "1886-06-29", + "414": "1953-05-31", + "415": "1935-07-30", + "416": "1939-02-28", + "417": "1924-08-30", + "418": "1911-09-29", + "419": "1950-03-31", + "420": "1915-08-16", + "421": "1936-12-30", + "422": "1933-09-29", + "423": "1930-01-24", + "424": "1924-12-30", + "425": "1937-04-23", + "426": null, + "427": "1953-05-31", + "428": "1941-03-28", + "429": "1947-01-16", + "430": "1950-01-31", + "431": "1949-11-29", + "432": "1950-11-17", + "433": "1961-12-31", + "434": null, + "435": null, + "436": "1992-12-31", + "437": null, + "438": null, + "439": null, + "440": "2015-12-31", + "441": "2014-12-31", + "442": null, + "443": null, + "444": null, + "445": "1988-03-30", + "446": null, + "447": null, + "448": "1976-12-31", + "449": null, + "450": null, + "451": "2009-04-30", + "452": "1955-12-31", + "453": null, + "454": null, + "455": null, + "456": null, + "457": "1970-03-28", + "458": null, + "459": "2004-04-29", + "460": "1958-10-07", + "461": null, + "462": null, + "463": null, + "464": null, + "465": null, + "466": "2004-04-29", + "467": null, + "468": "1958-10-17", + "469": "2014-12-31", + "470": null, + "471": null, + "472": "1973-12-31", + "473": "2015-06-30", + "474": null, + "475": "1995-12-31", + "476": "1995-12-31", + "477": null, + "478": "1996-02-29", + "479": null, + "480": null, + "481": null, + "482": null, + "483": null, + "484": null, + "509": null, + "510": null, + "511": "1927-01-30", + "512": "1877-05-30", + "513": "1868-12-30", + "514": null, + "515": "1948-03-31", + "516": null, + "517": "1883-07-30", + "518": "1886-11-29", + "519": "1888-01-30", + "520": "1894-12-30", + "521": "1900-01-30", + "522": "1902-07-30", + "523": null, + "524": "1932-03-30", + "525": "1929-03-25", + "526": "1921-01-30", + "527": "1941-04-30", + "528": "1928-12-30", + "529": "1934-12-30", + "530": "1947-12-31", + "531": "1952-12-31", + "533": "1939-12-31", + "534": "1940-04-30", + "535": "1935-12-21", + "536": "1940-11-30", + "538": "1949-10-17", + "539": null, + "540": "1959-07-30", + "541": null, + "542": null, + "543": null, + "544": "1954-05-31", + "546": null, + "547": null, + "548": null, + "549": null, + "550": null, + "551": "1963-12-31", + "552": "1955-06-30", + "553": "1953-01-31", + "555": "1955-05-31", + "556": "1994-12-31", + "557": null, + "558": null, + "559": null, + "560": null, + "561": null, + "562": null, + "563": null, + "564": null, + "565": null, + "566": "1954-07-31", + "567": null, + "568": "1969-11-30", + "569": null, + "570": null, + "571": "2021-04-30", + "572": null, + "573": null, + "574": "1953-09-30", + "575": "1992-12-31", + "576": null, + "577": "1963-04-30", + "578": null, + "579": null, + "580": null, + "581": "1970-12-31", + "582": null, + "583": null, + "584": null, + "586": "1986-05-08", + "588": null, + "589": null, + "590": "1995-12-31", + "591": null, + "592": "1990-12-31", + "593": null, + "594": "1996-04-04", + "595": null, + "596": null, + "597": "1879-09-29", + "600": "1950-12-31", + "645": null, + "646": "1871-03-30", + "647": "1939-09-30", + "648": "1932-05-30", + "649": "1950-12-31", + "650": "1887-07-30", + "651": "1899-01-30", + "652": "1938-07-23", + "653": "1890-04-29", + "654": "1912-03-30", + "655": "1939-12-31", + "656": "1910-07-30", + "657": "1949-11-30", + "658": "1944-02-29", + "659": "1924-12-30", + "660": "1944-08-31", + "661": "1949-09-30", + "662": "1948-04-29", + "663": null, + "664": null, + "665": null, + "666": null, + "667": null, + "668": null, + "669": null, + "670": null, + "671": "1954-09-30", + "672": null, + "673": null, + "674": null, + "675": null, + "676": "1995-12-31", + "677": "2006-01-31", + "678": null, + "679": null, + "680": null, + "681": null, + "682": null, + "683": null, + "684": null, + "685": "1995-12-31", + "686": null, + "687": null, + "688": null, + "689": null, + "690": "1888-05-30", + "691": "1937-12-31", + "705": null, + "706": "1922-04-29", + "707": "1883-07-30", + "708": "1873-11-29", + "709": "1904-12-30", + "710": "1988-04-29", + "711": "1887-06-29", + "712": "1929-02-27", + "713": "1883-05-30", + "714": "1907-02-19", + "715": "1881-03-01", + "716": "1888-08-31", + "718": "1914-12-30", + "719": "1906-11-29", + "721": "1903-10-30", + "722": "1903-12-30", + "723": "1904-11-29", + "724": "1942-11-30", + "725": "1919-11-29", + "726": "1915-12-30", + "727": "1939-09-30", + "728": "1941-09-30", + "729": "1935-05-30", + "730": "1950-12-31", + "731": "1953-09-30", + "733": null, + "734": "1955-10-31", + "735": null, + "736": null, + "737": null, + "738": null, + "739": "1977-12-31", + "740": null, + "741": null, + "742": null, + "743": null, + "744": null, + "745": "1995-12-31", + "746": null, + "747": null, + "748": "1963-12-31", + "749": null, + "750": null, + "751": null, + "752": null, + "754": null, + "755": null, + "756": null, + "757": null, + "758": null, + "759": "1995-12-31", + "760": null, + "761": null, + "762": null, + "763": null, + "764": null, + "767": null, + "768": "1973-12-31", + "769": "1973-12-31", + "770": null, + "772": "1877-12-30", + "773": "1881-07-30", + "810": "1905-03-30", + "811": "1924-07-11", + "812": "1878-11-29", + "813": "1870-07-30", + "814": "1942-04-05", + "815": "1891-12-30", + "816": "1899-12-30", + "817": "1883-11-29", + "818": "1898-03-01", + "819": "1912-09-29", + "820": "1904-05-31", + "821": "1919-05-23", + "822": "1913-09-03", + "823": "1919-11-29", + "824": "1914-06-29", + "825": "1950-07-31", + "826": "1953-01-31", + "827": null, + "828": null, + "829": null, + "830": null, + "831": null, + "832": null, + "833": null, + "834": null, + "835": null, + "836": null, + "837": null, + "838": null, + "839": null, + "840": null, + "841": null, + "843": null, + "844": null, + "845": "1874-02-08", + "871": "1876-04-29", + "872": "1920-04-17", + "873": "1885-03-30", + "875": "1909-06-29", + "876": "1886-10-30", + "877": "1888-09-29", + "878": "1888-08-31", + "879": "1887-11-29", + "880": "1891-04-29", + "881": "1928-03-30", + "882": "1916-08-30", + "883": null, + "884": "1934-06-29", + "885": "1948-11-30", + "886": "1948-12-31", + "887": "1950-11-30", + "888": "1944-08-31", + "889": "1949-06-30", + "890": "1944-08-31", + "891": "1949-05-30", + "892": null, + "893": "1949-07-31", + "894": "1949-08-31", + "896": null, + "897": null, + "898": "1972-12-31", + "899": null, + "900": "1954-01-31", + "901": null, + "902": "2010-12-31", + "903": null, + "904": null, + "905": null, + "906": null, + "907": null, + "908": null, + "909": null, + "910": null, + "911": null, + "912": null, + "913": null, + "914": null, + "915": null, + "917": null, + "918": null, + "919": null, + "920": null, + "921": null, + "922": null, + "923": null, + "926": null, + "950": "2004-02-29", + "951": "1952-12-31", + "952": "1888-05-29", + "953": "1889-12-30", + "954": "1940-12-31", + "955": "1948-08-31", + "956": "1908-10-30", + "957": "1919-01-31", + "959": "1952-11-30", + "960": "1948-11-30", + "961": null, + "962": null, + "963": null, + "964": null, + "965": null, + "966": null, + "967": null, + "968": null, + "969": null, + "970": null, + "971": null, + "973": null, + "974": null, + "975": "1980-12-31", + "976": "1985-12-31", + "977": "2013-03-31", + "978": "1949-06-14", + "979": null, + "980": null, + "981": null, + "982": null, + "983": null + }, "RD": { "1": true, "3": true, diff --git a/hydropandas/data/update_knmi_meteostations.py b/hydropandas/data/update_knmi_meteostations.py deleted file mode 100644 index 44fde43d..00000000 --- a/hydropandas/data/update_knmi_meteostations.py +++ /dev/null @@ -1,33 +0,0 @@ -# %% -import pandas as pd -import requests - -from hydropandas.io import knmi - -meteo_df = pd.read_json("knmi_meteostation.json") -meteo_dft = meteo_df.copy() - -didx = [pd.Timestamp.today().normalize() - pd.Timedelta(days=1)] + list( - reversed( - pd.date_range(pd.Timestamp("1900-01-01"), pd.Timestamp.today(), freq="10YS") - ) -) - -for stn in meteo_dft.index: - df_daily, _ = knmi.get_knmi_daily_meteo_url(stn) - vars_available_daily = ~df_daily.isna().all().drop(index=["STN"]) - meteo_dft.loc[stn, vars_available_daily.index] = vars_available_daily.values - - for end, start in zip(didx[:-1], didx[1:]): - try: - df_hourly, _ = knmi.get_knmi_hourly_meteo_api(stn, start=start, end=end) - vars_available_hourly = ~df_hourly.isna().all().drop(index=["STN"]) - vah = vars_available_hourly[vars_available_hourly] - meteo_dft.loc[stn, vah.index] = vah.values - except requests.ConnectionError: - print(f"Geen data {stn}, {start}, {end}") - except pd.errors.EmptyDataError: - print(f"Geen data {stn}, {start}, {end}") - -meteo_dft = meteo_dft.fillna(False).drop(columns=[""]) -meteo_dft.to_json("knmi_meteostation.json") diff --git a/hydropandas/data/update_knmi_stations.py b/hydropandas/data/update_knmi_stations.py new file mode 100644 index 00000000..66c43f76 --- /dev/null +++ b/hydropandas/data/update_knmi_stations.py @@ -0,0 +1,100 @@ +# %% +# packages +import numpy as np +import pandas as pd +import requests + +from hydropandas.io import knmi + + +class EmptyDataFrameError(Exception): + pass + + +# %% +# meteo stations +meteo_df = pd.read_json("knmi_meteostation.json") +meteo_dft = meteo_df.copy() + +didx = [pd.Timestamp.today().normalize() - pd.Timedelta(days=1)] + list( + reversed( + pd.date_range(pd.Timestamp("1900-01-01"), pd.Timestamp.today(), freq="10YS") + ) +) + + +for stn in meteo_dft.index: + df_daily, _ = knmi.get_knmi_daily_meteo_url(stn) + vars_available_daily = ~df_daily.isna().all().drop(index=["STN"]) + meteo_dft.loc[stn, vars_available_daily.index] = vars_available_daily.values + + for end, start in zip(didx[:-1], didx[1:]): + try: + df_hourly, _ = knmi.get_knmi_hourly_meteo_api(stn, start=start, end=end) + if df_hourly.empty: + raise EmptyDataFrameError + vars_available_hourly = ~df_hourly.isna().all().drop(index=["STN"]) + vah = vars_available_hourly[vars_available_hourly] + meteo_dft.loc[stn, vah.index] = vah.values + except ( + pd.errors.EmptyDataError, + requests.ConnectionError, + EmptyDataFrameError, + ): + print(f"Geen data {stn}, {start}, {end}") + +meteo_dft = meteo_dft.fillna(False).drop(columns=[""]) + +# add tmin, tmax +meteo_tminmax = ( + pd.read_html("https://www.knmi.nl/nederland-nu/klimatologie/daggegevens")[0] + .dropna(how="any") + .convert_dtypes() + .set_index("Nummer") +) +meteo_tminmax["tmin"] = [ + pd.to_datetime(x, format="%Y%m%d").strftime("%Y-%m-%d") + for x in meteo_tminmax["Vanaf"] +] +meteo_tminmax["tmax"] = [ + pd.to_datetime(x, format="%Y%m%d") for x in meteo_tminmax["Tot en met"] +] +meteo_tminmax["tmax"] = [ + None + if (pd.Timestamp.today() - x) < pd.Timedelta(days=365) + else x.strftime("%Y-%m-%d") + for x in meteo_tminmax["tmax"] +] +meteo_dft.loc[meteo_tminmax.index, "tmin"] = meteo_tminmax["tmin"].values +meteo_dft.loc[meteo_tminmax.index, "tmax"] = meteo_tminmax["tmax"].values + +meteo_dft.to_json("knmi_meteostation.json") +# %% +# neerslag stations +prec_df = pd.read_json("knmi_neerslagstation.json") +prec_dft = prec_df.copy() +prec_tminmax = ( + pd.concat( + pd.read_html( + "https://www.knmi.nl/nederland-nu/klimatologie/monv/reeksen", index_col=0 + ) + ) + .reset_index() + .set_index("Nr") +) +unknown_location = prec_tminmax.index[~np.isin(prec_tminmax.index, prec_dft.index)] +prec_tminmax = prec_tminmax.drop(unknown_location) +tminmax = [[x[0], x[-1]] for x in prec_tminmax["Periode"].str.split(" ")] +tmin = [pd.to_datetime(x[0], format="%Y%m%d").strftime("%Y-%m-%d") for x in tminmax] +tmax = [pd.to_datetime(x[1], format="%Y%m%d") for x in tminmax] +tmax = [ + None + if (pd.Timestamp.today() - x) < pd.Timedelta(days=365) + else x.strftime("%Y-%m-%d") + for x in tmax +] +prec_dft.loc[prec_tminmax.index, "tmin"] = tmin +prec_dft.loc[prec_tminmax.index, "tmax"] = tmax +prec_dft.sort_index().loc[ + :, ["lon", "lat", "name", "x", "y", "altitude", "tmin", "tmax", "RD"] +].to_json("knmi_neerslagstation.json") diff --git a/hydropandas/io/bro.py b/hydropandas/io/bro.py index c2aab72b..22b65573 100644 --- a/hydropandas/io/bro.py +++ b/hydropandas/io/bro.py @@ -11,7 +11,7 @@ import pandas as pd import requests from pyproj import Proj, Transformer -from requests.adapters import Retry, HTTPAdapter +from requests.adapters import HTTPAdapter, Retry from tqdm import tqdm from ..rcparams import rcParams diff --git a/hydropandas/io/knmi.py b/hydropandas/io/knmi.py index 4e50bbcb..c0a9044a 100644 --- a/hydropandas/io/knmi.py +++ b/hydropandas/io/knmi.py @@ -116,7 +116,13 @@ def get_knmi_obs( f"variable {meteo_var}" ) stns = get_n_nearest_stations_xy( - xy=xy, meteo_var=meteo_var, n=1, stations=None, ignore=None + xy=xy, + meteo_var=meteo_var, + start=start, + end=end, + n=1, + stations=None, + ignore=None, ) ts, meta = get_knmi_timeseries_stn( stn=stns[0], meteo_var=meteo_var, settings=settings, start=start, end=end @@ -333,7 +339,10 @@ def get_knmi_timeseries_stn( settings=settings, stn_name=stn_name, ) - + if knmi_df.empty: + logger.warning( + f"No data for {meteo_var=} at {stn=} between" f"{start=} and {end=}." + ) if str(stn) in station_meta.index: meta = station_meta.loc[f"{stn}"].to_dict() else: @@ -356,13 +365,21 @@ def get_knmi_timeseries_stn( return knmi_df, meta -def get_stations(meteo_var: str) -> pd.DataFrame: +def get_stations( + meteo_var: str, + start: Union[pd.Timestamp, str, None] = None, + end: Union[pd.Timestamp, str, None] = None, +) -> pd.DataFrame: """get knmi stations from json files according to variable. Parameters ---------- meteo_var : str, optional type of meteodata, by default 'RH' + start : str, datetime or None, optional + start date of observations. The default is None. + end : str, datetime or None, optional + end date of observations. The default is None. Returns ------- @@ -380,18 +397,62 @@ def get_stations(meteo_var: str) -> pd.DataFrame: stations = stations.where(~stations.isna(), False) if meteo_var in ("makkink", "penman", "hargreaves"): meteo_var = "EV24" - return stations.loc[ + + # select only stations with meteo_var + stations = stations.loc[ stations.loc[:, meteo_var], - [ - "lon", - "lat", - "name", - "x", - "y", - "altitude", - ], + ["lon", "lat", "name", "x", "y", "altitude", "tmin", "tmax"], ] + # select only stations with measurement + stations = _get_stations_tmin_tmax(stations, start, end) + + return stations + + +def _get_stations_tmin_tmax(stations_df, start, end): + """select stations within period defined by start and end. + + Parameters + ---------- + stations_df : pd.DataFrame + stations + start : datetime or None, optional + start date of observations. + end : datetime or None, optional + end date of observations. + + Returns + ------- + DataFrame with all station with measurements in selected period. + + Notes + ----- + Does not work on a DataFrames with duplicate indices + """ + if stations_df.index.duplicated().any(): + raise IndexError("function does not work for dataframe with duplicated index") + + if end is None: + tmin_stns = set(stations_df.index) + else: + # keep stations where tmin is unknonw (=False) + stns_unknown_tmin = set(stations_df.loc[stations_df["tmin"] == False].index) + tmin_available = stations_df.loc[stations_df["tmin"] != False, "tmin"] + tmin_within_range = pd.to_datetime(tmin_available) < end + tmin_stns = set(tmin_available.loc[tmin_within_range].index) | stns_unknown_tmin + + if start is None: + tmax_stns = set(stations_df.index) + else: + stns_unknown_tmax = set(stations_df.loc[stations_df["tmax"] == False].index) + tmax_available = stations_df.loc[stations_df["tmax"] != False, "tmax"] + tmax_available.loc[tmax_available.isnull()] = dt.datetime.now().date() + tmax_within_range = pd.to_datetime(tmax_available) > start + tmax_stns = set(tmax_available.loc[tmax_within_range].index) | stns_unknown_tmax + + return stations_df.loc[list(tmin_stns & tmax_stns)] + def get_station_name(stn: int, stations: Union[pd.DataFrame, None] = None) -> str: """Returns the station name from a KNMI station. @@ -467,7 +528,7 @@ def fill_missing_measurements( ) # get the location of the stations - stations = get_stations(meteo_var=meteo_var) + stations = get_stations(meteo_var=meteo_var, start=start, end=end) if stn_name is None: stn_name = get_station_name(stn=stn, stations=stations) @@ -479,11 +540,15 @@ def fill_missing_measurements( # if the first station cannot be read, read another station as the first ignore = [stn] while knmi_df.empty: - logger.info(f"station {stn} has no measurements between {start} and {end}") - logger.info("trying to get measurements from nearest station") + logger.debug(f"station {stn} has no measurements between {start} and {end}") + logger.debug("trying to get measurements from nearest station") stn_lst = get_nearest_station_df( - stations.loc[[ignore[0]]], meteo_var=meteo_var, ignore=ignore + stations.loc[[ignore[0]]], + meteo_var=meteo_var, + start=start, + end=end, + ignore=ignore, ) if stn_lst is None: logger.warning( @@ -509,7 +574,7 @@ def fill_missing_measurements( ) if new_end < end: end = new_end - logger.warning(f'changing end_date to {end.strftime("%Y-%m-%d")}') + logger.info(f'changing end_date to {end.strftime("%Y-%m-%d")}') # find missing values knmi_df = _add_missing_indices(knmi_df, stn, start, end) @@ -522,7 +587,11 @@ def fill_missing_measurements( # fill missing values while np.any(missing) and not np.all(missing): stn_comp = get_nearest_station_df( - stations.loc[[stn]], meteo_var=meteo_var, ignore=ignore + stations.loc[[stn]], + meteo_var=meteo_var, + start=start, + end=end, + ignore=ignore, ) if stn_comp is None: @@ -537,7 +606,7 @@ def fill_missing_measurements( stn_comp = stn_comp[0] n_missing = missing.sum() - logger.info( + logger.debug( f"Trying to fill {n_missing} missing measurements with station {stn_comp}" ) @@ -547,7 +616,7 @@ def fill_missing_measurements( ) if knmi_df_comp.empty: - logger.info(f"No data available for station {stn_comp}") + logger.debug(f"No data available for station {stn_comp}") else: # dropnans from new data @@ -658,12 +727,7 @@ def download_knmi_data( stn=stn, meteo_var=meteo_var, start=start, end=end ) add_day = True - if df.empty: - logger.warning( - f"No data for {meteo_var=} at {stn=} between" - f"{start=} and {end=}. Returning empty DataFrame." - ) - else: + if not df.empty: knmi_df, variables = interpret_knmi_file( df=df, meta=meta, @@ -693,12 +757,7 @@ def download_knmi_data( else: # daily data from meteorological stations df, meta = get_knmi_daily_meteo_url(stn=stn) - if df.empty: - logger.warning( - f"No data for {meteo_var=} at {stn=} between" - f"{start=} and {end=}. Returning empty DataFrame." - ) - else: + if not df.empty: knmi_df, variables = interpret_knmi_file( df=df, meta=meta, @@ -1344,6 +1403,8 @@ def get_nearest_station_df( ycol: str = "y", stations: Union[pd.DataFrame, None] = None, meteo_var: str = "RH", + start: Union[pd.Timestamp, str, None] = None, + end: Union[pd.Timestamp, str, None] = None, ignore: Union[List[str], None] = None, ) -> list[int]: """Find the KNMI stations that measure 'meteo_var' closest to the @@ -1362,6 +1423,10 @@ def get_nearest_station_df( The default is None. meteo_var : str measurement variable e.g. 'RH' or 'EV24' + start : str, datetime or None, optional + start date of observations. The default is None. + end : str, datetime or None, optional + end date of observations. The default is None. ignore : list, optional list of stations to ignore. The default is None. @@ -1371,7 +1436,7 @@ def get_nearest_station_df( station numbers. """ if stations is None: - stations = get_stations(meteo_var=meteo_var) + stations = get_stations(meteo_var=meteo_var, start=start, end=end) if ignore is not None: stations.drop(ignore, inplace=True) if stations.empty: @@ -1444,6 +1509,8 @@ def get_nearest_station_xy( def get_n_nearest_stations_xy( xy: List[List[float]], meteo_var: str, + start: Union[pd.Timestamp, str, None] = None, + end: Union[pd.Timestamp, str, None] = None, n: int = 1, stations: Union[pd.DataFrame, None] = None, ignore: Union[List[str], None] = None, @@ -1457,6 +1524,10 @@ def get_n_nearest_stations_xy( sinlge pair of xy coordinates. e.g. (150_000., 400_000.) meteo_var : str measurement variable e.g. 'RH' or 'EV24' + start : str, datetime or None, optional + start date of observations. The default is None. + end : str, datetime or None, optional + end date of observations. The default is None. n : int, optional number of stations you want to return. The default is 1. stations : pandas DataFrame, optional @@ -1472,7 +1543,7 @@ def get_n_nearest_stations_xy( """ if stations is None: - stations = get_stations(meteo_var=meteo_var) + stations = get_stations(meteo_var=meteo_var, start=start, end=end) if ignore is not None: stations.drop(ignore, inplace=True) if stations.empty: @@ -1640,14 +1711,18 @@ class of the observations, can be PrecipitationObs or # get stations if stns is None: - stations = get_stations(meteo_var=meteo_var) + stations = get_stations(meteo_var=meteo_var, start=start, end=end) if (locations is None) and (xy is not None): _stns = get_nearest_station_xy( xy, stations=stations, meteo_var=meteo_var ) elif locations is not None: _stns = get_nearest_station_df( - locations, stations=stations, meteo_var=meteo_var + locations, + stations=stations, + meteo_var=meteo_var, + start=start, + end=end, ) else: raise ValueError( diff --git a/pyproject.toml b/pyproject.toml index 59975910..b1186e73 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -88,6 +88,9 @@ lint.extend-select = ["I"] show-fixes = true fix = true +[tool.ruff.lint.per-file-ignores] +"knmi.py" = ["E712"] # see PR 239 + [tool.pytest.ini_options] addopts = "--durations=0" markers = ["slow: mark test as slow."] From f02849d9eeeaf39fd6df5d441627889a3998f5b4 Mon Sep 17 00:00:00 2001 From: OnnoEbbens Date: Mon, 23 Sep 2024 17:06:51 +0200 Subject: [PATCH 16/16] version bump --- hydropandas/version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hydropandas/version.py b/hydropandas/version.py index 4cc307b9..adb03b93 100644 --- a/hydropandas/version.py +++ b/hydropandas/version.py @@ -1,7 +1,7 @@ from importlib import metadata from sys import version as os_version -__version__ = "0.12.2" +__version__ = "0.12.3" def show_versions():