From 2a9f7d8403937839fbc2cb98ecd26e362675177f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Dav=C3=ADd=20Brakenhoff?= Date: Fri, 27 Sep 2024 11:12:07 +0200 Subject: [PATCH 01/12] allow get_stations with meteo_var == slice(None) --- hydropandas/io/knmi.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/hydropandas/io/knmi.py b/hydropandas/io/knmi.py index c0a9044a..7cb347b0 100644 --- a/hydropandas/io/knmi.py +++ b/hydropandas/io/knmi.py @@ -399,9 +399,12 @@ def get_stations( meteo_var = "EV24" # select only stations with meteo_var + if meteo_var == slice(None): + meteo_mask = stations.loc[:, meteo_var].any(axis=1) + else: + meteo_mask = stations.loc[:, meteo_var] stations = stations.loc[ - stations.loc[:, meteo_var], - ["lon", "lat", "name", "x", "y", "altitude", "tmin", "tmax"], + meteo_mask, ["lon", "lat", "name", "x", "y", "altitude", "tmin", "tmax"] ] # select only stations with measurement From a3cbdf4185a5a88d55141eac9fa4f56ebab1e69e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Dav=C3=ADd=20Brakenhoff?= Date: Fri, 27 Sep 2024 11:12:27 +0200 Subject: [PATCH 02/12] only filter stations if start/end is provided --- hydropandas/io/knmi.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/hydropandas/io/knmi.py b/hydropandas/io/knmi.py index 7cb347b0..46e27ebc 100644 --- a/hydropandas/io/knmi.py +++ b/hydropandas/io/knmi.py @@ -408,7 +408,8 @@ def get_stations( ] # select only stations with measurement - stations = _get_stations_tmin_tmax(stations, start, end) + if start is not None or end is not None: + stations = _get_stations_tmin_tmax(stations, start, end) return stations From c256914e1a2877012fcb60a0bd50fb6738a689f1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Dav=C3=ADd=20Brakenhoff?= Date: Fri, 27 Sep 2024 11:14:57 +0200 Subject: [PATCH 03/12] ensure current station is in dataframe so nearest station can be determined --- hydropandas/io/knmi.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/hydropandas/io/knmi.py b/hydropandas/io/knmi.py index 46e27ebc..012d31e7 100644 --- a/hydropandas/io/knmi.py +++ b/hydropandas/io/knmi.py @@ -536,10 +536,13 @@ def fill_missing_measurements( if stn_name is None: stn_name = get_station_name(stn=stn, stations=stations) - # download data from station + # download data from station if it has data between start-end knmi_df, variables, station_meta = download_knmi_data( stn, meteo_var, start, end, settings, stn_name ) + if stn not in stations.index: + # add current station to df to determine which station to fill data with + stations = pd.concat([stations, get_stations(meteo_var=meteo_var).loc[[stn]]]) # if the first station cannot be read, read another station as the first ignore = [stn] @@ -552,6 +555,7 @@ def fill_missing_measurements( meteo_var=meteo_var, start=start, end=end, + stations=stations, ignore=ignore, ) if stn_lst is None: From 17de5ab491bdb209fea51b12bc40f5eddc2e23e8 Mon Sep 17 00:00:00 2001 From: OnnoEbbens Date: Fri, 27 Sep 2024 11:54:55 +0200 Subject: [PATCH 04/12] fix for knmi precipitation data station attribute is sometimes a series with duplicate entries #241 (#242) --- hydropandas/io/knmi.py | 15 ++++++++++----- hydropandas/observation.py | 5 ++++- 2 files changed, 14 insertions(+), 6 deletions(-) diff --git a/hydropandas/io/knmi.py b/hydropandas/io/knmi.py index c0a9044a..3cb9983c 100644 --- a/hydropandas/io/knmi.py +++ b/hydropandas/io/knmi.py @@ -54,7 +54,10 @@ def get_knmi_obs( **kwargs: fill_missing_obs : bool, optional if True nan values in time series are filled with nearby time series. - The default is False. + The default is False. Note: if the given stn has no data between start and + end the data from nearby stations is used. In this case the metadata of the + Observation is the metadata from the nearest station that has any + measurement in the given period. interval : str, optional desired time interval for observations. Options are 'daily' and 'hourly'. The default is 'daily'. @@ -1214,8 +1217,7 @@ def interpret_knmi_file( raise ValueError( f"Cannot handle multiple stations {unique_stn} in single file" ) - else: - stn = df.at[df.index[0], "STN"] + stn = unique_stn[0] if add_day or add_hour: if add_day and add_hour: @@ -1659,7 +1661,10 @@ class of the observations, can be PrecipitationObs or **kwargs: fill_missing_obs : bool, optional if True nan values in time series are filled with nearby time series. - The default is False. + The default is False. Note: if the given stn has no data between start and + end the data from nearby stations is used. In this case the metadata of the + Observation is the metadata from the nearest station that has any + measurement in the given period. interval : str, optional desired time interval for observations. Options are 'daily' and 'hourly'. The default is 'daily'. @@ -1674,7 +1679,7 @@ class of the observations, can be PrecipitationObs or Returns ------- - obs_list : list of obsevation objects + obs_list : list of observation objects collection of multiple point observations """ diff --git a/hydropandas/observation.py b/hydropandas/observation.py index 71fcb0be..dbb3a8a9 100644 --- a/hydropandas/observation.py +++ b/hydropandas/observation.py @@ -1083,7 +1083,10 @@ def from_knmi( end date of observations. The default is None. fill_missing_obs : bool, optional if True nan values in time series are filled with nearby time series. - The default is False. + The default is False. Note: if the given stn has no data between start and + end the data from nearby stations is used. In this case the metadata of the + Observation is the metadata from the nearest station that has any + measurement in the given period. interval : str, optional desired time interval for observations. Options are 'daily' and 'hourly'. The default is 'daily'. From a17947b77f79790a3ca43746fb40ea938aa424b4 Mon Sep 17 00:00:00 2001 From: OnnoEbbens Date: Fri, 27 Sep 2024 12:41:08 +0200 Subject: [PATCH 05/12] reshuffle a bit --- hydropandas/io/knmi.py | 23 +++++++++++++++-------- 1 file changed, 15 insertions(+), 8 deletions(-) diff --git a/hydropandas/io/knmi.py b/hydropandas/io/knmi.py index 012d31e7..c3ea9b8c 100644 --- a/hydropandas/io/knmi.py +++ b/hydropandas/io/knmi.py @@ -483,6 +483,9 @@ def get_station_name(stn: int, stations: Union[pd.DataFrame, None] = None) -> st return None stn_name = stations.at[stn, "name"] + if isinstance(stn_name, pd.Series): + raise ValueError(f'station {stn} is a meteo- and a precipitation station, please indicate which one you want to use using a "meteo_var"') + stn_name = stn_name.upper().replace(" ", "-").replace("(", "").replace(")", "") return stn_name @@ -533,16 +536,20 @@ def fill_missing_measurements( # get the location of the stations stations = get_stations(meteo_var=meteo_var, start=start, end=end) - if stn_name is None: - stn_name = get_station_name(stn=stn, stations=stations) - - # download data from station if it has data between start-end - knmi_df, variables, station_meta = download_knmi_data( - stn, meteo_var, start, end, settings, stn_name - ) if stn not in stations.index: - # add current station to df to determine which station to fill data with + # no measurements in given period, continue with empty dataframe + knmi_df = pd.DataFrame() + + # add location of station without data to dataframe stations = pd.concat([stations, get_stations(meteo_var=meteo_var).loc[[stn]]]) + else: + if stn_name is None: + stn_name = get_station_name(stn=stn, stations=stations) + + # download data from station if it has data between start-end + knmi_df, variables, station_meta = download_knmi_data( + stn, meteo_var, start, end, settings, stn_name + ) # if the first station cannot be read, read another station as the first ignore = [stn] From 2765eceb108aa2694ddd81a486034f61a9a40514 Mon Sep 17 00:00:00 2001 From: OnnoEbbens Date: Fri, 27 Sep 2024 12:43:00 +0200 Subject: [PATCH 06/12] ruff --- hydropandas/io/knmi.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/hydropandas/io/knmi.py b/hydropandas/io/knmi.py index c3ea9b8c..74695b0a 100644 --- a/hydropandas/io/knmi.py +++ b/hydropandas/io/knmi.py @@ -484,7 +484,9 @@ def get_station_name(stn: int, stations: Union[pd.DataFrame, None] = None) -> st stn_name = stations.at[stn, "name"] if isinstance(stn_name, pd.Series): - raise ValueError(f'station {stn} is a meteo- and a precipitation station, please indicate which one you want to use using a "meteo_var"') + raise ValueError( + f'station {stn} is a meteo- and a precipitation station, please indicate which one you want to use using a "meteo_var"' + ) stn_name = stn_name.upper().replace(" ", "-").replace("(", "").replace(")", "") return stn_name From 4e124b61b71017580c77cdf3b405eb835c04e45b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Dav=C3=ADd=20Brakenhoff?= Date: Fri, 27 Sep 2024 15:35:00 +0200 Subject: [PATCH 07/12] up version for minor release --- hydropandas/version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hydropandas/version.py b/hydropandas/version.py index adb03b93..841fa6a4 100644 --- a/hydropandas/version.py +++ b/hydropandas/version.py @@ -1,7 +1,7 @@ from importlib import metadata from sys import version as os_version -__version__ = "0.12.3" +__version__ = "0.12.4" def show_versions(): From d8b46a4c56523760c73ac3fe3211aeb751f7cf79 Mon Sep 17 00:00:00 2001 From: OnnoEbbens Date: Fri, 27 Sep 2024 15:52:02 +0200 Subject: [PATCH 08/12] version bump --- hydropandas/version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hydropandas/version.py b/hydropandas/version.py index 841fa6a4..d163576c 100644 --- a/hydropandas/version.py +++ b/hydropandas/version.py @@ -1,7 +1,7 @@ from importlib import metadata from sys import version as os_version -__version__ = "0.12.4" +__version__ = "0.12.5b" def show_versions(): From cd81586d62d2a2781a6cc5d5b66c940558ab1615 Mon Sep 17 00:00:00 2001 From: OnnoEbbens Date: Tue, 8 Oct 2024 11:30:04 +0200 Subject: [PATCH 09/12] fix bug in fill_missing_obs --- hydropandas/io/knmi.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/hydropandas/io/knmi.py b/hydropandas/io/knmi.py index ecbc3cea..9e761fd2 100644 --- a/hydropandas/io/knmi.py +++ b/hydropandas/io/knmi.py @@ -249,8 +249,8 @@ def _get_default_settings(settings=None) -> Dict[str, Any]: if "fill_missing_obs" in settings.keys(): if "raise_exceptions" in settings.keys(): - logger.debug("set raise_exceptions=False because fill_missing_obs is True") if settings["fill_missing_obs"] and settings["raise_exceptions"]: + logger.debug("set raise_exceptions=False because fill_missing_obs is True") settings["raise_exceptions"] = False else: settings["raise_exceptions"] = False @@ -611,6 +611,7 @@ def fill_missing_measurements( meteo_var=meteo_var, start=start, end=end, + stations=stations, ignore=ignore, ) @@ -1457,7 +1458,7 @@ def get_nearest_station_df( if stations is None: stations = get_stations(meteo_var=meteo_var, start=start, end=end) if ignore is not None: - stations.drop(ignore, inplace=True) + stations = stations.drop(ignore) if stations.empty: return None From a75f0a0f59c00c72a93c9ef72921987d9fd19406 Mon Sep 17 00:00:00 2001 From: OnnoEbbens Date: Tue, 8 Oct 2024 14:55:42 +0200 Subject: [PATCH 10/12] version bump --- hydropandas/version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hydropandas/version.py b/hydropandas/version.py index d163576c..5e3bf002 100644 --- a/hydropandas/version.py +++ b/hydropandas/version.py @@ -1,7 +1,7 @@ from importlib import metadata from sys import version as os_version -__version__ = "0.12.5b" +__version__ = "0.12.5" def show_versions(): From 211c9582a92a4f1dd46ef733479762b6e0b26b3d Mon Sep 17 00:00:00 2001 From: OnnoEbbens Date: Tue, 8 Oct 2024 15:03:04 +0200 Subject: [PATCH 11/12] ruff --- hydropandas/io/knmi.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/hydropandas/io/knmi.py b/hydropandas/io/knmi.py index 9e761fd2..7fa48298 100644 --- a/hydropandas/io/knmi.py +++ b/hydropandas/io/knmi.py @@ -250,7 +250,9 @@ def _get_default_settings(settings=None) -> Dict[str, Any]: if "fill_missing_obs" in settings.keys(): if "raise_exceptions" in settings.keys(): if settings["fill_missing_obs"] and settings["raise_exceptions"]: - logger.debug("set raise_exceptions=False because fill_missing_obs is True") + logger.debug( + "set raise_exceptions=False because fill_missing_obs is True" + ) settings["raise_exceptions"] = False else: settings["raise_exceptions"] = False From e27f10ad02f0eef59f25f0e22fa09213c1337224 Mon Sep 17 00:00:00 2001 From: OnnoEbbens Date: Tue, 8 Oct 2024 19:00:18 +0200 Subject: [PATCH 12/12] ruff --- hydropandas/version.py | 1 + 1 file changed, 1 insertion(+) diff --git a/hydropandas/version.py b/hydropandas/version.py index 5607ba9f..5e3bf002 100644 --- a/hydropandas/version.py +++ b/hydropandas/version.py @@ -3,6 +3,7 @@ __version__ = "0.12.5" + def show_versions(): """Method to print the versions of dependencies.""" msg = (