diff --git a/hydropandas/__init__.py b/hydropandas/__init__.py index d3590f7d..3097634a 100644 --- a/hydropandas/__init__.py +++ b/hydropandas/__init__.py @@ -17,6 +17,7 @@ read_knmi, read_menyanthes, read_modflow, + read_pastastore, read_pickle, read_waterinfo, read_wiski, diff --git a/hydropandas/extensions/plots.py b/hydropandas/extensions/plots.py index 7694425c..491a552b 100644 --- a/hydropandas/extensions/plots.py +++ b/hydropandas/extensions/plots.py @@ -44,7 +44,7 @@ def interactive_plots( if True plot multiple tubes at the same monitoring_well in one figure **kwargs : - will be passed to the Obs.to_interactive_plot method, options + will be passed to the Obs.interactive_plot method, options include: - cols : list of str or None @@ -55,7 +55,7 @@ def interactive_plots( - hoover_names : list of str - plot_colors : list of str - ylabel : str - - add_filter_to_legend : boolean + - add_screen_to_legend : boolean """ _color_cycle = ( "blue", @@ -184,7 +184,7 @@ def interactive_map( if True interactive plots will be created, if False the iplot_fname in the meta ditctionary of the observations is used. **kwargs : - will be passed to the to_interactive_plots method options are: + will be passed to the interactive_plots method options are: - cols : list of str or None - hoover_names : list of str @@ -194,7 +194,7 @@ def interactive_map( - hoover_names : list of str - plot_colors : list of str - ylabel : str - - add_filter_to_legend : boolean + - add_screen_to_legend : boolean - tmin : dt.datetime - tmax : dt.datetime diff --git a/hydropandas/io/pastas.py b/hydropandas/io/pastas.py index 8ddd047d..80e83012 100644 --- a/hydropandas/io/pastas.py +++ b/hydropandas/io/pastas.py @@ -6,6 +6,10 @@ import logging import numbers +from tqdm.auto import tqdm + +from ..observation import GroundwaterObs + logger = logging.getLogger(__name__) @@ -113,3 +117,76 @@ def create_pastastore( ) return pstore + + +def read_pastastore_item(pstore, libname, name): + """Read item from pastastore library. + + Parameters + ---------- + pstore : pastastore.PastaStore + pastastore object + libname : str + name of library containing item + name : str + name of item + + Returns + ------- + series : pd.Series + time series for item + meta : dict + dictionary containing metadata + + Raises + ------ + ValueError + if library is not oseries or stresses + """ + if libname == "oseries": + series, meta = pstore.get_oseries(name, return_metadata=True) + elif libname == "stresses": + series, meta = pstore.get_stresses(name, return_metadata=True) + else: + raise ValueError( + f"Cannot store items from library '{libname}' in ObsCollection." + ) + return series, meta + + +def read_pastastore_library( + pstore, libname, ObsClass=GroundwaterObs, metadata_mapping=None +): + """Read pastastore library. + + Parameters + ---------- + pstore : pastastore.PastaStore + pastastore object + libname : str + name of library to read + ObsClass : Obs, optional + type of Obs to read data as, by default GroundwaterObs + metadata_mapping : dict, optional + dictionary containing map between metadata field names in pastastore and + metadata field names expected by hydropandas, by default None. + + Returns + ------- + obs_list : list of Obs + list of Obs containing data + """ + names = pstore.conn._parse_names(None, libname) + + obs_list = [] + for name in tqdm(names, desc=f"Read Pastastore '{libname}'"): + try: + o = ObsClass.from_pastastore( + pstore, libname, name, metadata_mapping=metadata_mapping + ) + except AttributeError: + series, meta = read_pastastore_item(pstore, libname, name) + o = ObsClass(series, meta=meta) + obs_list.append(o) + + return obs_list diff --git a/hydropandas/io/waterinfo.py b/hydropandas/io/waterinfo.py index e1451257..5c2aeef6 100644 --- a/hydropandas/io/waterinfo.py +++ b/hydropandas/io/waterinfo.py @@ -45,9 +45,6 @@ def read_waterinfo_file( "File type '{}' not supported!".format(os.path.splitext(path)[-1]) ) - if index_cols is None: - index_cols = ["WAARNEMINGDATUM", "WAARNEMINGTIJD"] - if value_col is None: value_col = "NUMERIEKEWAARDE" @@ -66,11 +63,25 @@ def read_waterinfo_file( sep=";", decimal=",", encoding="ISO-8859-1", - parse_dates=[index_cols], dayfirst=True, - index_col="_".join(index_cols), ) + if index_cols is None: + index_cols = ["WAARNEMINGDATUM"] + if "WAARNEMINGTIJD (MET/CET)" in df.columns: + index_cols += ["WAARNEMINGTIJD (MET/CET)"] + elif "WAARNEMINGTIJD" in df.columns: + index_cols += ["WAARNEMINGTIJD"] + else: + raise KeyError( + "expected column with WAARNEMINGSTIJD but could not find one" + ) + + df.index = pd.to_datetime( + df[index_cols[0]] + " " + df[index_cols[1]], dayfirst=True + ) + df.drop(columns=index_cols, inplace=True) + # do some conversions df.loc[df[value_col] == 999999999, value_col] = np.NaN df[value_col] = df[value_col] / 100.0 diff --git a/hydropandas/obs_collection.py b/hydropandas/obs_collection.py index 8acb3cf9..11d5173e 100644 --- a/hydropandas/obs_collection.py +++ b/hydropandas/obs_collection.py @@ -725,6 +725,39 @@ def read_wiski( return oc +def read_pastastore( + pstore, + libname, + ObsClass=obs.GroundwaterObs, + metadata_mapping=None, +): + """Read pastastore library. + + Parameters + ---------- + pstore : pastastore.PastaStore + PastaStore object + libname : str + name of library (e.g. oseries or stresses) + ObsClass : Obs, optional + type of Obs to read data as, by default obs.GroundwaterObs + metadata_mapping : dict, optional + dictionary containing map between metadata field names in pastastore and + metadata field names expected by hydropandas, by default None. + + Returns + ------- + ObsCollection + ObsCollection containing data + """ + return ObsCollection.from_pastastore( + pstore=pstore, + libname=libname, + ObsClass=ObsClass, + metadata_mapping=metadata_mapping, + ) + + class ObsCollection(pd.DataFrame): """class for a collection of point observations. @@ -1848,6 +1881,43 @@ def from_wiski( return cls(obs_df, name=name, meta=meta) + @classmethod + def from_pastastore( + cls, pstore, libname, ObsClass=obs.GroundwaterObs, metadata_mapping=None + ): + """Read pastastore library. + + Parameters + ---------- + pstore : pastastore.PastaStore + PastaStore object + libname : str + name of library (e.g. oseries or stresses) + ObsClass : Obs, optional + type of Obs to read data as, by default obs.GroundwaterObs + metadata_mapping : dict, optional + dictionary containing map between metadata field names in pastastore and + metadata field names expected by hydropandas, by default None. + + Returns + ------- + ObsCollection + ObsCollection containing data + """ + from .io import pastas + + obs_list = pastas.read_pastastore_library( + pstore, libname, ObsClass=ObsClass, metadata_mapping=metadata_mapping + ) + obs_df = util._obslist_to_frame(obs_list) + + meta = { + "name": pstore.name, + "conntype": pstore.conn.conn_type, + "library": libname, + } + return cls(obs_df, name=pstore.name, meta=meta) + def to_excel(self, path, meta_sheet_name="metadata"): """Write an ObsCollection to an excel, the first sheet in the excel contains the metadata, the other tabs are the timeseries of each @@ -1954,7 +2024,8 @@ def to_pastastore( Name of the column in the Obs dataframe to be used. If None the first numeric column in the Obs Dataframe is used. kind : str, optional - The kind of series that is added to the pastastore + The kind of series that is added to the pastastore. Use 'oseries' + for observations and anything else for stresses. add_metadata : boolean, optional If True metadata from the observations added to the pastastore conn : pastastore.connectors or None, optional diff --git a/hydropandas/observation.py b/hydropandas/observation.py index 7fc8d59e..730e3882 100644 --- a/hydropandas/observation.py +++ b/hydropandas/observation.py @@ -745,6 +745,38 @@ def from_wiski(cls, path, **kwargs): return cls(data, meta=metadata, **metadata) + @classmethod + def from_pastastore(cls, pstore, libname, name, metadata_mapping=None): + """Read item from pastastore library. + + Parameters + ---------- + pstore : pastastore.PastaStore + pastastore object + libname : str + name of library containinig item + name : str + name of item + metadata_mapping : dict, optional + dictionary containing map between metadata field names in pastastore (keys) + and metadata field names expected by hydropandas (values), by default None. + """ + from .io import pastas + + data, metadata = pastas.read_pastastore_item(pstore, libname, name) + + if metadata_mapping is not None: + for pstore_name, oc_name in metadata_mapping.items(): + metadata[oc_name] = metadata.get(pstore_name, None) + + metadata["source"] = "pastastore" + kwargs = {} + for key, value in metadata.items(): + if key in cls._metadata: + kwargs[key] = value + + return cls(data, meta=metadata, **kwargs) + class WaterQualityObs(Obs): """class for water quality ((grond)watersamenstelling) point diff --git a/hydropandas/version.py b/hydropandas/version.py index 7db59c43..c5981731 100644 --- a/hydropandas/version.py +++ b/hydropandas/version.py @@ -1 +1 @@ -__version__ = "0.9.3b" +__version__ = "0.9.3" diff --git a/pyproject.toml b/pyproject.toml index d0d992ac..26b7f91d 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -87,6 +87,7 @@ version = { attr = "hydropandas.version.__version__" } [tool.setuptools.package-data] "hydropandas.data" = ["*.json"] +"hydropandas.static" = ["*.html", "*.css"] [tool.black] line-length = 88 diff --git a/tests/data/waterinfo-test/20200128_044.csv b/tests/data/2023-waterinfo-test/20200128_044.csv similarity index 100% rename from tests/data/waterinfo-test/20200128_044.csv rename to tests/data/2023-waterinfo-test/20200128_044.csv diff --git a/tests/data/2023-waterinfo-test/20231023_017.csv b/tests/data/2023-waterinfo-test/20231023_017.csv new file mode 100644 index 00000000..f76bd93b --- /dev/null +++ b/tests/data/2023-waterinfo-test/20231023_017.csv @@ -0,0 +1 @@ +MONSTER_IDENTIFICATIE;MEETPUNT_IDENTIFICATIE;LOCATIE_CODE;TYPERING_OMSCHRIJVING;TYPERING_CODE;GROOTHEID_OMSCHRIJVING;GROOTHEID_ CODE;PARAMETER_OMSCHRIJVING;PARAMETER_ CODE;CAS_NR;EENHEID_CODE;HOEDANIGHEID_OMSCHRIJVING;HOEDANIGHEID_CODE;COMPARTIMENT_OMSCHRIJVING;COMPARTIMENT_CODE;WAARDEBEWERKINGSMETHODE_OMSCHRIJVING;WAARDEBEWERKINGSMETHODE_CODE;WAARDEBEPALINGSMETHODE_OMSCHRIJVING;WAARDEBEPALINGSMETHODE_CODE;BEMONSTERINGSSOORT_OMSCHRIJVING;BEMONSTERINGSSOORT_CODE;WAARNEMINGDATUM;WAARNEMINGTIJD (MET/CET);LIMIETSYMBOOL;NUMERIEKEWAARDE;ALFANUMERIEKEWAARDE;KWALITEITSOORDEEL_CODE;REFERENTIE;NOTITIE_CODE;NOTITIE_OMSCHRIJVING;STATUSWAARDE;OPDRACHTGEVENDE_INSTANTIE;MEETAPPARAAT_OMSCHRIJVING;MEETAPPARAAT_CODE;BEMONSTERINGSAPPARAAT_OMSCHRIJVING;BEMONSTERINGSAPPARAAT_CODE;PLAATSBEPALINGSAPPARAAT_OMSCHRIJVING;PLAATSBEPALINGSAPPARAAT_CODE;BEMONSTERINGSHOOGTE;REFERENTIEVLAK;EPSG;X;Y;ORGAAN_OMSCHRIJVING;ORGAAN_CODE;TAXON_NAME;GROEPERING_OMSCHRIJVING;GROEPERING_CODE;GROEPERING_KANAAL;GROEPERING_TYPE ;Krimpen a/d Lek;KRIMPADLK;;;Waterhoogte;WATHTE;;;;cm;t.o.v. Normaal Amsterdams Peil;NAP;Oppervlaktewater;OW;;;Rekenkundig gemiddelde waarde over vorige 5 en volgende 5 minuten;other:F007;Steekbemonstering;SB;01-01-2022;00:00:00;;43;43;Normale waarde;;;;Gecontroleerd;RIKZMON_WAT;Vlotter;127;;;;;-999999999;NVT;25831;612093,755309996;5750138,05579548;;;;;;; ;Krimpen a/d Lek;KRIMPADLK;;;Waterhoogte;WATHTE;;;;cm;t.o.v. Normaal Amsterdams Peil;NAP;Oppervlaktewater;OW;;;Rekenkundig gemiddelde waarde over vorige 5 en volgende 5 minuten;other:F007;Steekbemonstering;SB;01-01-2022;00:10:00;;51;51;Normale waarde;;;;Gecontroleerd;RIKZMON_WAT;Vlotter;127;;;;;-999999999;NVT;25831;612093,755309996;5750138,05579548;;;;;;; ;Krimpen a/d Lek;KRIMPADLK;;;Waterhoogte;WATHTE;;;;cm;t.o.v. Normaal Amsterdams Peil;NAP;Oppervlaktewater;OW;;;Rekenkundig gemiddelde waarde over vorige 5 en volgende 5 minuten;other:F007;Steekbemonstering;SB;01-01-2022;00:20:00;;60;60;Normale waarde;;;;Gecontroleerd;RIKZMON_WAT;Vlotter;127;;;;;-999999999;NVT;25831;612093,755309996;5750138,05579548;;;;;;; ;Krimpen a/d Lek;KRIMPADLK;;;Waterhoogte;WATHTE;;;;cm;t.o.v. Normaal Amsterdams Peil;NAP;Oppervlaktewater;OW;;;Rekenkundig gemiddelde waarde over vorige 5 en volgende 5 minuten;other:F007;Steekbemonstering;SB;01-01-2022;00:30:00;;70;70;Normale waarde;;;;Gecontroleerd;RIKZMON_WAT;Vlotter;127;;;;;-999999999;NVT;25831;612093,755309996;5750138,05579548;;;;;;; ;Krimpen a/d Lek;KRIMPADLK;;;Waterhoogte;WATHTE;;;;cm;t.o.v. Normaal Amsterdams Peil;NAP;Oppervlaktewater;OW;;;Rekenkundig gemiddelde waarde over vorige 5 en volgende 5 minuten;other:F007;Steekbemonstering;SB;01-01-2022;00:40:00;;82;82;Normale waarde;;;;Gecontroleerd;RIKZMON_WAT;Vlotter;127;;;;;-999999999;NVT;25831;612093,755309996;5750138,05579548;;;;;;; \ No newline at end of file diff --git a/tests/data/waterinfo-test/20200128_045.csv b/tests/data/waterinfo-test/20200128_045.csv deleted file mode 100644 index 284abff7..00000000 --- a/tests/data/waterinfo-test/20200128_045.csv +++ /dev/null @@ -1 +0,0 @@ -MONSTER_IDENTIFICATIE;MEETPUNT_IDENTIFICATIE;TYPERING_OMSCHRIJVING;TYPERING_CODE;GROOTHEID_OMSCHRIJVING;GROOTHEID_ CODE;PARAMETER_OMSCHRIJVING;PARAMETER_ CODE;EENHEID_CODE;HOEDANIGHEID_OMSCHRIJVING;HOEDANIGHEID_CODE;COMPARTIMENT_OMSCHRIJVING;COMPARTIMENT_CODE;WAARDEBEWERKINGSMETHODE_OMSCHRIJVING;WAARDEBEWERKINGSMETHODE_CODE;WAARDEBEPALINGSMETHODE_OMSCHRIJVING;WAARDEBEPALINGSMETHODE_CODE;BEMONSTERINGSSOORT_OMSCHRIJVING;BEMONSTERINGSSOORT_CODE;WAARNEMINGDATUM;WAARNEMINGTIJD;LIMIETSYMBOOL;NUMERIEKEWAARDE;ALFANUMERIEKEWAARDE;KWALITEITSOORDEEL_CODE;STATUSWAARDE;OPDRACHTGEVENDE_INSTANTIE;MEETAPPARAAT_OMSCHRIJVING;MEETAPPARAAT_CODE;BEMONSTERINGSAPPARAAT_OMSCHRIJVING;BEMONSTERINGSAPPARAAT_CODE;PLAATSBEPALINGSAPPARAAT_OMSCHRIJVING;PLAATSBEPALINGSAPPARAAT_CODE;BEMONSTERINGSHOOGTE;REFERENTIEVLAK;EPSG;X;Y;ORGAAN_OMSCHRIJVING;ORGAAN_CODE;TAXON_NAME ;Driel beneden;;;Waterhoogte;WATHTE;;;cm;t.o.v. Normaal Amsterdams Peil;NAP;Oppervlaktewater;OW;;;Rekenkundig gemiddelde waarde over vorige 10 minuten;other:F001;Rechtstreekse meting;01;01-01-1990;00:00:00;;717;;Normale waarde;Definitief;RIKZMON_WAT;Vlotter;127;;;;;-999999999;NVT;25831;692540,602099716;5760632,32322453;;; ;Driel beneden;;;Waterhoogte;WATHTE;;;cm;t.o.v. Normaal Amsterdams Peil;NAP;Oppervlaktewater;OW;;;Rekenkundig gemiddelde waarde over vorige 10 minuten;other:F001;Rechtstreekse meting;01;01-01-1990;01:00:00;;716;;Normale waarde;Definitief;RIKZMON_WAT;Vlotter;127;;;;;-999999999;NVT;25831;692540,602099716;5760632,32322453;;; ;Driel beneden;;;Waterhoogte;WATHTE;;;cm;t.o.v. Normaal Amsterdams Peil;NAP;Oppervlaktewater;OW;;;Rekenkundig gemiddelde waarde over vorige 10 minuten;other:F001;Rechtstreekse meting;01;01-01-1990;02:00:00;;715;;Normale waarde;Definitief;RIKZMON_WAT;Vlotter;127;;;;;-999999999;NVT;25831;692540,602099716;5760632,32322453;;; ;Driel beneden;;;Waterhoogte;WATHTE;;;cm;t.o.v. Normaal Amsterdams Peil;NAP;Oppervlaktewater;OW;;;Rekenkundig gemiddelde waarde over vorige 10 minuten;other:F001;Rechtstreekse meting;01;01-01-1990;03:00:00;;714;;Normale waarde;Definitief;RIKZMON_WAT;Vlotter;127;;;;;-999999999;NVT;25831;692540,602099716;5760632,32322453;;; \ No newline at end of file diff --git a/tests/test_001_to_from.py b/tests/test_001_to_from.py index fbcae2b1..811f65ea 100644 --- a/tests/test_001_to_from.py +++ b/tests/test_001_to_from.py @@ -1,4 +1,7 @@ +import os + import pandas as pd +import pastastore as pst import pytest from requests.exceptions import ConnectionError @@ -193,7 +196,17 @@ def test_to_pastastore(): dino_gw = test_obscollection_dinozip_gw() # drop duplicate dino_gw.drop("B22D0155-001", inplace=True) - dino_gw.to_pastastore() + pstore = dino_gw.to_pastastore() + # export to zip for read test + pstore.to_zip("test_pastastore.zip") + + +def test_from_pastastore(): + pstore = pst.PastaStore.from_zip( + "test_pastastore.zip", conn=pst.DictConnector("pastas_db") + ) + _ = hpd.read_pastastore(pstore, "oseries") + os.remove("test_pastastore.zip") # %% excel @@ -325,7 +338,7 @@ def test_knmi_collection_from_grid(): def test_waterinfo_from_dir(): - path = "./tests/data/waterinfo-test" + path = "./tests/data/2023-waterinfo-test" hpd.read_waterinfo(path)