From 988479be08d84cbc1bdfda72213dc855861cbfae Mon Sep 17 00:00:00 2001 From: Baudouin Raoult Date: Thu, 22 Feb 2024 18:19:43 +0000 Subject: [PATCH 1/4] update --- climetlab/readers/grib/output.py | 6 +- climetlab/readers/netcdf/__init__.py | 6 +- climetlab/readers/netcdf/dataset.py | 1 + climetlab/readers/netcdf/field.py | 8 +-- climetlab/readers/netcdf/fieldset.py | 87 ++++++++++++++++------------ climetlab/sources/opendap.py | 4 +- 6 files changed, 62 insertions(+), 50 deletions(-) diff --git a/climetlab/readers/grib/output.py b/climetlab/readers/grib/output.py index a2499f51..2714f722 100644 --- a/climetlab/readers/grib/output.py +++ b/climetlab/readers/grib/output.py @@ -199,9 +199,9 @@ def update_metadata(self, handle, metadata, compulsary): if "number" in metadata: compulsary += ("numberOfForecastsInEnsemble",) productDefinitionTemplateNumber = {"tp": 11} - metadata["productDefinitionTemplateNumber"] = ( - productDefinitionTemplateNumber.get(handle.get("shortName"), 1) - ) + metadata[ + "productDefinitionTemplateNumber" + ] = productDefinitionTemplateNumber.get(handle.get("shortName"), 1) if metadata.get("type") in ("pf", "cf"): metadata.setdefault("typeOfGeneratingProcess", 4) diff --git a/climetlab/readers/netcdf/__init__.py b/climetlab/readers/netcdf/__init__.py index bd4265c1..09e872fa 100644 --- a/climetlab/readers/netcdf/__init__.py +++ b/climetlab/readers/netcdf/__init__.py @@ -9,13 +9,13 @@ from .. import Reader -from .fieldset import NetCDFFieldSet +from .fieldset import NetCDFFieldSetFromFile -class NetCDFReader(Reader, NetCDFFieldSet): +class NetCDFReader(Reader, NetCDFFieldSetFromFile): def __init__(self, source, path): Reader.__init__(self, source, path) - NetCDFFieldSet.__init__(self, path) + NetCDFFieldSetFromFile.__init__(self, path) def reader(source, path, magic=None, deeper_check=False): diff --git a/climetlab/readers/netcdf/dataset.py b/climetlab/readers/netcdf/dataset.py index 5b4b6463..45644efc 100644 --- a/climetlab/readers/netcdf/dataset.py +++ b/climetlab/readers/netcdf/dataset.py @@ -17,6 +17,7 @@ class DataSet: """ def __init__(self, ds): + assert ds is not None self._ds = ds self._cache = {} diff --git a/climetlab/readers/netcdf/field.py b/climetlab/readers/netcdf/field.py index d8554419..3ff3a46b 100644 --- a/climetlab/readers/netcdf/field.py +++ b/climetlab/readers/netcdf/field.py @@ -95,20 +95,20 @@ def tidy(x): return x return tidy( - self.owner.dataset[self.owner.dataset[self.variable].grid_mapping].attrs + self.owner.xr_dataset[self.owner.xr_dataset[self.variable].grid_mapping].attrs ) # Compatibility to GRIb fields below def grid_points(self): - return DataSet(self.owner.dataset).grid_points(self.variable) + return DataSet(self.owner.xr_dataset).grid_points(self.variable) def grid_points_xy(self): - return DataSet(self.owner.dataset).grid_points_xy(self.variable) + return DataSet(self.owner.xr_dataset).grid_points_xy(self.variable) def to_numpy(self, reshape=True, dtype=None): dimensions = dict((s.name, s.index) for s in self.slices) - values = self.owner.dataset[self.variable].isel(dimensions).values + values = self.owner.xr_dataset[self.variable].isel(dimensions).values if not reshape: values = values.flatten() if dtype is not None: diff --git a/climetlab/readers/netcdf/fieldset.py b/climetlab/readers/netcdf/fieldset.py index b07896cc..8989b1da 100644 --- a/climetlab/readers/netcdf/fieldset.py +++ b/climetlab/readers/netcdf/fieldset.py @@ -21,17 +21,10 @@ class NetCDFFieldSet(FieldSet): - def __init__(self, path): - self.path = path - self.opendap = path.startswith("http") - @classmethod def new_mask_index(self, *args, **kwargs): return NetCDFMaskFieldSet(*args, **kwargs) - def __repr__(self): - return "NetCDFReader(%s)" % (self.path,) - def __iter__(self): return iter(self.fields) @@ -41,21 +34,9 @@ def __len__(self): def __getitem__(self, n): return self.fields[n] - @cached_property - def dataset(self): - import xarray as xr - - if ".zarr" in self.path: - return xr.open_zarr(self.path) - - if self.opendap: - return xr.open_dataset(self.path) - else: - return xr.open_mfdataset(self.path, combine="by_coords") - @cached_property def fields(self): - return self._get_fields(DataSet(self.dataset)) + return self._get_fields(DataSet(self.xr_dataset)) def _get_fields(self, ds): # noqa C901 # Select only geographical variables @@ -170,11 +151,6 @@ def to_xarray_multi_from_paths(cls, paths, **kwargs): **options, ) - def to_metview(self): - from climetlab.metview import mv_read - - return mv_read(self.path) - def plot_map(self, *args, **kwargs): return self.fields[0].plot_map(*args, **kwargs) @@ -201,19 +177,47 @@ def merge(cls, sources): return NetCDFMultiFieldSet(sources) +class NetCDFFieldSetFromFileOrURL(NetCDFFieldSet): + def __init__(self, path_or_url): + self.path_or_url = path_or_url + + +class NetCDFFieldSetFromFile(NetCDFFieldSetFromFileOrURL): + def __init__(self, path): + super().__init__(path) + + def __repr__(self): + return "NetCDFFieldSetFromFile(%s)" % (self.path_or_url,) + + def to_metview(self): + from climetlab.metview import mv_read + + return mv_read(self.path_or_url) + + +class NetCDFFieldSetFromURL(NetCDFFieldSetFromFileOrURL): + def __init__(self, url): + super().__init__(url) + + def __repr__(self): + return "NetCDFFieldSetFromURL(%s)" % (self.path_or_url,) + + @cached_property + def xr_dataset(self): + import xarray as xr + + return xr.open_dataset(self.path_or_url) + + class NetCDFMaskFieldSet(NetCDFFieldSet, MaskIndex): def __init__(self, *args, **kwargs): MaskIndex.__init__(self, *args, **kwargs) self.path = "" - def __iter__(self): - return MaskIndex.__iter__(self) - def __len__(self): - return MaskIndex.__len__(self) - - def __getitem__(self, n): - return MaskIndex.__getitem__(self, n) + @cached_property + def fields(self): + return list(self.index[i] for i in self.indices) class NetCDFMultiFieldSet(NetCDFFieldSet, MultiIndex): @@ -230,14 +234,21 @@ def to_xarray(self, **kwargs): return xr.open_mfdataset(self.paths, **kwargs) @cached_property - def dataset(self): - return self.to_xarray(combine="by_coords") + def fields(self): + result = [] + for s in self.indexes: + result.extend(s.fields) + return result - def __iter__(self): - return MaskIndex.__iter__(self) + # @cached_property + # def dataset(self): + # return self.to_xarray(combine="by_coords") + + # def __iter__(self): + # return MultiIndex.__iter__(self) def __len__(self): - return MaskIndex.__len__(self) + return MultiIndex.__len__(self) def __getitem__(self, n): - return MaskIndex.__getitem__(self, n) + return MultiIndex.__getitem__(self, n) diff --git a/climetlab/sources/opendap.py b/climetlab/sources/opendap.py index 45570f28..16f7cacd 100644 --- a/climetlab/sources/opendap.py +++ b/climetlab/sources/opendap.py @@ -7,7 +7,7 @@ # nor does it submit to any jurisdiction. # -from climetlab.readers.netcdf import NetCDFFieldSet +from climetlab.readers.netcdf.fieldset import NetCDFFieldSetFromURL from climetlab.sources import Source @@ -16,7 +16,7 @@ def __init__(self, url): self.url = url def mutate(self): - return NetCDFFieldSet(self.url) + return NetCDFFieldSetFromURL(self.url) source = OpenDAP From 4c05966432666997e048fba87f87f90a34cfa1e8 Mon Sep 17 00:00:00 2001 From: Baudouin Raoult Date: Thu, 22 Feb 2024 18:47:35 +0000 Subject: [PATCH 2/4] update --- climetlab/readers/netcdf/field.py | 4 +++- climetlab/readers/netcdf/fieldset.py | 20 ++++++-------------- tests/readers/test_netcdf_reader.py | 2 +- 3 files changed, 10 insertions(+), 16 deletions(-) diff --git a/climetlab/readers/netcdf/field.py b/climetlab/readers/netcdf/field.py index 3ff3a46b..6c535cfb 100644 --- a/climetlab/readers/netcdf/field.py +++ b/climetlab/readers/netcdf/field.py @@ -95,7 +95,9 @@ def tidy(x): return x return tidy( - self.owner.xr_dataset[self.owner.xr_dataset[self.variable].grid_mapping].attrs + self.owner.xr_dataset[ + self.owner.xr_dataset[self.variable].grid_mapping + ].attrs ) # Compatibility to GRIb fields below diff --git a/climetlab/readers/netcdf/fieldset.py b/climetlab/readers/netcdf/fieldset.py index 8989b1da..1527caf6 100644 --- a/climetlab/readers/netcdf/fieldset.py +++ b/climetlab/readers/netcdf/fieldset.py @@ -181,6 +181,12 @@ class NetCDFFieldSetFromFileOrURL(NetCDFFieldSet): def __init__(self, path_or_url): self.path_or_url = path_or_url + @cached_property + def xr_dataset(self): + import xarray as xr + + return xr.open_dataset(self.path_or_url) + class NetCDFFieldSetFromFile(NetCDFFieldSetFromFileOrURL): def __init__(self, path): @@ -202,19 +208,12 @@ def __init__(self, url): def __repr__(self): return "NetCDFFieldSetFromURL(%s)" % (self.path_or_url,) - @cached_property - def xr_dataset(self): - import xarray as xr - - return xr.open_dataset(self.path_or_url) - class NetCDFMaskFieldSet(NetCDFFieldSet, MaskIndex): def __init__(self, *args, **kwargs): MaskIndex.__init__(self, *args, **kwargs) self.path = "" - @cached_property def fields(self): return list(self.index[i] for i in self.indices) @@ -240,13 +239,6 @@ def fields(self): result.extend(s.fields) return result - # @cached_property - # def dataset(self): - # return self.to_xarray(combine="by_coords") - - # def __iter__(self): - # return MultiIndex.__iter__(self) - def __len__(self): return MultiIndex.__len__(self) diff --git a/tests/readers/test_netcdf_reader.py b/tests/readers/test_netcdf_reader.py index 664af369..a8ddbfa8 100644 --- a/tests/readers/test_netcdf_reader.py +++ b/tests/readers/test_netcdf_reader.py @@ -27,7 +27,7 @@ def test_netcdf(): def test_dummy_netcdf_reader_1(): s = load_source("file", climetlab_file("docs/examples/test.nc")) r = s._reader - assert str(r).startswith("NetCDFReader"), r + assert str(r).startswith("NetCDFFieldSetFromFile"), r assert len(r) == 2 assert isinstance(r[1], NetCDFField), r From eb9d42871feb1384ab20b7a6e2fe6a497513b5b9 Mon Sep 17 00:00:00 2001 From: Florian Pinault Date: Fri, 23 Feb 2024 07:40:34 +0000 Subject: [PATCH 3/4] qa --- climetlab/readers/grib/output.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/climetlab/readers/grib/output.py b/climetlab/readers/grib/output.py index 2714f722..a2499f51 100644 --- a/climetlab/readers/grib/output.py +++ b/climetlab/readers/grib/output.py @@ -199,9 +199,9 @@ def update_metadata(self, handle, metadata, compulsary): if "number" in metadata: compulsary += ("numberOfForecastsInEnsemble",) productDefinitionTemplateNumber = {"tp": 11} - metadata[ - "productDefinitionTemplateNumber" - ] = productDefinitionTemplateNumber.get(handle.get("shortName"), 1) + metadata["productDefinitionTemplateNumber"] = ( + productDefinitionTemplateNumber.get(handle.get("shortName"), 1) + ) if metadata.get("type") in ("pf", "cf"): metadata.setdefault("typeOfGeneratingProcess", 4) From f5c0ebcd199264ef7d65de37b82b6b330b327847 Mon Sep 17 00:00:00 2001 From: Florian Pinault Date: Fri, 23 Feb 2024 07:41:22 +0000 Subject: [PATCH 4/4] Bump version 0.21.0 --- climetlab/version | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/climetlab/version b/climetlab/version index c7f4ab0c..88541566 100644 --- a/climetlab/version +++ b/climetlab/version @@ -1 +1 @@ -0.20.15 +0.21.0