Merge branch 'release/0.21.0'

climetlab/readers/netcdf/__init__.py

@@ -9,13 +9,13 @@
 
 
 from .. import Reader
-from .fieldset import NetCDFFieldSet
+from .fieldset import NetCDFFieldSetFromFile
 
 
-class NetCDFReader(Reader, NetCDFFieldSet):
+class NetCDFReader(Reader, NetCDFFieldSetFromFile):
     def __init__(self, source, path):
         Reader.__init__(self, source, path)
-        NetCDFFieldSet.__init__(self, path)
+        NetCDFFieldSetFromFile.__init__(self, path)
 
 
 def reader(source, path, magic=None, deeper_check=False):

climetlab/readers/netcdf/dataset.py

@@ -17,6 +17,7 @@ class DataSet:
     """
 
     def __init__(self, ds):
+        assert ds is not None
         self._ds = ds
         self._cache = {}
 

climetlab/readers/netcdf/field.py

@@ -95,20 +95,22 @@ def tidy(x):
             return x
 
         return tidy(
-            self.owner.dataset[self.owner.dataset[self.variable].grid_mapping].attrs
+            self.owner.xr_dataset[
+                self.owner.xr_dataset[self.variable].grid_mapping
+            ].attrs
         )
 
     # Compatibility to GRIb fields below
 
     def grid_points(self):
-        return DataSet(self.owner.dataset).grid_points(self.variable)
+        return DataSet(self.owner.xr_dataset).grid_points(self.variable)
 
     def grid_points_xy(self):
-        return DataSet(self.owner.dataset).grid_points_xy(self.variable)
+        return DataSet(self.owner.xr_dataset).grid_points_xy(self.variable)
 
     def to_numpy(self, reshape=True, dtype=None):
         dimensions = dict((s.name, s.index) for s in self.slices)
-        values = self.owner.dataset[self.variable].isel(dimensions).values
+        values = self.owner.xr_dataset[self.variable].isel(dimensions).values
         if not reshape:
             values = values.flatten()
         if dtype is not None:

climetlab/readers/netcdf/fieldset.py

@@ -21,17 +21,10 @@
 
 
 class NetCDFFieldSet(FieldSet):
-    def __init__(self, path):
-        self.path = path
-        self.opendap = path.startswith("http")
-
     @classmethod
     def new_mask_index(self, *args, **kwargs):
         return NetCDFMaskFieldSet(*args, **kwargs)
 
-    def __repr__(self):
-        return "NetCDFReader(%s)" % (self.path,)
-
     def __iter__(self):
         return iter(self.fields)
 
@@ -41,21 +34,9 @@ def __len__(self):
     def __getitem__(self, n):
         return self.fields[n]
 
-    @cached_property
-    def dataset(self):
-        import xarray as xr
-
-        if ".zarr" in self.path:
-            return xr.open_zarr(self.path)
-
-        if self.opendap:
-            return xr.open_dataset(self.path)
-        else:
-            return xr.open_mfdataset(self.path, combine="by_coords")
-
     @cached_property
     def fields(self):
-        return self._get_fields(DataSet(self.dataset))
+        return self._get_fields(DataSet(self.xr_dataset))
 
     def _get_fields(self, ds):  # noqa C901
         # Select only geographical variables
@@ -170,11 +151,6 @@ def to_xarray_multi_from_paths(cls, paths, **kwargs):
             **options,
         )
 
-    def to_metview(self):
-        from climetlab.metview import mv_read
-
-        return mv_read(self.path)
-
     def plot_map(self, *args, **kwargs):
         return self.fields[0].plot_map(*args, **kwargs)
 
@@ -201,19 +177,46 @@ def merge(cls, sources):
         return NetCDFMultiFieldSet(sources)
 
 
+class NetCDFFieldSetFromFileOrURL(NetCDFFieldSet):
+    def __init__(self, path_or_url):
+        self.path_or_url = path_or_url
+
+    @cached_property
+    def xr_dataset(self):
+        import xarray as xr
+
+        return xr.open_dataset(self.path_or_url)
+
+
+class NetCDFFieldSetFromFile(NetCDFFieldSetFromFileOrURL):
+    def __init__(self, path):
+        super().__init__(path)
+
+    def __repr__(self):
+        return "NetCDFFieldSetFromFile(%s)" % (self.path_or_url,)
+
+    def to_metview(self):
+        from climetlab.metview import mv_read
+
+        return mv_read(self.path_or_url)
+
+
+class NetCDFFieldSetFromURL(NetCDFFieldSetFromFileOrURL):
+    def __init__(self, url):
+        super().__init__(url)
+
+    def __repr__(self):
+        return "NetCDFFieldSetFromURL(%s)" % (self.path_or_url,)
+
+
 class NetCDFMaskFieldSet(NetCDFFieldSet, MaskIndex):
     def __init__(self, *args, **kwargs):
         MaskIndex.__init__(self, *args, **kwargs)
         self.path = "<mask>"
 
-    def __iter__(self):
-        return MaskIndex.__iter__(self)
-
-    def __len__(self):
-        return MaskIndex.__len__(self)
-
-    def __getitem__(self, n):
-        return MaskIndex.__getitem__(self, n)
+    @cached_property
+    def fields(self):
+        return list(self.index[i] for i in self.indices)
 
 
 class NetCDFMultiFieldSet(NetCDFFieldSet, MultiIndex):
@@ -230,14 +233,14 @@ def to_xarray(self, **kwargs):
         return xr.open_mfdataset(self.paths, **kwargs)
 
     @cached_property
-    def dataset(self):
-        return self.to_xarray(combine="by_coords")
-
-    def __iter__(self):
-        return MaskIndex.__iter__(self)
+    def fields(self):
+        result = []
+        for s in self.indexes:
+            result.extend(s.fields)
+        return result
 
     def __len__(self):
-        return MaskIndex.__len__(self)
+        return MultiIndex.__len__(self)
 
     def __getitem__(self, n):
-        return MaskIndex.__getitem__(self, n)
+        return MultiIndex.__getitem__(self, n)

climetlab/sources/opendap.py

@@ -7,7 +7,7 @@
 # nor does it submit to any jurisdiction.
 #
 
-from climetlab.readers.netcdf import NetCDFFieldSet
+from climetlab.readers.netcdf.fieldset import NetCDFFieldSetFromURL
 from climetlab.sources import Source
 
 
@@ -16,7 +16,7 @@ def __init__(self, url):
         self.url = url
 
     def mutate(self):
-        return NetCDFFieldSet(self.url)
+        return NetCDFFieldSetFromURL(self.url)
 
 
 source = OpenDAP

climetlab/version

@@ -1 +1 @@
-0.20.15
+0.21.0

tests/readers/test_netcdf_reader.py

@@ -27,7 +27,7 @@ def test_netcdf():
 def test_dummy_netcdf_reader_1():
     s = load_source("file", climetlab_file("docs/examples/test.nc"))
     r = s._reader
-    assert str(r).startswith("NetCDFReader"), r
+    assert str(r).startswith("NetCDFFieldSetFromFile"), r
     assert len(r) == 2
     assert isinstance(r[1], NetCDFField), r