update

climetlab/readers/grib/output.py

@@ -199,9 +199,9 @@ def update_metadata(self, handle, metadata, compulsary):
  if "number" in metadata:
  compulsary += ("numberOfForecastsInEnsemble",)
  productDefinitionTemplateNumber = {"tp": 11}
- metadata["productDefinitionTemplateNumber"] = (
- productDefinitionTemplateNumber.get(handle.get("shortName"), 1)
- )
+ metadata[
+ "productDefinitionTemplateNumber"
+ ] = productDefinitionTemplateNumber.get(handle.get("shortName"), 1)
 
  if metadata.get("type") in ("pf", "cf"):
  metadata.setdefault("typeOfGeneratingProcess", 4)

climetlab/readers/netcdf/__init__.py

@@ -9,13 +9,13 @@
 
 
 from .. import Reader
-from .fieldset import NetCDFFieldSet
+from .fieldset import NetCDFFieldSetFromFile
 
 
-class NetCDFReader(Reader, NetCDFFieldSet):
+class NetCDFReader(Reader, NetCDFFieldSetFromFile):
  def __init__(self, source, path):
  Reader.__init__(self, source, path)
- NetCDFFieldSet.__init__(self, path)
+ NetCDFFieldSetFromFile.__init__(self, path)
 
 
 def reader(source, path, magic=None, deeper_check=False):

climetlab/readers/netcdf/dataset.py

@@ -17,6 +17,7 @@ class DataSet:
  """
 
  def __init__(self, ds):
+ assert ds is not None
  self._ds = ds
  self._cache = {}
 

climetlab/readers/netcdf/field.py

@@ -95,20 +95,20 @@ def tidy(x):
  return x
 
  return tidy(
- self.owner.dataset[self.owner.dataset[self.variable].grid_mapping].attrs
+ self.owner.xr_dataset[self.owner.xr_dataset[self.variable].grid_mapping].attrs
  )
 
  # Compatibility to GRIb fields below
 
  def grid_points(self):
- return DataSet(self.owner.dataset).grid_points(self.variable)
+ return DataSet(self.owner.xr_dataset).grid_points(self.variable)
 
  def grid_points_xy(self):
- return DataSet(self.owner.dataset).grid_points_xy(self.variable)
+ return DataSet(self.owner.xr_dataset).grid_points_xy(self.variable)
 
  def to_numpy(self, reshape=True, dtype=None):
  dimensions = dict((s.name, s.index) for s in self.slices)
- values = self.owner.dataset[self.variable].isel(dimensions).values
+ values = self.owner.xr_dataset[self.variable].isel(dimensions).values
  if not reshape:
  values = values.flatten()
  if dtype is not None:

climetlab/readers/netcdf/fieldset.py

@@ -21,17 +21,10 @@
 
 
 class NetCDFFieldSet(FieldSet):
- def __init__(self, path):
- self.path = path
- self.opendap = path.startswith("http")
-
  @classmethod
  def new_mask_index(self, *args, **kwargs):
  return NetCDFMaskFieldSet(*args, **kwargs)
 
- def __repr__(self):
- return "NetCDFReader(%s)" % (self.path,)
-
  def __iter__(self):
  return iter(self.fields)
 
@@ -41,21 +34,9 @@ def __len__(self):
  def __getitem__(self, n):
  return self.fields[n]
 
- @cached_property
- def dataset(self):
- import xarray as xr
-
- if ".zarr" in self.path:
- return xr.open_zarr(self.path)
-
- if self.opendap:
- return xr.open_dataset(self.path)
- else:
- return xr.open_mfdataset(self.path, combine="by_coords")
-
  @cached_property
  def fields(self):
- return self._get_fields(DataSet(self.dataset))
+ return self._get_fields(DataSet(self.xr_dataset))
 
  def _get_fields(self, ds): # noqa C901
  # Select only geographical variables
@@ -170,11 +151,6 @@ def to_xarray_multi_from_paths(cls, paths, **kwargs):
  **options,
  )
 
- def to_metview(self):
- from climetlab.metview import mv_read
-
- return mv_read(self.path)
-
  def plot_map(self, *args, **kwargs):
  return self.fields[0].plot_map(*args, **kwargs)
 
@@ -201,19 +177,47 @@ def merge(cls, sources):
  return NetCDFMultiFieldSet(sources)
 
 
+class NetCDFFieldSetFromFileOrURL(NetCDFFieldSet):
+ def __init__(self, path_or_url):
+ self.path_or_url = path_or_url
+
+
+class NetCDFFieldSetFromFile(NetCDFFieldSetFromFileOrURL):
+ def __init__(self, path):
+ super().__init__(path)
+
+ def __repr__(self):
+ return "NetCDFFieldSetFromFile(%s)" % (self.path_or_url,)
+
+ def to_metview(self):
+ from climetlab.metview import mv_read
+
+ return mv_read(self.path_or_url)
+
+
+class NetCDFFieldSetFromURL(NetCDFFieldSetFromFileOrURL):
+ def __init__(self, url):
+ super().__init__(url)
+
+ def __repr__(self):
+ return "NetCDFFieldSetFromURL(%s)" % (self.path_or_url,)
+
+ @cached_property
+ def xr_dataset(self):
+ import xarray as xr
+
+ return xr.open_dataset(self.path_or_url)
+
+
 class NetCDFMaskFieldSet(NetCDFFieldSet, MaskIndex):
  def __init__(self, *args, **kwargs):
  MaskIndex.__init__(self, *args, **kwargs)
  self.path = "<mask>"
 
- def __iter__(self):
- return MaskIndex.__iter__(self)
 
- def __len__(self):
- return MaskIndex.__len__(self)
-
- def __getitem__(self, n):
- return MaskIndex.__getitem__(self, n)
+ @cached_property
+ def fields(self):
+ return list(self.index[i] for i in self.indices)
 
 
 class NetCDFMultiFieldSet(NetCDFFieldSet, MultiIndex):
@@ -230,14 +234,21 @@ def to_xarray(self, **kwargs):
  return xr.open_mfdataset(self.paths, **kwargs)
 
  @cached_property
- def dataset(self):
- return self.to_xarray(combine="by_coords")
+ def fields(self):
+ result = []
+ for s in self.indexes:
+ result.extend(s.fields)
+ return result
 
- def __iter__(self):
- return MaskIndex.__iter__(self)
+ # @cached_property
+ # def dataset(self):
+ # return self.to_xarray(combine="by_coords")
+
+ # def __iter__(self):
+ # return MultiIndex.__iter__(self)
 
  def __len__(self):
- return MaskIndex.__len__(self)
+ return MultiIndex.__len__(self)
 
  def __getitem__(self, n):
- return MaskIndex.__getitem__(self, n)
+ return MultiIndex.__getitem__(self, n)

climetlab/sources/opendap.py

@@ -7,7 +7,7 @@
 # nor does it submit to any jurisdiction.
 #
 
-from climetlab.readers.netcdf import NetCDFFieldSet
+from climetlab.readers.netcdf.fieldset import NetCDFFieldSetFromURL
 from climetlab.sources import Source
 
 
@@ -16,7 +16,7 @@ def __init__(self, url):
  self.url = url
 
  def mutate(self):
- return NetCDFFieldSet(self.url)
+ return NetCDFFieldSetFromURL(self.url)
 
 
 source = OpenDAP