Merge branch 'release/0.20.3'

.pre-commit-config.yaml

@@ -1,12 +1,12 @@
 exclude: 'experiments'
 repos:
 -   repo: https://github.com/ambv/black
-    rev: 23.1.0
+    rev: 24.1.1
     hooks:
       - id: black
         language_version: python3.10
 -   repo: https://github.com/pycqa/isort
-    rev: 5.12.0
+    rev: 5.13.0
     hooks:
       - id: isort
         name: isort (python)
@@ -17,6 +17,6 @@ repos:
         name: isort (pyi)
         types: [pyi]
 -   repo: https://github.com/pycqa/flake8
-    rev: 6.0.0
+    rev: 7.0.0
     hooks:
     -   id: flake8

climetlab/indexing/cube.py

@@ -67,8 +67,22 @@ def __init__(
 
         self._field_shape = None
 
+        # Sort the source according to their
+        # internal_args = reduce(operator.add, [Separator.split(a) for a in args], [])
+        # for i in ds:
+        #   print(i)
+        # print("before")
+        # print(ds[0])
+        # print(ds[1])
+        # print(ds[2])
+        # print(ds[3])
         self.source = ds.order_by(*args, remapping=remapping, patches=patches)
         del ds
+        # print("after")
+        # print(self.source[0])
+        # print(self.source[1])
+        # print(self.source[2])
+        # print(self.source[3])
 
         # Get a mapping of user names to unique values
         # With possible reduce dimentionality if the user use 'level+param'
@@ -238,14 +252,17 @@ def __init__(self, cube, coords, coords_names=None):
         self.owner = cube
         assert all(isinstance(_, int) for _ in coords), coords
         self.coords = coords
-        self.extended_icoords = self.coords
         self.flatten_values = cube.flatten_values
 
     def __repr__(self):
         return (
             f"{self.__class__.__name__}({self.coords},index_names={self._coords_names})"
         )
 
+    @property
+    def extended_icoords(self):
+        return self.coords
+
     def to_numpy(self, **kwargs):
         return self.owner[self.coords].to_numpy(
             reshape=not self.flatten_values, **kwargs

climetlab/indexing/fieldset.py

@@ -10,12 +10,12 @@
 from climetlab.core import Base
 from climetlab.core.index import Index, MaskIndex, MultiIndex
 
+
 class Field(Base):
     pass
 
-class FieldSet(Index):
-
 
+class FieldSet(Index):
     @classmethod
     def new_mask_index(self, *args, **kwargs):
         return MaskFieldSet(*args, **kwargs)

climetlab/readers/grib/codes.py

@@ -18,7 +18,6 @@
 
 from climetlab.core import Base
 from climetlab.core.constants import DATETIME
-from climetlab.indexing.fieldset import Field
 from climetlab.profiling import call_counter
 from climetlab.utils.bbox import BoundingBox
 
@@ -322,7 +321,7 @@ def at_offset(self, offset):
 # count = defaultdict(int)
 
 
-class GribField(Field):
+class GribField(Base):
     def __init__(self, path, offset, length, handle_cache=None):
         self.path = path
         self._offset = offset

climetlab/readers/grib/index/__init__.py

@@ -14,6 +14,7 @@
 
 from lru import LRU
 
+from climetlab.core.index import Index, MaskIndex, MultiIndex
 from climetlab.decorators import normalize_grib_key_values, normalize_grib_keys
 from climetlab.indexing.database import (
     FILEPARTS_KEY_NAMES,
@@ -25,12 +26,11 @@
 from climetlab.readers.grib.fieldset import FieldSetMixin
 from climetlab.utils import progress_bar
 from climetlab.utils.availability import Availability
-from climetlab.indexing.fieldset import FieldSet
 
 LOG = logging.getLogger(__name__)
 
 
-class GribFieldSet(FieldSetMixin, FieldSet):
+class FieldSet(FieldSetMixin, Index):
     _availability = None
 
     def __init__(self, *args, **kwargs):
@@ -39,12 +39,21 @@ def __init__(self, *args, **kwargs):
         ):
             self._availability = Availability(self.availability_path)
 
-        FieldSet.__init__(self, *args, **kwargs)
+        Index.__init__(self, *args, **kwargs)
+
+    @classmethod
+    def new_mask_index(self, *args, **kwargs):
+        return MaskFieldSet(*args, **kwargs)
 
     @property
     def availability_path(self):
         return None
 
+    @classmethod
+    def merge(cls, sources):
+        assert all(isinstance(_, FieldSet) for _ in sources)
+        return MultiFieldSet(sources)
+
     def available(self, request, as_list_of_dicts=False):
         from climetlab.utils.availability import Availability
 
@@ -141,7 +150,18 @@ def is_full_hypercube(self):
     def _normalize_kwargs_names(self, **kwargs):
         return kwargs
 
-class FieldSetInFiles(GribFieldSet):
+
+class MaskFieldSet(FieldSet, MaskIndex):
+    def __init__(self, *args, **kwargs):
+        MaskIndex.__init__(self, *args, **kwargs)
+
+
+class MultiFieldSet(FieldSet, MultiIndex):
+    def __init__(self, *args, **kwargs):
+        MultiIndex.__init__(self, *args, **kwargs)
+
+
+class FieldSetInFiles(FieldSet):
     # Remote Fieldsets (with urls) are also here,
     # as the actual fieldset is accessed on a file in cache.
     # This class changes the interface (_getitem__ and __len__)

climetlab/readers/grib/output.py

@@ -199,9 +199,9 @@ def update_metadata(self, handle, metadata, compulsary):
         if "number" in metadata:
             compulsary += ("numberOfForecastsInEnsemble",)
             productDefinitionTemplateNumber = {"tp": 11}
-            metadata[
-                "productDefinitionTemplateNumber"
-            ] = productDefinitionTemplateNumber.get(handle.get("shortName"), 1)
+            metadata["productDefinitionTemplateNumber"] = (
+                productDefinitionTemplateNumber.get(handle.get("shortName"), 1)
+            )
 
         if metadata.get("type") in ("pf", "cf"):
             metadata.setdefault("typeOfGeneratingProcess", 4)

climetlab/readers/grib/reader.py

@@ -10,7 +10,7 @@
 import logging
 
 from climetlab.readers import Reader
-from climetlab.indexing.fieldset import MultiFieldSet
+from climetlab.readers.grib.index import MultiFieldSet
 from climetlab.readers.grib.index.file import FieldSetInOneFile
 
 LOG = logging.getLogger(__name__)

climetlab/readers/netcdf/__init__.py

@@ -15,7 +15,6 @@
 
 import numpy as np
 
-from climetlab.core import Base
 from climetlab.indexing.fieldset import Field
 from climetlab.readers.netcdf.flavours import get_flavour
 from climetlab.utils.bbox import BoundingBox

climetlab/readers/netcdf/flavours.py

@@ -32,5 +32,6 @@ def get_levelist(self, field):
     def get_number(self, field):
         return None
 
+
 def get_flavour(reader, flavour):
     return Flavour("default", reader)

climetlab/sources/constants.py

@@ -16,7 +16,7 @@
 
 from climetlab.decorators import cached_method, normalize
 from climetlab.indexing.cube import index_to_coords
-from climetlab.readers.grib.index import GribFieldSet
+from climetlab.readers.grib.index import FieldSet
 from climetlab.utils.dates import to_datetime
 
 LOG = logging.getLogger(__name__)
@@ -239,7 +239,7 @@ def make_datetime(date, time):
     return datetime.datetime(date.year, date.month, date.day, time)
 
 
-class Constants(GribFieldSet):
+class Constants(FieldSet):
     def __init__(self, source_or_dataset, request={}, **kwargs):
         request = dict(**request)
         request.update(kwargs)

climetlab/sources/era5_accumulations.py

@@ -95,7 +95,7 @@ class Era5Accumulations(Source):
     Note 2 : There are no overlap due to the way the date+time+step are computed
     """
 
-    def __init__(self, *args, accumulation_period=6, **kwargs):
+    def __init__(self, *args, **kwargs):
         request = {}
         for a in args:
             request.update(self.requests(**a))
@@ -111,7 +111,7 @@ def __init__(self, *args, accumulation_period=6, **kwargs):
         number = request.get("number", [0])
         assert isinstance(number, (list, tuple))
 
-        user_step = accumulation_period  # For now, we only support 6h accumulation
+        user_step = 6  # For now, we only support 6h accumulation
 
         user_dates = request["date"]
         user_times = request["time"]

climetlab/sources/indexed_urls.py

@@ -10,7 +10,7 @@
 import warnings
 
 from climetlab.indexing import PerUrlIndex
-from climetlab.indexing.fieldset import MultiFieldSet
+from climetlab.readers.grib.index import MultiFieldSet
 from climetlab.readers.grib.index.sql import FieldsetInFilesWithSqlIndex
 from climetlab.sources.indexed import IndexedSource
 from climetlab.utils.patterns import Pattern

climetlab/sources/loader.py

@@ -124,7 +124,7 @@ def mutate(self):
         source = sources[0]
         for s in sources[1:]:
             source += s
-        from climetlab.indexing.fieldset import FieldSet
+        from climetlab.readers.grib.index import FieldSet
 
         assert isinstance(source, FieldSet), type(source)
 

climetlab/sources/opendap.py

@@ -8,32 +8,19 @@
 #
 
 
-from climetlab.core.index import FieldSet
-from climetlab.readers.netcdf import NetCDFReader
-from climetlab.sources import Source
+import xarray as xr
 
+from .base import Source
 
-class OpenDAP(FieldSet):
-    def __init__(self, url, flavour=None, **kwargs):
-        super().__init__()
-
-        self._url = url
-        self._reader = NetCDFReader(self, url, opendap=True, flavour=flavour)
-        self._kwargs = kwargs
 
-    def __iter__(self):
-        return iter(self._reader)
-
-    def __len__(self):
-        return len(self._reader)
+class OpenDAP(Source):
+    def __init__(self, url):
+        super().__init__()
 
-    def __getitem__(self, n):
-        return self._reader[n]
+        self.url = url
 
-    def mutate(self):
-        if self._kwargs:
-            return self.sel(**self._kwargs)
-        return super().mutate()
+    def to_xarray(self):
+        return xr.open_dataset(self.url)
 
 
 source = OpenDAP

climetlab/sources/virtual.py

@@ -13,7 +13,7 @@
 import threading
 
 import climetlab as cml
-from climetlab.readers.grib.index import GribFieldSet
+from climetlab.readers.grib.index import FieldSet
 from climetlab.utils.serialise import register_serialisation
 
 LOG = logging.getLogger(__name__)
@@ -71,7 +71,7 @@ def __getitem__(self, key):
             return self[key]
 
 
-class Virtual(GribFieldSet):
+class Virtual(FieldSet):
     SIZE = int(365.25 * 24 * (2022 - 1959))
     # SIZE = 100
 

climetlab/utils/config.py

@@ -108,9 +108,9 @@ def do_load(self, partial=False):
                 else:
                     out += ds
 
-            from climetlab.readers.grib.index import GribFieldSet
+            from climetlab.readers.grib.index import FieldSet
 
-            assert isinstance(out, GribFieldSet), type(out)
+            assert isinstance(out, FieldSet), type(out)
             self._do_load = (out, partial)
 
         return self._do_load[0]
@@ -385,9 +385,11 @@ def __init__(self, loops, input, output, partial=False):
         inputs = Inputs(input)
         self.output = output
         self.loops = [
-            c
-            if isinstance(c, Loop) and c.inputs == inputs
-            else Loop(c, inputs, parent=self, partial=partial)
+            (
+                c
+                if isinstance(c, Loop) and c.inputs == inputs
+                else Loop(c, inputs, parent=self, partial=partial)
+            )
             for c in loops
         ]
         if not self.loops:

climetlab/version

@@ -1 +1 @@
-0.20.2
+0.20.3

tests/sources/test_merge.py

@@ -15,7 +15,6 @@
 import xarray as xr
 
 from climetlab import load_source
-from climetlab.testing import IN_GITHUB
 
 # These functionalities are variations around
 # http://xarray.pydata.org/en/stable/user-guide/combining.html#combining-multi