Merge branch 'release/0.15.1'

climetlab/decorators.py

@@ -91,6 +91,88 @@ def newfunc(*args, **kwargs):
 }
 
 
+def normalize_grib_keys(f):
+    f = alias_argument("levelist", ["level", "levellist"])(f)
+    f = alias_argument("levtype", ["leveltype"])(f)
+    f = alias_argument("param", ["variable", "parameter"])(f)
+    f = alias_argument("number", ["realization", "realisation"])(f)
+    f = alias_argument("class", ["klass", "class_"])(f)
+    return f
+
+
+def _normalize_time_as_tuple(time, type):
+    if isinstance(time, str):
+        time = (time,)
+    return tuple(_normalize_time(t, type) for t in time)
+
+
+def _normalize_time(time, type):
+    assert type in (int, str), type
+
+    if time is None or time == all:
+        return time
+
+    if isinstance(time, list):
+        return [_normalize_time(t, type) for t in time]
+    if isinstance(time, tuple):
+        return tuple([_normalize_time(t, type) for t in time])
+
+    try:
+        time = int(time)
+    except ValueError:
+        return time
+
+    if time % 100:  # not multiple of 100
+        time = time * 100
+
+    assert time <= 2400, time
+    assert time >= 0, time
+
+    if type is str:
+        return f"{time:04d}"
+    assert isinstance(time, int)
+    return time
+
+
+def _normalize_expver(expver):
+    if isinstance(expver, str):
+        return expver
+    assert isinstance(expver, int)
+    return f"{expver:04}"
+
+
+def _normalize_expver_as_tuple(expver):
+    lst = []
+    for x in expver:
+        if not isinstance(x, str):
+            x = "%04d" % x
+        lst.append(x)
+    return tuple(lst)
+
+
+def normalize_grib_key_values(kwargs, accept_none=True, as_tuple=False):
+    def f(**kwargs):
+        return kwargs
+
+    f = normalize_grib_keys(f)
+    f = normalize("param", "variable-list(mars)")(f)
+    f = normalize("date", "date-list(%Y%m%d)")(f)
+    f = normalize("area", "bounding-box(list)")(f)
+    f = normalize("levelist", "int-list")(f)
+    kwargs = f(**kwargs)
+
+    if "time" in kwargs:
+        kwargs["time"] = {False: _normalize_time, True: _normalize_time_as_tuple}[
+            as_tuple
+        ](kwargs["time"], int)
+    if "expver" in kwargs:
+        kwargs["expver"] = {False: _normalize_expver, True: _normalize_expver_as_tuple}[
+            as_tuple
+        ](kwargs["expver"])
+
+    return kwargs
+
+
 class alias_argument(Decorator):
     def __init__(
         self,

climetlab/exceptions.py

@@ -0,0 +1,12 @@
+# (C) Copyright 2023 ECMWF.
+#
+# This software is licensed under the terms of the Apache Licence Version 2.0
+# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0.
+# In applying this licence, ECMWF does not waive the privileges and immunities
+# granted to it by virtue of its status as an intergovernmental organisation
+# nor does it submit to any jurisdiction.
+#
+
+
+class NotIndexedDirectoryError(ValueError):
+    pass

climetlab/indexing/cube.py

@@ -43,12 +43,10 @@ def __init__(
         *args,
         remapping=None,
         flatten_values=False,
-        grid_points_first=False,
     ):
         assert len(ds), f"No data in {ds}"
 
         self.flatten_values = flatten_values
-        self.grid_points_first = grid_points_first
 
         if len(args) == 1 and isinstance(args[0], (list, tuple)):
             args = args[0]
@@ -112,10 +110,7 @@ def field_shape(self):
 
     @property
     def extended_user_shape(self):
-        if self.grid_points_first:
-            return self.field_shape + self.user_shape
-        else:
-            return self.user_shape + self.field_shape
+        return self.user_shape + self.field_shape
 
     def __str__(self):
         content = ", ".join([f"{k}:{len(v)}" for k, v in self.user_coords.items()])
@@ -171,11 +166,9 @@ def __getitem__(self, indexes):
             ds,
             *self.user_coords,
             flatten_values=self.flatten_values,
-            grid_points_first=self.grid_points_first,
         )
 
     def to_numpy(self, **kwargs):
-        assert not self.grid_points_first, "Not yes implemented"
         return self.source.to_numpy(**kwargs).reshape(*self.extended_user_shape)
 
     def _names(self, coords, reading_chunks=None, **kwargs):
@@ -201,10 +194,8 @@ def iterate_cubelets(self, reading_chunks=None, **kwargs):
         names = self._names(reading_chunks=reading_chunks, coords=self.user_coords)
         indexes = list(range(0, len(lst)) for lst in names)
 
-        CUBELT = GridPointFirstCubelet if self.grid_points_first else Cubelet
-
         return (
-            CUBELT(self, i, coords_names=n)
+            Cubelet(self, i, coords_names=n)
             for n, i in zip(itertools.product(*names), itertools.product(*indexes))
         )
 

climetlab/indexing/database/sql.py

@@ -13,6 +13,7 @@
 import logging
 import os
 import sqlite3
+import time
 from threading import local
 
 import numpy as np
@@ -53,7 +54,19 @@ def execute(connection, statement, *arg, **kwargs):
     if LOG.level == logging.DEBUG:
         assert False
         dump_sql(statement)
-    return connection.execute(statement, *arg, **kwargs)
+
+    delay = 1
+    while delay < 30 * 60:  # max delay 30 min
+        try:
+            return connection.execute(statement, *arg, **kwargs)
+        except sqlite3.OperationalError as e:
+            if not str(e).endswith("database is locked"):
+                raise e
+            time.sleep(delay)
+            dump_sql(statement)
+            print(f"{e}. Retrying in {delay} seconds.")
+            delay = delay * 1.5
+    raise e  # noqa: F821
 
 
 def entryname_to_dbname(n):