Merge branch 'develop' of github.com:ecmwf/anemoi-datasets into develop

.pre-commit-config.yaml

@@ -64,7 +64,7 @@ repos:
       exclude: 'cli/.*' # Because we use argparse
 
 - repo: https://github.com/b8raoult/pre-commit-docconvert
-  rev: "0.1.4"
+  rev: "0.1.5"
   hooks:
   - id: docconvert
     args: ["numpy"]

docs/test.ipynb

@@ -113,7 +113,7 @@
     }
    ],
    "source": [
-    "print(len(ds))"
+    "print(len(ds,"
    ]
   },
   {

pyproject.toml

@@ -64,6 +64,7 @@ optional-dependencies.all = [
   "climetlab>=0.23.2",
   "earthkit-meteo",
   "ecmwflibs>=0.6.3",
+  "entrypoints",
   "numpy",
   "pyproj",
   "pyyaml",
@@ -77,6 +78,7 @@ optional-dependencies.create = [
   "climetlab>=0.23.2", # "earthkit-data"
   "earthkit-meteo",
   "ecmwflibs>=0.6.3",
+  "entrypoints",
   "pyproj",
 ]
 optional-dependencies.dev = [

src/anemoi/datasets/create/functions/__init__.py

@@ -8,7 +8,30 @@
 #
 
 
+import importlib
+
+import entrypoints
+
+
 def assert_is_fieldset(obj):
     from climetlab.readers.grib.index import FieldSet
 
     assert isinstance(obj, FieldSet), type(obj)
+
+
+def import_function(name, kind):
+
+    name = name.replace("-", "_")
+
+    plugins = {}
+    for e in entrypoints.get_group_all(f"anemoi.datasets.{kind}s"):
+        plugins[e.name.replace("_", "-")] = e
+
+    if name in plugins:
+        return plugins[name].load()
+
+    module = importlib.import_module(
+        f".{kind}.{name}",
+        package=__name__,
+    )
+    return module.execute

src/anemoi/datasets/create/input.py

@@ -7,7 +7,6 @@
 # nor does it submit to any jurisdiction.
 #
 import datetime
-import importlib
 import logging
 import time
 from collections import defaultdict
@@ -21,6 +20,7 @@
 
 from anemoi.datasets.dates import Dates
 
+from .functions import import_function
 from .template import Context
 from .template import notify_result
 from .template import resolve
@@ -65,14 +65,6 @@ def time_delta_to_string(delta):
         return f"minus_{hours}h"
 
 
-def import_function(name, kind):
-    module = importlib.import_module(
-        f"..functions.{kind}.{name}",
-        package=__name__,
-    )
-    return module.execute
-
-
 def is_function(name, kind):
     name, delta = parse_function_name(name)  # noqa
     try: