Feature/params metadata (#79)

* add `variables_metadata` to metadata * fix bad params * use sanitise * updated tests and updated test data accordingly. this needs to be merged to fix the tests in "develop" --------- Co-authored-by: Baudouin Raoult <baudouin.raoult@ecmwf.int>
ecmwf · Oct 9, 2024 · 904e102 · 904e102
1 parent da7a9c7
commit 904e102
Show file tree

Hide file tree

Showing 3 changed files with 11 additions and 36 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -18,6 +18,10 @@ Keep it human-readable, your future self will thank you!
 
 - Add `variables_metadata` entry in the dataset metadata
 
+### Changed
+
+- Add `variables_metadata` entry in the dataset metadata
+
 ## [0.5.5](https://github.com/ecmwf/anemoi-datasets/compare/0.5.4...0.5.5) - 2024-10-04
 
 ### Changed

diff --git a/tests/create/data_sources.yaml b/tests/create/data_sources.yaml
diff --git a/tests/create/test_create.py b/tests/create/test_create.py
@@ -98,23 +98,22 @@ def compare_dot_zattrs(a, b, path, errors):
     if isinstance(a, dict):
         a_keys = list(a.keys())
         b_keys = list(b.keys())
-        for k in set(a_keys) & set(b_keys):
-            if k in [
-                "licence",
-                "total_number_of_files",
-            ]:
+        for k in set(a_keys) | set(b_keys):
+            if k not in a_keys:
+                errors.append(f"❌ {path}.{k} : missing key (only in reference)")
+                continue
+            if k not in b_keys:
+                errors.append(f"❌ {path}.{k} : additional key (missing in reference)")
                 continue
             if k in [
                 "timestamp",
                 "uuid",
                 "latest_write_timestamp",
-                "_create_yaml_config",
                 "history",
                 "provenance",
                 "provenance_load",
                 "description",
                 "config_path",
-                "dataset_status",
                 "total_size",
             ]:
                 if type(a[k]) is not type(b[k]):
@@ -217,8 +216,8 @@ def compare(self):
             raise AssertionError("Comparison failed")
 
         compare_datasets(self.ds_output, self.ds_reference)
-
         compare_statistics(self.ds_output, self.ds_reference)
+        # do not compare tendencies statistics yet, as we don't know yet if they should stay
 
 
 @pytest.mark.parametrize("name", NAMES)