From fe832c143ac7ee418b42013ac57f864ee4910190 Mon Sep 17 00:00:00 2001 From: Paul Madden <136389411+maddenp-noaa@users.noreply.github.com> Date: Wed, 4 Dec 2024 12:44:10 -0700 Subject: [PATCH] Omnibus 2024-11-27 (#663) --- .github/workflows/test.yaml | 4 +- .../cli/tools/config/realize-verbose.out | 56 +++++++------- .../cli/tools/config/validate-verbose.out | 42 +++++----- docs/sections/user_guide/yaml/tags.rst | 77 +++++++++++++------ recipe/meta.json | 3 +- recipe/meta.yaml | 1 + src/uwtools/config/formats/yaml.py | 12 +-- src/uwtools/config/jinja2.py | 24 +++--- src/uwtools/config/support.py | 11 +++ src/uwtools/tests/config/test_jinja2.py | 11 ++- src/uwtools/tests/config/test_tools.py | 62 +++++++++++++++ 11 files changed, 207 insertions(+), 96 deletions(-) diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index 3351ba97d..c34875b49 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -2,10 +2,10 @@ name: Test on: pull_request: branches: - - main + - '**' push: branches: - - main + - '**' workflow_dispatch: branches: - '**' diff --git a/docs/sections/user_guide/cli/tools/config/realize-verbose.out b/docs/sections/user_guide/cli/tools/config/realize-verbose.out index 3371e2070..cac8e0711 100644 --- a/docs/sections/user_guide/cli/tools/config/realize-verbose.out +++ b/docs/sections/user_guide/cli/tools/config/realize-verbose.out @@ -1,30 +1,30 @@ -[2024-05-23T19:39:16] DEBUG Command: uw config realize --input-format yaml --output-format yaml --verbose -[2024-05-23T19:39:16] DEBUG Reading input from stdin -[2024-05-23T19:39:16] DEBUG Dereferencing, current value: -[2024-05-23T19:39:16] DEBUG hello: '{{ recipient }}' -[2024-05-23T19:39:16] DEBUG recipient: world -[2024-05-23T19:39:16] DEBUG [dereference] Rendering: {{ recipient }} -[2024-05-23T19:39:16] DEBUG [dereference] Rendered: world -[2024-05-23T19:39:16] DEBUG [dereference] Rendering: hello -[2024-05-23T19:39:16] DEBUG [dereference] Rendered: hello -[2024-05-23T19:39:16] DEBUG [dereference] Rendering: world -[2024-05-23T19:39:16] DEBUG [dereference] Rendered: world -[2024-05-23T19:39:16] DEBUG [dereference] Rendering: recipient -[2024-05-23T19:39:16] DEBUG [dereference] Rendered: recipient -[2024-05-23T19:39:16] DEBUG Dereferencing, current value: -[2024-05-23T19:39:16] DEBUG hello: world -[2024-05-23T19:39:16] DEBUG recipient: world -[2024-05-23T19:39:16] DEBUG [dereference] Rendering: world -[2024-05-23T19:39:16] DEBUG [dereference] Rendered: world -[2024-05-23T19:39:16] DEBUG [dereference] Rendering: hello -[2024-05-23T19:39:16] DEBUG [dereference] Rendered: hello -[2024-05-23T19:39:16] DEBUG [dereference] Rendering: world -[2024-05-23T19:39:16] DEBUG [dereference] Rendered: world -[2024-05-23T19:39:16] DEBUG [dereference] Rendering: recipient -[2024-05-23T19:39:16] DEBUG [dereference] Rendered: recipient -[2024-05-23T19:39:16] DEBUG Dereferencing, final value: -[2024-05-23T19:39:16] DEBUG hello: world -[2024-05-23T19:39:16] DEBUG recipient: world -[2024-05-23T19:39:16] DEBUG Writing output to stdout +[2024-11-27T05:24:34] DEBUG Command: uw config realize --input-format yaml --output-format yaml --verbose +[2024-11-27T05:24:34] DEBUG Reading input from stdin +[2024-11-27T05:24:34] DEBUG Dereferencing, current value: +[2024-11-27T05:24:34] DEBUG hello: '{{ recipient }}' +[2024-11-27T05:24:34] DEBUG recipient: world +[2024-11-27T05:24:34] DEBUG [dereference] Rendering: hello +[2024-11-27T05:24:34] DEBUG [dereference] Rendered: hello +[2024-11-27T05:24:34] DEBUG [dereference] Rendering: {{ recipient }} +[2024-11-27T05:24:34] DEBUG [dereference] Rendered: world +[2024-11-27T05:24:34] DEBUG [dereference] Rendering: recipient +[2024-11-27T05:24:34] DEBUG [dereference] Rendered: recipient +[2024-11-27T05:24:34] DEBUG [dereference] Rendering: world +[2024-11-27T05:24:34] DEBUG [dereference] Rendered: world +[2024-11-27T05:24:34] DEBUG Dereferencing, current value: +[2024-11-27T05:24:34] DEBUG hello: world +[2024-11-27T05:24:34] DEBUG recipient: world +[2024-11-27T05:24:34] DEBUG [dereference] Rendering: hello +[2024-11-27T05:24:34] DEBUG [dereference] Rendered: hello +[2024-11-27T05:24:34] DEBUG [dereference] Rendering: world +[2024-11-27T05:24:34] DEBUG [dereference] Rendered: world +[2024-11-27T05:24:34] DEBUG [dereference] Rendering: recipient +[2024-11-27T05:24:34] DEBUG [dereference] Rendered: recipient +[2024-11-27T05:24:34] DEBUG [dereference] Rendering: world +[2024-11-27T05:24:34] DEBUG [dereference] Rendered: world +[2024-11-27T05:24:34] DEBUG Dereferencing, final value: +[2024-11-27T05:24:34] DEBUG hello: world +[2024-11-27T05:24:34] DEBUG recipient: world +[2024-11-27T05:24:34] DEBUG Writing output to stdout hello: world recipient: world diff --git a/docs/sections/user_guide/cli/tools/config/validate-verbose.out b/docs/sections/user_guide/cli/tools/config/validate-verbose.out index 17dc2a651..d48c7ea14 100644 --- a/docs/sections/user_guide/cli/tools/config/validate-verbose.out +++ b/docs/sections/user_guide/cli/tools/config/validate-verbose.out @@ -1,21 +1,21 @@ -[2024-08-26T22:54:28] DEBUG Command: uw config validate --schema-file schema.jsonschema --input-file values.yaml --verbose -[2024-08-26T22:54:28] DEBUG Using schema file: schema.jsonschema -[2024-08-26T22:54:28] DEBUG Dereferencing, current value: -[2024-08-26T22:54:28] DEBUG values: -[2024-08-26T22:54:28] DEBUG greeting: Hello -[2024-08-26T22:54:28] DEBUG recipient: World -[2024-08-26T22:54:28] DEBUG [dereference] Rendering: Hello -[2024-08-26T22:54:28] DEBUG [dereference] Rendered: Hello -[2024-08-26T22:54:28] DEBUG [dereference] Rendering: greeting -[2024-08-26T22:54:28] DEBUG [dereference] Rendered: greeting -[2024-08-26T22:54:28] DEBUG [dereference] Rendering: World -[2024-08-26T22:54:28] DEBUG [dereference] Rendered: World -[2024-08-26T22:54:28] DEBUG [dereference] Rendering: recipient -[2024-08-26T22:54:28] DEBUG [dereference] Rendered: recipient -[2024-08-26T22:54:28] DEBUG [dereference] Rendering: values -[2024-08-26T22:54:28] DEBUG [dereference] Rendered: values -[2024-08-26T22:54:28] DEBUG Dereferencing, final value: -[2024-08-26T22:54:28] DEBUG values: -[2024-08-26T22:54:28] DEBUG greeting: Hello -[2024-08-26T22:54:28] DEBUG recipient: World -[2024-08-26T22:54:29] INFO 0 UW schema-validation errors found in config +[2024-11-27T05:24:34] DEBUG Command: uw config validate --schema-file schema.jsonschema --input-file values.yaml --verbose +[2024-11-27T05:24:34] DEBUG Using schema file: schema.jsonschema +[2024-11-27T05:24:34] DEBUG Dereferencing, current value: +[2024-11-27T05:24:34] DEBUG values: +[2024-11-27T05:24:34] DEBUG greeting: Hello +[2024-11-27T05:24:34] DEBUG recipient: World +[2024-11-27T05:24:34] DEBUG [dereference] Rendering: values +[2024-11-27T05:24:34] DEBUG [dereference] Rendered: values +[2024-11-27T05:24:34] DEBUG [dereference] Rendering: greeting +[2024-11-27T05:24:34] DEBUG [dereference] Rendered: greeting +[2024-11-27T05:24:34] DEBUG [dereference] Rendering: Hello +[2024-11-27T05:24:34] DEBUG [dereference] Rendered: Hello +[2024-11-27T05:24:34] DEBUG [dereference] Rendering: recipient +[2024-11-27T05:24:34] DEBUG [dereference] Rendered: recipient +[2024-11-27T05:24:34] DEBUG [dereference] Rendering: World +[2024-11-27T05:24:34] DEBUG [dereference] Rendered: World +[2024-11-27T05:24:34] DEBUG Dereferencing, final value: +[2024-11-27T05:24:34] DEBUG values: +[2024-11-27T05:24:34] DEBUG greeting: Hello +[2024-11-27T05:24:34] DEBUG recipient: World +[2024-11-27T05:24:34] INFO 0 UW schema-validation errors found in config diff --git a/docs/sections/user_guide/yaml/tags.rst b/docs/sections/user_guide/yaml/tags.rst index 8acd99caa..e768d74e5 100644 --- a/docs/sections/user_guide/yaml/tags.rst +++ b/docs/sections/user_guide/yaml/tags.rst @@ -26,7 +26,7 @@ Additionally, UW defines the following tags to support use cases not covered by ``!bool`` ^^^^^^^^^ -Converts the tagged node to a Python ``boolean`` object. For example, given ``input.yaml``: +Converts the tagged node to a Python ``bool`` object. For example, given ``input.yaml``: .. code-block:: yaml @@ -35,7 +35,7 @@ Converts the tagged node to a Python ``boolean`` object. For example, given ``in .. code-block:: text - % uw config realize -i ../input.yaml --output-format yaml + $ uw config realize -i ../input.yaml --output-format yaml flag1: True flag2: True @@ -52,7 +52,7 @@ Converts the tagged node to a Python ``datetime`` object. For example, given ``i .. code-block:: text - % uw config realize -i ../input.yaml --output-format yaml + $ uw config realize -i ../input.yaml --output-format yaml date1: 2024-09-01 date2: 2024-09-01 00:00:00 @@ -69,49 +69,76 @@ Converts the tagged node to a Python ``float`` value. For example, given ``input .. code-block:: text - % uw config realize --input-file input.yaml --output-format yaml + $ uw config realize --input-file input.yaml --output-format yaml f2: 5.859 -``!int`` -^^^^^^^^ +``!include`` +^^^^^^^^^^^^ -Converts the tagged node to a Python ``int`` value. For example, given ``input.yaml``: +Load and parse the files specified in the tagged sequence value and insert their contents here. For example, given ``numbers.yaml``: .. code-block:: yaml - f1: 3 - f2: 11 - f3: !int "{{ (f1 + f2) * 10 }}" + values: !include [constants.yaml] + +and ``constants.yaml``: + +.. code-block:: yaml + + e: 2.718 + pi: 3.141 .. code-block:: text - % uw config realize --input-file input.yaml --output-format yaml - f1: 3 - f2: 11 - f2: 140 + $ uw config realize --input-file numbers.yaml --output-format yaml + values: + e: 2.718 + pi: 3.141 -``!include`` -^^^^^^^^^^^^ +Values from files later in the sequence overwrite their predecessors, and full-value replacement, not structural merging, is performed. For example, given ``numbers.yaml``: + +.. code-block:: yaml + + values: !include [e.yaml, pi.yaml] -Parse the tagged file and include its tags. For example, given ``input.yaml``: +``e.yaml``: .. code-block:: yaml - values: !include [./supplemental.yaml] + constants: + e: 2.718 -and ``supplemental.yaml``: +and ``pi.yaml``: .. code-block:: yaml - e: 2.718 - pi: 3.141 + constants: + pi: 3.141 .. code-block:: text - % uw config realize --input-file input.yaml --output-format yaml + $ uw config realize --input-file numbers.yaml --output-format yaml values: - e: 2.718 - pi: 3.141 + constants: + pi: 3.141 + +``!int`` +^^^^^^^^ + +Converts the tagged node to a Python ``int`` value. For example, given ``input.yaml``: + +.. code-block:: yaml + + f1: 3 + f2: 11 + f3: !int "{{ (f1 + f2) * 10 }}" + +.. code-block:: text + + $ uw config realize --input-file input.yaml --output-format yaml + f1: 3 + f2: 11 + f2: 140 ``!remove`` ^^^^^^^^^^^ @@ -131,5 +158,5 @@ and ``update.yaml``: .. code-block:: text - % uw config realize --input-file input.yaml --update-file update.yaml --output-format yaml + $ uw config realize --input-file input.yaml --update-file update.yaml --output-format yaml pi: 3.141 diff --git a/recipe/meta.json b/recipe/meta.json index 282173e37..8cfa7a58f 100644 --- a/recipe/meta.json +++ b/recipe/meta.json @@ -21,7 +21,8 @@ "pytest-cov =5.0.*", "pytest-xdist =3.6.*", "python >=3.9,<3.13", - "pyyaml =6.0.*" + "pyyaml =6.0.*", + "setuptools" ], "run": [ "f90nml =1.4.*", diff --git a/recipe/meta.yaml b/recipe/meta.yaml index 479b49330..9a9dfd0d9 100644 --- a/recipe/meta.yaml +++ b/recipe/meta.yaml @@ -12,6 +12,7 @@ build: requirements: build: - pip + - setuptools run: - f90nml 1.4.* - iotaa 0.8.* diff --git a/src/uwtools/config/formats/yaml.py b/src/uwtools/config/formats/yaml.py index 99c5b32a6..71bc870f8 100644 --- a/src/uwtools/config/formats/yaml.py +++ b/src/uwtools/config/formats/yaml.py @@ -10,9 +10,9 @@ from uwtools.config.support import ( INCLUDE_TAG, UWYAMLConvert, - UWYAMLRemove, from_od, log_and_error, + uw_yaml_loader, yaml_to_str, ) from uwtools.exceptions import UWConfigError @@ -94,10 +94,9 @@ def _load(self, config_file: Optional[Path]) -> dict: :param config_file: Path to config file to load. """ - loader = self._yaml_loader with readable(config_file) as f: try: - config = yaml.load(f.read(), Loader=loader) + config = yaml.load(f.read(), Loader=self._yaml_loader) if isinstance(config, dict): return config t = type(config).__name__ @@ -157,13 +156,10 @@ def _yaml_include(self, loader: yaml.Loader, node: yaml.SequenceNode) -> dict: @property def _yaml_loader(self) -> type[yaml.SafeLoader]: """ - The loader, with appropriate constructors added. + A loader with all UW constructors added. """ - loader = yaml.SafeLoader + loader = uw_yaml_loader() loader.add_constructor(INCLUDE_TAG, self._yaml_include) - for tag_class in (UWYAMLConvert, UWYAMLRemove): - for tag in getattr(tag_class, "TAGS"): - loader.add_constructor(tag, tag_class) return loader # Public methods diff --git a/src/uwtools/config/jinja2.py b/src/uwtools/config/jinja2.py index 8115e248b..cba03bae0 100644 --- a/src/uwtools/config/jinja2.py +++ b/src/uwtools/config/jinja2.py @@ -8,10 +8,12 @@ from pathlib import Path from typing import Optional, Union +import yaml from jinja2 import Environment, FileSystemLoader, StrictUndefined, Undefined, meta from jinja2.exceptions import UndefinedError -from uwtools.config.support import UWYAMLConvert, UWYAMLRemove, format_to_config +from uwtools.config.support import UWYAMLConvert, UWYAMLRemove, format_to_config, uw_yaml_loader +from uwtools.exceptions import UWConfigRealizeError from uwtools.logging import INDENT, MSGWIDTH, log from uwtools.utils.file import get_file_format, readable, writable @@ -122,19 +124,19 @@ def dereference( :param keys: The dict keys leading to this value. :return: The input value, with Jinja2 syntax rendered. """ - rendered: _ConfigVal = val # fall-back value + rendered: _ConfigVal if isinstance(val, dict): keys = keys or [] - new = {} + rendered = {} for k, v in val.items(): if isinstance(v, UWYAMLRemove): - _deref_debug("Removing value at", " > ".join([*keys, k])) + _deref_debug("Removing value at", ".".join([*keys, k])) else: - new[dereference(k, context)] = dereference(v, context, local=val, keys=[*keys, k]) - return new - if isinstance(val, list): - return [dereference(v, context) for v in val] - if isinstance(val, str): + kd, vd = [dereference(x, context, val, [*keys, k]) for x in (k, v)] + rendered[kd] = vd + elif isinstance(val, list): + rendered = [dereference(v, context) for v in val] + elif isinstance(val, str): _deref_debug("Rendering", val) rendered = _deref_render(val, context, local) elif isinstance(val, UWYAMLConvert): @@ -143,6 +145,7 @@ def dereference( rendered = _deref_convert(val) else: _deref_debug("Accepting", val) + rendered = val return rendered @@ -266,6 +269,9 @@ def _deref_render(val: str, context: dict, local: Optional[dict] = None) -> str: context = {**(local or {}), **context} try: rendered = _register_filters(env).from_string(val).render(context) + if isinstance(yaml.load(rendered, Loader=uw_yaml_loader()), UWYAMLConvert): + _deref_debug("Held", rendered) + raise UWConfigRealizeError() _deref_debug("Rendered", rendered) except Exception as e: # pylint: disable=broad-exception-caught rendered = val diff --git a/src/uwtools/config/support.py b/src/uwtools/config/support.py index f6337853c..393ed1a44 100644 --- a/src/uwtools/config/support.py +++ b/src/uwtools/config/support.py @@ -68,6 +68,17 @@ def log_and_error(msg: str) -> Exception: return UWConfigError(msg) +def uw_yaml_loader() -> type[yaml.SafeLoader]: + """ + A loader with basic UW constructors added. + """ + loader = yaml.SafeLoader + for tag_class in (UWYAMLConvert, UWYAMLRemove): + for tag in getattr(tag_class, "TAGS"): + loader.add_constructor(tag, tag_class) + return loader + + def yaml_to_str(cfg: dict) -> str: """ Return a uwtools-conventional YAML representation of the given dict. diff --git a/src/uwtools/tests/config/test_jinja2.py b/src/uwtools/tests/config/test_jinja2.py index 407e49f9c..2985addc1 100644 --- a/src/uwtools/tests/config/test_jinja2.py +++ b/src/uwtools/tests/config/test_jinja2.py @@ -17,7 +17,7 @@ from uwtools.config import jinja2 from uwtools.config.jinja2 import J2Template -from uwtools.config.support import UWYAMLConvert, UWYAMLRemove +from uwtools.config.support import UWYAMLConvert, UWYAMLRemove, uw_yaml_loader from uwtools.logging import log from uwtools.tests.support import logged, regex_logged @@ -141,7 +141,7 @@ def test_dereference_remove(caplog): remove = UWYAMLRemove(yaml.SafeLoader(""), yaml.ScalarNode(tag="!remove", value="")) val = {"a": {"b": {"c": "cherry", "d": remove}}} assert jinja2.dereference(val=val, context={}) == {"a": {"b": {"c": "cherry"}}} - assert regex_logged(caplog, "Removing value at: a > b > d") + assert regex_logged(caplog, "Removing value at: a.b.d") def test_dereference_str_expression_rendered(): @@ -315,6 +315,13 @@ def test__deref_debug(caplog): assert logged(caplog, "[dereference] Frobnicated: foo") +def test__deref_render_held(caplog): + val, context = "!int '{{ a }}'", yaml.load("a: !int '{{ 42 }}'", Loader=uw_yaml_loader()) + assert jinja2._deref_render(val=val, context=context) == val + assert not regex_logged(caplog, "Rendered") + assert regex_logged(caplog, "Held") + + def test__deref_render_no(caplog, deref_render_assets): val, context, _ = deref_render_assets assert jinja2._deref_render(val=val, context=context) == val diff --git a/src/uwtools/tests/config/test_tools.py b/src/uwtools/tests/config/test_tools.py index f7103628a..793af23db 100644 --- a/src/uwtools/tests/config/test_tools.py +++ b/src/uwtools/tests/config/test_tools.py @@ -59,6 +59,16 @@ def realize_config_yaml_input(tmp_path): # Helpers +def help_realize_config_double_tag(config, expected, tmp_path): + path_in = tmp_path / "in.yaml" + path_out = tmp_path / "out.yaml" + with open(path_in, "w", encoding="utf-8") as f: + print(dedent(config).strip(), file=f) + tools.realize_config(input_config=path_in, output_file=path_out) + with open(path_out, "r", encoding="utf-8") as f: + assert f.read().strip() == dedent(expected).strip() + + def help_realize_config_fmt2fmt(input_file, input_format, update_file, update_format, tmpdir): input_file = fixture_path(input_file) update_file = fixture_path(update_file) @@ -235,6 +245,58 @@ def test_realize_config_depth_mismatch_to_sh(realize_config_yaml_input): ) +def test_realize_config_double_tag_flat(tmp_path): + config = """ + a: 1 + b: 2 + foo: !int "{{ a + b }}" + bar: !int "{{ foo }}" + """ + expected = """ + a: 1 + b: 2 + foo: 3 + bar: 3 + """ + help_realize_config_double_tag(config, expected, tmp_path) + + +def test_realize_config_double_tag_nest(tmp_path): + config = """ + a: 1.0 + b: 2.0 + qux: + foo: !float "{{ a + b }}" + bar: !float "{{ foo }}" + """ + expected = """ + a: 1.0 + b: 2.0 + qux: + foo: 3.0 + bar: 3.0 + """ + help_realize_config_double_tag(config, expected, tmp_path) + + +def test_realize_config_double_tag_nest_forwrad_reference(tmp_path): + config = """ + a: true + b: false + bar: !bool "{{ qux.foo }}" + qux: + foo: !bool "{{ a or b }}" + """ + expected = """ + a: true + b: false + bar: true + qux: + foo: true + """ + help_realize_config_double_tag(config, expected, tmp_path) + + def test_realize_config_dry_run(caplog): """ Test that providing a YAML base file with a dry-run flag will print an YAML config file.