From b773035c53fe22fe6fa3da8e24da8659c7957d28 Mon Sep 17 00:00:00 2001 From: Thordur Matthiasson Date: Fri, 12 Apr 2024 09:16:23 +0000 Subject: [PATCH] Version 3.1.0 - Required Env Vars ### Added - A way to make evaluating environment variables require a value to be found by appending `!=` to the end of the expression like so `${__ENV__:API_KEY!=}`. Missing values will trigger a `ValueError` raised when loading config files. --- CHANGELOG.md | 8 +++ alviss/__init__.py | 2 +- alviss/loaders/base.py | 25 ++++++++- tests/basic/test_required.py | 63 ++++++++++++++++++++++ tests/res/rendering/expected_required.json | 60 +++++++++++++++++++++ tests/res/rendering/expected_required.yaml | 48 +++++++++++++++++ tests/res/rendering/json/required_env.json | 47 ++++++++++++++++ tests/res/rendering/yaml/required_env.yaml | 39 ++++++++++++++ 8 files changed, 289 insertions(+), 3 deletions(-) create mode 100644 tests/basic/test_required.py create mode 100644 tests/res/rendering/expected_required.json create mode 100644 tests/res/rendering/expected_required.yaml create mode 100644 tests/res/rendering/json/required_env.json create mode 100644 tests/res/rendering/yaml/required_env.yaml diff --git a/CHANGELOG.md b/CHANGELOG.md index 0854fbb..04ec33e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,14 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## [3.1.0] - 2024-04-12 + +### Added + +- A way to make evaluating environment variables require a value to be found by + appending `!=` to the end of the expression like so `${__ENV__:API_KEY!=}`. + Missing values will trigger a `ValueError` raised when loading config files. + ## [3.0.0] - 2024-04-11 ### Added diff --git a/alviss/__init__.py b/alviss/__init__.py index 4b78e1f..6598ca5 100644 --- a/alviss/__init__.py +++ b/alviss/__init__.py @@ -1,4 +1,4 @@ -__version__ = '3.0.0' +__version__ = '3.1.0' __author__ = 'Thordur Matthiasson ' __license__ = 'MIT License' diff --git a/alviss/loaders/base.py b/alviss/loaders/base.py index 56cb5e6..583c315 100644 --- a/alviss/loaders/base.py +++ b/alviss/loaders/base.py @@ -10,7 +10,7 @@ log = logging.getLogger(__file__) -_VAR_PATTERN = re.compile(r'\$\{(?P([^=}\s])+(=[^}]+)?)}') +_VAR_PATTERN = re.compile(r'\$\{(?P(?:[^=}\s]+)(?:!?=[^}]*)?)}') class BaseLoader(IAlvissLoader, abc.ABC): @@ -139,13 +139,29 @@ def _resolve(self): unresolved = list(self._unresolved.items()) for location, value in unresolved: res = self._check_for_var(value, location) - if res: + # if res: # Not sure if this will cause some unwanted side effects? + # (e.g. overwriting something with ''?) + if res is not None: iters.nested_set(self._data, location, res) if start_count == self._resolve_count: # Loop! break + # Check for required vars... + if self._unresolved: + required_list = [] + unresolved = list(self._unresolved.items()) + for location, value in unresolved: + for match in list(_VAR_PATTERN.finditer(value))[::-1]: + key = match.group(1) + if key.strip().endswith('!='): + required_list.append((location, value)) + if required_list: + for location, value in required_list: + log.error(f'Required variable config value is unresolved: {location}, {value}') + raise ValueError(f'Required variable config values were unresolved: {required_list!r}') + def _fetch_fidelius(self): if self._fidelius_keys: try: @@ -293,6 +309,9 @@ def _explode_path_strings(path_strings: Union[Sequence[str], str]) -> List[str]: def _get_env(key: str) -> str: if '=' in key: key, default = key.split('=', 2) + if isinstance(key, str): + if key.endswith('!'): + key = key[0:-1] return os.environ.get(key[8:], default) return os.environ.get(key[8:], None) @@ -305,6 +324,8 @@ def _try_to_resolve(self, match: re.Match, path: Sequence[str]) -> Optional[str] if not self._skip_env_loading and key.startswith('__ENV__:'): val = self._get_env(key) if val is not None: + if val == '' and key.strip().endswith('!='): + return None # Still unresolved self._resolve_count += 1 return val return match.group(0) diff --git a/tests/basic/test_required.py b/tests/basic/test_required.py new file mode 100644 index 0000000..d1f43b0 --- /dev/null +++ b/tests/basic/test_required.py @@ -0,0 +1,63 @@ +import os +import unittest + +from alviss import quickloader +import yaml +import json + +import logging +log = logging.getLogger(__name__) +logging.basicConfig(level=logging.DEBUG) + +_HERE = os.path.dirname(__file__) + + +class TestRendering(unittest.TestCase): + def setUp(self): + os.environ['OS_MOCK'] = 'MockDos' + if 'ALVISS_FIDELIUS_MODE' in os.environ: + del os.environ['ALVISS_FIDELIUS_MODE'] + if 'PRETEND_API_KEY' in os.environ: + del os.environ['PRETEND_API_KEY'] + + def test_yaml_required_should_fail(self): + if 'PRETEND_API_KEY' in os.environ: + del os.environ['PRETEND_API_KEY'] + with self.assertRaises(ValueError): + quickloader.autoload(os.path.join(_HERE, '../res/rendering/yaml/required_env.yaml')) + + def test_json_required_should_fail(self): + if 'PRETEND_API_KEY' in os.environ: + del os.environ['PRETEND_API_KEY'] + with self.assertRaises(ValueError): + quickloader.autoload(os.path.join(_HERE, '../res/rendering/json/required_env.json')) + + def test_yaml_required_env(self): + os.environ['PRETEND_API_KEY'] = 'veryimportantvalue' + + cfg = quickloader.autoload(os.path.join(_HERE, '../res/rendering/yaml/required_env.yaml')) + + self.assertEqual('veryimportantvalue', cfg.required_key) + + with open(os.path.join(_HERE, '../res/rendering/expected_required.yaml'), 'r') as fin: + expected_data = yaml.safe_load(fin) + + self.assertEqual(expected_data, cfg.as_dict(unmaksed=True)) + + if 'PRETEND_API_KEY' in os.environ: + del os.environ['PRETEND_API_KEY'] + + def test_json_required_env(self): + os.environ['PRETEND_API_KEY'] = 'veryimportantvalue' + + cfg = quickloader.autoload(os.path.join(_HERE, '../res/rendering/json/required_env.json')) + + self.assertEqual('veryimportantvalue', cfg.required_key) + + with open(os.path.join(_HERE, '../res/rendering/expected_required.json'), 'r') as fin: + expected_data = json.load(fin) + + self.assertEqual(expected_data, cfg.as_dict(unmaksed=True)) + + if 'PRETEND_API_KEY' in os.environ: + del os.environ['PRETEND_API_KEY'] diff --git a/tests/res/rendering/expected_required.json b/tests/res/rendering/expected_required.json new file mode 100644 index 0000000..af7fa1c --- /dev/null +++ b/tests/res/rendering/expected_required.json @@ -0,0 +1,60 @@ +{ + "foo": "I am foo", + "internal_ref": { + "deeper": { + "there": "Something", + "everywhere": "My heart will go on!" + } + }, + "base_key": "is basic", + "foo_inherited": "Should be static!", + "bar": "This is bar", + "group": { + "alpha": "AAAAA!", + "beta": "B", + "gamma_with_var": "Foo:I am foo", + "delta": "NewDeltaRocks!" + }, + "collapsed": { + "group": { + "successful": true, + "in": { + "one": { + "string": "Yes!" + } + } + } + }, + "my_list": [ + "one", + "two", + "three", + "I am foo", + "This is B group", + "No ${var}", + "${__ENV__:I_DONT_WANT_THIS_GUY_TO_RESOLVE_TO_ANYTHING}" + ], + "import_test": { + "generic": "Overridden", + "because": "that's cool", + "imported_key": { + "import2": "Tveir", + "import1": "one", + "import3": "three", + "included_env_stuff": "MockDos" + } + }, + "required_key": "veryimportantvalue", + "a_float": 3.5, + "a_bool": true, + "a_false_bool": false, + "a_none": null, + "a_zero": 0, + "a_default_env": "I'm a default value!", + "an_empty": "", + "new_delta": "NewDeltaRocks!", + "env_stuff": "MockDos", + "an_integer": 7, + "another_integer": 42, + "should_be_seven": 7 +} \ No newline at end of file diff --git a/tests/res/rendering/expected_required.yaml b/tests/res/rendering/expected_required.yaml new file mode 100644 index 0000000..312d9b5 --- /dev/null +++ b/tests/res/rendering/expected_required.yaml @@ -0,0 +1,48 @@ +foo: I am foo +internal_ref: + deeper: + there: Something + everywhere: My heart will go on! +base_key: is basic +foo_inherited: Should be static! +bar: This is bar +group: + alpha: AAAAA! + beta: B + gamma_with_var: Foo:I am foo + delta: NewDeltaRocks! +collapsed: + group: + successful: true + in: + one: + string: Yes! +my_list: +- one +- two +- three +- I am foo +- This is B group +- No ${var} +- ${__ENV__:I_DONT_WANT_THIS_GUY_TO_RESOLVE_TO_ANYTHING} +import_test: + generic: Overridden + because: that's cool + imported_key: + import2: Tveir + import1: one + import3: three + included_env_stuff: MockDos +required_key: veryimportantvalue +a_float: 3.5 +a_bool: true +a_false_bool: false +a_none: null +a_zero: 0 +a_default_env: I'm a default value! +an_empty: '' +new_delta: NewDeltaRocks! +env_stuff: MockDos +an_integer: 7 +another_integer: 42 +should_be_seven: 7 diff --git a/tests/res/rendering/json/required_env.json b/tests/res/rendering/json/required_env.json new file mode 100644 index 0000000..0da1b69 --- /dev/null +++ b/tests/res/rendering/json/required_env.json @@ -0,0 +1,47 @@ +{ + "__extends__": "_base_file.json", + "foo": "I am foo", + "bar": "This is bar", + "group": { + "alpha": "A", + "beta": "B", + "__extends__": "_subinclude.json" + }, + "group.delta": "${new_delta}", + "collapsed.group.in.one.string": "Yes!", + "collapsed.group.__include__": "_collapse_import.json", + "my_list": [ + "one", + "two", + "three", + "${foo}", + "This is ${group.beta} group", + "No ${var}", + "${__ENV__:I_DONT_WANT_THIS_GUY_TO_RESOLVE_TO_ANYTHING}" + ], + "import_test": { + "__extends__": [ + "_import.json", + "inc/import2.json" + ], + "generic": "Overridden", + "imported_key.import2": "Tveir" + }, + "required_key": "${__ENV__:PRETEND_API_KEY!=}", + "an_integer": 1, + "a_float": 3.5, + "a_bool": true, + "a_false_bool": false, + "a_none": null, + "a_zero": 0, + "a_default_env": "${__ENV__:I_DONT_WANT_THIS_GUY_TO_RESOLVE_TO_ANYTHING_EITHER=I'm a default value!}", + "an_empty": "", + "__include__": [ + "_include1.json", + "inc/include2.json" + ], + "foo_inherited": "Should be static!", + "should_be_seven": "Nine!", + "new_delta": "NewDeltaRocks!", + "env_stuff": "${__ENV__:OS_MOCK}" +} \ No newline at end of file diff --git a/tests/res/rendering/yaml/required_env.yaml b/tests/res/rendering/yaml/required_env.yaml new file mode 100644 index 0000000..1d8330b --- /dev/null +++ b/tests/res/rendering/yaml/required_env.yaml @@ -0,0 +1,39 @@ +# This is a cool Config File! +__extends__: _base_file.yaml +foo: I am foo +bar: This is bar +group: + alpha: A + beta: B + __extends__: _subinclude.yaml +group.delta: ${new_delta} +collapsed.group.in.one.string: Yes! +collapsed.group.__include__: _collapse_import.yaml +my_list: +- one +- two +- three +- ${foo} +- This is ${group.beta} group +- No ${var} +- ${__ENV__:I_DONT_WANT_THIS_GUY_TO_RESOLVE_TO_ANYTHING} +import_test: + __extends__: ["_import.yaml", "inc/import2.yaml"] + generic: Overridden + imported_key.import2: Tveir +an_integer: 1 +required_key: ${__ENV__:PRETEND_API_KEY!=} +a_float: 3.5 +a_bool: true +a_false_bool: false +a_none: null +a_zero: 0 +a_default_env: ${__ENV__:I_DONT_WANT_THIS_GUY_TO_RESOLVE_TO_ANYTHING_EITHER=I'm a default value!} +an_empty: '' +__include__: +- _include1.yaml +- inc/include2.yaml +foo_inherited: Should be static! +should_be_seven: Nine! +new_delta: NewDeltaRocks! +env_stuff: ${__ENV__:OS_MOCK} \ No newline at end of file