Merge pull request #2 from ccpgames/feature/required-env-vars

Version 3.1.0 - Required Env Vars

CHANGELOG.md

@@ -5,6 +5,14 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
+## [3.1.0] - 2024-04-12
+
+### Added
+
+- A way to make evaluating environment variables require a value to be found by 
+  appending `!=` to the end of the expression like so `${__ENV__:API_KEY!=}`. 
+  Missing values will trigger a `ValueError` raised when loading config files.
+
 ## [3.0.0] - 2024-04-11
 
 ### Added

alviss/__init__.py

@@ -1,4 +1,4 @@
-__version__ = '3.0.0'
+__version__ = '3.1.0'
 
 __author__ = 'Thordur Matthiasson <thordurm@ccpgames.com>'
 __license__ = 'MIT License'

alviss/loaders/base.py

@@ -10,7 +10,7 @@
 log = logging.getLogger(__file__)
 
 
-_VAR_PATTERN = re.compile(r'\$\{(?P<var>([^=}\s])+(=[^}]+)?)}')
+_VAR_PATTERN = re.compile(r'\$\{(?P<var>(?:[^=}\s]+)(?:!?=[^}]*)?)}')
 
 
 class BaseLoader(IAlvissLoader, abc.ABC):
@@ -139,13 +139,29 @@ def _resolve(self):
             unresolved = list(self._unresolved.items())
             for location, value in unresolved:
                 res = self._check_for_var(value, location)
-                if res:
+                # if res:  # Not sure if this will cause some unwanted side effects?
+                # (e.g. overwriting something with ''?)
+                if res is not None:
                     iters.nested_set(self._data, location, res)
 
             if start_count == self._resolve_count:
                 # Loop!
                 break
 
+        # Check for required vars...
+        if self._unresolved:
+            required_list = []
+            unresolved = list(self._unresolved.items())
+            for location, value in unresolved:
+                for match in list(_VAR_PATTERN.finditer(value))[::-1]:
+                    key = match.group(1)
+                    if key.strip().endswith('!='):
+                        required_list.append((location, value))
+            if required_list:
+                for location, value in required_list:
+                    log.error(f'Required variable config value is unresolved: {location}, {value}')
+                raise ValueError(f'Required variable config values were unresolved: {required_list!r}')
+
     def _fetch_fidelius(self):
         if self._fidelius_keys:
             try:
@@ -293,6 +309,9 @@ def _explode_path_strings(path_strings: Union[Sequence[str], str]) -> List[str]:
     def _get_env(key: str) -> str:
         if '=' in key:
             key, default = key.split('=', 2)
+            if isinstance(key, str):
+                if key.endswith('!'):
+                    key = key[0:-1]
             return os.environ.get(key[8:], default)
         return os.environ.get(key[8:], None)
 
@@ -305,6 +324,8 @@ def _try_to_resolve(self, match: re.Match, path: Sequence[str]) -> Optional[str]
         if not self._skip_env_loading and key.startswith('__ENV__:'):
             val = self._get_env(key)
             if val is not None:
+                if val == '' and key.strip().endswith('!='):
+                    return None  # Still unresolved
                 self._resolve_count += 1
                 return val
             return match.group(0)

tests/basic/test_required.py

@@ -0,0 +1,63 @@
+import os
+import unittest
+
+from alviss import quickloader
+import yaml
+import json
+
+import logging
+log = logging.getLogger(__name__)
+logging.basicConfig(level=logging.DEBUG)
+
+_HERE = os.path.dirname(__file__)
+
+
+class TestRendering(unittest.TestCase):
+    def setUp(self):
+        os.environ['OS_MOCK'] = 'MockDos'
+        if 'ALVISS_FIDELIUS_MODE' in os.environ:
+            del os.environ['ALVISS_FIDELIUS_MODE']
+        if 'PRETEND_API_KEY' in os.environ:
+            del os.environ['PRETEND_API_KEY']
+
+    def test_yaml_required_should_fail(self):
+        if 'PRETEND_API_KEY' in os.environ:
+            del os.environ['PRETEND_API_KEY']
+        with self.assertRaises(ValueError):
+            quickloader.autoload(os.path.join(_HERE, '../res/rendering/yaml/required_env.yaml'))
+
+    def test_json_required_should_fail(self):
+        if 'PRETEND_API_KEY' in os.environ:
+            del os.environ['PRETEND_API_KEY']
+        with self.assertRaises(ValueError):
+            quickloader.autoload(os.path.join(_HERE, '../res/rendering/json/required_env.json'))
+
+    def test_yaml_required_env(self):
+        os.environ['PRETEND_API_KEY'] = 'veryimportantvalue'
+
+        cfg = quickloader.autoload(os.path.join(_HERE, '../res/rendering/yaml/required_env.yaml'))
+
+        self.assertEqual('veryimportantvalue', cfg.required_key)
+
+        with open(os.path.join(_HERE, '../res/rendering/expected_required.yaml'), 'r') as fin:
+            expected_data = yaml.safe_load(fin)
+
+        self.assertEqual(expected_data, cfg.as_dict(unmaksed=True))
+
+        if 'PRETEND_API_KEY' in os.environ:
+            del os.environ['PRETEND_API_KEY']
+
+    def test_json_required_env(self):
+        os.environ['PRETEND_API_KEY'] = 'veryimportantvalue'
+
+        cfg = quickloader.autoload(os.path.join(_HERE, '../res/rendering/json/required_env.json'))
+
+        self.assertEqual('veryimportantvalue', cfg.required_key)
+
+        with open(os.path.join(_HERE, '../res/rendering/expected_required.json'), 'r') as fin:
+            expected_data = json.load(fin)
+
+        self.assertEqual(expected_data, cfg.as_dict(unmaksed=True))
+
+        if 'PRETEND_API_KEY' in os.environ:
+            del os.environ['PRETEND_API_KEY']

tests/res/rendering/expected_required.json

@@ -0,0 +1,60 @@
+{
+  "foo": "I am foo",
+  "internal_ref": {
+    "deeper": {
+      "there": "Something",
+      "everywhere": "My heart will go on!"
+    }
+  },
+  "base_key": "is basic",
+  "foo_inherited": "Should be static!",
+  "bar": "This is bar",
+  "group": {
+    "alpha": "AAAAA!",
+    "beta": "B",
+    "gamma_with_var": "Foo:I am foo",
+    "delta": "NewDeltaRocks!"
+  },
+  "collapsed": {
+    "group": {
+      "successful": true,
+      "in": {
+        "one": {
+          "string": "Yes!"
+        }
+      }
+    }
+  },
+  "my_list": [
+    "one",
+    "two",
+    "three",
+    "I am foo",
+    "This is B group",
+    "No ${var}",
+    "${__ENV__:I_DONT_WANT_THIS_GUY_TO_RESOLVE_TO_ANYTHING}"
+  ],
+  "import_test": {
+    "generic": "Overridden",
+    "because": "that's cool",
+    "imported_key": {
+      "import2": "Tveir",
+      "import1": "one",
+      "import3": "three",
+      "included_env_stuff": "MockDos"
+    }
+  },
+  "required_key": "veryimportantvalue",
+  "a_float": 3.5,
+  "a_bool": true,
+  "a_false_bool": false,
+  "a_none": null,
+  "a_zero": 0,
+  "a_default_env": "I'm a default value!",
+  "an_empty": "",
+  "new_delta": "NewDeltaRocks!",
+  "env_stuff": "MockDos",
+  "an_integer": 7,
+  "another_integer": 42,
+  "should_be_seven": 7
+}

tests/res/rendering/expected_required.yaml

@@ -0,0 +1,48 @@
+foo: I am foo
+internal_ref:
+  deeper:
+    there: Something
+    everywhere: My heart will go on!
+base_key: is basic
+foo_inherited: Should be static!
+bar: This is bar
+group:
+  alpha: AAAAA!
+  beta: B
+  gamma_with_var: Foo:I am foo
+  delta: NewDeltaRocks!
+collapsed:
+  group:
+    successful: true
+    in:
+      one:
+        string: Yes!
+my_list:
+- one
+- two
+- three
+- I am foo
+- This is B group
+- No ${var}
+- ${__ENV__:I_DONT_WANT_THIS_GUY_TO_RESOLVE_TO_ANYTHING}
+import_test:
+  generic: Overridden
+  because: that's cool
+  imported_key:
+    import2: Tveir
+    import1: one
+    import3: three
+    included_env_stuff: MockDos
+required_key: veryimportantvalue
+a_float: 3.5
+a_bool: true
+a_false_bool: false
+a_none: null
+a_zero: 0
+a_default_env: I'm a default value!
+an_empty: ''
+new_delta: NewDeltaRocks!
+env_stuff: MockDos
+an_integer: 7
+another_integer: 42
+should_be_seven: 7

tests/res/rendering/json/required_env.json

@@ -0,0 +1,47 @@
+{
+  "__extends__": "_base_file.json",
+  "foo": "I am foo",
+  "bar": "This is bar",
+  "group": {
+    "alpha": "A",
+    "beta": "B",
+    "__extends__": "_subinclude.json"
+  },
+  "group.delta": "${new_delta}",
+  "collapsed.group.in.one.string": "Yes!",
+  "collapsed.group.__include__": "_collapse_import.json",
+  "my_list": [
+    "one",
+    "two",
+    "three",
+    "${foo}",
+    "This is ${group.beta} group",
+    "No ${var}",
+    "${__ENV__:I_DONT_WANT_THIS_GUY_TO_RESOLVE_TO_ANYTHING}"
+  ],
+  "import_test": {
+    "__extends__": [
+      "_import.json",
+      "inc/import2.json"
+    ],
+    "generic": "Overridden",
+    "imported_key.import2": "Tveir"
+  },
+  "required_key": "${__ENV__:PRETEND_API_KEY!=}",
+  "an_integer": 1,
+  "a_float": 3.5,
+  "a_bool": true,
+  "a_false_bool": false,
+  "a_none": null,
+  "a_zero": 0,
+  "a_default_env": "${__ENV__:I_DONT_WANT_THIS_GUY_TO_RESOLVE_TO_ANYTHING_EITHER=I'm a default value!}",
+  "an_empty": "",
+  "__include__": [
+    "_include1.json",
+    "inc/include2.json"
+  ],
+  "foo_inherited": "Should be static!",
+  "should_be_seven": "Nine!",
+  "new_delta": "NewDeltaRocks!",
+  "env_stuff": "${__ENV__:OS_MOCK}"
+}

tests/res/rendering/yaml/required_env.yaml

@@ -0,0 +1,39 @@
+# This is a cool Config File!
+__extends__: _base_file.yaml
+foo: I am foo
+bar: This is bar
+group:
+  alpha: A
+  beta: B
+  __extends__: _subinclude.yaml
+group.delta: ${new_delta}
+collapsed.group.in.one.string: Yes!
+collapsed.group.__include__: _collapse_import.yaml
+my_list:
+- one
+- two
+- three
+- ${foo}
+- This is ${group.beta} group
+- No ${var}
+- ${__ENV__:I_DONT_WANT_THIS_GUY_TO_RESOLVE_TO_ANYTHING}
+import_test:
+  __extends__: ["_import.yaml", "inc/import2.yaml"]
+  generic: Overridden
+  imported_key.import2: Tveir
+an_integer: 1
+required_key: ${__ENV__:PRETEND_API_KEY!=}
+a_float: 3.5
+a_bool: true
+a_false_bool: false
+a_none: null
+a_zero: 0
+a_default_env: ${__ENV__:I_DONT_WANT_THIS_GUY_TO_RESOLVE_TO_ANYTHING_EITHER=I'm a default value!}
+an_empty: ''
+__include__:
+- _include1.yaml
+- inc/include2.yaml
+foo_inherited: Should be static!
+should_be_seven: Nine!
+new_delta: NewDeltaRocks!
+env_stuff: ${__ENV__:OS_MOCK}