From b80f800a8f26208c14f8b2f659fd2cc3f8fae1b9 Mon Sep 17 00:00:00 2001
From: Vaughn Kottler <vaughnkottler@gmail.com>
Date: Tue, 5 Sep 2023 01:11:08 -0500
Subject: [PATCH] 2.5.0 - Automatic build regeneration support

---
 .github/workflows/python-package.yml          |  2 +-
 README.md                                     | 41 ++++++++--
 local/configs/package.yaml                    |  2 +
 local/includes/sub_commands.yaml              |  2 +-
 local/variables/package.yaml                  |  4 +-
 manifest.yaml                                 |  1 +
 pyproject.toml                                |  2 +-
 tests/commands/test_compile_config.py         | 27 +++++++
 tests/data/valid/compile_config_in1.yaml      |  4 +
 tests/data/valid/compile_config_in2.yaml      |  4 +
 tests/data/valid/compile_config_out.json      |  5 ++
 tests/data/valid/scenarios/native/yambs.yaml  |  3 +
 yambs/__init__.py                             |  4 +-
 yambs/commands/all.py                         |  8 +-
 yambs/commands/compile_config.py              | 76 +++++++++++++++++++
 yambs/config/common.py                        |  5 ++
 .../data/templates/compile_commands.ninja.j2  |  2 +-
 yambs/data/templates/native_build.ninja.j2    |  3 +
 yambs/data/templates/native_rules.ninja.j2    |  2 +-
 yambs/data/templates/regenerate.ninja.j2      | 17 +++++
 yambs/data/templates/variant.ninja.j2         |  2 +-
 yambs/dependency/handlers/yambs/__init__.py   | 11 ++-
 22 files changed, 209 insertions(+), 18 deletions(-)
 create mode 100644 tests/commands/test_compile_config.py
 create mode 100644 tests/data/valid/compile_config_in1.yaml
 create mode 100644 tests/data/valid/compile_config_in2.yaml
 create mode 100644 tests/data/valid/compile_config_out.json
 create mode 100644 yambs/commands/compile_config.py
 create mode 100644 yambs/data/templates/regenerate.ninja.j2

diff --git a/.github/workflows/python-package.yml b/.github/workflows/python-package.yml
index ba4e6a3..0bdfb7f 100644
--- a/.github/workflows/python-package.yml
+++ b/.github/workflows/python-package.yml
@@ -75,7 +75,7 @@ jobs:
 
       - run: |
           mk python-release owner=vkottler \
-                            repo=yambs version=2.4.2
+                            repo=yambs version=2.5.0
         if: |
           matrix.python-version == '3.11'
           && matrix.system == 'ubuntu-latest'
diff --git a/README.md b/README.md
index 78551fd..271e4d4 100644
--- a/README.md
+++ b/README.md
@@ -2,11 +2,11 @@
     =====================================
     generator=datazen
     version=3.1.3
-    hash=a00dd48dceea2789fda90dde3cb83fa9
+    hash=bc7cb8521b2bef4718193efe5c136f80
     =====================================
 -->
 
-# yambs ([2.4.2](https://pypi.org/project/yambs/))
+# yambs ([2.5.0](https://pypi.org/project/yambs/))
 
 [![python](https://img.shields.io/pypi/pyversions/yambs.svg)](https://pypi.org/project/yambs/)
 ![Build Status](https://github.com/vkottler/yambs/workflows/Python%20Package/badge.svg)
@@ -132,7 +132,8 @@ following a specific convention), put your configuration data here.
 ```
 $ ./venv3.11/bin/mbs -h
 
-usage: mbs [-h] [--version] [-v] [-C DIR] {dist,gen,native,uf2conv,noop} ...
+usage: mbs [-h] [--version] [-v] [-C DIR]
+           {compile_config,dist,gen,native,uf2conv,noop} ...
 
 Yet another meta build-system.
 
@@ -143,8 +144,9 @@ options:
   -C DIR, --dir DIR     execute from a specific directory
 
 commands:
-  {dist,gen,native,uf2conv,noop}
+  {compile_config,dist,gen,native,uf2conv,noop}
                         set of available commands
+    compile_config      load configuration data and write results to a file
     dist                create a source distribution
     gen                 poll the source tree and generate any new build files
     native              generate build files for native-only target projects
@@ -155,6 +157,31 @@ commands:
 
 ## Sub-command Options
 
+### `compile_config`
+
+```
+$ ./venv3.11/bin/mbs compile_config -h
+
+usage: mbs compile_config [-h] [-i INCLUDES_KEY] [-u] [-e]
+                          output inputs [inputs ...]
+
+positional arguments:
+  output                file to write
+  inputs                files to read
+
+options:
+  -h, --help            show this help message and exit
+  -i INCLUDES_KEY, --includes-key INCLUDES_KEY
+                        top-level key to use for included files (default:
+                        includes)
+  -u, --update          whether or not to use the 'update' merge strategy
+                        (instead of 'recursive')
+  -e, --expect-overwrite
+                        allow configuration files to overwrite data when
+                        loaded
+
+```
+
 ### `dist`
 
 ```
@@ -176,7 +203,7 @@ options:
 ```
 $ ./venv3.11/bin/mbs gen -h
 
-usage: mbs gen [-h] [-c CONFIG] [-i] [-w] [-s]
+usage: mbs gen [-h] [-c CONFIG] [-i] [-w] [-s] [-n]
 
 options:
   -h, --help            show this help message and exit
@@ -187,6 +214,7 @@ options:
   -w, --watch           whether or not to continue watching for source tree
                         changes
   -s, --sources         whether or not to only re-generate source manifests
+  -n, --no-build        whether or not to skip running 'ninja'
 
 ```
 
@@ -195,7 +223,7 @@ options:
 ```
 $ ./venv3.11/bin/mbs native -h
 
-usage: mbs native [-h] [-c CONFIG] [-i] [-w] [-s]
+usage: mbs native [-h] [-c CONFIG] [-i] [-w] [-s] [-n]
 
 options:
   -h, --help            show this help message and exit
@@ -206,6 +234,7 @@ options:
   -w, --watch           whether or not to continue watching for source tree
                         changes
   -s, --sources         whether or not to only re-generate source manifests
+  -n, --no-build        whether or not to skip running 'ninja'
 
 ```
 
diff --git a/local/configs/package.yaml b/local/configs/package.yaml
index 424816f..9678d92 100644
--- a/local/configs/package.yaml
+++ b/local/configs/package.yaml
@@ -24,6 +24,8 @@ dev_requirements:
   - types-requests
 
 commands:
+  - name: compile_config
+    description: load configuration data and write results to a file
   - name: dist
     description: create a source distribution
   - name: gen
diff --git a/local/includes/sub_commands.yaml b/local/includes/sub_commands.yaml
index f416cff..c40bbf3 100644
--- a/local/includes/sub_commands.yaml
+++ b/local/includes/sub_commands.yaml
@@ -3,7 +3,7 @@
 default_dirs: false
 
 commands:
-{% for command in ["dist", "gen", "native", "uf2conv"] %}
+{% for command in ["compile_config", "dist", "gen", "native", "uf2conv"] %}
   - name: help-{{command}}
     command: "./venv{{python_version}}/bin/{{entry}}"
     force: true
diff --git a/local/variables/package.yaml b/local/variables/package.yaml
index b682e04..7db3497 100644
--- a/local/variables/package.yaml
+++ b/local/variables/package.yaml
@@ -1,5 +1,5 @@
 ---
 major: 2
-minor: 4
-patch: 2
+minor: 5
+patch: 0
 entry: mbs
diff --git a/manifest.yaml b/manifest.yaml
index 0ee9568..cbe1fa0 100644
--- a/manifest.yaml
+++ b/manifest.yaml
@@ -30,6 +30,7 @@ renders:
       - renders-python_readme_header.md
       - renders-python_readme_dep_graph.md
       - commands-help
+      - commands-help-compile_config
       - commands-help-dist
       - commands-help-gen
       - commands-help-native
diff --git a/pyproject.toml b/pyproject.toml
index 6cb8b3c..7da54b6 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta:__legacy__"
 
 [project]
 name = "yambs"
-version = "2.4.2"
+version = "2.5.0"
 description = "Yet another meta build-system."
 readme = "README.md"
 requires-python = ">=3.11"
diff --git a/tests/commands/test_compile_config.py b/tests/commands/test_compile_config.py
new file mode 100644
index 0000000..76341f8
--- /dev/null
+++ b/tests/commands/test_compile_config.py
@@ -0,0 +1,27 @@
+"""
+Test the 'commands.compile_config' module.
+"""
+
+# internal
+from tests.resources import resource
+
+# module under test
+from yambs import PKG_NAME
+from yambs.entry import main as yambs_main
+
+
+def test_compile_config_command_basic():
+    """Test the 'compile_config' command."""
+
+    base = [
+        PKG_NAME,
+        "compile_config",
+        str(resource("compile_config_out.json")),
+    ]
+
+    in1 = str(resource("compile_config_in1.yaml"))
+
+    assert yambs_main(base + [in1]) == 0
+    assert yambs_main(base + ["-e", in1]) == 0
+    assert yambs_main(base + ["-e", "-u", in1]) == 0
+    assert yambs_main(base + ["-u", in1]) == 0
diff --git a/tests/data/valid/compile_config_in1.yaml b/tests/data/valid/compile_config_in1.yaml
new file mode 100644
index 0000000..4021d74
--- /dev/null
+++ b/tests/data/valid/compile_config_in1.yaml
@@ -0,0 +1,4 @@
+---
+a: 1
+b: 2
+c: 3
diff --git a/tests/data/valid/compile_config_in2.yaml b/tests/data/valid/compile_config_in2.yaml
new file mode 100644
index 0000000..db47e75
--- /dev/null
+++ b/tests/data/valid/compile_config_in2.yaml
@@ -0,0 +1,4 @@
+---
+d: 4
+e: 5
+f: 6
diff --git a/tests/data/valid/compile_config_out.json b/tests/data/valid/compile_config_out.json
new file mode 100644
index 0000000..5527ab8
--- /dev/null
+++ b/tests/data/valid/compile_config_out.json
@@ -0,0 +1,5 @@
+{
+  "a": 1,
+  "b": 2,
+  "c": 3
+}
\ No newline at end of file
diff --git a/tests/data/valid/scenarios/native/yambs.yaml b/tests/data/valid/scenarios/native/yambs.yaml
index 80e43c6..3322882 100644
--- a/tests/data/valid/scenarios/native/yambs.yaml
+++ b/tests/data/valid/scenarios/native/yambs.yaml
@@ -11,6 +11,9 @@ project:
     owner: &self vkottler
 
 variants:
+  # Toggle this to test regeneration.
+  # new_variant: {}
+
   clang:
     cflag_groups: [debug]
 
diff --git a/yambs/__init__.py b/yambs/__init__.py
index db909df..54af25a 100644
--- a/yambs/__init__.py
+++ b/yambs/__init__.py
@@ -1,7 +1,7 @@
 # =====================================
 # generator=datazen
 # version=3.1.3
-# hash=8850eb976847e4398acb7f31506bb897
+# hash=b744ad1050c0889bacd918e8f2a33ad7
 # =====================================
 
 """
@@ -10,4 +10,4 @@
 
 DESCRIPTION = "Yet another meta build-system."
 PKG_NAME = "yambs"
-VERSION = "2.4.2"
+VERSION = "2.5.0"
diff --git a/yambs/commands/all.py b/yambs/commands/all.py
index 3132ed3..410a5fd 100644
--- a/yambs/commands/all.py
+++ b/yambs/commands/all.py
@@ -1,7 +1,7 @@
 # =====================================
 # generator=datazen
 # version=3.1.3
-# hash=f12a77b15954045668945389858afc26
+# hash=e10efb5b655cacd368d023aacf84f288
 # =====================================
 
 """
@@ -16,6 +16,7 @@
 from vcorelib.args import CommandRegister as _CommandRegister
 
 # internal
+from yambs.commands.compile_config import add_compile_config_cmd
 from yambs.commands.dist import add_dist_cmd
 from yambs.commands.gen import add_gen_cmd
 from yambs.commands.native import add_native_cmd
@@ -26,6 +27,11 @@ def commands() -> _List[_Tuple[str, str, _CommandRegister]]:
     """Get this package's commands."""
 
     return [
+        (
+            "compile_config",
+            "load configuration data and write results to a file",
+            add_compile_config_cmd,
+        ),
         (
             "dist",
             "create a source distribution",
diff --git a/yambs/commands/compile_config.py b/yambs/commands/compile_config.py
new file mode 100644
index 0000000..49ca95b
--- /dev/null
+++ b/yambs/commands/compile_config.py
@@ -0,0 +1,76 @@
+"""
+An entry-point for the 'compile_config' command.
+"""
+
+# built-in
+from argparse import ArgumentParser as _ArgumentParser
+from argparse import Namespace as _Namespace
+from pathlib import Path
+
+# third-party
+from vcorelib.args import CommandFunction as _CommandFunction
+from vcorelib.dict import MergeStrategy, merge_dicts
+from vcorelib.io import ARBITER, DEFAULT_INCLUDES_KEY
+
+
+def compile_config_cmd(args: _Namespace) -> int:
+    """Execute the compile_config command."""
+
+    merge_strat = MergeStrategy.RECURSIVE
+    if args.update:
+        merge_strat = MergeStrategy.UPDATE
+
+    return (
+        0
+        if ARBITER.encode(
+            args.output,
+            merge_dicts(
+                [
+                    ARBITER.decode(
+                        file,
+                        require_success=True,
+                        includes_key=args.includes_key,
+                        expect_overwrite=args.expect_overwrite,
+                        strategy=merge_strat,
+                    ).data
+                    for file in args.inputs
+                ],
+                expect_overwrite=args.expect_overwrite,
+                strategy=merge_strat,
+            ),
+        )[0]
+        else 1
+    )
+
+
+def add_compile_config_cmd(parser: _ArgumentParser) -> _CommandFunction:
+    """Add dist-command arguments to its parser."""
+
+    parser.add_argument(
+        "-i",
+        "--includes-key",
+        default=DEFAULT_INCLUDES_KEY,
+        help="top-level key to use for included files (default: %(default)s)",
+    )
+
+    parser.add_argument(
+        "-u",
+        "--update",
+        action="store_true",
+        help=(
+            "whether or not to use the 'update' merge strategy "
+            "(instead of 'recursive')"
+        ),
+    )
+
+    parser.add_argument(
+        "-e",
+        "--expect-overwrite",
+        action="store_true",
+        help="allow configuration files to overwrite data when loaded",
+    )
+
+    parser.add_argument("output", type=Path, help="file to write")
+    parser.add_argument("inputs", nargs="+", type=Path, help="files to read")
+
+    return compile_config_cmd
diff --git a/yambs/config/common.py b/yambs/config/common.py
index f56d6c1..82d3209 100644
--- a/yambs/config/common.py
+++ b/yambs/config/common.py
@@ -4,6 +4,7 @@
 
 # built-in
 from pathlib import Path
+from sys import executable
 from typing import Any, Dict, NamedTuple, Optional, Set, Type, TypeVar
 
 # third-party
@@ -100,6 +101,9 @@ def init(self, data: _JsonObject) -> None:
         """Initialize this instance."""
 
         self.data = data
+        self.data["entry"] = f"{executable} -m {PKG_NAME}"
+        self.data["config_file"] = str(DEFAULT_CONFIG)
+
         self.root = Path(data["root"])  # type: ignore
 
         self.src_root = self.directory("src_root")
@@ -157,5 +161,6 @@ def load(
 
         if path.is_file():
             result.file = path
+            result.data["config_file"] = str(path)
 
         return result
diff --git a/yambs/data/templates/compile_commands.ninja.j2 b/yambs/data/templates/compile_commands.ninja.j2
index 3b13d50..958435c 100644
--- a/yambs/data/templates/compile_commands.ninja.j2
+++ b/yambs/data/templates/compile_commands.ninja.j2
@@ -1,3 +1,3 @@
 rule compile_commands
   command = ninja -t compdb > compile_commands.json
-build compdb: compile_commands
+build compdb: compile_commands | regen
diff --git a/yambs/data/templates/native_build.ninja.j2 b/yambs/data/templates/native_build.ninja.j2
index 95cc86f..5eb6c37 100644
--- a/yambs/data/templates/native_build.ninja.j2
+++ b/yambs/data/templates/native_build.ninja.j2
@@ -6,6 +6,9 @@ generated_dir = $src_dir/generated
 common_cflags = -I${src_dir}{% for flag in common_cflags %} {{flag}}{% endfor %}
 
 
+{% include "regenerate.ninja.j2" %}
+
+
 include $include_dir/all.ninja
 
 {% include "compile_commands.ninja.j2" %}
diff --git a/yambs/data/templates/native_rules.ninja.j2 b/yambs/data/templates/native_rules.ninja.j2
index 1b3a6af..10ab85a 100644
--- a/yambs/data/templates/native_rules.ninja.j2
+++ b/yambs/data/templates/native_rules.ninja.j2
@@ -27,6 +27,6 @@ build_dir = {{build_root}}/$variant
 rule script
   command = /bin/bash $in $out
 
-build $build_dir/third_party.txt: script $include_dir/third_party.sh
+build $build_dir/third_party.txt: script $include_dir/third_party.sh | regen
 
 build ${variant}_third_party: phony $build_dir/third_party.txt
diff --git a/yambs/data/templates/regenerate.ninja.j2 b/yambs/data/templates/regenerate.ninja.j2
new file mode 100644
index 0000000..679e414
--- /dev/null
+++ b/yambs/data/templates/regenerate.ninja.j2
@@ -0,0 +1,17 @@
+###############################################################################
+
+# regeneration logic
+
+rule compile_config
+  command = {{entry}} compile_config $out $in
+
+build $include_dir/compiled_config.json: compile_config {{config_file}}
+build check_config: phony $include_dir/compiled_config.json
+
+rule native
+  command = {{entry}} native -n -c $in
+
+build | build.ninja: native {{config_file}} | check_config
+build regen: phony build.ninja
+
+###############################################################################
diff --git a/yambs/data/templates/variant.ninja.j2 b/yambs/data/templates/variant.ninja.j2
index b4c8261..c7a5922 100644
--- a/yambs/data/templates/variant.ninja.j2
+++ b/yambs/data/templates/variant.ninja.j2
@@ -20,4 +20,4 @@ include $include_dir/rules.ninja
 include $include_dir/sources.ninja
 include $include_dir/apps.ninja
 
-build $variant: phony{% for task in targets %} ${variant}_{{task}}{% endfor %}
+build $variant: phony{% for task in targets %} ${variant}_{{task}}{% endfor %} | regen
diff --git a/yambs/dependency/handlers/yambs/__init__.py b/yambs/dependency/handlers/yambs/__init__.py
index fffabac..7fce602 100644
--- a/yambs/dependency/handlers/yambs/__init__.py
+++ b/yambs/dependency/handlers/yambs/__init__.py
@@ -77,7 +77,16 @@ def handle_static_lib(directory: Path, task: DependencyTask) -> None:
         # if the desired library isn't already built.
         if task.dep.source == DependencySource.DIRECTORY:
             cmd.extend(
-                [executable, "-m", PKG_NAME, "-C", rel_dir_arg, "native", "&&"]
+                [
+                    executable,
+                    "-m",
+                    PKG_NAME,
+                    "-C",
+                    rel_dir_arg,
+                    "native",
+                    "-n",
+                    "&&",
+                ]
             )
 
         # Add a build command if the library still needs to be built.