2.5.0 - Automatic build regeneration support

.github/workflows/python-package.yml

@@ -75,7 +75,7 @@ jobs:
 
       - run: |
           mk python-release owner=vkottler \
-                            repo=yambs version=2.4.2
+                            repo=yambs version=2.5.0
         if: |
           matrix.python-version == '3.11'
           && matrix.system == 'ubuntu-latest'

README.md

@@ -2,11 +2,11 @@
     =====================================
     generator=datazen
     version=3.1.3
-    hash=a00dd48dceea2789fda90dde3cb83fa9
+    hash=bc7cb8521b2bef4718193efe5c136f80
     =====================================
 -->
 
-# yambs ([2.4.2](https://pypi.org/project/yambs/))
+# yambs ([2.5.0](https://pypi.org/project/yambs/))
 
 [![python](https://img.shields.io/pypi/pyversions/yambs.svg)](https://pypi.org/project/yambs/)
 ![Build Status](https://github.com/vkottler/yambs/workflows/Python%20Package/badge.svg)
@@ -132,7 +132,8 @@ following a specific convention), put your configuration data here.
 ```
 $ ./venv3.11/bin/mbs -h
 
-usage: mbs [-h] [--version] [-v] [-C DIR] {dist,gen,native,uf2conv,noop} ...
+usage: mbs [-h] [--version] [-v] [-C DIR]
+           {compile_config,dist,gen,native,uf2conv,noop} ...
 
 Yet another meta build-system.
 
@@ -143,8 +144,9 @@ options:
   -C DIR, --dir DIR     execute from a specific directory
 
 commands:
-  {dist,gen,native,uf2conv,noop}
+  {compile_config,dist,gen,native,uf2conv,noop}
                         set of available commands
+    compile_config      load configuration data and write results to a file
     dist                create a source distribution
     gen                 poll the source tree and generate any new build files
     native              generate build files for native-only target projects
@@ -155,6 +157,31 @@ commands:
 
 ## Sub-command Options
 
+### `compile_config`
+
+```
+$ ./venv3.11/bin/mbs compile_config -h
+
+usage: mbs compile_config [-h] [-i INCLUDES_KEY] [-u] [-e]
+                          output inputs [inputs ...]
+
+positional arguments:
+  output                file to write
+  inputs                files to read
+
+options:
+  -h, --help            show this help message and exit
+  -i INCLUDES_KEY, --includes-key INCLUDES_KEY
+                        top-level key to use for included files (default:
+                        includes)
+  -u, --update          whether or not to use the 'update' merge strategy
+                        (instead of 'recursive')
+  -e, --expect-overwrite
+                        allow configuration files to overwrite data when
+                        loaded
+
+```
+
 ### `dist`
 
 ```
@@ -176,7 +203,7 @@ options:
 ```
 $ ./venv3.11/bin/mbs gen -h
 
-usage: mbs gen [-h] [-c CONFIG] [-i] [-w] [-s]
+usage: mbs gen [-h] [-c CONFIG] [-i] [-w] [-s] [-n]
 
 options:
   -h, --help            show this help message and exit
@@ -187,6 +214,7 @@ options:
   -w, --watch           whether or not to continue watching for source tree
                         changes
   -s, --sources         whether or not to only re-generate source manifests
+  -n, --no-build        whether or not to skip running 'ninja'
 
 ```
 
@@ -195,7 +223,7 @@ options:
 ```
 $ ./venv3.11/bin/mbs native -h
 
-usage: mbs native [-h] [-c CONFIG] [-i] [-w] [-s]
+usage: mbs native [-h] [-c CONFIG] [-i] [-w] [-s] [-n]
 
 options:
   -h, --help            show this help message and exit
@@ -206,6 +234,7 @@ options:
   -w, --watch           whether or not to continue watching for source tree
                         changes
   -s, --sources         whether or not to only re-generate source manifests
+  -n, --no-build        whether or not to skip running 'ninja'
 
 ```
 

local/configs/package.yaml

@@ -24,6 +24,8 @@ dev_requirements:
   - types-requests
 
 commands:
+  - name: compile_config
+    description: load configuration data and write results to a file
   - name: dist
     description: create a source distribution
   - name: gen

local/includes/sub_commands.yaml

@@ -3,7 +3,7 @@
 default_dirs: false
 
 commands:
-{% for command in ["dist", "gen", "native", "uf2conv"] %}
+{% for command in ["compile_config", "dist", "gen", "native", "uf2conv"] %}
   - name: help-{{command}}
     command: "./venv{{python_version}}/bin/{{entry}}"
     force: true

local/variables/package.yaml

@@ -1,5 +1,5 @@
 ---
 major: 2
-minor: 4
-patch: 2
+minor: 5
+patch: 0
 entry: mbs

manifest.yaml

@@ -30,6 +30,7 @@ renders:
       - renders-python_readme_header.md
       - renders-python_readme_dep_graph.md
       - commands-help
+      - commands-help-compile_config
       - commands-help-dist
       - commands-help-gen
       - commands-help-native

pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta:__legacy__"
 
 [project]
 name = "yambs"
-version = "2.4.2"
+version = "2.5.0"
 description = "Yet another meta build-system."
 readme = "README.md"
 requires-python = ">=3.11"

tests/commands/test_compile_config.py

@@ -0,0 +1,27 @@
+"""
+Test the 'commands.compile_config' module.
+"""
+
+# internal
+from tests.resources import resource
+
+# module under test
+from yambs import PKG_NAME
+from yambs.entry import main as yambs_main
+
+
+def test_compile_config_command_basic():
+    """Test the 'compile_config' command."""
+
+    base = [
+        PKG_NAME,
+        "compile_config",
+        str(resource("compile_config_out.json")),
+    ]
+
+    in1 = str(resource("compile_config_in1.yaml"))
+
+    assert yambs_main(base + [in1]) == 0
+    assert yambs_main(base + ["-e", in1]) == 0
+    assert yambs_main(base + ["-e", "-u", in1]) == 0
+    assert yambs_main(base + ["-u", in1]) == 0

tests/data/valid/compile_config_in1.yaml

@@ -0,0 +1,4 @@
+---
+a: 1
+b: 2
+c: 3

tests/data/valid/compile_config_in2.yaml

@@ -0,0 +1,4 @@
+---
+d: 4
+e: 5
+f: 6

tests/data/valid/compile_config_out.json

@@ -0,0 +1,5 @@
+{
+  "a": 1,
+  "b": 2,
+  "c": 3
+}

tests/data/valid/scenarios/native/yambs.yaml

@@ -11,6 +11,9 @@ project:
     owner: &self vkottler
 
 variants:
+  # Toggle this to test regeneration.
+  # new_variant: {}
+
   clang:
     cflag_groups: [debug]
 

yambs/__init__.py

@@ -1,7 +1,7 @@
 # =====================================
 # generator=datazen
 # version=3.1.3
-# hash=8850eb976847e4398acb7f31506bb897
+# hash=b744ad1050c0889bacd918e8f2a33ad7
 # =====================================
 
 """
@@ -10,4 +10,4 @@
 
 DESCRIPTION = "Yet another meta build-system."
 PKG_NAME = "yambs"
-VERSION = "2.4.2"
+VERSION = "2.5.0"

yambs/commands/all.py

@@ -1,7 +1,7 @@
 # =====================================
 # generator=datazen
 # version=3.1.3
-# hash=f12a77b15954045668945389858afc26
+# hash=e10efb5b655cacd368d023aacf84f288
 # =====================================
 
 """
@@ -16,6 +16,7 @@
 from vcorelib.args import CommandRegister as _CommandRegister
 
 # internal
+from yambs.commands.compile_config import add_compile_config_cmd
 from yambs.commands.dist import add_dist_cmd
 from yambs.commands.gen import add_gen_cmd
 from yambs.commands.native import add_native_cmd
@@ -26,6 +27,11 @@ def commands() -> _List[_Tuple[str, str, _CommandRegister]]:
     """Get this package's commands."""
 
     return [
+        (
+            "compile_config",
+            "load configuration data and write results to a file",
+            add_compile_config_cmd,
+        ),
         (
             "dist",
             "create a source distribution",

yambs/commands/compile_config.py

@@ -0,0 +1,76 @@
+"""
+An entry-point for the 'compile_config' command.
+"""
+
+# built-in
+from argparse import ArgumentParser as _ArgumentParser
+from argparse import Namespace as _Namespace
+from pathlib import Path
+
+# third-party
+from vcorelib.args import CommandFunction as _CommandFunction
+from vcorelib.dict import MergeStrategy, merge_dicts
+from vcorelib.io import ARBITER, DEFAULT_INCLUDES_KEY
+
+
+def compile_config_cmd(args: _Namespace) -> int:
+    """Execute the compile_config command."""
+
+    merge_strat = MergeStrategy.RECURSIVE
+    if args.update:
+        merge_strat = MergeStrategy.UPDATE
+
+    return (
+        0
+        if ARBITER.encode(
+            args.output,
+            merge_dicts(
+                [
+                    ARBITER.decode(
+                        file,
+                        require_success=True,
+                        includes_key=args.includes_key,
+                        expect_overwrite=args.expect_overwrite,
+                        strategy=merge_strat,
+                    ).data
+                    for file in args.inputs
+                ],
+                expect_overwrite=args.expect_overwrite,
+                strategy=merge_strat,
+            ),
+        )[0]
+        else 1
+    )
+
+
+def add_compile_config_cmd(parser: _ArgumentParser) -> _CommandFunction:
+    """Add dist-command arguments to its parser."""
+
+    parser.add_argument(
+        "-i",
+        "--includes-key",
+        default=DEFAULT_INCLUDES_KEY,
+        help="top-level key to use for included files (default: %(default)s)",
+    )
+
+    parser.add_argument(
+        "-u",
+        "--update",
+        action="store_true",
+        help=(
+            "whether or not to use the 'update' merge strategy "
+            "(instead of 'recursive')"
+        ),
+    )
+
+    parser.add_argument(
+        "-e",
+        "--expect-overwrite",
+        action="store_true",
+        help="allow configuration files to overwrite data when loaded",
+    )
+
+    parser.add_argument("output", type=Path, help="file to write")
+    parser.add_argument("inputs", nargs="+", type=Path, help="files to read")
+
+    return compile_config_cmd

yambs/config/common.py

@@ -4,6 +4,7 @@
 
 # built-in
 from pathlib import Path
+from sys import executable
 from typing import Any, Dict, NamedTuple, Optional, Set, Type, TypeVar
 
 # third-party
@@ -100,6 +101,9 @@ def init(self, data: _JsonObject) -> None:
         """Initialize this instance."""
 
         self.data = data
+        self.data["entry"] = f"{executable} -m {PKG_NAME}"
+        self.data["config_file"] = str(DEFAULT_CONFIG)
+
         self.root = Path(data["root"])  # type: ignore
 
         self.src_root = self.directory("src_root")
@@ -157,5 +161,6 @@ def load(
 
         if path.is_file():
             result.file = path
+            result.data["config_file"] = str(path)
 
         return result

yambs/data/templates/compile_commands.ninja.j2

@@ -1,3 +1,3 @@
 rule compile_commands
   command = ninja -t compdb > compile_commands.json
-build compdb: compile_commands
+build compdb: compile_commands | regen

yambs/data/templates/native_build.ninja.j2

@@ -6,6 +6,9 @@ generated_dir = $src_dir/generated
 common_cflags = -I${src_dir}{% for flag in common_cflags %} {{flag}}{% endfor %}
 
 
+{% include "regenerate.ninja.j2" %}
+
+
 include $include_dir/all.ninja
 
 {% include "compile_commands.ninja.j2" %}

yambs/data/templates/native_rules.ninja.j2

@@ -27,6 +27,6 @@ build_dir = {{build_root}}/$variant
 rule script
   command = /bin/bash $in $out
 
-build $build_dir/third_party.txt: script $include_dir/third_party.sh
+build $build_dir/third_party.txt: script $include_dir/third_party.sh | regen
 
 build ${variant}_third_party: phony $build_dir/third_party.txt

yambs/data/templates/regenerate.ninja.j2

@@ -0,0 +1,17 @@
+###############################################################################
+
+# regeneration logic
+
+rule compile_config
+  command = {{entry}} compile_config $out $in
+
+build $include_dir/compiled_config.json: compile_config {{config_file}}
+build check_config: phony $include_dir/compiled_config.json
+
+rule native
+  command = {{entry}} native -n -c $in
+
+build | build.ninja: native {{config_file}} | check_config
+build regen: phony build.ninja
+
+###############################################################################