Merge pull request #66 from vkottler/dev/2.5.0

2.5.0 - Automatic build regeneration support

.github/workflows/python-package.yml

@@ -75,7 +75,7 @@ jobs:
 
  - run: |
  mk python-release owner=vkottler \
- repo=yambs version=2.4.2
+ repo=yambs version=2.5.0
  if: |
  matrix.python-version == '3.11'
  && matrix.system == 'ubuntu-latest'

README.md

@@ -2,11 +2,11 @@
  =====================================
  generator=datazen
  version=3.1.3
- hash=a00dd48dceea2789fda90dde3cb83fa9
+ hash=bc7cb8521b2bef4718193efe5c136f80
  =====================================
 -->
 
-# yambs ([2.4.2](https://pypi.org/project/yambs/))
+# yambs ([2.5.0](https://pypi.org/project/yambs/))
 
 [![python](https://img.shields.io/pypi/pyversions/yambs.svg)](https://pypi.org/project/yambs/)
 ![Build Status](https://github.com/vkottler/yambs/workflows/Python%20Package/badge.svg)
@@ -132,7 +132,8 @@ following a specific convention), put your configuration data here.
 ```
 $ ./venv3.11/bin/mbs -h
 
-usage: mbs [-h] [--version] [-v] [-C DIR] {dist,gen,native,uf2conv,noop} ...
+usage: mbs [-h] [--version] [-v] [-C DIR]
+ {compile_config,dist,gen,native,uf2conv,noop} ...
 
 Yet another meta build-system.
 
@@ -143,8 +144,9 @@ options:
  -C DIR, --dir DIR execute from a specific directory
 
 commands:
- {dist,gen,native,uf2conv,noop}
+ {compile_config,dist,gen,native,uf2conv,noop}
  set of available commands
+ compile_config load configuration data and write results to a file
  dist create a source distribution
  gen poll the source tree and generate any new build files
  native generate build files for native-only target projects
@@ -155,6 +157,31 @@ commands:
 
 ## Sub-command Options
 
+### `compile_config`
+
+```
+$ ./venv3.11/bin/mbs compile_config -h
+
+usage: mbs compile_config [-h] [-i INCLUDES_KEY] [-u] [-e]
+ output inputs [inputs ...]
+
+positional arguments:
+ output file to write
+ inputs files to read
+
+options:
+ -h, --help show this help message and exit
+ -i INCLUDES_KEY, --includes-key INCLUDES_KEY
+ top-level key to use for included files (default:
+ includes)
+ -u, --update whether or not to use the 'update' merge strategy
+ (instead of 'recursive')
+ -e, --expect-overwrite
+ allow configuration files to overwrite data when
+ loaded
+
+```
+
 ### `dist`
 
 ```
@@ -176,7 +203,7 @@ options:
 ```
 $ ./venv3.11/bin/mbs gen -h
 
-usage: mbs gen [-h] [-c CONFIG] [-i] [-w] [-s]
+usage: mbs gen [-h] [-c CONFIG] [-i] [-w] [-s] [-n]
 
 options:
  -h, --help show this help message and exit
@@ -187,6 +214,7 @@ options:
  -w, --watch whether or not to continue watching for source tree
  changes
  -s, --sources whether or not to only re-generate source manifests
+ -n, --no-build whether or not to skip running 'ninja'
 
 ```
 
@@ -195,7 +223,7 @@ options:
 ```
 $ ./venv3.11/bin/mbs native -h
 
-usage: mbs native [-h] [-c CONFIG] [-i] [-w] [-s]
+usage: mbs native [-h] [-c CONFIG] [-i] [-w] [-s] [-n]
 
 options:
  -h, --help show this help message and exit
@@ -206,6 +234,7 @@ options:
  -w, --watch whether or not to continue watching for source tree
  changes
  -s, --sources whether or not to only re-generate source manifests
+ -n, --no-build whether or not to skip running 'ninja'
 
 ```
 

local/configs/package.yaml

@@ -24,6 +24,8 @@ dev_requirements:
  - types-requests
 
 commands:
+ - name: compile_config
+ description: load configuration data and write results to a file
  - name: dist
  description: create a source distribution
  - name: gen

local/includes/sub_commands.yaml

@@ -3,7 +3,7 @@
 default_dirs: false
 
 commands:
-{% for command in ["dist", "gen", "native", "uf2conv"] %}
+{% for command in ["compile_config", "dist", "gen", "native", "uf2conv"] %}
  - name: help-{{command}}
  command: "./venv{{python_version}}/bin/{{entry}}"
  force: true

local/variables/package.yaml

@@ -1,5 +1,5 @@
 ---
 major: 2
-minor: 4
-patch: 2
+minor: 5
+patch: 0
 entry: mbs

manifest.yaml

@@ -30,6 +30,7 @@ renders:
  - renders-python_readme_header.md
  - renders-python_readme_dep_graph.md
  - commands-help
+ - commands-help-compile_config
  - commands-help-dist
  - commands-help-gen
  - commands-help-native

pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta:__legacy__"
 
 [project]
 name = "yambs"
-version = "2.4.2"
+version = "2.5.0"
 description = "Yet another meta build-system."
 readme = "README.md"
 requires-python = ">=3.11"

tests/commands/test_compile_config.py

@@ -0,0 +1,27 @@
+"""
+Test the 'commands.compile_config' module.
+"""
+
+# internal
+from tests.resources import resource
+
+# module under test
+from yambs import PKG_NAME
+from yambs.entry import main as yambs_main
+
+
+def test_compile_config_command_basic():
+ """Test the 'compile_config' command."""
+
+ base = [
+ PKG_NAME,
+ "compile_config",
+ str(resource("compile_config_out.json")),
+ ]
+
+ in1 = str(resource("compile_config_in1.yaml"))
+
+ assert yambs_main(base + [in1]) == 0
+ assert yambs_main(base + ["-e", in1]) == 0
+ assert yambs_main(base + ["-e", "-u", in1]) == 0
+ assert yambs_main(base + ["-u", in1]) == 0

tests/data/valid/compile_config_in1.yaml

@@ -0,0 +1,4 @@
+---
+a: 1
+b: 2
+c: 3

tests/data/valid/compile_config_in2.yaml

@@ -0,0 +1,4 @@
+---
+d: 4
+e: 5
+f: 6

tests/data/valid/compile_config_out.json

@@ -0,0 +1,5 @@
+{
+ "a": 1,
+ "b": 2,
+ "c": 3
+}

tests/data/valid/scenarios/native/yambs.yaml

@@ -11,6 +11,9 @@ project:
  owner: &self vkottler
 
 variants:
+ # Toggle this to test regeneration.
+ # new_variant: {}
+
  clang:
  cflag_groups: [debug]
 

yambs/__init__.py

@@ -1,7 +1,7 @@
 # =====================================
 # generator=datazen
 # version=3.1.3
-# hash=8850eb976847e4398acb7f31506bb897
+# hash=b744ad1050c0889bacd918e8f2a33ad7
 # =====================================
 
 """
@@ -10,4 +10,4 @@
 
 DESCRIPTION = "Yet another meta build-system."
 PKG_NAME = "yambs"
-VERSION = "2.4.2"
+VERSION = "2.5.0"

yambs/commands/all.py

@@ -1,7 +1,7 @@
 # =====================================
 # generator=datazen
 # version=3.1.3
-# hash=f12a77b15954045668945389858afc26
+# hash=e10efb5b655cacd368d023aacf84f288
 # =====================================
 
 """
@@ -16,6 +16,7 @@
 from vcorelib.args import CommandRegister as _CommandRegister
 
 # internal
+from yambs.commands.compile_config import add_compile_config_cmd
 from yambs.commands.dist import add_dist_cmd
 from yambs.commands.gen import add_gen_cmd
 from yambs.commands.native import add_native_cmd
@@ -26,6 +27,11 @@ def commands() -> _List[_Tuple[str, str, _CommandRegister]]:
  """Get this package's commands."""
 
  return [
+ (
+ "compile_config",
+ "load configuration data and write results to a file",
+ add_compile_config_cmd,
+ ),
  (
  "dist",
  "create a source distribution",

yambs/commands/compile_config.py

@@ -0,0 +1,76 @@
+"""
+An entry-point for the 'compile_config' command.
+"""
+
+# built-in
+from argparse import ArgumentParser as _ArgumentParser
+from argparse import Namespace as _Namespace
+from pathlib import Path
+
+# third-party
+from vcorelib.args import CommandFunction as _CommandFunction
+from vcorelib.dict import MergeStrategy, merge_dicts
+from vcorelib.io import ARBITER, DEFAULT_INCLUDES_KEY
+
+
+def compile_config_cmd(args: _Namespace) -> int:
+ """Execute the compile_config command."""
+
+ merge_strat = MergeStrategy.RECURSIVE
+ if args.update:
+ merge_strat = MergeStrategy.UPDATE
+
+ return (
+ 0
+ if ARBITER.encode(
+ args.output,
+ merge_dicts(
+ [
+ ARBITER.decode(
+ file,
+ require_success=True,
+ includes_key=args.includes_key,
+ expect_overwrite=args.expect_overwrite,
+ strategy=merge_strat,
+ ).data
+ for file in args.inputs
+ ],
+ expect_overwrite=args.expect_overwrite,
+ strategy=merge_strat,
+ ),
+ )[0]
+ else 1
+ )
+
+
+def add_compile_config_cmd(parser: _ArgumentParser) -> _CommandFunction:
+ """Add dist-command arguments to its parser."""
+
+ parser.add_argument(
+ "-i",
+ "--includes-key",
+ default=DEFAULT_INCLUDES_KEY,
+ help="top-level key to use for included files (default: %(default)s)",
+ )
+
+ parser.add_argument(
+ "-u",
+ "--update",
+ action="store_true",
+ help=(
+ "whether or not to use the 'update' merge strategy "
+ "(instead of 'recursive')"
+ ),
+ )
+
+ parser.add_argument(
+ "-e",
+ "--expect-overwrite",
+ action="store_true",
+ help="allow configuration files to overwrite data when loaded",
+ )
+
+ parser.add_argument("output", type=Path, help="file to write")
+ parser.add_argument("inputs", nargs="+", type=Path, help="files to read")
+
+ return compile_config_cmd

yambs/config/common.py

@@ -4,6 +4,7 @@
 
 # built-in
 from pathlib import Path
+from sys import executable
 from typing import Any, Dict, NamedTuple, Optional, Set, Type, TypeVar
 
 # third-party
@@ -100,6 +101,9 @@ def init(self, data: _JsonObject) -> None:
  """Initialize this instance."""
 
  self.data = data
+ self.data["entry"] = f"{executable} -m {PKG_NAME}"
+ self.data["config_file"] = str(DEFAULT_CONFIG)
+
  self.root = Path(data["root"]) # type: ignore
 
  self.src_root = self.directory("src_root")
@@ -157,5 +161,6 @@ def load(
 
  if path.is_file():
  result.file = path
+ result.data["config_file"] = str(path)
 
  return result

yambs/data/templates/compile_commands.ninja.j2

@@ -1,3 +1,3 @@
 rule compile_commands
  command = ninja -t compdb > compile_commands.json
-build compdb: compile_commands
+build compdb: compile_commands | regen

yambs/data/templates/native_build.ninja.j2

@@ -6,6 +6,9 @@ generated_dir = $src_dir/generated
 common_cflags = -I${src_dir}{% for flag in common_cflags %} {{flag}}{% endfor %}
 
 
+{% include "regenerate.ninja.j2" %}
+
+
 include $include_dir/all.ninja
 
 {% include "compile_commands.ninja.j2" %}

yambs/data/templates/native_rules.ninja.j2

@@ -27,6 +27,6 @@ build_dir = {{build_root}}/$variant
 rule script
  command = /bin/bash $in $out
 
-build $build_dir/third_party.txt: script $include_dir/third_party.sh
+build $build_dir/third_party.txt: script $include_dir/third_party.sh | regen
 
 build ${variant}_third_party: phony $build_dir/third_party.txt

yambs/data/templates/regenerate.ninja.j2

@@ -0,0 +1,17 @@
+###############################################################################
+
+# regeneration logic
+
+rule compile_config
+ command = {{entry}} compile_config $out $in
+
+build $include_dir/compiled_config.json: compile_config {{config_file}}
+build check_config: phony $include_dir/compiled_config.json
+
+rule native
+ command = {{entry}} native -n -c $in
+
+build | build.ninja: native {{config_file}} | check_config
+build regen: phony build.ninja
+
+###############################################################################