From b88e298f9adbd2bf17f84b76f044fb77710196da Mon Sep 17 00:00:00 2001 From: Vaughn Kottler Date: Sun, 16 Jul 2023 12:52:48 -0500 Subject: [PATCH 1/4] Add fPIC arg to everything --- yambs/data/native.yaml | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/yambs/data/native.yaml b/yambs/data/native.yaml index 7882cba..fd6d332 100644 --- a/yambs/data/native.yaml +++ b/yambs/data/native.yaml @@ -3,6 +3,9 @@ includes: - includes/common.yaml cflag_groups: + # Compile position-independent so we can build shared objects. + common: [-fPIC] + # Instrument builds for coverage and other analysis. debug: [-Og, -g, --coverage] opt: [-O2] @@ -12,18 +15,19 @@ cflag_groups: variants: debug: - cflag_groups: [asan] + cflag_groups: [common, asan] - opt: {} + opt: + cflag_groups: [common] clang: - cflag_groups: [debug, msan] + cflag_groups: [common, debug, msan] cc: clang cxx: clang++ ld: clang++ clang-opt: - cflag_groups: [opt] + cflag_groups: [common, opt] cc: clang cxx: clang++ ld: clang++ From f2b5f2b24a18e6d76ead0062e634e4dddd4ab3a1 Mon Sep 17 00:00:00 2001 From: Vaughn Kottler Date: Sun, 16 Jul 2023 21:11:23 -0500 Subject: [PATCH 2/4] 2.0.0 - 'dist' improvements --- .github/workflows/python-package.yml | 2 +- .pylintrc | 1 + README.md | 7 +- local/variables/package.yaml | 6 +- pyproject.toml | 2 +- tests/commands/test_dist.py | 13 ++++ .../valid/scenarios/native/extra/test.txt | 1 + tests/data/valid/scenarios/native/yambs.yaml | 2 + yambs/__init__.py | 4 +- yambs/commands/dist.py | 52 +++++++-------- yambs/config/common.py | 9 +++ yambs/data/native.yaml | 13 ++-- yambs/data/schemas/Config.yaml | 5 ++ yambs/data/schemas/config_common.yaml | 2 +- yambs/data/templates/native_rules.ninja.j2 | 3 + yambs/dist/__init__.py | 64 +++++++++++++++++++ yambs/environment/native.py | 32 +++++++++- yambs/translation/__init__.py | 1 + 18 files changed, 172 insertions(+), 47 deletions(-) create mode 100644 tests/data/valid/scenarios/native/extra/test.txt create mode 100644 yambs/dist/__init__.py diff --git a/.github/workflows/python-package.yml b/.github/workflows/python-package.yml index 7d8788f..072053b 100644 --- a/.github/workflows/python-package.yml +++ b/.github/workflows/python-package.yml @@ -65,7 +65,7 @@ jobs: - run: | mk python-release owner=vkottler \ - repo=yambs version=1.10.5 + repo=yambs version=2.0.0 if: | matrix.python-version == '3.11' && matrix.system == 'ubuntu-latest' diff --git a/.pylintrc b/.pylintrc index 93d8ace..3a23c51 100644 --- a/.pylintrc +++ b/.pylintrc @@ -1,5 +1,6 @@ [DESIGN] max-args=8 +max-attributes=8 [MESSAGES CONTROL] disable=too-few-public-methods diff --git a/README.md b/README.md index 19e995a..b34c7a3 100644 --- a/README.md +++ b/README.md @@ -2,11 +2,11 @@ ===================================== generator=datazen version=3.1.2 - hash=7b491a6ae08910955ce5d652d62d907f + hash=002bf1ec758c597d37da825e56ca08c0 ===================================== --> -# yambs ([1.10.5](https://pypi.org/project/yambs/)) +# yambs ([2.0.0](https://pypi.org/project/yambs/)) [![python](https://img.shields.io/pypi/pyversions/yambs.svg)](https://pypi.org/project/yambs/) ![Build Status](https://github.com/vkottler/yambs/workflows/Python%20Package/badge.svg) @@ -157,13 +157,14 @@ commands: ``` $ ./venv3.11/bin/mbs dist -h -usage: mbs dist [-h] [-c CONFIG] +usage: mbs dist [-h] [-c CONFIG] [-s] options: -h, --help show this help message and exit -c CONFIG, --config CONFIG the path to the top-level configuration file (default: 'yambs.yaml') + -s, --sources set this flag to only capture source files ``` diff --git a/local/variables/package.yaml b/local/variables/package.yaml index b01c63c..ba34427 100644 --- a/local/variables/package.yaml +++ b/local/variables/package.yaml @@ -1,5 +1,5 @@ --- -major: 1 -minor: 10 -patch: 5 +major: 2 +minor: 0 +patch: 0 entry: mbs diff --git a/pyproject.toml b/pyproject.toml index 9e85f46..7cb621f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta:__legacy__" [project] name = "yambs" -version = "1.10.5" +version = "2.0.0" description = "Yet another meta build-system." readme = "README.md" requires-python = ">=3.8" diff --git a/tests/commands/test_dist.py b/tests/commands/test_dist.py index 898e11b..5f2451c 100644 --- a/tests/commands/test_dist.py +++ b/tests/commands/test_dist.py @@ -14,6 +14,19 @@ def test_dist_command_basic(): """Test the 'dist' command.""" for scenario in ["sample", "native"]: + assert ( + yambs_main( + [ + PKG_NAME, + "-C", + str(resource("scenarios", scenario)), + "dist", + "-s", + ] + ) + == 0 + ) + assert ( yambs_main( [ diff --git a/tests/data/valid/scenarios/native/extra/test.txt b/tests/data/valid/scenarios/native/extra/test.txt new file mode 100644 index 0000000..190a180 --- /dev/null +++ b/tests/data/valid/scenarios/native/extra/test.txt @@ -0,0 +1 @@ +123 diff --git a/tests/data/valid/scenarios/native/yambs.yaml b/tests/data/valid/scenarios/native/yambs.yaml index b69612b..2fea231 100644 --- a/tests/data/valid/scenarios/native/yambs.yaml +++ b/tests/data/valid/scenarios/native/yambs.yaml @@ -3,6 +3,8 @@ cflag_groups: debug: [-Og, -g] opt: [-O2] +extra_dist: [extra] + project: name: yambs github: diff --git a/yambs/__init__.py b/yambs/__init__.py index 3b1d609..9d65ca6 100644 --- a/yambs/__init__.py +++ b/yambs/__init__.py @@ -1,7 +1,7 @@ # ===================================== # generator=datazen # version=3.1.2 -# hash=e45ca62e08fc5342a3dc3c43ca433ee4 +# hash=8dd6e5c8fb283e23758f59bf4a0844de # ===================================== """ @@ -10,4 +10,4 @@ DESCRIPTION = "Yet another meta build-system." PKG_NAME = "yambs" -VERSION = "1.10.5" +VERSION = "2.0.0" diff --git a/yambs/commands/dist.py b/yambs/commands/dist.py index 28e6092..65bf5d2 100644 --- a/yambs/commands/dist.py +++ b/yambs/commands/dist.py @@ -5,16 +5,18 @@ # built-in from argparse import ArgumentParser as _ArgumentParser from argparse import Namespace as _Namespace -from shutil import make_archive, rmtree +from pathlib import Path +from shutil import copytree, rmtree +from tempfile import TemporaryDirectory # third-party from vcorelib.args import CommandFunction as _CommandFunction from vcorelib.paths import create_hex_digest -from vcorelib.paths.context import in_dir # internal from yambs.commands.common import add_config_arg from yambs.config.common import CommonConfig +from yambs.dist import copy_source_tree, make_archives def dist_cmd(args: _Namespace) -> int: @@ -22,36 +24,28 @@ def dist_cmd(args: _Namespace) -> int: config = CommonConfig.load(path=args.config, root=args.dir) - # Remove and re-create the dist directory. - dist = config.dist_root - rmtree(dist, ignore_errors=True) - dist.mkdir() + # Prepare a temporary directory with project sources. + with TemporaryDirectory() as tmp: + path = Path(tmp) - slug = str(config.project) - src = config.src_root - archives = [] + # Put everything under a directory named after the project. + base = path.joinpath(config.project.name) - for ext, kind in [ - ("tar.gz", "gztar"), - ("tar.xz", "xztar"), - ("zip", "zip"), - ]: - out = src.joinpath(f"{slug}.{ext}") - out.unlink(missing_ok=True) + if args.sources: + copytree(config.src_root, base) + else: + base.mkdir() + copy_source_tree(config, base) - with in_dir(src): - make_archive(slug, kind) + # Remove and re-create the dist directory. + dist = config.dist_root + rmtree(dist, ignore_errors=True) + dist.mkdir() - assert out.is_file(), out - - final = dist.joinpath(out.name) - out.rename(final) - archives.append(final) - - print(f"Created '{final}'.") + make_archives(path, config) # Produce a hex digest. - print(f"Wrote '{create_hex_digest(dist, slug)}'.") + print(f"Wrote '{create_hex_digest(dist, str(config.project))}'.") return 0 @@ -60,4 +54,10 @@ def add_dist_cmd(parser: _ArgumentParser) -> _CommandFunction: """Add dist-command arguments to its parser.""" add_config_arg(parser) + parser.add_argument( + "-s", + "--sources", + action="store_true", + help="set this flag to only capture source files", + ) return dist_cmd diff --git a/yambs/config/common.py b/yambs/config/common.py index f6315ad..66b497d 100644 --- a/yambs/config/common.py +++ b/yambs/config/common.py @@ -74,6 +74,8 @@ class CommonConfig(_YambsDictCodec, _BasicDictCodec): ninja_root: Path dist_root: Path + file: Optional[Path] + def directory(self, name: str, mkdir: bool = True) -> Path: """Get a configurable directory.""" @@ -97,6 +99,8 @@ def init(self, data: _JsonObject) -> None: self.ninja_root = self.directory("ninja_out") self.dist_root = self.directory("dist_out") + self.file = None + self.project = Project.create(data["project"]) # type: ignore @classmethod @@ -111,6 +115,8 @@ def load( src_config = find_file(package_config, package=PKG_NAME) assert src_config is not None + path = normalize(path) + data = merge( _ARBITER.decode( src_config, @@ -125,6 +131,9 @@ def load( result = cls.create(data) + if path.is_file(): + result.file = path + if root is not None: result.root = normalize(root) result.root.mkdir(parents=True, exist_ok=True) diff --git a/yambs/data/native.yaml b/yambs/data/native.yaml index fd6d332..4d8cc3d 100644 --- a/yambs/data/native.yaml +++ b/yambs/data/native.yaml @@ -3,8 +3,8 @@ includes: - includes/common.yaml cflag_groups: - # Compile position-independent so we can build shared objects. - common: [-fPIC] + # Compile position-independent to build shared objects. + pic: [-fPIC] # Instrument builds for coverage and other analysis. debug: [-Og, -g, --coverage] @@ -15,19 +15,18 @@ cflag_groups: variants: debug: - cflag_groups: [common, asan] + cflag_groups: [asan] - opt: - cflag_groups: [common] + opt: {} clang: - cflag_groups: [common, debug, msan] + cflag_groups: [debug, msan] cc: clang cxx: clang++ ld: clang++ clang-opt: - cflag_groups: [common, opt] + cflag_groups: [opt] cc: clang cxx: clang++ ld: clang++ diff --git a/yambs/data/schemas/Config.yaml b/yambs/data/schemas/Config.yaml index 0ad6315..de3f6f0 100644 --- a/yambs/data/schemas/Config.yaml +++ b/yambs/data/schemas/Config.yaml @@ -43,3 +43,8 @@ properties: patternProperties: "^[a-zA-Z0-9-_.+]+$": $ref: package://yambs/schemas/Architecture.yaml + + extra_third_party: + type: array + items: + type: string diff --git a/yambs/data/schemas/config_common.yaml b/yambs/data/schemas/config_common.yaml index 213f535..ba46a23 100644 --- a/yambs/data/schemas/config_common.yaml +++ b/yambs/data/schemas/config_common.yaml @@ -22,7 +22,7 @@ properties: items: type: string - extra_third_party: + extra_dist: type: array items: type: string diff --git a/yambs/data/templates/native_rules.ninja.j2 b/yambs/data/templates/native_rules.ninja.j2 index a8ce01a..6961d2a 100644 --- a/yambs/data/templates/native_rules.ninja.j2 +++ b/yambs/data/templates/native_rules.ninja.j2 @@ -18,4 +18,7 @@ ldflags ={% for flag in common_ldflags %} {{flag}}{% endfor %} rule link command = $ld $cflags $ldflags -Wl,-Map=$out.map $in -o $out +rule ar + command = ar rcs $out $in + build_dir = {{build_root}}/$variant diff --git a/yambs/dist/__init__.py b/yambs/dist/__init__.py new file mode 100644 index 0000000..4701877 --- /dev/null +++ b/yambs/dist/__init__.py @@ -0,0 +1,64 @@ +""" +A module implementing interfaces for facilitating project distribution. +""" + +# built-in +from pathlib import Path +from shutil import copy2, copytree, make_archive + +# third-party +from vcorelib.paths.context import in_dir + +# internal +from yambs.config.common import CommonConfig + +ARCHIVES = [ + ("tar.gz", "gztar"), + ("tar.xz", "xztar"), + ("zip", "zip"), +] + + +def make_archives(tmp: Path, config: CommonConfig) -> None: + """Create a distribution that only contains sources.""" + + slug = str(config.project) + + for ext, kind in ARCHIVES: + out = tmp.joinpath(f"{slug}.{ext}") + out.unlink(missing_ok=True) + + with in_dir(tmp): + make_archive(slug, kind) + + assert out.is_file(), out + + final = config.dist_root.joinpath(out.name) + out.rename(final) + + print(f"Created '{final}'.") + + +def copy_source_tree(config: CommonConfig, dest: Path) -> None: + """ + Copy necessary parts of the project source tree to some destination + directory. + """ + + root = config.root + + for item in [ + x + for x in [ + config.src_root, + config.ninja_root, + root.joinpath("build.ninja"), + config.file, + ] + + [root.joinpath(x) for x in config.data.get("extra_dist", [])] + if x and x.exists() + ]: + if item.is_dir(): + copytree(item, dest.joinpath(item.name)) + else: + copy2(item, dest.joinpath(item.name)) diff --git a/yambs/environment/native.py b/yambs/environment/native.py index 59b6659..b8b253f 100644 --- a/yambs/environment/native.py +++ b/yambs/environment/native.py @@ -18,12 +18,12 @@ from yambs.generate.ninja import write_continuation from yambs.generate.ninja.format import render_format from yambs.generate.variants import generate as generate_variants -from yambs.translation import BUILD_DIR_VAR, get_translator +from yambs.translation import BUILD_DIR_PATH, get_translator def resolve_build_dir(build_root: Path, variant: str, path: Path) -> Path: """Resolve the build-directory variable in a path.""" - return build_root.joinpath(variant, path.relative_to(BUILD_DIR_VAR)) + return build_root.joinpath(variant, path.relative_to(BUILD_DIR_PATH)) class NativeBuildEnvironment(LoggerMixin): @@ -71,6 +71,25 @@ def write_source_rules(self, stream: TextIO) -> Set[Path]: """Write source rules.""" return {self.write_compile_line(stream, path) for path in self.regular} + def write_static_library_rule( + self, stream: TextIO, outputs: Set[Path] + ) -> Path: + """Create a rule for a static library output.""" + + lib = BUILD_DIR_PATH.joinpath(f"{self.config.project}.a") + line = f"build {lib}: ar " + offset = " " * len(line) + + list_outputs = list(outputs) + stream.write(line + str(list_outputs[0])) + for file in list_outputs[1:]: + write_continuation(stream, offset) + stream.write(str(file)) + + stream.write(linesep + linesep) + + return lib + def write_app_rules( self, stream: TextIO, outputs: Set[Path] ) -> Dict[Path, Path]: @@ -78,11 +97,12 @@ def write_app_rules( elfs: Dict[Path, Path] = {} + # Create rules for linked executables. for path in self.apps: out = self.write_compile_line(stream, path) from_src = path.relative_to(self.config.src_root) - elf = Path(BUILD_DIR_VAR, from_src.with_suffix(".elf")) + elf = BUILD_DIR_PATH.joinpath(from_src.with_suffix(".elf")) elfs[path] = elf line = f"build {elf}: link " offset = " " * len(line) @@ -99,6 +119,12 @@ def write_app_rules( offset = " " * len(line) elfs_list = list(elfs.values()) + + # Also update the static library (if necessary) when linking + # applications. + if outputs: + elfs_list.append(self.write_static_library_rule(stream, outputs)) + stream.write(line + str(elfs_list[0])) for elf in elfs_list[1:]: write_continuation(stream, offset) diff --git a/yambs/translation/__init__.py b/yambs/translation/__init__.py index 02a5478..f76e4f1 100644 --- a/yambs/translation/__init__.py +++ b/yambs/translation/__init__.py @@ -10,6 +10,7 @@ HEADER_EXTENSIONS = {".h", ".hpp"} BUILD_DIR_VAR = "$build_dir" +BUILD_DIR_PATH = Path(BUILD_DIR_VAR) class SourceTranslator(NamedTuple): From 71bf70a15299cac24e683042dfd24b0976577dc1 Mon Sep 17 00:00:00 2001 From: Vaughn Kottler Date: Mon, 17 Jul 2023 01:09:38 -0500 Subject: [PATCH 3/4] Use absolute path for renaming file --- yambs/dist/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/yambs/dist/__init__.py b/yambs/dist/__init__.py index 4701877..e0b2d26 100644 --- a/yambs/dist/__init__.py +++ b/yambs/dist/__init__.py @@ -34,7 +34,7 @@ def make_archives(tmp: Path, config: CommonConfig) -> None: assert out.is_file(), out final = config.dist_root.joinpath(out.name) - out.rename(final) + out.rename(final.resolve()) print(f"Created '{final}'.") From 93518db22e60619d51086277696ed701669e649f Mon Sep 17 00:00:00 2001 From: Vaughn Kottler Date: Mon, 17 Jul 2023 01:14:57 -0500 Subject: [PATCH 4/4] Try copy and remove instead of move file --- yambs/dist/__init__.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/yambs/dist/__init__.py b/yambs/dist/__init__.py index e0b2d26..edb16ec 100644 --- a/yambs/dist/__init__.py +++ b/yambs/dist/__init__.py @@ -34,7 +34,8 @@ def make_archives(tmp: Path, config: CommonConfig) -> None: assert out.is_file(), out final = config.dist_root.joinpath(out.name) - out.rename(final.resolve()) + copy2(out, final.resolve()) + out.unlink() print(f"Created '{final}'.")