From 8533fe6a46fabb8fe3ba1ea49477fc30c2da4090 Mon Sep 17 00:00:00 2001 From: Vaughn Kottler Date: Thu, 14 Mar 2024 12:06:27 -0500 Subject: [PATCH 1/2] Initial sync. --- .github/workflows/python-package.yml | 3 ++- README.md | 3 ++- config | 2 +- local/configs/python.yaml | 3 ++- pyproject.toml | 1 + setup.py | 3 ++- tasks/conf.py | 14 +++++++------- yambs/environment/__init__.py | 3 +-- yambs/github/__init__.py | 6 +++--- 9 files changed, 21 insertions(+), 17 deletions(-) diff --git a/.github/workflows/python-package.yml b/.github/workflows/python-package.yml index 67500f0..0cd9314 100644 --- a/.github/workflows/python-package.yml +++ b/.github/workflows/python-package.yml @@ -19,6 +19,7 @@ jobs: matrix: python-version: - "3.11" + - "3.12" system: - ubuntu-latest - macos-latest @@ -72,7 +73,7 @@ jobs: env: PY_TEST_EXTRA_ARGS: --cov-report=xml - - uses: codecov/codecov-action@v3 + - uses: codecov/codecov-action@v3.1.5 - run: mk pypi-upload-ci env: diff --git a/README.md b/README.md index 1d3cd30..3b85327 100644 --- a/README.md +++ b/README.md @@ -2,7 +2,7 @@ ===================================== generator=datazen version=3.1.4 - hash=4976235943725fd73fdb02c40af8f1cb + hash=06928573fc54b4eef50b5f45d1476972 ===================================== --> @@ -30,6 +30,7 @@ This package is tested with the following Python minor versions: * [`python3.11`](https://docs.python.org/3.11/) +* [`python3.12`](https://docs.python.org/3.12/) ## Platform Support diff --git a/config b/config index 273b089..6b9977e 160000 --- a/config +++ b/config @@ -1 +1 @@ -Subproject commit 273b08905dae33e058c016de6681c77284cadcf5 +Subproject commit 6b9977e7d2132e33d09b6f5bd45b661cf66802d5 diff --git a/local/configs/python.yaml b/local/configs/python.yaml index 924bf91..5d4d989 100644 --- a/local/configs/python.yaml +++ b/local/configs/python.yaml @@ -3,8 +3,9 @@ author_info: name: Vaughn Kottler email: vaughnkottler@gmail.com username: vkottler -versions: ["3.11"] +# CI matrix. +versions: ["3.11", "3.12"] systems: - macos-latest - windows-latest diff --git a/pyproject.toml b/pyproject.toml index 7f80b9d..ce70eda 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -16,6 +16,7 @@ maintainers = [ ] classifiers = [ "Programming Language :: Python :: 3.11", + "Programming Language :: Python :: 3.12", "Operating System :: Microsoft :: Windows", "Operating System :: MacOS", "Operating System :: POSIX :: Linux", diff --git a/setup.py b/setup.py index db96b9d..68c698d 100644 --- a/setup.py +++ b/setup.py @@ -1,7 +1,7 @@ # ===================================== # generator=datazen # version=3.1.4 -# hash=f3af34a4b5815c617489419b194b30b5 +# hash=ce50fe613526c5b8b8b7fd8527e3566d # ===================================== """ @@ -29,6 +29,7 @@ "description": DESCRIPTION, "versions": [ "3.11", + "3.12", ], } setup( diff --git a/tasks/conf.py b/tasks/conf.py index 7356deb..0aebbed 100644 --- a/tasks/conf.py +++ b/tasks/conf.py @@ -1,7 +1,7 @@ # ===================================== # generator=datazen # version=3.1.4 -# hash=9f62028523c3b5a953733ca89dcc3018 +# hash=7d378a1752611508007a77d4ca39a5af # ===================================== """ A module for project-specific task registration. @@ -20,14 +20,9 @@ def audit_local_tasks() -> None: """Ensure that shared task infrastructure is present.""" local = Path(__file__).parent.joinpath("mklocal") - - # Also link a top-level file. top_level = local.parent.parent.joinpath("mklocal") - if not top_level.is_symlink(): - assert not top_level.exists() - top_level.symlink_to(local) - if local.is_symlink(): + if local.is_symlink() and top_level.is_symlink(): return # If it's not a symlink, it shouldn't be any other kind of file. @@ -48,6 +43,11 @@ def audit_local_tasks() -> None: # Create the link. local.symlink_to(vmklib) + # Also link a top-level file. + if not top_level.is_symlink(): + assert not top_level.exists() + top_level.symlink_to(local) + def register( manager: TaskManager, diff --git a/yambs/environment/__init__.py b/yambs/environment/__init__.py index ec3004d..40c02b5 100644 --- a/yambs/environment/__init__.py +++ b/yambs/environment/__init__.py @@ -92,8 +92,7 @@ def first_party_sources_headers(self) -> Iterator[Path]: yield source visited.add(source) - for item in self.first_party_headers: - yield item + yield from self.first_party_headers def set_board_sources( self, board: Board, regular: Set[Path], apps: Set[Path] diff --git a/yambs/github/__init__.py b/yambs/github/__init__.py index d9e89f3..3319575 100644 --- a/yambs/github/__init__.py +++ b/yambs/github/__init__.py @@ -70,9 +70,9 @@ def check_api_token() -> None: if "Authorization" not in GIHTUB_HEADERS: if "GITHUB_API_TOKEN" in os.environ: - GIHTUB_HEADERS[ - "Authorization" - ] = f"Bearer {os.environ['GITHUB_API_TOKEN']}" + GIHTUB_HEADERS["Authorization"] = ( + f"Bearer {os.environ['GITHUB_API_TOKEN']}" + ) def release_data( From fa23d247c7e82bad734e4e9c59ab5a092ba4b678 Mon Sep 17 00:00:00 2001 From: Vaughn Kottler Date: Thu, 14 Mar 2024 13:51:15 -0500 Subject: [PATCH 2/2] Some initial WASM integration Try to fix CI build --- .github/workflows/python-package.yml | 6 +- README.md | 4 +- local/configs/package.yaml | 8 ++ local/variables/package.yaml | 6 +- pyproject.toml | 2 +- tests/commands/test_native.py | 12 ++ tests/data/valid/scenarios/native3/yambs.yaml | 3 + yambs/__init__.py | 4 +- yambs/config/native.py | 4 + yambs/data/includes/wasm.yaml | 8 ++ yambs/data/templates/native_rules.ninja.j2 | 3 + yambs/environment/native.py | 103 +++++++----------- yambs/generate/ninja/__init__.py | 65 +++++++++++ yambs/paths.py | 24 ++++ yambs/translation/__init__.py | 23 +++- 15 files changed, 200 insertions(+), 75 deletions(-) create mode 100644 yambs/data/includes/wasm.yaml create mode 100644 yambs/paths.py diff --git a/.github/workflows/python-package.yml b/.github/workflows/python-package.yml index 0cd9314..5f538d1 100644 --- a/.github/workflows/python-package.yml +++ b/.github/workflows/python-package.yml @@ -55,6 +55,10 @@ jobs: if: matrix.system == 'ubuntu-latest' - run: sudo apt-get install gcc-13 g++-13 clang-17 clang-format-17 if: matrix.system == 'ubuntu-latest' + - run: sudo apt-get install emscripten + if: matrix.system == 'ubuntu-latest' + - run: em++ --version + if: matrix.system == 'ubuntu-latest' # End project-specific setup. - run: mk python-sa-types @@ -86,7 +90,7 @@ jobs: - run: | mk python-release owner=vkottler \ - repo=yambs version=2.8.1 + repo=yambs version=3.0.0 if: | matrix.python-version == '3.11' && matrix.system == 'ubuntu-latest' diff --git a/README.md b/README.md index 3b85327..e6a3040 100644 --- a/README.md +++ b/README.md @@ -2,11 +2,11 @@ ===================================== generator=datazen version=3.1.4 - hash=06928573fc54b4eef50b5f45d1476972 + hash=807e0a42a738120cbe4629477dbbafe5 ===================================== --> -# yambs ([2.8.1](https://pypi.org/project/yambs/)) +# yambs ([3.0.0](https://pypi.org/project/yambs/)) [![python](https://img.shields.io/pypi/pyversions/yambs.svg)](https://pypi.org/project/yambs/) ![Build Status](https://github.com/vkottler/yambs/workflows/Python%20Package/badge.svg) diff --git a/local/configs/package.yaml b/local/configs/package.yaml index 549b4c0..c0915b4 100644 --- a/local/configs/package.yaml +++ b/local/configs/package.yaml @@ -14,6 +14,7 @@ ci_local: - "- run: mk python-editable" - " if: matrix.system == 'ubuntu-latest'" + # Get latest clang. - "- name: setup clang 17" - " run: |" - " wget https://apt.llvm.org/llvm.sh" @@ -22,9 +23,16 @@ ci_local: - " rm llvm.sh" - " if: matrix.system == 'ubuntu-latest'" + # Native toolchains. - "- run: sudo apt-get install gcc-13 g++-13 clang-17 clang-format-17" - " if: matrix.system == 'ubuntu-latest'" + # WASM. + - "- run: sudo apt-get install emscripten" + - " if: matrix.system == 'ubuntu-latest'" + - "- run: em++ --version" + - " if: matrix.system == 'ubuntu-latest'" + requirements: - datazen - vcorelib>=2.4.2 diff --git a/local/variables/package.yaml b/local/variables/package.yaml index 9aa4eae..cfbccc9 100644 --- a/local/variables/package.yaml +++ b/local/variables/package.yaml @@ -1,5 +1,5 @@ --- -major: 2 -minor: 8 -patch: 1 +major: 3 +minor: 0 +patch: 0 entry: mbs diff --git a/pyproject.toml b/pyproject.toml index ce70eda..fe1249f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta:__legacy__" [project] name = "yambs" -version = "2.8.1" +version = "3.0.0" description = "Yet another meta build-system." readme = "README.md" requires-python = ">=3.11" diff --git a/tests/commands/test_native.py b/tests/commands/test_native.py index 273dad1..fc1f424 100644 --- a/tests/commands/test_native.py +++ b/tests/commands/test_native.py @@ -18,6 +18,17 @@ from yambs.entry import main as yambs_main +def test_native_command_wasm(): + """Test the 'native' command with WASM variants.""" + + with in_dir(clean_scenario("native3")): + assert yambs_main([PKG_NAME, "native"]) == 0 + + # Try to build (if we can). + if platform == "linux" and which("ninja"): + run(["ninja", "wasm"], check=True) + + def test_native_command_basic(): """Test the 'native' command.""" @@ -25,6 +36,7 @@ def test_native_command_basic(): # Ensure the directory dependency is cleaned as well. clean_scenario("native2") + clean_scenario("native3") with in_dir(path): assert yambs_main([PKG_NAME, "native", "-w", "-i"]) == 0 diff --git a/tests/data/valid/scenarios/native3/yambs.yaml b/tests/data/valid/scenarios/native3/yambs.yaml index 5ed301c..3b74433 100644 --- a/tests/data/valid/scenarios/native3/yambs.yaml +++ b/tests/data/valid/scenarios/native3/yambs.yaml @@ -1,3 +1,6 @@ --- +includes: + - package://yambs/includes/wasm.yaml + project: name: yambs3 diff --git a/yambs/__init__.py b/yambs/__init__.py index 3decc5f..f8f6a8e 100644 --- a/yambs/__init__.py +++ b/yambs/__init__.py @@ -1,7 +1,7 @@ # ===================================== # generator=datazen # version=3.1.4 -# hash=62d7b56325a95261f3a945bf98673529 +# hash=eb43e10924888ea3ff0d4f7e0f7089f0 # ===================================== """ @@ -10,4 +10,4 @@ DESCRIPTION = "Yet another meta build-system." PKG_NAME = "yambs" -VERSION = "2.8.1" +VERSION = "3.0.0" diff --git a/yambs/config/native.py b/yambs/config/native.py index 9598047..1787665 100644 --- a/yambs/config/native.py +++ b/yambs/config/native.py @@ -12,6 +12,10 @@ class Native(CommonConfig): """The top-level configuration object for the package.""" + def has_variant(self, name: str) -> bool: + """Determine whether or not""" + return name in self.data.get("variants", {}) + def load_native( path: Pathlike = DEFAULT_CONFIG, root: Pathlike = None diff --git a/yambs/data/includes/wasm.yaml b/yambs/data/includes/wasm.yaml new file mode 100644 index 0000000..4c4640a --- /dev/null +++ b/yambs/data/includes/wasm.yaml @@ -0,0 +1,8 @@ +--- +variants: + wasm: + cc: emcc + cxx: em++ + ld: em++ + + targets: [htmls] diff --git a/yambs/data/templates/native_rules.ninja.j2 b/yambs/data/templates/native_rules.ninja.j2 index 22476e5..e0e7a15 100644 --- a/yambs/data/templates/native_rules.ninja.j2 +++ b/yambs/data/templates/native_rules.ninja.j2 @@ -19,6 +19,9 @@ ldflags = $common_ldflags $variant_ldflags rule link command = $ld $cflags -Wl,-Map=$out.map $in $ldflags -o $out +rule link_no_map + command = $ld $cflags $in $ldflags -o $out + rule bin command = ${toolchain_prefix}objcopy -O binary $in $out diff --git a/yambs/environment/native.py b/yambs/environment/native.py index 4e5e9d6..2105b3e 100644 --- a/yambs/environment/native.py +++ b/yambs/environment/native.py @@ -10,31 +10,19 @@ # third-party from vcorelib.io import ARBITER from vcorelib.logging import LoggerMixin -from vcorelib.paths import Pathlike, normalize # internal from yambs.aggregation import collect_files, populate_sources, sources_headers from yambs.config.native import Native from yambs.dependency.manager import DependencyManager from yambs.generate.common import APP_ROOT, get_jinja, render_template -from yambs.generate.ninja import write_continuation +from yambs.generate.ninja import variant_phony, write_continuation, write_link from yambs.generate.ninja.format import render_format from yambs.generate.variants import generate as generate_variants +from yambs.paths import combine_if_not_absolute, resolve_build_dir from yambs.translation import BUILD_DIR_PATH, get_translator -def resolve_build_dir(build_root: Path, variant: str, path: Path) -> Path: - """Resolve the build-directory variable in a path.""" - return build_root.joinpath(variant, path.relative_to(BUILD_DIR_PATH)) - - -def combine_if_not_absolute(root: Path, candidate: Pathlike) -> Path: - """https://github.com/vkottler/ifgen/blob/master/ifgen/paths.py""" - - candidate = normalize(candidate) - return candidate if candidate.is_absolute() else root.joinpath(candidate) - - class NativeBuildEnvironment(LoggerMixin): """A class implementing a native-build environment.""" @@ -65,7 +53,9 @@ def render(self, root: Path, name: str) -> None: self.jinja, root, f"native_{name}", self.config.data, out=name ) - def write_compile_line(self, stream: TextIO, path: Path) -> Path: + def write_compile_line( + self, stream: TextIO, path: Path, wasm: bool = False + ) -> Path: """Write a single source-compile line.""" translator = get_translator(path) @@ -73,13 +63,12 @@ def write_compile_line(self, stream: TextIO, path: Path) -> Path: out = translator.output(from_src) - stream.write(f"build {out}: {translator.rule} $src_dir/{from_src}") - stream.write(linesep) + translator.write(stream, out, f"$src_dir/{from_src}", wasm=wasm) return out def write_third_party_line( - self, stream: TextIO, path: Path + self, stream: TextIO, path: Path, wasm: bool = False ) -> Optional[Path]: """Write a single source-compile line for a third-party source.""" @@ -98,23 +87,25 @@ def write_third_party_line( Path("$build_dir", "third-party", rel_part) ) - stream.write( - f"build {out}: {translator.rule} $third_party_dir/{rel_part}" + translator.write( + stream, out, f"$third_party_dir/{rel_part}", wasm=wasm ) - stream.write(linesep) return out - def write_source_rules(self, stream: TextIO) -> Set[Path]: + def write_source_rules( + self, stream: TextIO, wasm: bool = False + ) -> Set[Path]: """Write source rules.""" result = { - self.write_compile_line(stream, path) for path in self.regular + self.write_compile_line(stream, path, wasm=wasm) + for path in self.regular } # Add third-party sources. for path in self.third_party: - path_result = self.write_third_party_line(stream, path) + path_result = self.write_third_party_line(stream, path, wasm=wasm) if path_result is not None: result.add(path_result) @@ -140,37 +131,24 @@ def write_static_library_rule( return lib def _write_app_phony_targets( - self, stream: TextIO, elfs: Dict[Path, Path], uf2_family: str = None + self, + stream: TextIO, + elfs: Dict[Path, Path], + uf2_family: str = None, + wasm: bool = False, ) -> None: """Write phony targets for all variants.""" elfs_list = list(elfs.values()) - if elfs_list: - line = "build ${variant}_apps: phony " - offset = " " * len(line) - - stream.write(line + str(elfs_list[0])) - for elf in elfs_list[1:]: - write_continuation(stream, offset) - stream.write(str(elf)) - stream.write(linesep) - - if uf2_family: - # Create uf2 phony. - line = "build ${variant}_uf2s: phony " - offset = " " * len(line) - - uf2s = [x.with_suffix(".uf2") for x in elfs_list] - - stream.write(line + str(uf2s[0])) - for elf in uf2s[1:]: - write_continuation(stream, offset) - stream.write(str(elf)) - stream.write(linesep) + variant_phony(stream, elfs_list, uf2_family=uf2_family, wasm=wasm) def write_app_rules( - self, stream: TextIO, outputs: Set[Path], uf2_family: str = None + self, + stream: TextIO, + outputs: Set[Path], + uf2_family: str = None, + wasm: bool = False, ) -> Dict[Path, Path]: """Write app rules.""" @@ -178,25 +156,13 @@ def write_app_rules( # Create rules for linked executables. for path in self.apps: - out = self.write_compile_line(stream, path) + out = self.write_compile_line(stream, path, wasm=wasm) from_src = path.relative_to(self.config.src_root) elf = BUILD_DIR_PATH.joinpath(from_src.with_suffix(".elf")) elfs[path] = elf - line = f"build {elf}: link " - offset = " " * len(line) - stream.write(line + str(out)) - - for file in outputs: - write_continuation(stream, offset) - stream.write(str(file)) - - # Executables can't be linked until third-party dependencies are - # actually built. - stream.write(" | ${variant}_third_party") - - stream.write(linesep + linesep) + write_link(stream, elf, out, outputs, wasm=wasm) # Write rules for other kinds of outputs. for output in ["bin", "hex", "dump"]: @@ -225,7 +191,9 @@ def write_app_rules( + linesep ) - self._write_app_phony_targets(stream, elfs, uf2_family=uf2_family) + self._write_app_phony_targets( + stream, elfs, uf2_family=uf2_family, wasm=wasm + ) return elfs @@ -284,6 +252,8 @@ def _handle_extra_source_dirs(self) -> None: def generate(self, sources_only: bool = False) -> None: """Generate ninja files.""" + wasm = self.config.has_variant("wasm") + if not sources_only: # Audit dependencies. for dep in self.config.dependencies: @@ -326,14 +296,17 @@ def generate(self, sources_only: bool = False) -> None: path = self.config.ninja_root.joinpath("sources.ninja") with self.log_time("Write '%s'", path): with path.open("w") as path_fd: - outputs = self.write_source_rules(path_fd) + outputs = self.write_source_rules(path_fd, wasm=wasm) # Render apps file. path = self.config.ninja_root.joinpath("apps.ninja") with self.log_time("Write '%s'", path): with path.open("w") as path_fd: elfs = self.write_app_rules( - path_fd, outputs, self.config.data.get("uf2_family") + path_fd, + outputs, + self.config.data.get("uf2_family"), + wasm=wasm, ) # Render format file. diff --git a/yambs/generate/ninja/__init__.py b/yambs/generate/ninja/__init__.py index a52afed..46fd3f0 100644 --- a/yambs/generate/ninja/__init__.py +++ b/yambs/generate/ninja/__init__.py @@ -169,3 +169,68 @@ def write_phony( stream.write(f"{BUILD_DIR_VAR}/{src.with_suffix(suffix)}") stream.write(linesep) + + +def write_link( + stream: TextIO, + output: Path, + entry_object: Path, + outputs: Set[Path], + wasm: bool = False, +) -> None: + """Write a 'link' rule.""" + + pairs = [(output.suffix, entry_object.suffix, "link")] + + if wasm: + pairs.append((".html", ".wasm", "link_no_map")) + + for out_suffix, entry_suffx, rule in pairs: + line = f"build {output.with_suffix(out_suffix)}: {rule} " + offset = " " * len(line) + + stream.write(line + str(entry_object.with_suffix(entry_suffx))) + + for file in outputs: + write_continuation(stream, offset) + stream.write(str(file.with_suffix(entry_suffx))) + + # Executables can't be linked until third-party dependencies are + # actually built. + stream.write(" | ${variant}_third_party" + linesep + linesep) + + +def variant_phony( + stream: TextIO, + elfs_list: list[Path], + uf2_family: str = None, + wasm: bool = False, +): + """Write variant-specific phony targets.""" + + line = "build ${variant}_apps: phony " + offset = " " * len(line) + + stream.write(line + str(elfs_list[0])) + for elf in elfs_list[1:]: + write_continuation(stream, offset) + stream.write(str(elf)) + stream.write(linesep) + + suffixes = [] + if wasm: + suffixes.append("html") + if uf2_family: + suffixes.append("uf2") + + for suffix in suffixes: + line = "build ${variant}_" + f"{suffix}s: phony " + offset = " " * len(line) + + outputs = [x.with_suffix("." + suffix) for x in elfs_list] + + stream.write(line + str(outputs[0])) + for elf in outputs[1:]: + write_continuation(stream, offset) + stream.write(str(elf)) + stream.write(linesep) diff --git a/yambs/paths.py b/yambs/paths.py new file mode 100644 index 0000000..cabd50d --- /dev/null +++ b/yambs/paths.py @@ -0,0 +1,24 @@ +""" +A module implementing some file-system path utilities. +""" + +# built-in +from pathlib import Path + +# third-party +from vcorelib.paths import Pathlike, normalize + +# internal +from yambs.translation import BUILD_DIR_PATH + + +def resolve_build_dir(build_root: Path, variant: str, path: Path) -> Path: + """Resolve the build-directory variable in a path.""" + return build_root.joinpath(variant, path.relative_to(BUILD_DIR_PATH)) + + +def combine_if_not_absolute(root: Path, candidate: Pathlike) -> Path: + """https://github.com/vkottler/ifgen/blob/master/ifgen/paths.py""" + + candidate = normalize(candidate) + return candidate if candidate.is_absolute() else root.joinpath(candidate) diff --git a/yambs/translation/__init__.py b/yambs/translation/__init__.py index 3560677..008f67c 100644 --- a/yambs/translation/__init__.py +++ b/yambs/translation/__init__.py @@ -5,8 +5,9 @@ # built-in from functools import lru_cache +from os import linesep from pathlib import Path -from typing import NamedTuple, Optional +from typing import NamedTuple, Optional, TextIO HEADER_EXTENSIONS = {".h", ".hpp"} BUILD_DIR_VAR = "$build_dir" @@ -44,6 +45,26 @@ def generated_header(self) -> bool: """Determine if this translation produces a header file.""" return self.output_extension in HEADER_EXTENSIONS + def write( + self, + stream: TextIO, + out: Path, + source: str, + rule: str = "build", + wasm: bool = False, + ) -> None: + """Write a ninja rule to the stream.""" + + stream.write(f"{rule} {out}: {self.rule} {source}") + stream.write(linesep) + + # Also add a '.wasm' variant. + if wasm: + stream.write( + (f"build {out.with_suffix('.wasm')}: " f"{self.rule} {source}") + ) + stream.write(linesep) + DEFAULT = SourceTranslator()