diff --git a/.github/workflows/python-package.yml b/.github/workflows/python-package.yml index 0d764af..778456a 100644 --- a/.github/workflows/python-package.yml +++ b/.github/workflows/python-package.yml @@ -18,9 +18,6 @@ jobs: strategy: matrix: python-version: - - "3.8" - - "3.9" - - "3.10" - "3.11" system: - ubuntu-latest @@ -65,7 +62,7 @@ jobs: - run: | mk python-release owner=vkottler \ - repo=yambs version=2.1.1 + repo=yambs version=2.2.0 if: | matrix.python-version == '3.11' && matrix.system == 'ubuntu-latest' diff --git a/.gitignore b/.gitignore index 601b968..0504843 100644 --- a/.gitignore +++ b/.gitignore @@ -13,3 +13,9 @@ htmlcov *-stubs coverage*.xml tags +third-party + +ninja +build.ninja +.ninja* +compile_commands.json diff --git a/.pylintrc b/.pylintrc index 3a23c51..eed8f06 100644 --- a/.pylintrc +++ b/.pylintrc @@ -1,6 +1,6 @@ [DESIGN] max-args=8 -max-attributes=8 +max-attributes=10 [MESSAGES CONTROL] disable=too-few-public-methods diff --git a/README.md b/README.md index 970ea4e..554af8c 100644 --- a/README.md +++ b/README.md @@ -2,11 +2,11 @@ ===================================== generator=datazen version=3.1.2 - hash=f95824d927cb8814a559ce41e03ebd38 + hash=9fd78d02f6bd1c0baddfb2fa7ba65d51 ===================================== --> -# yambs ([2.1.1](https://pypi.org/project/yambs/)) +# yambs ([2.2.0](https://pypi.org/project/yambs/)) [![python](https://img.shields.io/pypi/pyversions/yambs.svg)](https://pypi.org/project/yambs/) ![Build Status](https://github.com/vkottler/yambs/workflows/Python%20Package/badge.svg) @@ -23,9 +23,6 @@ See also: [generated documentation](https://vkottler.github.io/python/pydoc/yamb This package is tested with the following Python minor versions: -* [`python3.8`](https://docs.python.org/3.8/) -* [`python3.9`](https://docs.python.org/3.9/) -* [`python3.10`](https://docs.python.org/3.10/) * [`python3.11`](https://docs.python.org/3.11/) ## Platform Support diff --git a/local/configs/package.yaml b/local/configs/package.yaml index 7b19956..6d887e3 100644 --- a/local/configs/package.yaml +++ b/local/configs/package.yaml @@ -7,11 +7,13 @@ time_command: true requirements: - datazen - - vcorelib>=2.3.1 + - vcorelib>=2.4.2 - rcmpy>=1.5.0 + - requests dev_requirements: - setuptools-wrapper - types-setuptools + - types-requests commands: - name: dist diff --git a/local/configs/python.yaml b/local/configs/python.yaml index a641966..924bf91 100644 --- a/local/configs/python.yaml +++ b/local/configs/python.yaml @@ -3,7 +3,7 @@ author_info: name: Vaughn Kottler email: vaughnkottler@gmail.com username: vkottler -versions: ["3.8", "3.9", "3.10", "3.11"] +versions: ["3.11"] systems: - macos-latest diff --git a/local/variables/package.yaml b/local/variables/package.yaml index 039237a..19e51d1 100644 --- a/local/variables/package.yaml +++ b/local/variables/package.yaml @@ -1,5 +1,5 @@ --- major: 2 -minor: 1 -patch: 1 +minor: 2 +patch: 0 entry: mbs diff --git a/pyproject.toml b/pyproject.toml index 4793add..3da3dae 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,10 +4,10 @@ build-backend = "setuptools.build_meta:__legacy__" [project] name = "yambs" -version = "2.1.1" +version = "2.2.0" description = "Yet another meta build-system." readme = "README.md" -requires-python = ">=3.8" +requires-python = ">=3.11" authors = [ {name = "Vaughn Kottler", email = "vaughnkottler@gmail.com"} ] @@ -15,9 +15,6 @@ maintainers = [ {name = "Vaughn Kottler", email = "vaughnkottler@gmail.com"} ] classifiers = [ - "Programming Language :: Python :: 3.8", - "Programming Language :: Python :: 3.9", - "Programming Language :: Python :: 3.10", "Programming Language :: Python :: 3.11", "Operating System :: Microsoft :: Windows", "Operating System :: MacOS", @@ -38,7 +35,8 @@ test = [ "isort", "yamllint", "setuptools-wrapper", - "types-setuptools" + "types-setuptools", + "types-requests" ] [project.scripts] diff --git a/scripts/.gitignore b/scripts/.gitignore new file mode 100644 index 0000000..a6c57f5 --- /dev/null +++ b/scripts/.gitignore @@ -0,0 +1 @@ +*.json diff --git a/scripts/common.sh b/scripts/common.sh new file mode 100644 index 0000000..b664be8 --- /dev/null +++ b/scripts/common.sh @@ -0,0 +1,29 @@ +#!/bin/bash + +CURL_ARGS=(curl -L) + +add_header() { + CURL_ARGS+=(-H "$1: $2") +} + +add_header Accept application/vnd.github+json + +API_VERSION=2022-11-28 +add_header X-GitHub-Api-Version $API_VERSION + +if [ -n "$API_TOKEN" ]; then + add_header Authorization "Bearer $API_TOKEN" +fi + +run_curl() { + echo "${CURL_ARGS[@]}" "$@" >&2 + "${CURL_ARGS[@]}" "$@" +} + +repo_api_url() { + echo "https://api.github.com/repos/$OWNER/$REPO" +} + +latest_release_url() { + echo "$(repo_api_url "$1" "$2")/releases/latest" +} diff --git a/scripts/latest_release.sh b/scripts/latest_release.sh new file mode 100755 index 0000000..9ded928 --- /dev/null +++ b/scripts/latest_release.sh @@ -0,0 +1,13 @@ +#!/bin/bash + +REPO=$(git rev-parse --show-toplevel) +CWD=$REPO/scripts + +# Set API_TOKEN= to enable header argument. +. "$CWD/common.sh" + +OWNER=vkottler +REPO=yambs-sample + +run_curl "$(latest_release_url $OWNER $REPO)" > "$CWD/output.json" +cat "$CWD/output.json" diff --git a/setup.py b/setup.py index 2985d86..855d1c0 100644 --- a/setup.py +++ b/setup.py @@ -1,7 +1,7 @@ # ===================================== # generator=datazen # version=3.1.2 -# hash=7cd5513daf171d1a95f64dc81343a73b +# hash=f3af34a4b5815c617489419b194b30b5 # ===================================== """ @@ -28,9 +28,6 @@ "version": VERSION, "description": DESCRIPTION, "versions": [ - "3.8", - "3.9", - "3.10", "3.11", ], } diff --git a/tests/commands/test_gen.py b/tests/commands/test_gen.py index 9ebd119..a5130a4 100644 --- a/tests/commands/test_gen.py +++ b/tests/commands/test_gen.py @@ -3,7 +3,7 @@ """ # internal -from tests.resources import resource +from tests.resources import clean_scenario # module under test from yambs import PKG_NAME @@ -13,16 +13,7 @@ def test_gen_command_basic(): """Test the 'gen' command.""" - assert ( - yambs_main( - [ - PKG_NAME, - "-C", - str(resource("scenarios", "sample")), - "gen", - "-w", - "-i", - ] - ) - == 0 - ) + path = str(clean_scenario("sample")) + + assert yambs_main([PKG_NAME, "-C", path, "gen", "-w", "-i"]) == 0 + assert yambs_main([PKG_NAME, "-C", path, "gen"]) == 0 diff --git a/tests/commands/test_native.py b/tests/commands/test_native.py index b6f0895..af74f48 100644 --- a/tests/commands/test_native.py +++ b/tests/commands/test_native.py @@ -2,8 +2,16 @@ Test the 'commands.native' module. """ +# built-in +from shutil import which +from subprocess import run +from sys import platform + +# third-party +from vcorelib.paths.context import in_dir + # internal -from tests.resources import resource +from tests.resources import clean_scenario # module under test from yambs import PKG_NAME @@ -13,16 +21,12 @@ def test_native_command_basic(): """Test the 'native' command.""" - assert ( - yambs_main( - [ - PKG_NAME, - "-C", - str(resource("scenarios", "native")), - "native", - "-w", - "-i", - ] - ) - == 0 - ) + path = str(clean_scenario("native")) + + with in_dir(path): + assert yambs_main([PKG_NAME, "native", "-w", "-i"]) == 0 + assert yambs_main([PKG_NAME, "native"]) == 0 + + # Try to build (if we can). + if platform == "linux" and which("ninja"): + run("ninja", check=True) diff --git a/tests/data/valid/scenarios/native/.gitignore b/tests/data/valid/scenarios/native/.gitignore deleted file mode 100644 index a02eb1c..0000000 --- a/tests/data/valid/scenarios/native/.gitignore +++ /dev/null @@ -1,4 +0,0 @@ -ninja -build.ninja -.ninja* -compile_commands.json diff --git a/tests/data/valid/scenarios/native/build.ninja b/tests/data/valid/scenarios/native/build.ninja deleted file mode 100644 index 73f60b8..0000000 --- a/tests/data/valid/scenarios/native/build.ninja +++ /dev/null @@ -1,19 +0,0 @@ -include_dir = ninja -src_dir = src -generated_dir = $src_dir/generated - -# Flags common to all builds, regardless of variant. -common_cflags = -I${src_dir} -Wall -Werror -Wextra -Wpedantic -ffunction-sections - -include $include_dir/all.ninja - -rule compile_commands - command = ninja -t compdb > compile_commands.json -build compdb: compile_commands - -build all: phony all_variants compdb - -default_target = debug -build single: phony $default_target compdb - -default single diff --git a/tests/data/valid/scenarios/native/src/apps/app1.cc b/tests/data/valid/scenarios/native/src/apps/app1.cc index 3d5536f..ddd82d6 100644 --- a/tests/data/valid/scenarios/native/src/apps/app1.cc +++ b/tests/data/valid/scenarios/native/src/apps/app1.cc @@ -1,6 +1,9 @@ /* toolchain */ #include +/* third-party */ +#include "yambs-sample/example/sample.h" + int test1(int a, int b) { return a + b; } int main(void) { @@ -13,5 +16,9 @@ int main(void) { std::cout << a << std::endl; } + Example::method1(); + Example::method2(); + Example::method3(); + return 0; } diff --git a/tests/data/valid/scenarios/native/yambs.yaml b/tests/data/valid/scenarios/native/yambs.yaml index 2fea231..6d3db61 100644 --- a/tests/data/valid/scenarios/native/yambs.yaml +++ b/tests/data/valid/scenarios/native/yambs.yaml @@ -8,7 +8,7 @@ extra_dist: [extra] project: name: yambs github: - owner: vkottler + owner: &self vkottler variants: clang: @@ -17,3 +17,6 @@ variants: clang-opt: extra_cflags: [--coverage] cflag_groups: [opt] + +dependencies: + - github: {repo: yambs-sample, owner: *self} diff --git a/tests/data/valid/scenarios/sample/.gitignore b/tests/data/valid/scenarios/sample/.gitignore deleted file mode 100644 index fd0f6f9..0000000 --- a/tests/data/valid/scenarios/sample/.gitignore +++ /dev/null @@ -1,2 +0,0 @@ -ninja -build.ninja diff --git a/tests/dependency/__init__.py b/tests/dependency/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/dependency/test_github.py b/tests/dependency/test_github.py new file mode 100644 index 0000000..ec59e90 --- /dev/null +++ b/tests/dependency/test_github.py @@ -0,0 +1,15 @@ +""" +Test the 'dependency.github' module. +""" + +# internal +from tests.resources import OWNER, REPO + +# module under test +from yambs.dependency.github import GithubDependency + + +def test_github_dependency_basic(): + """Test basic interactions with a GitHub dependency.""" + + assert GithubDependency(OWNER, REPO) diff --git a/tests/dependency/test_manager.py b/tests/dependency/test_manager.py new file mode 100644 index 0000000..b1248f0 --- /dev/null +++ b/tests/dependency/test_manager.py @@ -0,0 +1,17 @@ +""" +Test the 'dependency.manager' module. +""" + +# built-in +from pathlib import Path +from tempfile import TemporaryDirectory + +# module under test +from yambs.dependency.manager import DependencyManager + + +def test_dependency_manager_basic(): + """Test basic interactions with a dependency manager.""" + + with TemporaryDirectory() as tmp: + assert DependencyManager(Path(tmp)) diff --git a/tests/github/__init__.py b/tests/github/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/github/test_github.py b/tests/github/test_github.py new file mode 100644 index 0000000..093563b --- /dev/null +++ b/tests/github/test_github.py @@ -0,0 +1,25 @@ +""" +Test the 'github' module. +""" + +# internal +from tests.resources import OWNER, REPO + +# module under test +from yambs.github import ( + github_url, + latest_release_data, + latest_repo_release_api_url, +) + + +def test_github_url_basic(): + """Test URL encoding.""" + + assert github_url().geturl() == "https://github.com" + assert github_url(netloc_prefix="api").geturl() == "https://api.github.com" + assert ( + latest_repo_release_api_url(OWNER, REPO) + == "https://api.github.com/repos/vkottler/yambs-sample/releases/latest" + ) + assert latest_release_data(OWNER, REPO) diff --git a/tests/resources.py b/tests/resources.py index e198c01..0f4b4ec 100644 --- a/tests/resources.py +++ b/tests/resources.py @@ -4,6 +4,7 @@ # built-in from pathlib import Path +from shutil import rmtree def resource(resource_name: str, *parts: str, valid: bool = True) -> Path: @@ -12,3 +13,22 @@ def resource(resource_name: str, *parts: str, valid: bool = True) -> Path: return Path(__file__).parent.joinpath( "data", "valid" if valid else "invalid", resource_name, *parts ) + + +def clean_scenario(name: str) -> Path: + """ + Get the path to a scenario directory and ensure outputs are cleaned before + running. + """ + + base = resource("scenarios", name) + + # Clean things that can affect tests. + for path in ["third-party", "build", "ninja"]: + rmtree(base.joinpath(path), ignore_errors=True) + + return base + + +OWNER = "vkottler" +REPO = "yambs-sample" diff --git a/yambs/__init__.py b/yambs/__init__.py index fefbf89..ae2808e 100644 --- a/yambs/__init__.py +++ b/yambs/__init__.py @@ -1,7 +1,7 @@ # ===================================== # generator=datazen # version=3.1.2 -# hash=415bd9ff93df6a22da8865350f34d17d +# hash=fcb2cb10f781fa83974f986d304e773c # ===================================== """ @@ -10,4 +10,4 @@ DESCRIPTION = "Yet another meta build-system." PKG_NAME = "yambs" -VERSION = "2.1.1" +VERSION = "2.2.0" diff --git a/yambs/config/common.py b/yambs/config/common.py index 66b497d..2ff5b60 100644 --- a/yambs/config/common.py +++ b/yambs/config/common.py @@ -4,18 +4,19 @@ # built-in from pathlib import Path -from typing import Any, Dict, NamedTuple, Optional, Type, TypeVar +from typing import Any, Dict, NamedTuple, Optional, Set, Type, TypeVar # third-party from vcorelib.dict import merge from vcorelib.dict.codec import BasicDictCodec as _BasicDictCodec from vcorelib.io import ARBITER as _ARBITER from vcorelib.io import DEFAULT_INCLUDES_KEY -from vcorelib.io.types import JsonObject as _JsonObject +from vcorelib.io import JsonObject as _JsonObject from vcorelib.paths import Pathlike, find_file, normalize # internal from yambs import PKG_NAME +from yambs.dependency.config import Dependency from yambs.schemas import YambsDictCodec as _YambsDictCodec T = TypeVar("T", bound="CommonConfig") @@ -73,6 +74,7 @@ class CommonConfig(_YambsDictCodec, _BasicDictCodec): build_root: Path ninja_root: Path dist_root: Path + third_party_root: Path file: Optional[Path] @@ -98,11 +100,19 @@ def init(self, data: _JsonObject) -> None: self.build_root = self.directory("build_root") self.ninja_root = self.directory("ninja_out") self.dist_root = self.directory("dist_out") + self.third_party_root = self.directory("third_party_root") self.file = None self.project = Project.create(data["project"]) # type: ignore + # Collect project dependency data. + self.dependencies: Set[Dependency] = set() + for dep in data.get("dependencies", []): # type: ignore + new_dep = Dependency(data=dep, verify=False) # type: ignore + new_dep.github.setdefault("owner", self.project.owner) + self.dependencies.add(new_dep) + @classmethod def load( cls: Type[T], diff --git a/yambs/data/schemas/Dependency.yaml b/yambs/data/schemas/Dependency.yaml new file mode 100644 index 0000000..06efa1c --- /dev/null +++ b/yambs/data/schemas/Dependency.yaml @@ -0,0 +1,20 @@ +--- +type: object +additionalProperties: false +properties: + kind: + type: string + enum: [yambs] + default: yambs + + source: + type: string + enum: [github] + default: github + + target: + type: string + default: opt + + github: + $ref: package://yambs/schemas/Github.yaml diff --git a/yambs/data/schemas/Github.yaml b/yambs/data/schemas/Github.yaml new file mode 100644 index 0000000..c9f2920 --- /dev/null +++ b/yambs/data/schemas/Github.yaml @@ -0,0 +1,10 @@ +--- +type: object +additionalProperties: false +required: [owner] +properties: + # If not set, fallback to project.name (for top-level project). + repo: + type: string + owner: + type: string diff --git a/yambs/data/schemas/Project.yaml b/yambs/data/schemas/Project.yaml new file mode 100644 index 0000000..69943b5 --- /dev/null +++ b/yambs/data/schemas/Project.yaml @@ -0,0 +1,27 @@ +--- +type: object +additionalProperties: false +properties: + name: + type: string + default: yambs-sample + + github: + $ref: package://yambs/schemas/Github.yaml + + version: + type: object + additionalProperties: false + properties: + major: + type: integer + minimum: 0 + default: 0 + minor: + type: integer + minimum: 1 + default: 1 + patch: + type: integer + minimum: 0 + default: 0 diff --git a/yambs/data/schemas/config_common.yaml b/yambs/data/schemas/config_common.yaml index ba46a23..baa0aac 100644 --- a/yambs/data/schemas/config_common.yaml +++ b/yambs/data/schemas/config_common.yaml @@ -12,6 +12,10 @@ properties: type: string default: dist + third_party_root: + type: string + default: third-party + build_root: type: string default: build @@ -28,37 +32,9 @@ properties: type: string project: - type: object - additionalProperties: false - properties: - name: - type: string - default: yambs-sample - - github: - type: object - additionalProperties: false - required: [owner] - properties: - # If not set, fallback to project.name. - repo: - type: string - owner: - type: string + $ref: package://yambs/schemas/Project.yaml - version: - type: object - additionalProperties: false - properties: - major: - type: integer - minimum: 0 - default: 0 - minor: - type: integer - minimum: 1 - default: 1 - patch: - type: integer - minimum: 0 - default: 0 + dependencies: + type: array + items: + $ref: package://yambs/schemas/Dependency.yaml diff --git a/yambs/data/templates/native_build.ninja.j2 b/yambs/data/templates/native_build.ninja.j2 index 95cc86f..f4f5b31 100644 --- a/yambs/data/templates/native_build.ninja.j2 +++ b/yambs/data/templates/native_build.ninja.j2 @@ -1,5 +1,6 @@ include_dir = {{ninja_out}} src_dir = {{src_root}} +third_party_dir = {{third_party_root}} generated_dir = $src_dir/generated # Flags common to all builds, regardless of variant. diff --git a/yambs/data/templates/native_rules.ninja.j2 b/yambs/data/templates/native_rules.ninja.j2 index 6961d2a..6eeb4fe 100644 --- a/yambs/data/templates/native_rules.ninja.j2 +++ b/yambs/data/templates/native_rules.ninja.j2 @@ -16,9 +16,16 @@ ldflags ={% for flag in common_ldflags %} {{flag}}{% endfor %} {% endif %} rule link - command = $ld $cflags $ldflags -Wl,-Map=$out.map $in -o $out + command = $ld $cflags -Wl,-Map=$out.map $in $ldflags -o $out rule ar command = ar rcs $out $in build_dir = {{build_root}}/$variant + +rule script + command = /bin/bash $in $out + +build $build_dir/third_party.txt: script $third_party_dir/third_party.sh + +build ${variant}_third_party: phony $build_dir/third_party.txt diff --git a/yambs/dependency/__init__.py b/yambs/dependency/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/yambs/dependency/config.py b/yambs/dependency/config.py new file mode 100644 index 0000000..cda4ab1 --- /dev/null +++ b/yambs/dependency/config.py @@ -0,0 +1,55 @@ +""" +A module for working with dependency configurations. +""" + +# built-in +from enum import StrEnum, auto +from typing import Any, Dict, Optional, cast + +# third-party +from vcorelib.dict.codec import BasicDictCodec as _BasicDictCodec +from vcorelib.io import JsonObject as _JsonObject + +# internal +from yambs.schemas import YambsDictCodec as _YambsDictCodec + + +class DependencyKind(StrEnum): + """All dependency kind options.""" + + YAMBS = auto() + + +class DependencySource(StrEnum): + """All dependency source options.""" + + GITHUB = auto() + + +DependencyData = Dict[str, Any] + + +class Dependency(_YambsDictCodec, _BasicDictCodec): + """A class for describing project dependencies.""" + + def __str__(self) -> str: + """Get this dependency as a string.""" + + # Change this when other sources are supported. + assert self.source == DependencySource.GITHUB + return f"{self.github['owner']}-{self.github['repo']}" + + def __hash__(self) -> int: + """Compute a hash for this dependency.""" + return hash(str(self)) + + def init(self, data: _JsonObject) -> None: + """Initialize this instance.""" + + self.kind = DependencyKind(cast(str, data["kind"])) + self.source = DependencySource(cast(str, data["source"])) + self.github: Dict[str, Optional[str]] = data.get( + "github", + {}, # type: ignore + ) + self.target: str = data["target"] # type: ignore diff --git a/yambs/dependency/github.py b/yambs/dependency/github.py new file mode 100644 index 0000000..0259164 --- /dev/null +++ b/yambs/dependency/github.py @@ -0,0 +1,39 @@ +""" +A module implementing GitHub dependency interactions. +""" + +# built-in +from typing import Dict + +# third-party +from vcorelib.logging import LoggerMixin + +# internal +from yambs.github import ReleaseData, latest_release_data + + +class GithubDependency(LoggerMixin): + """A class for managing GitHub dependencies.""" + + def __init__( + self, owner: str, repo: str, *args, data: ReleaseData = None, **kwargs + ) -> None: + """Initialize this instance.""" + + super().__init__(logger_name=f"{owner}.{repo}") + + if data is None: + data = latest_release_data(owner, repo, *args, **kwargs) + self.data = data + + self.logger.info( + "Loaded release '%s' (%s).", + self.data["name"], + self.data["html_url"], + ) + + # Collect URLs for release content. + self.download_urls: Dict[str, str] = { + item["name"]: item["browser_download_url"] + for item in self.data["assets"] + } diff --git a/yambs/dependency/handlers/__init__.py b/yambs/dependency/handlers/__init__.py new file mode 100644 index 0000000..d5cd0af --- /dev/null +++ b/yambs/dependency/handlers/__init__.py @@ -0,0 +1,15 @@ +""" +A module for aggregating dependency handlers. +""" + +# built-in +from typing import Dict + +# internal +from yambs.dependency.config import DependencyKind +from yambs.dependency.handlers.types import DependencyHandler +from yambs.dependency.handlers.yambs import yambs_handler + +HANDLERS: Dict[DependencyKind, DependencyHandler] = { + DependencyKind.YAMBS: yambs_handler +} diff --git a/yambs/dependency/handlers/types.py b/yambs/dependency/handlers/types.py new file mode 100644 index 0000000..911d9d2 --- /dev/null +++ b/yambs/dependency/handlers/types.py @@ -0,0 +1,34 @@ +""" +A module declaring shared types. +""" + +# built-in +from pathlib import Path +from typing import Callable, List, NamedTuple, Set + +# internal +from yambs.dependency.config import Dependency, DependencyData +from yambs.dependency.state import DependencyState + + +class DependencyTask(NamedTuple): + """A container for dependency handler invocation data.""" + + # Useful paths. + root: Path + include: Path + static: Path + + build_commands: List[List[str]] + + compile_flags: List[str] + link_flags: List[str] + + dep: Dependency + current: DependencyState + data: DependencyData + + nested: Set[Dependency] + + +DependencyHandler = Callable[[DependencyTask], DependencyState] diff --git a/yambs/dependency/handlers/yambs.py b/yambs/dependency/handlers/yambs.py new file mode 100644 index 0000000..0531b6f --- /dev/null +++ b/yambs/dependency/handlers/yambs.py @@ -0,0 +1,156 @@ +""" +A module implementing a dependency handler for other yambs projects. +""" + +# built-in +from pathlib import Path + +# third-party +import requests +from vcorelib.io.archive import extractall +from vcorelib.paths import validate_hex_digest + +# internal +from yambs.dependency.config import ( + Dependency, + DependencyData, + DependencySource, +) +from yambs.dependency.github import GithubDependency +from yambs.dependency.handlers.types import DependencyTask +from yambs.dependency.state import DependencyState + +TARBALL = ".tar.xz" + + +def audit_downloads( + root: Path, data: DependencyData, github: GithubDependency +) -> None: + """Ensure release assets are downloaded.""" + + to_download = ["sum", TARBALL] + for asset in github.data["assets"]: + name = asset["name"] + + dest = root.joinpath(name) + + if dest.is_file(): + continue + + for suffix in to_download: + if name.endswith(suffix): + # Download the file. + req = requests.get(asset["browser_download_url"], timeout=10) + with dest.open("wb") as dest_fd: + for chunk in req.iter_content(chunk_size=4096): + dest_fd.write(chunk) + + data["assets"][suffix] = str(dest) + break + + +def github_release(dep: Dependency, data: DependencyData) -> GithubDependency: + """Obtain GitHub release metadata from the project if necessary.""" + + # Ensure repository parameters are set. + assert "owner" in dep.github and dep.github["owner"], dep + assert "repo" in dep.github and dep.github["repo"], dep + + # Load GitHub release data. + github = GithubDependency( + dep.github["owner"], + dep.github["repo"], + data=data.get("latest_release"), + ) + + # Initialize data. + data["latest_release"] = github.data + data["version"] = github.data["tag_name"] + data.setdefault("assets", {}) + + return github + + +def audit_extract(root: Path, data: DependencyData) -> Path: + """ + Ensure the release is extracted (and the archive contents are verified). + """ + + if "directory" not in data: + # The expected directory is just the name of the tarball with no + # suffix. + expected = Path(data["assets"][TARBALL].replace(TARBALL, "")) + + # The name of the project is the directory name without the version + # suffix. + data["slug"] = expected.name + data["name"] = data["slug"].replace(f"-{data['version']}", "") + + # check if need to un-archive, if so, verify checksum. + if not expected.is_dir(): + validate_hex_digest(data["assets"]["sum"], root=root) + assert extractall( + data["assets"][TARBALL], + dst=root, + maxsplit=1 + expected.name.count("."), + )[0] + assert expected.is_dir() + + data["directory"] = str(expected) + + assert "name" in data + + # Link 'name' to the destination directory. + dest_dir = Path(data["directory"]) + name_link = dest_dir.with_name(data["name"]) + if ( + not name_link.is_symlink() + or str(name_link.readlink()) != dest_dir.name + ): + name_link.unlink(missing_ok=True) + name_link.symlink_to(dest_dir.name) + + return name_link + + +def yambs_handler(task: DependencyTask) -> DependencyState: + """Handle a yambs dependency.""" + + # No other source implementations currently. + assert task.dep.source == DependencySource.GITHUB + + github = github_release(task.dep, task.data) + audit_downloads(task.root, task.data, github) + directory = audit_extract(task.root, task.data) + + static_lib = directory.joinpath( + "build", task.dep.target, f"{task.data['slug']}.a" + ) + task.data["static_library"] = str(static_lib) + + if not static_lib.is_file(): + # Add a build command if the library still needs to be built. + task.build_commands.append( + ["ninja", "-C", str(directory), f"{task.dep.target}_lib"] + ) + + # Ensure the final static library is linked within the static directory. + static_include = task.static.joinpath(f"lib{static_lib.name}") + if not static_include.is_symlink(): + static_include.symlink_to( + Path("..", static_lib.relative_to(task.root)) + ) + + # Ensure this dependency's static library gets linked. + task.link_flags.append(f"-l{task.data['slug']}") + + # Ensure the 'src' directory is linked within the include directory. + src_include = task.include.joinpath(task.data["name"]) + if not src_include.is_symlink(): + src_include.symlink_to(Path("..", task.data["name"], "src")) + + # Read the project's configuration data to find any nested dependencies. + # task.root.joinpath("yambs.yaml"), look for data file? + # task.nested.add() + + return task.current diff --git a/yambs/dependency/manager.py b/yambs/dependency/manager.py new file mode 100644 index 0000000..fd30fad --- /dev/null +++ b/yambs/dependency/manager.py @@ -0,0 +1,117 @@ +""" +A module implementing a dependency manager. +""" + +# built-in +from pathlib import Path +from typing import List, Set + +# third-party +from vcorelib.io import ARBITER +from vcorelib.logging import LoggerType +from vcorelib.paths import set_exec_flags + +# internal +from yambs.dependency.config import Dependency, DependencyData +from yambs.dependency.handlers import HANDLERS +from yambs.dependency.handlers.types import DependencyTask +from yambs.dependency.state import DependencyState + + +class DependencyManager: + """A class for managing project dependencies.""" + + def __init__(self, root: Path) -> None: + """Initialize this instance.""" + + self.root = root + self.state_path = self.root.joinpath("state.json") + self.state = ARBITER.decode(self.state_path).data + + # A place for third-party include roots to be linked. + self.include = self.root.joinpath("include") + self.include.mkdir(parents=True, exist_ok=True) + + # A place for third-party static libraries to be linked. + self.static = self.root.joinpath("static") + self.static.mkdir(parents=True, exist_ok=True) + + # A list of commands to run that should build dependencies. + self.build_commands: List[List[str]] = [] + + # Aggregate compiler flags. + self.compile_flags = ["-iquote", str(self.include)] + self.link_flags = [f"-L{self.static}"] + + def info(self, logger: LoggerType) -> None: + """Log some information.""" + + if self.build_commands: + logger.info("Build commands: %s.", self.build_commands) + logger.info("Third-party compile flags: %s.", self.compile_flags) + logger.info("Third-party link flags: %s.", self.link_flags) + + def save(self, logger: LoggerType = None) -> None: + """Save state data and create the third-party build script.""" + + ARBITER.encode(self.state_path, self.state) + if logger is not None: + self.info(logger) + + script = self.root.joinpath("third_party.sh") + with script.open("w") as script_fd: + script_fd.write("#!/bin/bash\n\n") + + # Add build commands. + for command in self.build_commands: + script_fd.write(" ".join(command)) + script_fd.write("\n") + + script_fd.write("\ndate > $1\n") + + set_exec_flags(script) + + def _create_task(self, dep: Dependency) -> DependencyTask: + """Create a new task object.""" + + dep_data: DependencyData = self.state.setdefault( + str(dep), + {}, + ) # type: ignore + + return DependencyTask( + self.root, + self.include, + self.static, + self.build_commands, + self.compile_flags, + self.link_flags, + dep, + DependencyState(dep_data.setdefault("state", "init")), + dep_data.setdefault("handler", {}), + set(), + ) + + def audit(self, dep: Dependency) -> DependencyState: + """Interact with a dependency if needed.""" + + tasks = [self._create_task(dep)] + resolved: Set[Dependency] = set() + + while tasks: + task = tasks.pop() + state = HANDLERS[dep.kind](task) + resolved.add(task.dep) + + # Update state. + task.data["state"] = str(state.value) + + # Handle any nested dependencies. + # + # Enable this soon! + # + # for nested in task.nested: + # if nested not in resolved: + # tasks.append(self._create_task(nested)) + + return state diff --git a/yambs/dependency/state.py b/yambs/dependency/state.py new file mode 100644 index 0000000..5324180 --- /dev/null +++ b/yambs/dependency/state.py @@ -0,0 +1,12 @@ +""" +A module implementing interfaces for working with dependency states. +""" + +# built-in +from enum import StrEnum, auto + + +class DependencyState(StrEnum): + """States that a dependency can be in.""" + + INIT = auto() diff --git a/yambs/dev_requirements.txt b/yambs/dev_requirements.txt index 67fd651..377eb6b 100644 --- a/yambs/dev_requirements.txt +++ b/yambs/dev_requirements.txt @@ -7,3 +7,4 @@ isort yamllint setuptools-wrapper types-setuptools +types-requests diff --git a/yambs/environment/native.py b/yambs/environment/native.py index 77e7b5f..024ae68 100644 --- a/yambs/environment/native.py +++ b/yambs/environment/native.py @@ -14,6 +14,7 @@ # internal from yambs.aggregation import collect_files, populate_sources, sources_headers from yambs.config.native import Native +from yambs.dependency.manager import DependencyManager from yambs.generate.common import get_jinja, render_template from yambs.generate.ninja import write_continuation from yambs.generate.ninja.format import render_format @@ -36,6 +37,10 @@ def __init__(self, config: Native) -> None: self.config = config + self.dependency_manager = DependencyManager( + self.config.third_party_root + ) + # Collect sources. self.sources = collect_files(config.src_root) self.apps: Set[Path] = set() @@ -113,6 +118,10 @@ def write_app_rules( write_continuation(stream, offset) stream.write(str(file)) + # Executables can't be linked until third-party dependencies are + # actually built. + stream.write(" | ${variant}_third_party") + stream.write(linesep + linesep) line = "build ${variant}_apps: phony " @@ -174,6 +183,21 @@ def generate(self, sources_only: bool = False) -> None: """Generate ninja files.""" if not sources_only: + # Audit dependencies. + for dep in self.config.dependencies: + self.dependency_manager.audit(dep) + + # Create build script. + self.dependency_manager.save(logger=self.logger) + + # Handle compile and link flags generated by the third-party pass. + self.config.data["common_cflags"].extend( + self.dependency_manager.compile_flags + ) + self.config.data["common_ldflags"].extend( + self.dependency_manager.link_flags + ) + # Render templates. generate_variants( self.jinja, self.config, self.config.data["cflag_groups"] diff --git a/yambs/github/__init__.py b/yambs/github/__init__.py new file mode 100644 index 0000000..18bc262 --- /dev/null +++ b/yambs/github/__init__.py @@ -0,0 +1,85 @@ +""" +A module for working with GitHub releases. +""" + +# built-in +import os +from typing import Any, Dict +from urllib.parse import ParseResult + +# third-party +import requests +from vcorelib.dict.codec import BasicDictCodec as _BasicDictCodec + +# internal +from yambs.schemas import YambsDictCodec as _YambsDictCodec + + +class Github(_YambsDictCodec, _BasicDictCodec): + """GitHub repository information.""" + + +def github_url( + netloc_prefix: str = "", + scheme: str = "https", + path: str = "", + params: str = "", + query: str = "", + fragment: str = "", +) -> ParseResult: + """See: https://docs.python.org/3/library/urllib.parse.html.""" + + netloc = "github.com" + if netloc_prefix: + netloc = f"{netloc_prefix}." + netloc + + return ParseResult( + scheme=scheme, + netloc=netloc, + path=path, + params=params, + query=query, + fragment=fragment, + ) + + +def latest_repo_release_api_url(owner: str, repo: str) -> str: + """Get a URL string for a repository's latest release.""" + return github_url( + netloc_prefix="api", path=f"repos/{owner}/{repo}/releases/latest" + ).geturl() + + +GIHTUB_HEADERS = { + "Accept": "application/vnd.github+json", + "X-GitHub-Api-Version": "2022-11-28", +} + + +def check_api_token() -> None: + """Check for a GitHub API token set via the environment.""" + + if "Authorization" not in GIHTUB_HEADERS: + if "GITHUB_API_TOKEN" in os.environ: + GIHTUB_HEADERS[ + "Authorization" + ] = f"Bearer {os.environ['GITHUB_API_TOKEN']}" + + +ReleaseData = Dict[str, Any] + + +def latest_release_data( + owner: str, repo: str, *args, timeout: float = None, **kwargs +) -> ReleaseData: + """Get latest-release data.""" + + check_api_token() + + return requests.get( # type: ignore + latest_repo_release_api_url(owner, repo), + *args, + timeout=timeout, + headers=GIHTUB_HEADERS, + **kwargs, + ).json() diff --git a/yambs/requirements.txt b/yambs/requirements.txt index eab3a00..cd617bf 100644 --- a/yambs/requirements.txt +++ b/yambs/requirements.txt @@ -1,3 +1,4 @@ datazen -vcorelib>=2.3.1 +vcorelib>=2.4.2 rcmpy>=1.5.0 +requests