From 471a9ce10e628ee4a86cfa375fb748b806916ae8 Mon Sep 17 00:00:00 2001 From: Vaughn Kottler Date: Sun, 1 Oct 2023 19:50:04 -0500 Subject: [PATCH] 2.7.0 - Add 'download' command --- .github/workflows/python-package.yml | 2 +- README.md | 29 ++++++-- local/configs/package.yaml | 2 + local/includes/sub_commands.yaml | 2 +- local/variables/package.yaml | 2 +- manifest.yaml | 1 + pyproject.toml | 2 +- tests/commands/test_download.py | 17 +++++ yambs/__init__.py | 4 +- yambs/commands/all.py | 8 ++- yambs/commands/download.py | 62 +++++++++++++++++ yambs/dependency/github.py | 83 ++++++++++++++++++++++- yambs/dependency/handlers/yambs/github.py | 26 ++++--- 13 files changed, 213 insertions(+), 27 deletions(-) create mode 100644 tests/commands/test_download.py create mode 100644 yambs/commands/download.py diff --git a/.github/workflows/python-package.yml b/.github/workflows/python-package.yml index 21e08fe..8ff86fd 100644 --- a/.github/workflows/python-package.yml +++ b/.github/workflows/python-package.yml @@ -75,7 +75,7 @@ jobs: - run: | mk python-release owner=vkottler \ - repo=yambs version=2.6.0 + repo=yambs version=2.7.0 if: | matrix.python-version == '3.11' && matrix.system == 'ubuntu-latest' diff --git a/README.md b/README.md index da79fc1..85638eb 100644 --- a/README.md +++ b/README.md @@ -2,11 +2,11 @@ ===================================== generator=datazen version=3.1.3 - hash=5385bdb3ad34d67455cd0d0cd2349d4e + hash=7c5f56f5b4ab2cfc9391105a8b2dc902 ===================================== --> -# yambs ([2.6.0](https://pypi.org/project/yambs/)) +# yambs ([2.7.0](https://pypi.org/project/yambs/)) [![python](https://img.shields.io/pypi/pyversions/yambs.svg)](https://pypi.org/project/yambs/) ![Build Status](https://github.com/vkottler/yambs/workflows/Python%20Package/badge.svg) @@ -133,7 +133,7 @@ following a specific convention), put your configuration data here. $ ./venv3.11/bin/mbs -h usage: mbs [-h] [--version] [-v] [-q] [--curses] [--no-uvloop] [-C DIR] - {compile_config,dist,gen,native,uf2conv,noop} ... + {compile_config,dist,download,gen,native,uf2conv,noop} ... Yet another meta build-system. @@ -147,10 +147,11 @@ options: -C DIR, --dir DIR execute from a specific directory commands: - {compile_config,dist,gen,native,uf2conv,noop} + {compile_config,dist,download,gen,native,uf2conv,noop} set of available commands compile_config load configuration data and write results to a file dist create a source distribution + download download GitHub release assets gen poll the source tree and generate any new build files native generate build files for native-only target projects uf2conv convert to UF2 or flash directly @@ -201,6 +202,26 @@ options: ``` +### `download` + +``` +$ ./venv3.11/bin/mbs download -h + +usage: mbs download [-h] [-o OWNER] [-r REPO] [-O OUTPUT] [-p PATTERN] + +options: + -h, --help show this help message and exit + -o OWNER, --owner OWNER + repository owner (default: 'vkottler') + -r REPO, --repo REPO repository name (default: 'toolchains') + -O OUTPUT, --output OUTPUT + output directory (default: 'toolchains') + -p PATTERN, --pattern PATTERN + a pattern to use to select project specifications + filtered by name + +``` + ### `gen` ``` diff --git a/local/configs/package.yaml b/local/configs/package.yaml index 9678d92..98cd95c 100644 --- a/local/configs/package.yaml +++ b/local/configs/package.yaml @@ -28,6 +28,8 @@ commands: description: load configuration data and write results to a file - name: dist description: create a source distribution + - name: download + description: download GitHub release assets - name: gen description: poll the source tree and generate any new build files - name: native diff --git a/local/includes/sub_commands.yaml b/local/includes/sub_commands.yaml index c40bbf3..15afc78 100644 --- a/local/includes/sub_commands.yaml +++ b/local/includes/sub_commands.yaml @@ -3,7 +3,7 @@ default_dirs: false commands: -{% for command in ["compile_config", "dist", "gen", "native", "uf2conv"] %} +{% for command in ["compile_config", "dist", "download", "gen", "native", "uf2conv"] %} - name: help-{{command}} command: "./venv{{python_version}}/bin/{{entry}}" force: true diff --git a/local/variables/package.yaml b/local/variables/package.yaml index e61835f..f130e27 100644 --- a/local/variables/package.yaml +++ b/local/variables/package.yaml @@ -1,5 +1,5 @@ --- major: 2 -minor: 6 +minor: 7 patch: 0 entry: mbs diff --git a/manifest.yaml b/manifest.yaml index cbe1fa0..68aaef6 100644 --- a/manifest.yaml +++ b/manifest.yaml @@ -32,6 +32,7 @@ renders: - commands-help - commands-help-compile_config - commands-help-dist + - commands-help-download - commands-help-gen - commands-help-native - commands-help-uf2conv diff --git a/pyproject.toml b/pyproject.toml index 7fe1fd9..9aa4d69 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta:__legacy__" [project] name = "yambs" -version = "2.6.0" +version = "2.7.0" description = "Yet another meta build-system." readme = "README.md" requires-python = ">=3.11" diff --git a/tests/commands/test_download.py b/tests/commands/test_download.py new file mode 100644 index 0000000..c587b82 --- /dev/null +++ b/tests/commands/test_download.py @@ -0,0 +1,17 @@ +""" +Test the 'commands.download' module. +""" + +# built-in +from tempfile import TemporaryDirectory + +# module under test +from yambs import PKG_NAME +from yambs.entry import main as yambs_main + + +def test_download_basic(): + """Test the 'download' command.""" + + with TemporaryDirectory() as tmpdir: + assert yambs_main([PKG_NAME, "-C", str(tmpdir), "download"]) == 0 diff --git a/yambs/__init__.py b/yambs/__init__.py index 29ba182..c858c88 100644 --- a/yambs/__init__.py +++ b/yambs/__init__.py @@ -1,7 +1,7 @@ # ===================================== # generator=datazen # version=3.1.3 -# hash=c440d523035573f23897dbb501a02261 +# hash=63f46d83195b193de72a6bbd1bcc7c47 # ===================================== """ @@ -10,4 +10,4 @@ DESCRIPTION = "Yet another meta build-system." PKG_NAME = "yambs" -VERSION = "2.6.0" +VERSION = "2.7.0" diff --git a/yambs/commands/all.py b/yambs/commands/all.py index 410a5fd..e75792c 100644 --- a/yambs/commands/all.py +++ b/yambs/commands/all.py @@ -1,7 +1,7 @@ # ===================================== # generator=datazen # version=3.1.3 -# hash=e10efb5b655cacd368d023aacf84f288 +# hash=a579de7276b0c8f219e976ab8cc6a008 # ===================================== """ @@ -18,6 +18,7 @@ # internal from yambs.commands.compile_config import add_compile_config_cmd from yambs.commands.dist import add_dist_cmd +from yambs.commands.download import add_download_cmd from yambs.commands.gen import add_gen_cmd from yambs.commands.native import add_native_cmd from yambs.commands.uf2conv import add_uf2conv_cmd @@ -37,6 +38,11 @@ def commands() -> _List[_Tuple[str, str, _CommandRegister]]: "create a source distribution", add_dist_cmd, ), + ( + "download", + "download GitHub release assets", + add_download_cmd, + ), ( "gen", "poll the source tree and generate any new build files", diff --git a/yambs/commands/download.py b/yambs/commands/download.py new file mode 100644 index 0000000..be0d951 --- /dev/null +++ b/yambs/commands/download.py @@ -0,0 +1,62 @@ +""" +An entry-point for the 'download' command. +""" + +# built-in +from argparse import ArgumentParser as _ArgumentParser +from argparse import Namespace as _Namespace +from pathlib import Path + +# third-party +from vcorelib.args import CommandFunction as _CommandFunction + +# internal +from yambs.dependency.github import GithubDependency, default_filt + + +def download_cmd(args: _Namespace) -> int: + """Execute the download command.""" + + dep = GithubDependency(args.owner, args.repo) + + # Download and extract things. + dep.download_release_assets( + default_filt(args.output, pattern=args.pattern) + ) + + return 0 + + +def add_download_cmd(parser: _ArgumentParser) -> _CommandFunction: + """Add download-command arguments to its parser.""" + + parser.add_argument( + "-o", + "--owner", + default="vkottler", + help="repository owner (default: '%(default)s')", + ) + parser.add_argument( + "-r", + "--repo", + default="toolchains", + help="repository name (default: '%(default)s')", + ) + parser.add_argument( + "-O", + "--output", + type=Path, + default=Path("toolchains"), + help="output directory (default: '%(default)s')", + ) + parser.add_argument( + "-p", + "--pattern", + default=".*", + help=( + "a pattern to use to select project " + "specifications filtered by name" + ), + ) + + return download_cmd diff --git a/yambs/dependency/github.py b/yambs/dependency/github.py index 0259164..a5e9974 100644 --- a/yambs/dependency/github.py +++ b/yambs/dependency/github.py @@ -3,14 +3,80 @@ """ # built-in -from typing import Dict +from pathlib import Path +from re import search +from typing import Any, Callable, Dict, Optional # third-party -from vcorelib.logging import LoggerMixin +import requests +from vcorelib.io.archive import extractall +from vcorelib.io.types import FileExtension +from vcorelib.logging import LoggerMixin, LoggerType +from vcorelib.math import nano_str # internal from yambs.github import ReleaseData, latest_release_data +AssetFilter = Callable[[dict[str, Any]], Optional[Path]] + + +def download_file_if_missing( + uri: str, dest: Path, timeout: float = 10.0, chunk_size: int = 4096 +) -> None: + """Download a file if necessary.""" + + if not dest.is_file(): + req = requests.get(uri, timeout=timeout) + with dest.open("wb") as dest_fd: + for chunk in req.iter_content(chunk_size=chunk_size): + dest_fd.write(chunk) + + +def default_filt( + output: Path, pattern: str = ".*", mkdir: bool = True +) -> AssetFilter: + """Create a default release-asset filter method.""" + + if mkdir: + output.mkdir(parents=True, exist_ok=True) + + def filt(asset: dict[str, Any]) -> Optional[Path]: + """Determine if the release asset should be downloaded.""" + + result = None + + name = asset["name"] + if search(pattern, name) is not None: + result = output.joinpath(name) + + return result + + return filt + + +def ensure_extracted(path: Path, logger: LoggerType = None) -> None: + """Ensure that all archive files in a directory are extracted.""" + + for item in path.iterdir(): + ext = FileExtension.from_path(item) + if ext is not None and ext.is_archive() and item.is_file(): + dest = item.parent.joinpath(item.name.replace(f".{ext}", "")) + if not dest.is_dir(): + if logger is not None: + logger.info("Extracting '%s' -> '%s'.", item, dest) + + result = extractall(item, dst=item.parent) + assert result[0] + + if logger is not None: + logger.info( + "Extracted '%s' in %s.", + dest, + nano_str(result[1], is_time=True), + ) + + assert dest.is_dir(), dest + class GithubDependency(LoggerMixin): """A class for managing GitHub dependencies.""" @@ -37,3 +103,16 @@ def __init__( item["name"]: item["browser_download_url"] for item in self.data["assets"] } + + def download_release_assets( + self, filt: AssetFilter, extract: bool = True + ) -> None: + """Ensure release assets are downloaded.""" + + for asset in self.data["assets"]: + dest = filt(asset) + if dest is not None: + download_file_if_missing(asset["browser_download_url"], dest) + + if extract: + ensure_extracted(dest.parent, logger=self.logger) diff --git a/yambs/dependency/handlers/yambs/github.py b/yambs/dependency/handlers/yambs/github.py index e57fa3b..d66f46e 100644 --- a/yambs/dependency/handlers/yambs/github.py +++ b/yambs/dependency/handlers/yambs/github.py @@ -4,9 +4,9 @@ # built-in from pathlib import Path +from typing import Any, Optional # third-party -import requests from vcorelib.io.archive import extractall from vcorelib.paths import validate_hex_digest @@ -45,25 +45,23 @@ def audit_downloads( """Ensure release assets are downloaded.""" to_download = ["sum", TARBALL] - for asset in github.data["assets"]: - name = asset["name"] + def filt(asset: dict[str, Any]) -> Optional[Path]: + """Determine if the release asset should be downloaded.""" + + result = None + + name = asset["name"] dest = root.joinpath(name) for suffix in to_download: if name.endswith(suffix): - # Download if necessary. - if not dest.is_file(): - # Download the file. - req = requests.get( - asset["browser_download_url"], timeout=10 - ) - with dest.open("wb") as dest_fd: - for chunk in req.iter_content(chunk_size=4096): - dest_fd.write(chunk) - data["assets"][suffix] = str(dest) - break + result = dest + + return result + + github.download_release_assets(filt) def audit_extract(root: Path, data: DependencyData) -> Path: