Merge pull request #71 from vkottler/dev/2.7.0

2.7.0 - Add 'download' command

.github/workflows/python-package.yml

@@ -75,7 +75,7 @@ jobs:
 
       - run: |
           mk python-release owner=vkottler \
-                            repo=yambs version=2.6.0
+                            repo=yambs version=2.7.0
         if: |
           matrix.python-version == '3.11'
           && matrix.system == 'ubuntu-latest'

README.md

@@ -2,11 +2,11 @@
     =====================================
     generator=datazen
     version=3.1.3
-    hash=5385bdb3ad34d67455cd0d0cd2349d4e
+    hash=7c5f56f5b4ab2cfc9391105a8b2dc902
     =====================================
 -->
 
-# yambs ([2.6.0](https://pypi.org/project/yambs/))
+# yambs ([2.7.0](https://pypi.org/project/yambs/))
 
 [![python](https://img.shields.io/pypi/pyversions/yambs.svg)](https://pypi.org/project/yambs/)
 ![Build Status](https://github.com/vkottler/yambs/workflows/Python%20Package/badge.svg)
@@ -133,7 +133,7 @@ following a specific convention), put your configuration data here.
 $ ./venv3.11/bin/mbs -h
 
 usage: mbs [-h] [--version] [-v] [-q] [--curses] [--no-uvloop] [-C DIR]
-           {compile_config,dist,gen,native,uf2conv,noop} ...
+           {compile_config,dist,download,gen,native,uf2conv,noop} ...
 
 Yet another meta build-system.
 
@@ -147,10 +147,11 @@ options:
   -C DIR, --dir DIR     execute from a specific directory
 
 commands:
-  {compile_config,dist,gen,native,uf2conv,noop}
+  {compile_config,dist,download,gen,native,uf2conv,noop}
                         set of available commands
     compile_config      load configuration data and write results to a file
     dist                create a source distribution
+    download            download GitHub release assets
     gen                 poll the source tree and generate any new build files
     native              generate build files for native-only target projects
     uf2conv             convert to UF2 or flash directly
@@ -201,6 +202,26 @@ options:
 
 ```
 
+### `download`
+
+```
+$ ./venv3.11/bin/mbs download -h
+
+usage: mbs download [-h] [-o OWNER] [-r REPO] [-O OUTPUT] [-p PATTERN]
+
+options:
+  -h, --help            show this help message and exit
+  -o OWNER, --owner OWNER
+                        repository owner (default: 'vkottler')
+  -r REPO, --repo REPO  repository name (default: 'toolchains')
+  -O OUTPUT, --output OUTPUT
+                        output directory (default: 'toolchains')
+  -p PATTERN, --pattern PATTERN
+                        a pattern to use to select project specifications
+                        filtered by name
+
+```
+
 ### `gen`
 
 ```

local/configs/package.yaml

@@ -28,6 +28,8 @@ commands:
     description: load configuration data and write results to a file
   - name: dist
     description: create a source distribution
+  - name: download
+    description: download GitHub release assets
   - name: gen
     description: poll the source tree and generate any new build files
   - name: native

local/includes/sub_commands.yaml

@@ -3,7 +3,7 @@
 default_dirs: false
 
 commands:
-{% for command in ["compile_config", "dist", "gen", "native", "uf2conv"] %}
+{% for command in ["compile_config", "dist", "download", "gen", "native", "uf2conv"] %}
   - name: help-{{command}}
     command: "./venv{{python_version}}/bin/{{entry}}"
     force: true

local/variables/package.yaml

@@ -1,5 +1,5 @@
 ---
 major: 2
-minor: 6
+minor: 7
 patch: 0
 entry: mbs

manifest.yaml

@@ -32,6 +32,7 @@ renders:
       - commands-help
       - commands-help-compile_config
       - commands-help-dist
+      - commands-help-download
       - commands-help-gen
       - commands-help-native
       - commands-help-uf2conv

pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta:__legacy__"
 
 [project]
 name = "yambs"
-version = "2.6.0"
+version = "2.7.0"
 description = "Yet another meta build-system."
 readme = "README.md"
 requires-python = ">=3.11"

tests/commands/test_download.py

@@ -0,0 +1,17 @@
+"""
+Test the 'commands.download' module.
+"""
+
+# built-in
+from tempfile import TemporaryDirectory
+
+# module under test
+from yambs import PKG_NAME
+from yambs.entry import main as yambs_main
+
+
+def test_download_basic():
+    """Test the 'download' command."""
+
+    with TemporaryDirectory() as tmpdir:
+        assert yambs_main([PKG_NAME, "-C", str(tmpdir), "download"]) == 0

yambs/__init__.py

@@ -1,7 +1,7 @@
 # =====================================
 # generator=datazen
 # version=3.1.3
-# hash=c440d523035573f23897dbb501a02261
+# hash=63f46d83195b193de72a6bbd1bcc7c47
 # =====================================
 
 """
@@ -10,4 +10,4 @@
 
 DESCRIPTION = "Yet another meta build-system."
 PKG_NAME = "yambs"
-VERSION = "2.6.0"
+VERSION = "2.7.0"

yambs/commands/all.py

@@ -1,7 +1,7 @@
 # =====================================
 # generator=datazen
 # version=3.1.3
-# hash=e10efb5b655cacd368d023aacf84f288
+# hash=a579de7276b0c8f219e976ab8cc6a008
 # =====================================
 
 """
@@ -18,6 +18,7 @@
 # internal
 from yambs.commands.compile_config import add_compile_config_cmd
 from yambs.commands.dist import add_dist_cmd
+from yambs.commands.download import add_download_cmd
 from yambs.commands.gen import add_gen_cmd
 from yambs.commands.native import add_native_cmd
 from yambs.commands.uf2conv import add_uf2conv_cmd
@@ -37,6 +38,11 @@ def commands() -> _List[_Tuple[str, str, _CommandRegister]]:
             "create a source distribution",
             add_dist_cmd,
         ),
+        (
+            "download",
+            "download GitHub release assets",
+            add_download_cmd,
+        ),
         (
             "gen",
             "poll the source tree and generate any new build files",

yambs/commands/download.py

@@ -0,0 +1,62 @@
+"""
+An entry-point for the 'download' command.
+"""
+
+# built-in
+from argparse import ArgumentParser as _ArgumentParser
+from argparse import Namespace as _Namespace
+from pathlib import Path
+
+# third-party
+from vcorelib.args import CommandFunction as _CommandFunction
+
+# internal
+from yambs.dependency.github import GithubDependency, default_filt
+
+
+def download_cmd(args: _Namespace) -> int:
+    """Execute the download command."""
+
+    dep = GithubDependency(args.owner, args.repo)
+
+    # Download and extract things.
+    dep.download_release_assets(
+        default_filt(args.output, pattern=args.pattern)
+    )
+
+    return 0
+
+
+def add_download_cmd(parser: _ArgumentParser) -> _CommandFunction:
+    """Add download-command arguments to its parser."""
+
+    parser.add_argument(
+        "-o",
+        "--owner",
+        default="vkottler",
+        help="repository owner (default: '%(default)s')",
+    )
+    parser.add_argument(
+        "-r",
+        "--repo",
+        default="toolchains",
+        help="repository name (default: '%(default)s')",
+    )
+    parser.add_argument(
+        "-O",
+        "--output",
+        type=Path,
+        default=Path("toolchains"),
+        help="output directory (default: '%(default)s')",
+    )
+    parser.add_argument(
+        "-p",
+        "--pattern",
+        default=".*",
+        help=(
+            "a pattern to use to select project "
+            "specifications filtered by name"
+        ),
+    )
+
+    return download_cmd

yambs/dependency/github.py

@@ -3,14 +3,80 @@
 """
 
 # built-in
-from typing import Dict
+from pathlib import Path
+from re import search
+from typing import Any, Callable, Dict, Optional
 
 # third-party
-from vcorelib.logging import LoggerMixin
+import requests
+from vcorelib.io.archive import extractall
+from vcorelib.io.types import FileExtension
+from vcorelib.logging import LoggerMixin, LoggerType
+from vcorelib.math import nano_str
 
 # internal
 from yambs.github import ReleaseData, latest_release_data
 
+AssetFilter = Callable[[dict[str, Any]], Optional[Path]]
+
+
+def download_file_if_missing(
+    uri: str, dest: Path, timeout: float = 10.0, chunk_size: int = 4096
+) -> None:
+    """Download a file if necessary."""
+
+    if not dest.is_file():
+        req = requests.get(uri, timeout=timeout)
+        with dest.open("wb") as dest_fd:
+            for chunk in req.iter_content(chunk_size=chunk_size):
+                dest_fd.write(chunk)
+
+
+def default_filt(
+    output: Path, pattern: str = ".*", mkdir: bool = True
+) -> AssetFilter:
+    """Create a default release-asset filter method."""
+
+    if mkdir:
+        output.mkdir(parents=True, exist_ok=True)
+
+    def filt(asset: dict[str, Any]) -> Optional[Path]:
+        """Determine if the release asset should be downloaded."""
+
+        result = None
+
+        name = asset["name"]
+        if search(pattern, name) is not None:
+            result = output.joinpath(name)
+
+        return result
+
+    return filt
+
+
+def ensure_extracted(path: Path, logger: LoggerType = None) -> None:
+    """Ensure that all archive files in a directory are extracted."""
+
+    for item in path.iterdir():
+        ext = FileExtension.from_path(item)
+        if ext is not None and ext.is_archive() and item.is_file():
+            dest = item.parent.joinpath(item.name.replace(f".{ext}", ""))
+            if not dest.is_dir():
+                if logger is not None:
+                    logger.info("Extracting '%s' -> '%s'.", item, dest)
+
+                result = extractall(item, dst=item.parent)
+                assert result[0]
+
+                if logger is not None:
+                    logger.info(
+                        "Extracted '%s' in %s.",
+                        dest,
+                        nano_str(result[1], is_time=True),
+                    )
+
+                assert dest.is_dir(), dest
+
 
 class GithubDependency(LoggerMixin):
     """A class for managing GitHub dependencies."""
@@ -37,3 +103,16 @@ def __init__(
             item["name"]: item["browser_download_url"]
             for item in self.data["assets"]
         }
+
+    def download_release_assets(
+        self, filt: AssetFilter, extract: bool = True
+    ) -> None:
+        """Ensure release assets are downloaded."""
+
+        for asset in self.data["assets"]:
+            dest = filt(asset)
+            if dest is not None:
+                download_file_if_missing(asset["browser_download_url"], dest)
+
+                if extract:
+                    ensure_extracted(dest.parent, logger=self.logger)

yambs/dependency/handlers/yambs/github.py

@@ -4,9 +4,9 @@
 
 # built-in
 from pathlib import Path
+from typing import Any, Optional
 
 # third-party
-import requests
 from vcorelib.io.archive import extractall
 from vcorelib.paths import validate_hex_digest
 
@@ -45,25 +45,23 @@ def audit_downloads(
     """Ensure release assets are downloaded."""
 
     to_download = ["sum", TARBALL]
-    for asset in github.data["assets"]:
-        name = asset["name"]
 
+    def filt(asset: dict[str, Any]) -> Optional[Path]:
+        """Determine if the release asset should be downloaded."""
+
+        result = None
+
+        name = asset["name"]
         dest = root.joinpath(name)
 
         for suffix in to_download:
             if name.endswith(suffix):
-                # Download if necessary.
-                if not dest.is_file():
-                    # Download the file.
-                    req = requests.get(
-                        asset["browser_download_url"], timeout=10
-                    )
-                    with dest.open("wb") as dest_fd:
-                        for chunk in req.iter_content(chunk_size=4096):
-                            dest_fd.write(chunk)
-
                 data["assets"][suffix] = str(dest)
-                break
+                result = dest
+
+        return result
+
+    github.download_release_assets(filt)
 
 
 def audit_extract(root: Path, data: DependencyData) -> Path: