diff --git a/requirements.txt b/requirements.txt index 37b4d50..22853a1 100644 --- a/requirements.txt +++ b/requirements.txt @@ -21,5 +21,5 @@ pyyaml_env_tag==0.1 regex==2023.6.3 requests==2.31.0 six==1.16.0 -urllib3==2.0.3 +urllib3==2.0.6 watchdog==3.0.0 diff --git a/softpack_core/app.py b/softpack_core/app.py index 6ed361a..93e56ac 100644 --- a/softpack_core/app.py +++ b/softpack_core/app.py @@ -107,13 +107,25 @@ def url(path: str = "/", scheme: str = "http") -> str: ) return str(url) - def main(self) -> Any: + def main(self, package_update_interval: float) -> Any: """Main command line entrypoint. + Args: + package_update_interval: interval between updates of the spack + package list. Setting 0 disables the automatic updating. + Returns: Any: The return value from running Typer commands. """ - return self.commands() + if package_update_interval > 0: + self.spack.keep_packages_updated(package_update_interval) + + ret = self.commands() + + if package_update_interval > 0: + self.spack.stop_package_timer() + + return ret app = Application() diff --git a/softpack_core/artifacts.py b/softpack_core/artifacts.py index 3d9d9b9..3cf4426 100644 --- a/softpack_core/artifacts.py +++ b/softpack_core/artifacts.py @@ -16,14 +16,14 @@ from box import Box from fastapi import UploadFile -from softpack_core.spack import Spack +from softpack_core.spack import PackageBase from .app import app from .ldapapi import LDAP @strawberry.type -class Package(Spack.PackageBase): +class Package(PackageBase): """A Strawberry model representing a package.""" version: Optional[str] = None diff --git a/softpack_core/config/conf/config.yml b/softpack_core/config/conf/config.yml index e61c037..b41ae77 100644 --- a/softpack_core/config/conf/config.yml +++ b/softpack_core/config/conf/config.yml @@ -17,6 +17,10 @@ artifacts: email: softpack@sanger.ac.uk path: ./softpack-artifacts +spack: + repo: https://github.com/custom-spack/repo + bin: /path/to/spack + # Vault Config # vault: # url: @@ -31,4 +35,4 @@ artifacts: # filter: # group: # attr: -# pattern: +# pattern: \ No newline at end of file diff --git a/softpack_core/config/models.py b/softpack_core/config/models.py index d054fef..7a67193 100644 --- a/softpack_core/config/models.py +++ b/softpack_core/config/models.py @@ -63,3 +63,10 @@ class GroupConfig(BaseModel): base: str filter: str group: GroupConfig + + +class SpackConfig(BaseModel): + """Spack config model.""" + + repo: str + bin: str diff --git a/softpack_core/config/settings.py b/softpack_core/config/settings.py index 539cda6..5421360 100644 --- a/softpack_core/config/settings.py +++ b/softpack_core/config/settings.py @@ -14,7 +14,13 @@ from pydantic import BaseSettings from pydantic.env_settings import SettingsSourceCallable -from .models import ArtifactsConfig, LDAPConfig, ServerConfig, VaultConfig +from .models import ( + ArtifactsConfig, + LDAPConfig, + ServerConfig, + SpackConfig, + VaultConfig, +) class Settings(BaseSettings): @@ -25,6 +31,7 @@ class Settings(BaseSettings): vault: Optional[VaultConfig] ldap: Optional[LDAPConfig] artifacts: ArtifactsConfig + spack: SpackConfig class Config: """Configuration loader.""" diff --git a/softpack_core/main.py b/softpack_core/main.py index cc89e04..55803c6 100644 --- a/softpack_core/main.py +++ b/softpack_core/main.py @@ -14,9 +14,9 @@ ServiceAPI.register() -def main() -> Any: +def main(package_update_interval: float = 600) -> Any: """Main entrypoint.""" - return app.main() + return app.main(package_update_interval) if __name__ == "__main__": diff --git a/softpack_core/schemas/package_collection.py b/softpack_core/schemas/package_collection.py index 24093de..a83febc 100644 --- a/softpack_core/schemas/package_collection.py +++ b/softpack_core/schemas/package_collection.py @@ -11,11 +11,11 @@ import strawberry from softpack_core.app import app -from softpack_core.spack import Spack +from softpack_core.spack import Package @strawberry.type -class PackageMultiVersion(Spack.Package): +class PackageMultiVersion(Package): """A Strawberry model representing a package in a collection.""" @@ -28,35 +28,17 @@ class PackageCollection: packages: list[PackageMultiVersion] @classmethod - def iter(cls) -> Iterable["PackageCollection"]: + def iter(cls) -> Iterable["PackageMultiVersion"]: """Get an iterator over PackageCollection objects. Returns: Iterable[PackageCollection]: An iterator of PackageCollection objects. """ - return map(cls.from_collection, app.spack.collections) + return map(cls.from_package, app.spack.packages()) @classmethod - def from_collection( - cls, collection: Spack.Collection - ) -> "PackageCollection": - """Create a PackageCollection object from Spack.Collection. - - Args: - collection: A Spack.Collection - - Returns: - PackageCollection: A Spack package collection. - """ - return PackageCollection( - id=collection.id, - name=collection.name, - packages=list(map(cls.from_package, collection.packages)), - ) # type: ignore [call-arg] - - @classmethod - def from_package(cls, package: Spack.Package) -> PackageMultiVersion: + def from_package(cls, package: Package) -> PackageMultiVersion: """Create a PackageMultiVersion object. Args: diff --git a/softpack_core/spack.py b/softpack_core/spack.py index 6bc13fc..601e473 100644 --- a/softpack_core/spack.py +++ b/softpack_core/spack.py @@ -4,118 +4,130 @@ LICENSE file in the root directory of this source tree. """ - -import importlib -import itertools -import re -import shutil -import sys -import uuid +import json +import subprocess +import tempfile +import threading from dataclasses import dataclass -from pathlib import Path -from types import ModuleType -from uuid import UUID -class Spack: - """Spack interface class.""" +@dataclass +class PackageBase: + """Wrapper for a spack package.""" + + name: str + + +@dataclass +class Package(PackageBase): + """Wrapper for a spack package.""" + + versions: list[str] - @dataclass - class Modules: - """Spack modules.""" - config: ModuleType - repo: ModuleType +class Spack: + """Spack interface class.""" - def __init__(self) -> None: + def __init__( + self, spack_exe: str = "spack", custom_repo: str = "" + ) -> None: """Constructor.""" - self.modules = self.load_modules() - self.repos = self.load_repo_list() - self.packages = self.load_package_list() - self.collections = self.load_collections() - - def load_modules(self) -> Modules: - """Loads all required packages.""" - spack = shutil.which("spack") - if spack: - spack_root = Path(spack).resolve().parent.parent - else: - spack_root = Path.cwd() / "spack" + self.stored_packages: list[Package] = [] + self.checkout_path = "" + self.spack_exe = spack_exe + self.custom_repo = custom_repo - lib_path = spack_root / "lib/spack" + def load_package_list(self, spack_exe: str, custom_repo: str) -> None: + """Load a list of all packages.""" + checkout_path = "" - for path in [lib_path, lib_path / "external"]: - if path not in sys.path: - sys.path.append(str(path)) + if custom_repo != "": + tmp_dir = tempfile.TemporaryDirectory() + checkout_path = tmp_dir.name + self.checkout_custom_repo(custom_repo, checkout_path) - return self.Modules( - config=importlib.import_module('spack.config'), - repo=importlib.import_module('spack.repo'), - ) + self.store_packages_from_spack(spack_exe, checkout_path) + + def checkout_custom_repo( + self, custom_repo: str, checkout_path: str + ) -> None: + """Clones the custom spack package repo to a local path. - def load_repo_list(self) -> list: - """Load a list of all repos.""" - return list( - map(self.modules.repo.Repo, self.modules.config.get("repos")) + Args: + custom_repo (str): URL to custom spack package repo. + checkout_path (str): Path to clone custom spack repo to. + """ + result = subprocess.run( + ["git", "clone", "--depth", "1", custom_repo, checkout_path], + capture_output=True, ) - @dataclass - class PackageBase: - """Wrapper for a spack package.""" + result.check_returncode() - name: str + def store_packages_from_spack( + self, spack_exe: str, checkout_path: str + ) -> None: + """Reads the full list of available packages in spack and stores them. - @dataclass - class Package(PackageBase): - """Wrapper for a spack package.""" + Args: + spack_exe (str): Path to the spack executable. + checkout_path (str): Path to the cloned custom spack repo. + """ + if checkout_path == "": + result = subprocess.run( + [spack_exe, "list", "--format", "version_json"], + capture_output=True, + ) + else: + result = subprocess.run( + [ + spack_exe, + "--config", + "repos:[" + checkout_path + "]", + "list", + "--format", + "version_json", + ], + capture_output=True, + ) - versions: list[str] + pkgs = json.loads(result.stdout) - def load_package_list(self) -> list[Package]: - """Load a list of all packages.""" - return list( + self.stored_packages = list( map( - lambda package: self.Package( - name=package.name, + lambda package: Package( + name=package.get("name"), versions=[ - str(ver) for ver in list(package.versions.keys()) + str(ver) for ver in list(package.get("versions")) ], ), - itertools.chain.from_iterable( - list( - map( - lambda repo: repo.all_package_classes(), self.repos - ) - ) - ), + pkgs, ) ) - def filter_packages(self, prefix: str) -> list[Package]: - """Filter packages based on a prefix.""" - regex = re.compile(fr"^{prefix}.*$") - return list(filter(lambda p: regex.match(p.name), self.packages)) - - @dataclass - class Collection: - """Spack package collection.""" + def packages(self) -> list[Package]: + """Returns the list of stored packages. - id: UUID - name: str - packages: list["Spack.Package"] - - def load_collections(self) -> list[Collection]: - """Load package collections from Spack repo. + First generates the list if it is None. Returns: - list[Collection]: A list of package collections. + list[Package]: The stored list of spack packages. """ - collections = {"Python": "py-", "R": "r-"} - return [ - self.Collection( - id=uuid.uuid4(), - name=name, - packages=self.filter_packages(prefix), - ) - for name, prefix in collections.items() - ] + if len(self.stored_packages) == 0: + self.load_package_list(self.spack_exe, self.custom_repo) + + return self.stored_packages + + def keep_packages_updated(self, interval: float) -> None: + """Runs package list retireval on a timer.""" + self.load_package_list(self.spack_exe, self.custom_repo) + + self.timer = threading.Timer( + interval, self.keep_packages_updated, [interval] + ) + self.timer.start() + + def stop_package_timer(self) -> None: + """Stops any running timer threads.""" + if self.timer is not None: + self.timer.cancel() diff --git a/tests/integration/test_spack.py b/tests/integration/test_spack.py new file mode 100644 index 0000000..5a7f788 --- /dev/null +++ b/tests/integration/test_spack.py @@ -0,0 +1,82 @@ +"""Copyright (c) 2023 Genome Research Ltd. + +This source code is licensed under the MIT license found in the +LICENSE file in the root directory of this source tree. +""" + +import time + +import pytest + +from softpack_core.app import app +from softpack_core.schemas.package_collection import ( + PackageCollection, + PackageMultiVersion, +) +from softpack_core.spack import Package, Spack + + +def test_spack_packages(): + spack = Spack() + spack.packages() + + pkgs = spack.stored_packages + + assert len(pkgs) > 1 + + assert isinstance(pkgs[0], Package) + + assert pkgs[0].name != "" + + assert len(pkgs[0].versions) > 0 + + assert pkgs[0].versions[0] != "" + + packages = list(PackageCollection.iter()) + + assert len(packages) == len(pkgs) + + assert isinstance(packages[0], PackageMultiVersion) + + assert packages[0].name != "" + + assert len(packages[0].versions) != 0 + + if app.settings.spack.repo == "https://github.com/custom-spack/repo": + pytest.skip("skipped due to missing custom repo") + + spack = Spack("spack", app.settings.spack.repo) + spack.packages() + + assert len(spack.stored_packages) > len(pkgs) + + +def test_spack_package_updater(): + spack = Spack() + + assert len(spack.stored_packages) == 0 + + spack.keep_packages_updated(1) + + pkgs = spack.stored_packages + + assert len(pkgs) > 0 + + if app.settings.spack.repo == "https://github.com/custom-spack/repo": + pytest.skip("skipped due to missing custom repo") + + spack.custom_repo = app.settings.spack.repo + + timeout = time.time() + 60 * 2 + + while True: + new_pkgs = spack.stored_packages + + if len(new_pkgs) > len(pkgs) or time.time() > timeout: + break + + time.sleep(0.1) + + assert len(new_pkgs) > len(pkgs) + + spack.stop_package_timer() diff --git a/tests/unit/test_main.py b/tests/unit/test_main.py index 6a5f11e..3f1ba62 100644 --- a/tests/unit/test_main.py +++ b/tests/unit/test_main.py @@ -14,7 +14,7 @@ def test_main(capsys) -> None: with pytest.raises(SystemExit): - main() + main(0) captured = capsys.readouterr() command = Path(sys.argv[0]) assert f"{command.name} [OPTIONS] COMMAND [ARGS]" in captured.err