Merge pull request #30 from wtsi-hgi/update-spack-repo

Automatically update spack package repo

requirements.txt

@@ -21,5 +21,5 @@ pyyaml_env_tag==0.1
 regex==2023.6.3
 requests==2.31.0
 six==1.16.0
-urllib3==2.0.3
+urllib3==2.0.6
 watchdog==3.0.0

softpack_core/app.py

@@ -107,13 +107,25 @@ def url(path: str = "/", scheme: str = "http") -> str:
         )
         return str(url)
 
-    def main(self) -> Any:
+    def main(self, package_update_interval: float) -> Any:
         """Main command line entrypoint.
 
+        Args:
+            package_update_interval: interval between updates of the spack
+        package list. Setting 0 disables the automatic updating.
+
         Returns:
             Any: The return value from running Typer commands.
         """
-        return self.commands()
+        if package_update_interval > 0:
+            self.spack.keep_packages_updated(package_update_interval)
+
+        ret = self.commands()
+
+        if package_update_interval > 0:
+            self.spack.stop_package_timer()
+
+        return ret
 
 
 app = Application()

softpack_core/artifacts.py

@@ -16,14 +16,14 @@
 from box import Box
 from fastapi import UploadFile
 
-from softpack_core.spack import Spack
+from softpack_core.spack import PackageBase
 
 from .app import app
 from .ldapapi import LDAP
 
 
 @strawberry.type
-class Package(Spack.PackageBase):
+class Package(PackageBase):
     """A Strawberry model representing a package."""
 
     version: Optional[str] = None

softpack_core/config/conf/config.yml

@@ -17,6 +17,10 @@ artifacts:
     email: softpack@sanger.ac.uk
   path: ./softpack-artifacts
 
+spack:
+  repo: https://github.com/custom-spack/repo
+  bin: /path/to/spack
+
 # Vault Config
 #  vault:
 #      url:
@@ -31,4 +35,4 @@ artifacts:
 #    filter:
 #    group:
 #      attr:
-#      pattern:
+#      pattern:

softpack_core/config/models.py

@@ -63,3 +63,10 @@ class GroupConfig(BaseModel):
     base: str
     filter: str
     group: GroupConfig
+
+
+class SpackConfig(BaseModel):
+    """Spack config model."""
+
+    repo: str
+    bin: str

softpack_core/config/settings.py

@@ -14,7 +14,13 @@
 from pydantic import BaseSettings
 from pydantic.env_settings import SettingsSourceCallable
 
-from .models import ArtifactsConfig, LDAPConfig, ServerConfig, VaultConfig
+from .models import (
+    ArtifactsConfig,
+    LDAPConfig,
+    ServerConfig,
+    SpackConfig,
+    VaultConfig,
+)
 
 
 class Settings(BaseSettings):
@@ -25,6 +31,7 @@ class Settings(BaseSettings):
     vault: Optional[VaultConfig]
     ldap: Optional[LDAPConfig]
     artifacts: ArtifactsConfig
+    spack: SpackConfig
 
     class Config:
         """Configuration loader."""

softpack_core/main.py

@@ -14,9 +14,9 @@
 ServiceAPI.register()
 
 
-def main() -> Any:
+def main(package_update_interval: float = 600) -> Any:
     """Main entrypoint."""
-    return app.main()
+    return app.main(package_update_interval)
 
 
 if __name__ == "__main__":

softpack_core/schemas/package_collection.py

@@ -11,11 +11,11 @@
 import strawberry
 
 from softpack_core.app import app
-from softpack_core.spack import Spack
+from softpack_core.spack import Package
 
 
 @strawberry.type
-class PackageMultiVersion(Spack.Package):
+class PackageMultiVersion(Package):
     """A Strawberry model representing a package in a collection."""
 
 
@@ -28,35 +28,17 @@ class PackageCollection:
     packages: list[PackageMultiVersion]
 
     @classmethod
-    def iter(cls) -> Iterable["PackageCollection"]:
+    def iter(cls) -> Iterable["PackageMultiVersion"]:
         """Get an iterator over PackageCollection objects.
 
         Returns:
             Iterable[PackageCollection]: An iterator of PackageCollection
             objects.
         """
-        return map(cls.from_collection, app.spack.collections)
+        return map(cls.from_package, app.spack.packages())
 
     @classmethod
-    def from_collection(
-        cls, collection: Spack.Collection
-    ) -> "PackageCollection":
-        """Create a PackageCollection object from Spack.Collection.
-
-        Args:
-            collection: A Spack.Collection
-
-        Returns:
-            PackageCollection: A Spack package collection.
-        """
-        return PackageCollection(
-            id=collection.id,
-            name=collection.name,
-            packages=list(map(cls.from_package, collection.packages)),
-        )  # type: ignore [call-arg]
-
-    @classmethod
-    def from_package(cls, package: Spack.Package) -> PackageMultiVersion:
+    def from_package(cls, package: Package) -> PackageMultiVersion:
         """Create a PackageMultiVersion object.
 
         Args:

softpack_core/spack.py

@@ -4,118 +4,130 @@
 LICENSE file in the root directory of this source tree.
 """
 
-
-import importlib
-import itertools
-import re
-import shutil
-import sys
-import uuid
+import json
+import subprocess
+import tempfile
+import threading
 from dataclasses import dataclass
-from pathlib import Path
-from types import ModuleType
-from uuid import UUID
 
 
-class Spack:
-    """Spack interface class."""
+@dataclass
+class PackageBase:
+    """Wrapper for a spack package."""
+
+    name: str
+
+
+@dataclass
+class Package(PackageBase):
+    """Wrapper for a spack package."""
+
+    versions: list[str]
 
-    @dataclass
-    class Modules:
-        """Spack modules."""
 
-        config: ModuleType
-        repo: ModuleType
+class Spack:
+    """Spack interface class."""
 
-    def __init__(self) -> None:
+    def __init__(
+        self, spack_exe: str = "spack", custom_repo: str = ""
+    ) -> None:
         """Constructor."""
-        self.modules = self.load_modules()
-        self.repos = self.load_repo_list()
-        self.packages = self.load_package_list()
-        self.collections = self.load_collections()
-
-    def load_modules(self) -> Modules:
-        """Loads all required packages."""
-        spack = shutil.which("spack")
-        if spack:
-            spack_root = Path(spack).resolve().parent.parent
-        else:
-            spack_root = Path.cwd() / "spack"
+        self.stored_packages: list[Package] = []
+        self.checkout_path = ""
+        self.spack_exe = spack_exe
+        self.custom_repo = custom_repo
 
-        lib_path = spack_root / "lib/spack"
+    def load_package_list(self, spack_exe: str, custom_repo: str) -> None:
+        """Load a list of all packages."""
+        checkout_path = ""
 
-        for path in [lib_path, lib_path / "external"]:
-            if path not in sys.path:
-                sys.path.append(str(path))
+        if custom_repo != "":
+            tmp_dir = tempfile.TemporaryDirectory()
+            checkout_path = tmp_dir.name
+            self.checkout_custom_repo(custom_repo, checkout_path)
 
-        return self.Modules(
-            config=importlib.import_module('spack.config'),
-            repo=importlib.import_module('spack.repo'),
-        )
+        self.store_packages_from_spack(spack_exe, checkout_path)
+
+    def checkout_custom_repo(
+        self, custom_repo: str, checkout_path: str
+    ) -> None:
+        """Clones the custom spack package repo to a local path.
 
-    def load_repo_list(self) -> list:
-        """Load a list of all repos."""
-        return list(
-            map(self.modules.repo.Repo, self.modules.config.get("repos"))
+        Args:
+            custom_repo (str): URL to custom spack package repo.
+            checkout_path (str): Path to clone custom spack repo to.
+        """
+        result = subprocess.run(
+            ["git", "clone", "--depth", "1", custom_repo, checkout_path],
+            capture_output=True,
         )
 
-    @dataclass
-    class PackageBase:
-        """Wrapper for a spack package."""
+        result.check_returncode()
 
-        name: str
+    def store_packages_from_spack(
+        self, spack_exe: str, checkout_path: str
+    ) -> None:
+        """Reads the full list of available packages in spack and stores them.
 
-    @dataclass
-    class Package(PackageBase):
-        """Wrapper for a spack package."""
+        Args:
+            spack_exe (str): Path to the spack executable.
+            checkout_path (str): Path to the cloned custom spack repo.
+        """
+        if checkout_path == "":
+            result = subprocess.run(
+                [spack_exe, "list", "--format", "version_json"],
+                capture_output=True,
+            )
+        else:
+            result = subprocess.run(
+                [
+                    spack_exe,
+                    "--config",
+                    "repos:[" + checkout_path + "]",
+                    "list",
+                    "--format",
+                    "version_json",
+                ],
+                capture_output=True,
+            )
 
-        versions: list[str]
+        pkgs = json.loads(result.stdout)
 
-    def load_package_list(self) -> list[Package]:
-        """Load a list of all packages."""
-        return list(
+        self.stored_packages = list(
             map(
-                lambda package: self.Package(
-                    name=package.name,
+                lambda package: Package(
+                    name=package.get("name"),
                     versions=[
-                        str(ver) for ver in list(package.versions.keys())
+                        str(ver) for ver in list(package.get("versions"))
                     ],
                 ),
-                itertools.chain.from_iterable(
-                    list(
-                        map(
-                            lambda repo: repo.all_package_classes(), self.repos
-                        )
-                    )
-                ),
+                pkgs,
             )
         )
 
-    def filter_packages(self, prefix: str) -> list[Package]:
-        """Filter packages based on a prefix."""
-        regex = re.compile(fr"^{prefix}.*$")
-        return list(filter(lambda p: regex.match(p.name), self.packages))
-
-    @dataclass
-    class Collection:
-        """Spack package collection."""
+    def packages(self) -> list[Package]:
+        """Returns the list of stored packages.
 
-        id: UUID
-        name: str
-        packages: list["Spack.Package"]
-
-    def load_collections(self) -> list[Collection]:
-        """Load package collections from Spack repo.
+        First generates the list if it is None.
 
         Returns:
-            list[Collection]: A list of package collections.
+            list[Package]: The stored list of spack packages.
         """
-        collections = {"Python": "py-", "R": "r-"}
-        return [
-            self.Collection(
-                id=uuid.uuid4(),
-                name=name,
-                packages=self.filter_packages(prefix),
-            )
-            for name, prefix in collections.items()
-        ]
+        if len(self.stored_packages) == 0:
+            self.load_package_list(self.spack_exe, self.custom_repo)
+
+        return self.stored_packages
+
+    def keep_packages_updated(self, interval: float) -> None:
+        """Runs package list retireval on a timer."""
+        self.load_package_list(self.spack_exe, self.custom_repo)
+
+        self.timer = threading.Timer(
+            interval, self.keep_packages_updated, [interval]
+        )
+        self.timer.start()
+
+    def stop_package_timer(self) -> None:
+        """Stops any running timer threads."""
+        if self.timer is not None:
+            self.timer.cancel()