From 0ca722e2cc6372cc1557edaea29de5f4b5dcebe6 Mon Sep 17 00:00:00 2001
From: Altaf Ali <7231912+altaf-ali@users.noreply.github.com>
Date: Tue, 4 Jul 2023 12:04:34 +0100
Subject: [PATCH 001/129] =?UTF-8?q?=F0=9F=90=9B=20fix(artifacts.py):=20rem?=
 =?UTF-8?q?ove=20unused=20import=20and=20unused=20code=20related=20to=20Cr?=
 =?UTF-8?q?edentials=20model=20=F0=9F=90=9B=20fix(config.yml):=20add=20use?=
 =?UTF-8?q?rname=20field=20to=20artifacts.repo=20to=20fix=20missing=20user?=
 =?UTF-8?q?name=20error=20=F0=9F=90=9B=20fix(models.py):=20remove=20Creden?=
 =?UTF-8?q?tials=20model=20as=20it=20is=20no=20longer=20used=20?=
 =?UTF-8?q?=F0=9F=90=9B=20fix(settings.py):=20remove=20unused=20import=20a?=
 =?UTF-8?q?nd=20unused=20code=20related=20to=20merging=20secrets?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

---
 softpack_core/artifacts.py           | 10 +++-------
 softpack_core/config/conf/config.yml |  3 ++-
 softpack_core/config/models.py       | 12 +++---------
 softpack_core/config/settings.py     | 17 +++++++----------
 4 files changed, 15 insertions(+), 27 deletions(-)

diff --git a/softpack_core/artifacts.py b/softpack_core/artifacts.py
index 90c19dc..10f4405 100644
--- a/softpack_core/artifacts.py
+++ b/softpack_core/artifacts.py
@@ -7,13 +7,12 @@
 import itertools
 from dataclasses import dataclass
 from pathlib import Path
-from typing import Iterable, Iterator, Optional, cast
+from typing import Iterable, Iterator, Optional
 
 import pygit2
 from box import Box
 
 from .app import app
-from .config.models import Credentials
 from .ldapapi import LDAP
 
 
@@ -97,12 +96,9 @@ def __init__(self) -> None:
         path = self.settings.artifacts.path.expanduser() / ".git"
         credentials = None
         try:
-            credentials = cast(
-                Credentials, self.settings.artifacts.repo.reader
-            )
             credentials = pygit2.UserPass(
-                credentials.username,
-                credentials.password,
+                self.settings.artifacts.repo.username,
+                self.settings.artifacts.repo.reader,
             )
         except Exception as e:
             print(e)
diff --git a/softpack_core/config/conf/config.yml b/softpack_core/config/conf/config.yml
index dce7203..4e7bc7d 100644
--- a/softpack_core/config/conf/config.yml
+++ b/softpack_core/config/conf/config.yml
@@ -11,9 +11,10 @@ server:
 artifacts:
   repo:
     url: https://github.com/softpack-io/softpack-artifacts.git
+    username: softpack-core
   path: ./softpack-artifacts
 
-# LVault Config
+# Vault Config
 #  vault:
 #      url:
 #      path:
diff --git a/softpack_core/config/models.py b/softpack_core/config/models.py
index b0cd305..e1f0f33 100644
--- a/softpack_core/config/models.py
+++ b/softpack_core/config/models.py
@@ -31,13 +31,6 @@ class VaultConfig(BaseModel):
     token: str
 
 
-class Credentials(BaseModel):
-    """Credentials model."""
-
-    username: str
-    password: str
-
-
 class ArtifactsConfig(BaseModel):
     """Artifacts config model."""
 
@@ -45,8 +38,9 @@ class Repo(BaseModel):
         """Repo model."""
 
         url: AnyUrl
-        reader: Optional[Credentials]
-        writer: Optional[Credentials]
+        username: Optional[str]
+        reader: Optional[str]
+        writer: Optional[str]
 
     path: Path
     repo: Repo
diff --git a/softpack_core/config/settings.py b/softpack_core/config/settings.py
index c109265..539cda6 100644
--- a/softpack_core/config/settings.py
+++ b/softpack_core/config/settings.py
@@ -4,7 +4,6 @@
 LICENSE file in the root directory of this source tree.
 """
 
-import itertools
 import sys
 from pathlib import Path
 from typing import Any, Optional, Tuple
@@ -111,15 +110,13 @@ def get_secret(path: Path, key: str) -> dict[str, Any]:
                 secrets = client.secrets.kv.v1.list_secrets(
                     path=str(vault.path), mount_point="/"
                 )
-                merged_secrets = dict(
-                    itertools.chain.from_iterable(
-                        [
-                            get_secret(vault.path, key).items()
-                            for key in secrets["data"]["keys"]
-                        ]
-                    )
-                )
-                return {vault.path.name: merged_secrets}
+
+                return {
+                    vault.path.name: {
+                        key: get_secret(vault.path, key)
+                        for key in secrets["data"]["keys"]
+                    }
+                }
 
             except Exception as e:
                 print(e, file=sys.stderr)

From 423ee7be804612ea666d72087a83a066665dcadd Mon Sep 17 00:00:00 2001
From: Samuel Ogg <56794292+SamO135@users.noreply.github.com>
Date: Wed, 31 May 2023 13:07:54 +0100
Subject: [PATCH 002/129] skeleton code for environment mutations

---
 softpack_core/schemas/environment.py | 53 ++++++++++++++++++++++++++--
 1 file changed, 51 insertions(+), 2 deletions(-)

diff --git a/softpack_core/schemas/environment.py b/softpack_core/schemas/environment.py
index ba6d9e0..070cb84 100644
--- a/softpack_core/schemas/environment.py
+++ b/softpack_core/schemas/environment.py
@@ -22,6 +22,16 @@ class Package(Spack.PackageBase):
 
     version: Optional[str] = None
 
+@strawberry.input
+class PackageInput(Package):
+    """A Strawberry input model representing a pacakge."""
+
+    def to_package(self):
+        """Create a Package object from a PackageInput object.
+        
+        Return: a Package object
+        """
+        return Package(**self.__dict__)
 
 @strawberry.type
 class Environment:
@@ -81,12 +91,49 @@ def create(cls, name: str) -> "Environment":
         Returns:
             Environment: A newly created Environment.
         """
+        for env in Environment.iter():
+            if name == env.name:
+                return EnvironmentAlreadyExistsError(**env.__dict__)
         return Environment(
             id=uuid.uuid4().hex,
             name=name,
-            packges=[Package(id="unknown", name="unknown-package")],
+            path="users/username",
+            description="description",
+            packages=[Package(id="unknown", name="unknown-package")],
         )  # type: ignore [call-arg]
 
+    @classmethod
+    def update(cls, name: str, path: Optional[str]=None, description: Optional[str]=None, packages: Optional[list[PackageInput]]=None):
+        for env in Environment.iter():
+            if env.name == name:
+                if path != None:
+                    env.path = path
+                if description != None:
+                    env.description = description
+                if packages != None:
+                    env.packages = map(lambda pkg: pkg.to_package(), packages)
+                return env
+            return EnvironmentNotFoundError(name=name)
+    
+    @classmethod
+    def delete(cls, name: str):
+        for env in Environment.iter():
+            if env.name == name:
+                return f"Deleted {name}"
+        return "An environment with that name was not found"
+    
+#Error types
+@strawberry.type
+class EnvironmentNotFoundError:
+    """Environment not found"""
+    name: str
+
+@strawberry.type
+class EnvironmentAlreadyExistsError(Environment):
+    """Environment name already exists"""
+
+UpdateEnvironmentResponse = strawberry.union("UpdateEnvironmentResponse", [Environment, EnvironmentNotFoundError])
+CreateEnvironmentResponse = strawberry.union("CreateEnvironmentResponse", [Environment, EnvironmentAlreadyExistsError])
 
 class EnvironmentSchema(BaseSchema):
     """Environment schema."""
@@ -101,4 +148,6 @@ class Query:
     class Mutation:
         """GraphQL mutation schema."""
 
-        createEnvironment: Environment = Environment.create  # type: ignore
+        createEnvironment: CreateEnvironmentResponse = Environment.create  # type: ignore
+        updateEnvironment: UpdateEnvironmentResponse = Environment.update  # type: ignore
+        deleteEnvironment: str = Environment.delete # type: ignore

From 410c203fed3b54f50770bca7bc3425a0df847adf Mon Sep 17 00:00:00 2001
From: Samuel Ogg <56794292+SamO135@users.noreply.github.com>
Date: Mon, 5 Jun 2023 16:58:31 +0100
Subject: [PATCH 003/129] send build request to softpack-builder Error types
 and docstrings also added

---
 softpack_core/schemas/environment.py | 106 +++++++++++++++++++++++----
 1 file changed, 90 insertions(+), 16 deletions(-)

diff --git a/softpack_core/schemas/environment.py b/softpack_core/schemas/environment.py
index 070cb84..4531fbc 100644
--- a/softpack_core/schemas/environment.py
+++ b/softpack_core/schemas/environment.py
@@ -5,10 +5,11 @@
 """
 
 import os
-import uuid
 from dataclasses import dataclass
+from datetime import datetime
 from typing import Iterable, Optional
 
+import httpx
 import strawberry
 
 from softpack_core.artifacts import Artifacts
@@ -22,17 +23,19 @@ class Package(Spack.PackageBase):
 
     version: Optional[str] = None
 
+
 @strawberry.input
 class PackageInput(Package):
     """A Strawberry input model representing a pacakge."""
 
     def to_package(self):
         """Create a Package object from a PackageInput object.
-        
+
         Return: a Package object
         """
         return Package(**self.__dict__)
 
+
 @strawberry.type
 class Environment:
     """A Strawberry model representing a single environment."""
@@ -42,6 +45,8 @@ class Environment:
     path: str
     description: str
     packages: list[Package]
+    created: Optional[datetime]
+    state: Optional[str]
     artifacts = Artifacts()
 
     @classmethod
@@ -79,14 +84,25 @@ def from_artifact(cls, obj: Artifacts.Object) -> "Environment":
                 lambda package: Package(id=package, name=package),
                 spec.packages,
             ),  # type: ignore [call-arg]
+            created=None,
+            state=None,
         )
 
     @classmethod
-    def create(cls, name: str) -> "Environment":
+    def create(
+        cls,
+        name: str,
+        path: str,
+        description: Optional[str] = None,
+        packages: Optional[list[PackageInput]] = None,
+    ) -> "Environment":
         """Create an Environment object.
 
         Args:
             name: Name for an environment.
+            path: Path for an environment.
+            description: Description for an environment.
+            packages: List of packages in the environment.
 
         Returns:
             Environment: A newly created Environment.
@@ -94,46 +110,104 @@ def create(cls, name: str) -> "Environment":
         for env in Environment.iter():
             if name == env.name:
                 return EnvironmentAlreadyExistsError(**env.__dict__)
+        response = httpx.post(
+            "http://0.0.0.0:7080/environments/build",
+            json={
+                "name": name,
+                "model": {
+                    "description": description,
+                    "packages": [f"{pkg.name}" for pkg in packages],
+                },
+            },
+        ).json()
+        print(f"Create: {response}")
         return Environment(
-            id=uuid.uuid4().hex,
-            name=name,
-            path="users/username",
-            description="description",
-            packages=[Package(id="unknown", name="unknown-package")],
+            id=response['id'],
+            name=response['name'],
+            path=path,
+            description=description,
+            packages=map(lambda pkg: pkg.to_package(), packages),
+            created=datetime.strptime(
+                response['created'], '%Y-%m-%dT%H:%M:%S.%f%z'
+            ),
+            state=response['state']['type'],
         )  # type: ignore [call-arg]
 
     @classmethod
-    def update(cls, name: str, path: Optional[str]=None, description: Optional[str]=None, packages: Optional[list[PackageInput]]=None):
+    def update(
+        cls,
+        name: str,
+        path: Optional[str] = None,
+        description: Optional[str] = None,
+        packages: Optional[list[PackageInput]] = None,
+    ):
+        """Update an Environment object.
+
+        Args:
+            name: Name for an environment.
+            path: Path for an environment.
+            description: Description for an environment.
+            packages: List of packages in the environment.
+
+        Returns:
+            Environment: An updated Environment.
+        """
         for env in Environment.iter():
             if env.name == name:
+                response = httpx.post(
+                    "http://0.0.0.0:7080/environments/build",
+                    json={
+                        "name": name,
+                        "model": {
+                            "description": description,
+                            "packages": [pkg.name for pkg in packages],
+                        },
+                    },
+                ).json()
+                print(f"Update: {response}")
                 if path != None:
                     env.path = path
                 if description != None:
                     env.description = description
                 if packages != None:
                     env.packages = map(lambda pkg: pkg.to_package(), packages)
+                env.state = response['state']['type']
                 return env
-            return EnvironmentNotFoundError(name=name)
-    
+        return EnvironmentNotFoundError(name=name)
+
     @classmethod
     def delete(cls, name: str):
+        """Delete an Environment object.
+
+        Returns:
+            A string confirming the deletion of the Environment
+        """
         for env in Environment.iter():
             if env.name == name:
                 return f"Deleted {name}"
         return "An environment with that name was not found"
-    
-#Error types
+
+
+# Error types
 @strawberry.type
 class EnvironmentNotFoundError:
     """Environment not found"""
+
     name: str
 
+
 @strawberry.type
 class EnvironmentAlreadyExistsError(Environment):
     """Environment name already exists"""
 
-UpdateEnvironmentResponse = strawberry.union("UpdateEnvironmentResponse", [Environment, EnvironmentNotFoundError])
-CreateEnvironmentResponse = strawberry.union("CreateEnvironmentResponse", [Environment, EnvironmentAlreadyExistsError])
+
+UpdateEnvironmentResponse = strawberry.union(
+    "UpdateEnvironmentResponse", [Environment, EnvironmentNotFoundError]
+)
+CreateEnvironmentResponse = strawberry.union(
+    "CreateEnvironmentResponse", [Environment, EnvironmentAlreadyExistsError]
+)
+
 
 class EnvironmentSchema(BaseSchema):
     """Environment schema."""
@@ -150,4 +224,4 @@ class Mutation:
 
         createEnvironment: CreateEnvironmentResponse = Environment.create  # type: ignore
         updateEnvironment: UpdateEnvironmentResponse = Environment.update  # type: ignore
-        deleteEnvironment: str = Environment.delete # type: ignore
+        deleteEnvironment: str = Environment.delete  # type: ignore

From 04b559d2a1521808d60885859cb8a6ab7bd43ac8 Mon Sep 17 00:00:00 2001
From: Samuel Ogg <56794292+SamO135@users.noreply.github.com>
Date: Thu, 8 Jun 2023 13:55:01 +0100
Subject: [PATCH 004/129] added mutation to upload a file also modified the
 create mutation

---
 softpack_core/artifacts.py           |  3 ++
 softpack_core/schemas/environment.py | 62 +++++++++++++++++-----------
 2 files changed, 40 insertions(+), 25 deletions(-)

diff --git a/softpack_core/artifacts.py b/softpack_core/artifacts.py
index 10f4405..51aa8a3 100644
--- a/softpack_core/artifacts.py
+++ b/softpack_core/artifacts.py
@@ -240,3 +240,6 @@ def iter(self, user: Optional[str] = None) -> Iterable:
 
         except KeyError:
             return iter(())
+
+    def get(self, path: Path, name: str) -> pygit2.Tree:
+        return self.tree(self.environments_folder(path.as_posix(), name))
diff --git a/softpack_core/schemas/environment.py b/softpack_core/schemas/environment.py
index 4531fbc..065df97 100644
--- a/softpack_core/schemas/environment.py
+++ b/softpack_core/schemas/environment.py
@@ -7,10 +7,12 @@
 import os
 from dataclasses import dataclass
 from datetime import datetime
+from pathlib import Path
 from typing import Iterable, Optional
 
 import httpx
 import strawberry
+from strawberry.file_uploads import Upload
 
 from softpack_core.artifacts import Artifacts
 from softpack_core.schemas.base import BaseSchema
@@ -107,31 +109,33 @@ def create(
         Returns:
             Environment: A newly created Environment.
         """
-        for env in Environment.iter():
-            if name == env.name:
-                return EnvironmentAlreadyExistsError(**env.__dict__)
-        response = httpx.post(
-            "http://0.0.0.0:7080/environments/build",
-            json={
-                "name": name,
-                "model": {
-                    "description": description,
-                    "packages": [f"{pkg.name}" for pkg in packages],
+        try:
+            cls.artifacts.get(Path(path), name)
+            return EnvironmentAlreadyExistsError(path=path, name=name)
+        except KeyError as e:
+            print(f"Error {type(e)}: {e}")
+            response = httpx.post(
+                "http://0.0.0.0:7080/environments/build",
+                json={
+                    "name": name,
+                    "model": {
+                        "description": description,
+                        "packages": [f"{pkg.name}" for pkg in packages],
+                    },
                 },
-            },
-        ).json()
-        print(f"Create: {response}")
-        return Environment(
-            id=response['id'],
-            name=response['name'],
-            path=path,
-            description=description,
-            packages=map(lambda pkg: pkg.to_package(), packages),
-            created=datetime.strptime(
-                response['created'], '%Y-%m-%dT%H:%M:%S.%f%z'
-            ),
-            state=response['state']['type'],
-        )  # type: ignore [call-arg]
+            ).json()
+            print(f"Create: {response}")
+            return Environment(
+                id=response['id'],
+                name=response['name'],
+                path=path,
+                description=description,
+                packages=map(lambda pkg: pkg.to_package(), packages),
+                created=datetime.strptime(
+                    response['created'], '%Y-%m-%dT%H:%M:%S.%f%z'
+                ),
+                state=response['state']['type'],
+            )  # type: ignore [call-arg]
 
     @classmethod
     def update(
@@ -187,6 +191,10 @@ def delete(cls, name: str):
                 return f"Deleted {name}"
         return "An environment with that name was not found"
 
+    @classmethod
+    async def upload_file(cls, file: Upload):
+        return (await file.read()).decode("utf-8")
+
 
 # Error types
 @strawberry.type
@@ -197,9 +205,12 @@ class EnvironmentNotFoundError:
 
 
 @strawberry.type
-class EnvironmentAlreadyExistsError(Environment):
+class EnvironmentAlreadyExistsError:
     """Environment name already exists"""
 
+    path: str
+    name: str
+
 
 UpdateEnvironmentResponse = strawberry.union(
     "UpdateEnvironmentResponse", [Environment, EnvironmentNotFoundError]
@@ -225,3 +236,4 @@ class Mutation:
         createEnvironment: CreateEnvironmentResponse = Environment.create  # type: ignore
         updateEnvironment: UpdateEnvironmentResponse = Environment.update  # type: ignore
         deleteEnvironment: str = Environment.delete  # type: ignore
+        upload_file: str = Environment.upload_file  # type: ignore

From a62ba542449a08455576f7529ea7766e0021547d Mon Sep 17 00:00:00 2001
From: Samuel Ogg <56794292+SamO135@users.noreply.github.com>
Date: Mon, 12 Jun 2023 12:06:38 +0100
Subject: [PATCH 005/129] created environment input class for mutations

---
 softpack_core/artifacts.py           |   8 +-
 softpack_core/schemas/environment.py | 105 ++++++++++++++-------------
 2 files changed, 60 insertions(+), 53 deletions(-)

diff --git a/softpack_core/artifacts.py b/softpack_core/artifacts.py
index 51aa8a3..fd447a1 100644
--- a/softpack_core/artifacts.py
+++ b/softpack_core/artifacts.py
@@ -241,5 +241,9 @@ def iter(self, user: Optional[str] = None) -> Iterable:
         except KeyError:
             return iter(())
 
-    def get(self, path: Path, name: str) -> pygit2.Tree:
-        return self.tree(self.environments_folder(path.as_posix(), name))
+    def get(self, path: Path, name: str) -> Optional[pygit2.Tree]:
+        try: 
+            return self.tree(self.environments_folder(path.as_posix(), name).as_posix())
+        except KeyError:
+            return None
+
diff --git a/softpack_core/schemas/environment.py b/softpack_core/schemas/environment.py
index 065df97..d36d650 100644
--- a/softpack_core/schemas/environment.py
+++ b/softpack_core/schemas/environment.py
@@ -10,6 +10,7 @@
 from pathlib import Path
 from typing import Iterable, Optional
 
+import uuid
 import httpx
 import strawberry
 from strawberry.file_uploads import Upload
@@ -36,7 +37,13 @@ def to_package(self):
         Return: a Package object
         """
         return Package(**self.__dict__)
-
+    
+@strawberry.input
+class EnvironmentInput:
+    name: Optional[str]
+    path: Optional[str]
+    description: Optional[str]
+    packages: Optional[list[PackageInput]]
 
 @strawberry.type
 class Environment:
@@ -93,11 +100,8 @@ def from_artifact(cls, obj: Artifacts.Object) -> "Environment":
     @classmethod
     def create(
         cls,
-        name: str,
-        path: str,
-        description: Optional[str] = None,
-        packages: Optional[list[PackageInput]] = None,
-    ) -> "Environment":
+        env: EnvironmentInput
+    ):
         """Create an Environment object.
 
         Args:
@@ -109,41 +113,40 @@ def create(
         Returns:
             Environment: A newly created Environment.
         """
-        try:
-            cls.artifacts.get(Path(path), name)
-            return EnvironmentAlreadyExistsError(path=path, name=name)
-        except KeyError as e:
-            print(f"Error {type(e)}: {e}")
-            response = httpx.post(
-                "http://0.0.0.0:7080/environments/build",
-                json={
-                    "name": name,
-                    "model": {
-                        "description": description,
-                        "packages": [f"{pkg.name}" for pkg in packages],
-                    },
+        if not (env.name and env.path and env.description and env.packages):
+            raise ValueError("Environment name, path, description and packages are required")
+        
+        # Check if an env with same name already exists at given path
+        if cls.artifacts.get(Path(env.path), env.name):
+            return EnvironmentAlreadyExistsError(path=env.path, name=env.name)
+
+        response = httpx.post(
+            "http://0.0.0.0:7080/environments/build",
+            json={
+                "name": env.name,
+                "model": {
+                    "description": env.description,
+                    "packages": [f"{pkg.name}" for pkg in env.packages],
                 },
-            ).json()
-            print(f"Create: {response}")
-            return Environment(
-                id=response['id'],
-                name=response['name'],
-                path=path,
-                description=description,
-                packages=map(lambda pkg: pkg.to_package(), packages),
-                created=datetime.strptime(
-                    response['created'], '%Y-%m-%dT%H:%M:%S.%f%z'
-                ),
-                state=response['state']['type'],
-            )  # type: ignore [call-arg]
+            },
+        ).json()
+        print(f"Create: {response}")
+        return Environment(
+            id=uuid.uuid4().hex,
+            name=response['name'],
+            path=env.path,
+            description=env.description,
+            packages=list(map(lambda pkg: pkg.to_package(), env.packages)),
+            created=datetime.strptime(
+                response['created'], '%Y-%m-%dT%H:%M:%S.%f%z'
+            ),
+            state=response['state']['type'],
+        )  # type: ignore [call-arg]
 
     @classmethod
     def update(
         cls,
-        name: str,
-        path: Optional[str] = None,
-        description: Optional[str] = None,
-        packages: Optional[list[PackageInput]] = None,
+        env: EnvironmentInput,
     ):
         """Update an Environment object.
 
@@ -156,28 +159,28 @@ def update(
         Returns:
             Environment: An updated Environment.
         """
-        for env in Environment.iter():
-            if env.name == name:
+        for current in Environment.iter():
+            if current.name == env.name:
                 response = httpx.post(
                     "http://0.0.0.0:7080/environments/build",
                     json={
-                        "name": name,
+                        "name": env.name,
                         "model": {
-                            "description": description,
-                            "packages": [pkg.name for pkg in packages],
+                            "description": env.description,
+                            "packages": [pkg.name for pkg in env.packages],
                         },
                     },
                 ).json()
                 print(f"Update: {response}")
-                if path != None:
-                    env.path = path
-                if description != None:
-                    env.description = description
-                if packages != None:
-                    env.packages = map(lambda pkg: pkg.to_package(), packages)
-                env.state = response['state']['type']
-                return env
-        return EnvironmentNotFoundError(name=name)
+                if env.path != None:
+                    current.path = env.path
+                if env.description != None:
+                    current.description = env.description
+                if env.packages != None:
+                    current.packages = map(lambda pkg: pkg.to_package(), env.packages)
+                current.state = response['state']['type']
+                return current
+        return EnvironmentNotFoundError(name=env.name)
 
     @classmethod
     def delete(cls, name: str):
@@ -236,4 +239,4 @@ class Mutation:
         createEnvironment: CreateEnvironmentResponse = Environment.create  # type: ignore
         updateEnvironment: UpdateEnvironmentResponse = Environment.update  # type: ignore
         deleteEnvironment: str = Environment.delete  # type: ignore
-        upload_file: str = Environment.upload_file  # type: ignore
+        upload_file: str = Environment.upload_file   # type: ignore

From e0d7515d18a25703d5fe086201e9c29fe320ccd8 Mon Sep 17 00:00:00 2001
From: Samuel Ogg <56794292+SamO135@users.noreply.github.com>
Date: Tue, 4 Jul 2023 14:29:25 +0100
Subject: [PATCH 006/129] modified 'create' and 'update' query parameters

---
 softpack_core/artifacts.py           | 12 ++++-
 softpack_core/schemas/environment.py | 73 +++++++++++++---------------
 2 files changed, 43 insertions(+), 42 deletions(-)

diff --git a/softpack_core/artifacts.py b/softpack_core/artifacts.py
index fd447a1..af6772e 100644
--- a/softpack_core/artifacts.py
+++ b/softpack_core/artifacts.py
@@ -215,7 +215,7 @@ def environments(self, path: Path) -> Iterable:
             return iter(())
 
     def iter(self, user: Optional[str] = None) -> Iterable:
-        """Return am iterator for the specified user.
+        """Return an iterator for the specified user.
 
         Args:
             user: a username
@@ -242,8 +242,16 @@ def iter(self, user: Optional[str] = None) -> Iterable:
             return iter(())
 
     def get(self, path: Path, name: str) -> Optional[pygit2.Tree]:
+        """Return the environment at the specified name and path.
+        
+        Args:
+            path: the path containing the environment folder
+            name: the name of the environment folder
+            
+        Returns:
+            pygit2.Tree: a pygit2.Tree or None"""
         try: 
-            return self.tree(self.environments_folder(path.as_posix(), name).as_posix())
+            return self.tree(str(self.environments_folder(str(path), name)))
         except KeyError:
             return None
 
diff --git a/softpack_core/schemas/environment.py b/softpack_core/schemas/environment.py
index d36d650..5151162 100644
--- a/softpack_core/schemas/environment.py
+++ b/softpack_core/schemas/environment.py
@@ -40,10 +40,10 @@ def to_package(self):
     
 @strawberry.input
 class EnvironmentInput:
-    name: Optional[str]
-    path: Optional[str]
-    description: Optional[str]
-    packages: Optional[list[PackageInput]]
+    name: str
+    path: str
+    description: str
+    packages: list[PackageInput]
 
 @strawberry.type
 class Environment:
@@ -54,7 +54,6 @@ class Environment:
     path: str
     description: str
     packages: list[Package]
-    created: Optional[datetime]
     state: Optional[str]
     artifacts = Artifacts()
 
@@ -93,7 +92,6 @@ def from_artifact(cls, obj: Artifacts.Object) -> "Environment":
                 lambda package: Package(id=package, name=package),
                 spec.packages,
             ),  # type: ignore [call-arg]
-            created=None,
             state=None,
         )
 
@@ -105,16 +103,11 @@ def create(
         """Create an Environment object.
 
         Args:
-            name: Name for an environment.
-            path: Path for an environment.
-            description: Description for an environment.
-            packages: List of packages in the environment.
+            env: Details of the new environment
 
         Returns:
             Environment: A newly created Environment.
         """
-        if not (env.name and env.path and env.description and env.packages):
-            raise ValueError("Environment name, path, description and packages are required")
         
         # Check if an env with same name already exists at given path
         if cls.artifacts.get(Path(env.path), env.name):
@@ -137,9 +130,6 @@ def create(
             path=env.path,
             description=env.description,
             packages=list(map(lambda pkg: pkg.to_package(), env.packages)),
-            created=datetime.strptime(
-                response['created'], '%Y-%m-%dT%H:%M:%S.%f%z'
-            ),
             state=response['state']['type'],
         )  # type: ignore [call-arg]
 
@@ -147,39 +137,42 @@ def create(
     def update(
         cls,
         env: EnvironmentInput,
+        path: str,
+        name: str,
     ):
         """Update an Environment object.
 
         Args:
-            name: Name for an environment.
-            path: Path for an environment.
-            description: Description for an environment.
-            packages: List of packages in the environment.
+            env: Details of the updated environment
+            path: The path of the current environment
+            name: The name of the current environment
 
         Returns:
             Environment: An updated Environment.
         """
-        for current in Environment.iter():
-            if current.name == env.name:
-                response = httpx.post(
-                    "http://0.0.0.0:7080/environments/build",
-                    json={
-                        "name": env.name,
-                        "model": {
-                            "description": env.description,
-                            "packages": [pkg.name for pkg in env.packages],
-                        },
+        current_env = cls.artifacts.get(Path(path), name)
+        print(current_env)
+        if current_env:
+            response = httpx.post(
+                "http://0.0.0.0:7080/environments/build",
+                json={
+                    "name": env.name,
+                    "model": {
+                        "description": env.description,
+                        "packages": [pkg.name for pkg in env.packages or []],
                     },
-                ).json()
-                print(f"Update: {response}")
-                if env.path != None:
-                    current.path = env.path
-                if env.description != None:
-                    current.description = env.description
-                if env.packages != None:
-                    current.packages = map(lambda pkg: pkg.to_package(), env.packages)
-                current.state = response['state']['type']
-                return current
+                },
+            ).json()
+            print(f"Update: {response}")
+
+            return Environment(
+                id=uuid.uuid4().hex,
+                name=env.name,
+                path=env.path,
+                description=env.description,
+                packages=[pkg.to_package() for pkg in env.packages],
+                state=response['state']['type'],
+            )
         return EnvironmentNotFoundError(name=env.name)
 
     @classmethod
@@ -196,7 +189,7 @@ def delete(cls, name: str):
 
     @classmethod
     async def upload_file(cls, file: Upload):
-        return (await file.read()).decode("utf-8")
+        return (await file.read()).decode("utf-8") # type: ignore
 
 
 # Error types

From fbc786c3b01802459c35656a97818b99dcf7d316 Mon Sep 17 00:00:00 2001
From: Samuel Ogg <56794292+SamO135@users.noreply.github.com>
Date: Mon, 10 Jul 2023 15:14:46 +0100
Subject: [PATCH 007/129] allow port 5173 for the client

---
 softpack_core/config/conf/config.yml | 1 +
 1 file changed, 1 insertion(+)

diff --git a/softpack_core/config/conf/config.yml b/softpack_core/config/conf/config.yml
index 4e7bc7d..168516b 100644
--- a/softpack_core/config/conf/config.yml
+++ b/softpack_core/config/conf/config.yml
@@ -7,6 +7,7 @@ server:
       - http://localhost
       - http://localhost:8080
       - http://localhost:3000
+      - http://localhost:5173
 
 artifacts:
   repo:

From 931acdef9ee52dedc2c3e649adead3c29fcb6400 Mon Sep 17 00:00:00 2001
From: Samuel Ogg <56794292+SamO135@users.noreply.github.com>
Date: Thu, 13 Jul 2023 10:40:46 +0100
Subject: [PATCH 008/129] automatically push to the artifacts repo when
 creating a new environment

---
 softpack_core/artifacts.py           | 179 ++++++++++++++++++++++++++-
 softpack_core/config/conf/config.yml |   2 +
 softpack_core/config/models.py       |   2 +
 softpack_core/schemas/environment.py |  34 ++---
 4 files changed, 197 insertions(+), 20 deletions(-)

diff --git a/softpack_core/artifacts.py b/softpack_core/artifacts.py
index af6772e..21b6dad 100644
--- a/softpack_core/artifacts.py
+++ b/softpack_core/artifacts.py
@@ -11,6 +11,7 @@
 
 import pygit2
 from box import Box
+from pygit2 import Signature
 
 from .app import app
 from .ldapapi import LDAP
@@ -20,6 +21,7 @@ class Artifacts:
     """Artifacts repo access class."""
 
     environments_root = "environments"
+    environments_file = "softpack.yml"
 
     @dataclass
     class Object:
@@ -72,7 +74,7 @@ def spec(self) -> Box:
             Returns:
                 Box: A boxed dictionary.
             """
-            spec = self.obj["softpack.yml"]
+            spec = self.obj[self.environments_file]
             return Box.from_yaml(spec.data)
 
         def __iter__(self) -> Iterator["Artifacts.Object"]:
@@ -98,7 +100,7 @@ def __init__(self) -> None:
         try:
             credentials = pygit2.UserPass(
                 self.settings.artifacts.repo.username,
-                self.settings.artifacts.repo.reader,
+                self.settings.artifacts.repo.writer,
             )
         except Exception as e:
             print(e)
@@ -243,15 +245,180 @@ def iter(self, user: Optional[str] = None) -> Iterable:
 
     def get(self, path: Path, name: str) -> Optional[pygit2.Tree]:
         """Return the environment at the specified name and path.
-        
+
         Args:
             path: the path containing the environment folder
             name: the name of the environment folder
-            
+ 
         Returns:
-            pygit2.Tree: a pygit2.Tree or None"""
-        try: 
+            pygit2.Tree: a pygit2.Tree or None
+        """
+        try:
             return self.tree(str(self.environments_folder(str(path), name)))
         except KeyError:
             return None
 
+    def commit(
+        self, repo: pygit2.Repository, tree_oid: pygit2.Oid, message: str
+    ) -> pygit2.Commit:
+        """Create and return a commit.
+
+        Args:
+            repo: the repository to commit to
+            tree_oid: the oid of the tree object that will be committed. The
+            tree this refers to will replace the entire contents of the repo.
+            message: the commit message
+
+        Returns:
+            pygit2.Commit: the commit oid
+        """
+        ref = repo.head.name
+        author = committer = Signature(
+            self.settings.artifacts.repo.author,
+            self.settings.artifacts.repo.email
+            )
+        parents = [repo.lookup_reference(ref).target]
+        return repo.create_commit(
+            ref, author, committer, message, tree_oid, parents
+        )
+
+    def error_callback(self, refname: str, message: str) -> None:
+        """Push update reference callback.
+
+        Args:
+            refname: the name of the reference (on the remote)
+            message: rejection message from the remote. If None, the update was
+            accepted
+        """
+        if message is not None:
+            print(
+                f"An error occurred during push to ref '{refname}': {message}"
+            )
+
+    def push(self, repo: pygit2.Repository) -> None:
+        """Push all commits to a repository.
+
+        Args:
+            repo: the repository to push to
+        """
+        remote = self.repo.remotes[0]
+        credentials = None
+        try:
+            credentials = pygit2.UserPass(
+                self.settings.artifacts.repo.username,
+                self.settings.artifacts.repo.writer,
+            )
+        except Exception as e:
+            print(e)
+        callbacks = pygit2.RemoteCallbacks(credentials=credentials)
+        callbacks.push_update_reference = self.error_callback
+        remote.push([repo.head.name], callbacks=callbacks)
+
+    def build_tree(
+        self,
+        repo: pygit2.Repository,
+        root_tree: pygit2.Tree,
+        new_tree: pygit2.Oid,
+        path: Path,
+    ) -> pygit2.Oid:
+        """Expand new/updated sub tree to include the entire repository.
+
+        Args:
+            repo: a bare repository
+            root_tree: the tree containing the entire repository
+            new_tree: the oid of the new/updated sub tree to be added to the
+            repository
+            path: the path from root_tree root to new_tree root
+        """
+        while str(path) != ".":
+            try:
+                sub_tree = (
+                    root_tree[str(path.parent)]
+                    if str(path.parent) != "."
+                    else root_tree
+                )
+            except KeyError:
+                raise KeyError(
+                    f"{path.parent} does not exist in the repository"
+                )
+            sub_treebuilder = repo.TreeBuilder(sub_tree)
+            sub_treebuilder.insert(
+                path.name, new_tree, pygit2.GIT_FILEMODE_TREE
+            )
+            new_tree = sub_treebuilder.write()
+            path = path.parent
+        return new_tree
+
+    def generate_yaml_contents(self, env):
+        """Generate the softpack.yml file contents.
+
+        Args:
+            env: an Environment object
+        """
+        packages = [
+            f"- {pkg.name}@{pkg.version}" if pkg.version else f"- {pkg.name}"
+            for pkg in env.packages
+        ]
+        packages = "\n".join(packages)
+        contents = f"description: {env.description}\npackages:\n{packages}\n"
+        return contents
+
+    def create_environment(
+        self,
+        repo: pygit2.Repository,
+        env,
+        commit_message: str,
+        target_tree: Optional[pygit2.Tree] = None,
+    ):
+        """Create, commit and push a new environment file to GitLab.
+
+        Args:
+            repo: a bare repository
+            env: an Environment object
+            commit_message: the commit message
+            target_tree: pygit2.Tree object with the environment folder you
+            want to update as its root
+        """
+        root_tree = repo.head.peel(pygit2.Tree)
+
+        # Create new file
+        contents = self.generate_yaml_contents(env)
+        file_oid = repo.create_blob(contents.encode())
+
+        # Put new file into new env folder
+        if target_tree:
+            new_treebuilder = repo.TreeBuilder(target_tree)
+        else:
+            new_treebuilder = repo.TreeBuilder()
+        new_treebuilder.insert(
+            self.environments_file, file_oid, pygit2.GIT_FILEMODE_BLOB
+        )
+        new_tree = new_treebuilder.write()
+
+        # Expand tree to include the whole repo
+        full_path = (
+            Path(self.environments_root)
+            / env.path
+            / env.name
+            / self.environments_file
+        )
+        full_tree = self.build_tree(
+            repo, root_tree, new_tree, full_path.parent
+        )
+        new_tree = repo.get(full_tree)
+
+        # Check for errors in the new tree
+        diff = repo.diff(new_tree, root_tree)
+        if len(diff) > 1:
+            raise RuntimeError("Too many changes to the repo")
+        new_file = diff[0].delta.new_file
+        if new_file.path != str(full_path):
+            raise RuntimeError(
+                f"New file added to incorrect path: \
+                    {new_file.path} instead of {full_path}"
+            )
+        
+        # Commit and push
+        self.commit(repo, full_tree, commit_message)
+        self.push(repo)
+
diff --git a/softpack_core/config/conf/config.yml b/softpack_core/config/conf/config.yml
index 168516b..e61c037 100644
--- a/softpack_core/config/conf/config.yml
+++ b/softpack_core/config/conf/config.yml
@@ -13,6 +13,8 @@ artifacts:
   repo:
     url: https://github.com/softpack-io/softpack-artifacts.git
     username: softpack-core
+    author: softpack
+    email: softpack@sanger.ac.uk
   path: ./softpack-artifacts
 
 # Vault Config
diff --git a/softpack_core/config/models.py b/softpack_core/config/models.py
index e1f0f33..89bdbb3 100644
--- a/softpack_core/config/models.py
+++ b/softpack_core/config/models.py
@@ -39,6 +39,8 @@ class Repo(BaseModel):
 
         url: AnyUrl
         username: Optional[str]
+        author: str
+        email: str
         reader: Optional[str]
         writer: Optional[str]
 
diff --git a/softpack_core/schemas/environment.py b/softpack_core/schemas/environment.py
index 5151162..9b83379 100644
--- a/softpack_core/schemas/environment.py
+++ b/softpack_core/schemas/environment.py
@@ -5,12 +5,11 @@
 """
 
 import os
+import uuid
 from dataclasses import dataclass
-from datetime import datetime
 from pathlib import Path
 from typing import Iterable, Optional
 
-import uuid
 import httpx
 import strawberry
 from strawberry.file_uploads import Upload
@@ -29,7 +28,7 @@ class Package(Spack.PackageBase):
 
 @strawberry.input
 class PackageInput(Package):
-    """A Strawberry input model representing a pacakge."""
+    """A Strawberry input model representing a package."""
 
     def to_package(self):
         """Create a Package object from a PackageInput object.
@@ -37,14 +36,18 @@ def to_package(self):
         Return: a Package object
         """
         return Package(**self.__dict__)
-    
+
+
 @strawberry.input
 class EnvironmentInput:
+    """A Strawberry input model representing an environment."""
+
     name: str
     path: str
     description: str
     packages: list[PackageInput]
 
+
 @strawberry.type
 class Environment:
     """A Strawberry model representing a single environment."""
@@ -96,10 +99,7 @@ def from_artifact(cls, obj: Artifacts.Object) -> "Environment":
         )
 
     @classmethod
-    def create(
-        cls,
-        env: EnvironmentInput
-    ):
+    def create(cls, env: EnvironmentInput):
         """Create an Environment object.
 
         Args:
@@ -108,7 +108,6 @@ def create(
         Returns:
             Environment: A newly created Environment.
         """
-        
         # Check if an env with same name already exists at given path
         if cls.artifacts.get(Path(env.path), env.name):
             return EnvironmentAlreadyExistsError(path=env.path, name=env.name)
@@ -124,7 +123,7 @@ def create(
             },
         ).json()
         print(f"Create: {response}")
-        return Environment(
+        new_env = Environment(
             id=uuid.uuid4().hex,
             name=response['name'],
             path=env.path,
@@ -133,6 +132,13 @@ def create(
             state=response['state']['type'],
         )  # type: ignore [call-arg]
 
+        cls.artifacts.create_environment(
+            cls.artifacts.repo,
+            new_env,
+            "create new environment",
+        )
+        return new_env
+
     @classmethod
     def update(
         cls,
@@ -189,20 +195,20 @@ def delete(cls, name: str):
 
     @classmethod
     async def upload_file(cls, file: Upload):
-        return (await file.read()).decode("utf-8") # type: ignore
+        return (await file.read()).decode("utf-8")  # type: ignore
 
 
 # Error types
 @strawberry.type
 class EnvironmentNotFoundError:
-    """Environment not found"""
+    """Environment not found."""
 
     name: str
 
 
 @strawberry.type
 class EnvironmentAlreadyExistsError:
-    """Environment name already exists"""
+    """Environment name already exists."""
 
     path: str
     name: str
@@ -232,4 +238,4 @@ class Mutation:
         createEnvironment: CreateEnvironmentResponse = Environment.create  # type: ignore
         updateEnvironment: UpdateEnvironmentResponse = Environment.update  # type: ignore
         deleteEnvironment: str = Environment.delete  # type: ignore
-        upload_file: str = Environment.upload_file   # type: ignore
+        upload_file: str = Environment.upload_file  # type: ignore

From 4b68f5b3ab8201ab12a083d61b891e0ce3461d84 Mon Sep 17 00:00:00 2001
From: Samuel Ogg <56794292+SamO135@users.noreply.github.com>
Date: Thu, 13 Jul 2023 16:08:51 +0100
Subject: [PATCH 009/129] fix environments query not working

---
 softpack_core/artifacts.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/softpack_core/artifacts.py b/softpack_core/artifacts.py
index 21b6dad..e8adf4d 100644
--- a/softpack_core/artifacts.py
+++ b/softpack_core/artifacts.py
@@ -74,7 +74,7 @@ def spec(self) -> Box:
             Returns:
                 Box: A boxed dictionary.
             """
-            spec = self.obj[self.environments_file]
+            spec = self.obj[Artifacts.environments_file]
             return Box.from_yaml(spec.data)
 
         def __iter__(self) -> Iterator["Artifacts.Object"]:

From 3eb01d81608f855df929f0171d53852280a3644d Mon Sep 17 00:00:00 2001
From: Samuel Ogg <56794292+SamO135@users.noreply.github.com>
Date: Mon, 31 Jul 2023 11:59:18 +0100
Subject: [PATCH 010/129] update existing environment changing name and/or path
 is not yet implemented

---
 softpack_core/artifacts.py           | 52 ++++++++++++++++++++++------
 softpack_core/schemas/environment.py | 15 +++++---
 2 files changed, 52 insertions(+), 15 deletions(-)

diff --git a/softpack_core/artifacts.py b/softpack_core/artifacts.py
index e8adf4d..e4ffd12 100644
--- a/softpack_core/artifacts.py
+++ b/softpack_core/artifacts.py
@@ -249,7 +249,7 @@ def get(self, path: Path, name: str) -> Optional[pygit2.Tree]:
         Args:
             path: the path containing the environment folder
             name: the name of the environment folder
- 
+
         Returns:
             pygit2.Tree: a pygit2.Tree or None
         """
@@ -275,8 +275,8 @@ def commit(
         ref = repo.head.name
         author = committer = Signature(
             self.settings.artifacts.repo.author,
-            self.settings.artifacts.repo.email
-            )
+            self.settings.artifacts.repo.email,
+        )
         parents = [repo.lookup_reference(ref).target]
         return repo.create_commit(
             ref, author, committer, message, tree_oid, parents
@@ -370,7 +370,7 @@ def create_environment(
         commit_message: str,
         target_tree: Optional[pygit2.Tree] = None,
     ):
-        """Create, commit and push a new environment file to GitLab.
+        """Create, commit and push a new environment folder to GitLab.
 
         Args:
             repo: a bare repository
@@ -411,14 +411,44 @@ def create_environment(
         diff = repo.diff(new_tree, root_tree)
         if len(diff) > 1:
             raise RuntimeError("Too many changes to the repo")
-        new_file = diff[0].delta.new_file
-        if new_file.path != str(full_path):
-            raise RuntimeError(
-                f"New file added to incorrect path: \
-                    {new_file.path} instead of {full_path}"
-            )
-        
+        elif len(diff) < 1:
+            raise RuntimeError("No changes made")
+        elif len(diff) == 1:
+            new_file = diff[0].delta.new_file
+            if new_file.path != str(full_path):
+                raise RuntimeError(
+                    f"New file added to incorrect path: \
+                        {new_file.path} instead of {full_path}"
+                )
+
         # Commit and push
         self.commit(repo, full_tree, commit_message)
         self.push(repo)
 
+    def update_environment(
+        self, new_env, current_name: str, current_path: str
+    ):
+        """Update an existing environment folder in GitLab.
+
+        Args:
+            repo: a bare repository
+            new_env: an updated Environment object
+            current_name: the current name of the environment
+            current_path: the current path of the environment
+        """
+        if new_env.name == current_name and new_env.path == current_path:
+            # Update environment in the same location
+            root_tree = self.repo.head.peel(pygit2.Tree)
+            path = Path(self.environments_root) / current_path / current_name
+            target_tree = root_tree[path]
+            self.create_environment(
+                self.repo,
+                new_env,
+                "update existing environment",
+                target_tree,
+            )
+        else:
+            # Update environment in a new location
+            raise KeyError("not matching name or path")
+            # self.delete_environment()
+            # self.create_environment()
diff --git a/softpack_core/schemas/environment.py b/softpack_core/schemas/environment.py
index 9b83379..0a7875f 100644
--- a/softpack_core/schemas/environment.py
+++ b/softpack_core/schemas/environment.py
@@ -143,8 +143,8 @@ def create(cls, env: EnvironmentInput):
     def update(
         cls,
         env: EnvironmentInput,
-        path: str,
-        name: str,
+        current_path: str,
+        current_name: str,
     ):
         """Update an Environment object.
 
@@ -156,7 +156,8 @@ def update(
         Returns:
             Environment: An updated Environment.
         """
-        current_env = cls.artifacts.get(Path(path), name)
+        # Check if an environment exists at the specified path and name
+        current_env = cls.artifacts.get(Path(current_path), current_name)
         print(current_env)
         if current_env:
             response = httpx.post(
@@ -171,7 +172,7 @@ def update(
             ).json()
             print(f"Update: {response}")
 
-            return Environment(
+            new_env = Environment(
                 id=uuid.uuid4().hex,
                 name=env.name,
                 path=env.path,
@@ -179,6 +180,12 @@ def update(
                 packages=[pkg.to_package() for pkg in env.packages],
                 state=response['state']['type'],
             )
+
+            cls.artifacts.update_environment(
+                new_env, current_name, current_path
+            )
+            return new_env
+
         return EnvironmentNotFoundError(name=env.name)
 
     @classmethod

From eadf62c2346688bc2f1370a0869e62c812ba28e9 Mon Sep 17 00:00:00 2001
From: Samuel Ogg <56794292+SamO135@users.noreply.github.com>
Date: Tue, 1 Aug 2023 10:43:13 +0100
Subject: [PATCH 011/129] optional package ids when inputting package
 information to graphql mutations, the id field is optional

---
 softpack_core/schemas/environment.py | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/softpack_core/schemas/environment.py b/softpack_core/schemas/environment.py
index 0a7875f..eb07a59 100644
--- a/softpack_core/schemas/environment.py
+++ b/softpack_core/schemas/environment.py
@@ -30,6 +30,8 @@ class Package(Spack.PackageBase):
 class PackageInput(Package):
     """A Strawberry input model representing a package."""
 
+    id: Optional[str] = None
+
     def to_package(self):
         """Create a Package object from a PackageInput object.
 

From ad5a4b4185a79ef2af4dc871c865324111172d9d Mon Sep 17 00:00:00 2001
From: Samuel Ogg <56794292+SamO135@users.noreply.github.com>
Date: Tue, 1 Aug 2023 16:42:40 +0100
Subject: [PATCH 012/129] update error handling and responses given by graphql

---
 softpack_core/artifacts.py           |   4 +-
 softpack_core/schemas/environment.py | 103 ++++++++++++++++++++-------
 2 files changed, 80 insertions(+), 27 deletions(-)

diff --git a/softpack_core/artifacts.py b/softpack_core/artifacts.py
index e4ffd12..a7d2ea2 100644
--- a/softpack_core/artifacts.py
+++ b/softpack_core/artifacts.py
@@ -412,12 +412,12 @@ def create_environment(
         if len(diff) > 1:
             raise RuntimeError("Too many changes to the repo")
         elif len(diff) < 1:
-            raise RuntimeError("No changes made")
+            raise RuntimeError("No changes made to the environment")
         elif len(diff) == 1:
             new_file = diff[0].delta.new_file
             if new_file.path != str(full_path):
                 raise RuntimeError(
-                    f"New file added to incorrect path: \
+                    f"Attempted to add new file added to incorrect path: \
                         {new_file.path} instead of {full_path}"
                 )
 
diff --git a/softpack_core/schemas/environment.py b/softpack_core/schemas/environment.py
index eb07a59..4e02b59 100644
--- a/softpack_core/schemas/environment.py
+++ b/softpack_core/schemas/environment.py
@@ -110,9 +110,11 @@ def create(cls, env: EnvironmentInput):
         Returns:
             Environment: A newly created Environment.
         """
+        if env.name == "":
+            return InvalidInputError(message="name cannot be empty")
         # Check if an env with same name already exists at given path
         if cls.artifacts.get(Path(env.path), env.name):
-            return EnvironmentAlreadyExistsError(path=env.path, name=env.name)
+            return EnvironmentAlreadyExistsError(message="An environment of this name already exists in this location", path=env.path, name=env.name)
 
         response = httpx.post(
             "http://0.0.0.0:7080/environments/build",
@@ -127,19 +129,23 @@ def create(cls, env: EnvironmentInput):
         print(f"Create: {response}")
         new_env = Environment(
             id=uuid.uuid4().hex,
-            name=response['name'],
+            name=env.name,
             path=env.path,
             description=env.description,
             packages=list(map(lambda pkg: pkg.to_package(), env.packages)),
             state=response['state']['type'],
         )  # type: ignore [call-arg]
 
-        cls.artifacts.create_environment(
-            cls.artifacts.repo,
-            new_env,
-            "create new environment",
-        )
-        return new_env
+        try:
+            cls.artifacts.create_environment(
+                cls.artifacts.repo,
+                new_env,
+                "create new environment",
+            )
+        except RuntimeError as e:
+            return InvalidInputError(message=str(e))
+        
+        return CreateEnvironmentSuccess(message="Successfully scheduled environment creation", environment=new_env)
 
     @classmethod
     def update(
@@ -159,9 +165,7 @@ def update(
             Environment: An updated Environment.
         """
         # Check if an environment exists at the specified path and name
-        current_env = cls.artifacts.get(Path(current_path), current_name)
-        print(current_env)
-        if current_env:
+        if cls.artifacts.get(Path(current_path), current_name):
             response = httpx.post(
                 "http://0.0.0.0:7080/environments/build",
                 json={
@@ -183,12 +187,16 @@ def update(
                 state=response['state']['type'],
             )
 
-            cls.artifacts.update_environment(
-                new_env, current_name, current_path
-            )
-            return new_env
+            try:
+                cls.artifacts.update_environment(
+                    new_env, current_name, current_path
+                )
+            except RuntimeError as e:
+                return InvalidInputError(message=str(e))
 
-        return EnvironmentNotFoundError(name=env.name)
+            return UpdateEnvironmentSuccess(message="Successfully updated environment", environment=new_env)
+
+        return EnvironmentNotFoundError(message="Unable to find an environment of this name in this location", path=env.path, name=env.name)
 
     @classmethod
     def delete(cls, name: str):
@@ -207,27 +215,72 @@ async def upload_file(cls, file: Upload):
         return (await file.read()).decode("utf-8")  # type: ignore
 
 
+# Interfaces
+@strawberry.interface
+class Success:
+    """Interface for successful results."""
+
+    message: str
+
+@strawberry.interface
+class Error:
+    """Interface for errors."""
+
+    message: str
+
+
+# Success types
+@strawberry.type
+class CreateEnvironmentSuccess(Success):
+    """Environment successfully scheduled."""
+
+    message: str
+    environment: Environment
+
+@strawberry.type
+class UpdateEnvironmentSuccess(Success):
+    """Environment successfully updated."""
+
+    message: str
+    environment: Environment
+
+
 # Error types
 @strawberry.type
-class EnvironmentNotFoundError:
+class InvalidInputError(Error):
+    """Invalid input data"""
+
+    message: str
+
+@strawberry.type
+class EnvironmentNotFoundError(Error):
     """Environment not found."""
 
+    message: str
+    path: str
     name: str
 
-
 @strawberry.type
-class EnvironmentAlreadyExistsError:
+class EnvironmentAlreadyExistsError(Error):
     """Environment name already exists."""
 
+    message:str
     path: str
     name: str
 
 
-UpdateEnvironmentResponse = strawberry.union(
-    "UpdateEnvironmentResponse", [Environment, EnvironmentNotFoundError]
+CreateResponse = strawberry.union(
+    "CreateResponse", [CreateEnvironmentSuccess,
+                       InvalidInputError,
+                       EnvironmentAlreadyExistsError,
+                        ]
 )
-CreateEnvironmentResponse = strawberry.union(
-    "CreateEnvironmentResponse", [Environment, EnvironmentAlreadyExistsError]
+
+UpdateResponse = strawberry.union(
+    "UpdateResponse", [UpdateEnvironmentSuccess,
+                       InvalidInputError,
+                       EnvironmentNotFoundError,
+                        ]
 )
 
 
@@ -244,7 +297,7 @@ class Query:
     class Mutation:
         """GraphQL mutation schema."""
 
-        createEnvironment: CreateEnvironmentResponse = Environment.create  # type: ignore
-        updateEnvironment: UpdateEnvironmentResponse = Environment.update  # type: ignore
+        createEnvironment: CreateResponse = Environment.create  # type: ignore
+        updateEnvironment: UpdateResponse = Environment.update  # type: ignore
         deleteEnvironment: str = Environment.delete  # type: ignore
         upload_file: str = Environment.upload_file  # type: ignore

From 331d147fce297a9589d43cc99792d82364dbe141 Mon Sep 17 00:00:00 2001
From: Samuel Ogg <56794292+SamO135@users.noreply.github.com>
Date: Wed, 2 Aug 2023 10:36:39 +0100
Subject: [PATCH 013/129] check all fields are provided and the path is valid

---
 softpack_core/schemas/environment.py | 14 +++++++++++---
 1 file changed, 11 insertions(+), 3 deletions(-)

diff --git a/softpack_core/schemas/environment.py b/softpack_core/schemas/environment.py
index 4e02b59..ce9d72a 100644
--- a/softpack_core/schemas/environment.py
+++ b/softpack_core/schemas/environment.py
@@ -110,8 +110,13 @@ def create(cls, env: EnvironmentInput):
         Returns:
             Environment: A newly created Environment.
         """
-        if env.name == "":
-            return InvalidInputError(message="name cannot be empty")
+        # Check if any field has been left empty
+        if any(len(value) == 0 for value in vars(env).values()):
+            return InvalidInputError(message="all fields must be filled in")
+        # Check if a valid path has been provided
+        user = os.environ["USER"]
+        if env.path not in ["groups/hgi", f"users/{user}"]:
+            return InvalidInputError(message="Invalid path")
         # Check if an env with same name already exists at given path
         if cls.artifacts.get(Path(env.path), env.name):
             return EnvironmentAlreadyExistsError(message="An environment of this name already exists in this location", path=env.path, name=env.name)
@@ -164,6 +169,9 @@ def update(
         Returns:
             Environment: An updated Environment.
         """
+        # Check if any field has been left empty
+        if any(len(value) == 0 for value in vars(env).values()) or current_name == "" or current_path == "":
+            return InvalidInputError(message="all fields must be filled in")
         # Check if an environment exists at the specified path and name
         if cls.artifacts.get(Path(current_path), current_name):
             response = httpx.post(
@@ -196,7 +204,7 @@ def update(
 
             return UpdateEnvironmentSuccess(message="Successfully updated environment", environment=new_env)
 
-        return EnvironmentNotFoundError(message="Unable to find an environment of this name in this location", path=env.path, name=env.name)
+        return EnvironmentNotFoundError(message="Unable to find an environment of this name in this location", path=current_path, name=current_name)
 
     @classmethod
     def delete(cls, name: str):

From 15a2e67e8fba8d41f6d264bb38a397a3e784a3b4 Mon Sep 17 00:00:00 2001
From: Samuel Ogg <56794292+SamO135@users.noreply.github.com>
Date: Wed, 2 Aug 2023 14:43:11 +0100
Subject: [PATCH 014/129] delete environment mutation

---
 softpack_core/artifacts.py           | 48 +++++++++++++++++++---------
 softpack_core/schemas/environment.py | 43 +++++++++++++++++--------
 2 files changed, 62 insertions(+), 29 deletions(-)

diff --git a/softpack_core/artifacts.py b/softpack_core/artifacts.py
index a7d2ea2..17ee5f2 100644
--- a/softpack_core/artifacts.py
+++ b/softpack_core/artifacts.py
@@ -365,7 +365,6 @@ def generate_yaml_contents(self, env):
 
     def create_environment(
         self,
-        repo: pygit2.Repository,
         env,
         commit_message: str,
         target_tree: Optional[pygit2.Tree] = None,
@@ -373,23 +372,22 @@ def create_environment(
         """Create, commit and push a new environment folder to GitLab.
 
         Args:
-            repo: a bare repository
             env: an Environment object
             commit_message: the commit message
             target_tree: pygit2.Tree object with the environment folder you
             want to update as its root
         """
-        root_tree = repo.head.peel(pygit2.Tree)
+        root_tree = self.repo.head.peel(pygit2.Tree)
 
         # Create new file
         contents = self.generate_yaml_contents(env)
-        file_oid = repo.create_blob(contents.encode())
+        file_oid = self.repo.create_blob(contents.encode())
 
         # Put new file into new env folder
         if target_tree:
-            new_treebuilder = repo.TreeBuilder(target_tree)
+            new_treebuilder = self.repo.TreeBuilder(target_tree)
         else:
-            new_treebuilder = repo.TreeBuilder()
+            new_treebuilder = self.repo.TreeBuilder()
         new_treebuilder.insert(
             self.environments_file, file_oid, pygit2.GIT_FILEMODE_BLOB
         )
@@ -403,12 +401,12 @@ def create_environment(
             / self.environments_file
         )
         full_tree = self.build_tree(
-            repo, root_tree, new_tree, full_path.parent
+            self.repo, root_tree, new_tree, full_path.parent
         )
-        new_tree = repo.get(full_tree)
 
+        new_tree = self.repo.get(full_tree)
         # Check for errors in the new tree
-        diff = repo.diff(new_tree, root_tree)
+        diff = self.repo.diff(new_tree, root_tree)
         if len(diff) > 1:
             raise RuntimeError("Too many changes to the repo")
         elif len(diff) < 1:
@@ -422,16 +420,15 @@ def create_environment(
                 )
 
         # Commit and push
-        self.commit(repo, full_tree, commit_message)
-        self.push(repo)
+        self.commit(self.repo, full_tree, commit_message)
+        self.push(self.repo)
 
     def update_environment(
-        self, new_env, current_name: str, current_path: str
+        self, new_env, current_name: str, current_path: str, commit_message: str
     ):
         """Update an existing environment folder in GitLab.
 
         Args:
-            repo: a bare repository
             new_env: an updated Environment object
             current_name: the current name of the environment
             current_path: the current path of the environment
@@ -442,9 +439,8 @@ def update_environment(
             path = Path(self.environments_root) / current_path / current_name
             target_tree = root_tree[path]
             self.create_environment(
-                self.repo,
                 new_env,
-                "update existing environment",
+                commit_message,
                 target_tree,
             )
         else:
@@ -452,3 +448,25 @@ def update_environment(
             raise KeyError("not matching name or path")
             # self.delete_environment()
             # self.create_environment()
+
+    def delete_environment(self, name, path, commit_message):
+        """Delete an environment folder in GitLab.
+        
+        Args:
+            name: the name of the environment
+            path: the path of the environment
+        """
+        # Get repository tree
+        root_tree = self.repo.head.peel(pygit2.Tree)
+        # Find environment in the tree
+        full_path = Path(self.environments_root) / path 
+        target_tree = root_tree[full_path]
+        # Remove the environment
+        tree_builder = self.repo.TreeBuilder(target_tree)
+        tree_builder.remove(name)
+        new_tree = tree_builder.write()
+        full_tree = self.build_tree(self.repo, root_tree, new_tree, full_path)
+
+        # Commit and push
+        self.commit(self.repo, full_tree, commit_message)
+        self.push(self.repo)
diff --git a/softpack_core/schemas/environment.py b/softpack_core/schemas/environment.py
index ce9d72a..2dbceb8 100644
--- a/softpack_core/schemas/environment.py
+++ b/softpack_core/schemas/environment.py
@@ -102,13 +102,13 @@ def from_artifact(cls, obj: Artifacts.Object) -> "Environment":
 
     @classmethod
     def create(cls, env: EnvironmentInput):
-        """Create an Environment object.
+        """Create an Environment.
 
         Args:
             env: Details of the new environment
 
         Returns:
-            Environment: A newly created Environment.
+            A message confirming the success or failure of the operation.
         """
         # Check if any field has been left empty
         if any(len(value) == 0 for value in vars(env).values()):
@@ -143,7 +143,6 @@ def create(cls, env: EnvironmentInput):
 
         try:
             cls.artifacts.create_environment(
-                cls.artifacts.repo,
                 new_env,
                 "create new environment",
             )
@@ -159,7 +158,7 @@ def update(
         current_path: str,
         current_name: str,
     ):
-        """Update an Environment object.
+        """Update an Environment.
 
         Args:
             env: Details of the updated environment
@@ -167,7 +166,7 @@ def update(
             name: The name of the current environment
 
         Returns:
-            Environment: An updated Environment.
+            A message confirming the success or failure of the operation.
         """
         # Check if any field has been left empty
         if any(len(value) == 0 for value in vars(env).values()) or current_name == "" or current_path == "":
@@ -197,7 +196,7 @@ def update(
 
             try:
                 cls.artifacts.update_environment(
-                    new_env, current_name, current_path
+                    new_env, current_name, current_path, "update existing environment"
                 )
             except RuntimeError as e:
                 return InvalidInputError(message=str(e))
@@ -207,16 +206,21 @@ def update(
         return EnvironmentNotFoundError(message="Unable to find an environment of this name in this location", path=current_path, name=current_name)
 
     @classmethod
-    def delete(cls, name: str):
-        """Delete an Environment object.
+    def delete(cls, name: str, path: str):
+        """Delete an Environment.
+
+        Args:
+            name: the name of of environment
+            path: the path of the environment
 
         Returns:
-            A string confirming the deletion of the Environment
+            A message confirming the success or failure of the operation.
         """
-        for env in Environment.iter():
-            if env.name == name:
-                return f"Deleted {name}"
-        return "An environment with that name was not found"
+        if cls.artifacts.get(Path(path), name):
+            cls.artifacts.delete_environment(name, path, "delete environment")
+            return DeleteEnvironmentSuccess(message="Successfully deleted the environment")
+
+        return EnvironmentNotFoundError(message="An environment with that name was not found", path=path, name=name)
 
     @classmethod
     async def upload_file(cls, file: Upload):
@@ -252,6 +256,11 @@ class UpdateEnvironmentSuccess(Success):
     message: str
     environment: Environment
 
+@strawberry.type
+class DeleteEnvironmentSuccess(Success):
+    """Environment successfully deleted."""
+
+    message: str
 
 # Error types
 @strawberry.type
@@ -291,6 +300,12 @@ class EnvironmentAlreadyExistsError(Error):
                         ]
 )
 
+DeleteResponse = strawberry.union(
+    "DeleteResponse", [DeleteEnvironmentSuccess,
+                       EnvironmentNotFoundError,
+                       ]
+)
+
 
 class EnvironmentSchema(BaseSchema):
     """Environment schema."""
@@ -307,5 +322,5 @@ class Mutation:
 
         createEnvironment: CreateResponse = Environment.create  # type: ignore
         updateEnvironment: UpdateResponse = Environment.update  # type: ignore
-        deleteEnvironment: str = Environment.delete  # type: ignore
+        deleteEnvironment: DeleteResponse = Environment.delete  # type: ignore
         upload_file: str = Environment.upload_file  # type: ignore

From a6c40a3763b13af2fd9c84a3f5271761856f0bc2 Mon Sep 17 00:00:00 2001
From: Samuel Ogg <56794292+SamO135@users.noreply.github.com>
Date: Thu, 3 Aug 2023 11:24:04 +0100
Subject: [PATCH 015/129] prevent replacing environments when changing
 locations

---
 softpack_core/artifacts.py           | 51 ++++++++++++++++++----------
 softpack_core/schemas/environment.py |  5 ++-
 2 files changed, 38 insertions(+), 18 deletions(-)

diff --git a/softpack_core/artifacts.py b/softpack_core/artifacts.py
index 17ee5f2..d29945c 100644
--- a/softpack_core/artifacts.py
+++ b/softpack_core/artifacts.py
@@ -349,7 +349,7 @@ def build_tree(
             path = path.parent
         return new_tree
 
-    def generate_yaml_contents(self, env):
+    def generate_yaml_contents(self, env) -> str:
         """Generate the softpack.yml file contents.
 
         Args:
@@ -365,10 +365,12 @@ def generate_yaml_contents(self, env):
 
     def create_environment(
         self,
-        env,
+        new_env,
         commit_message: str,
         target_tree: Optional[pygit2.Tree] = None,
-    ):
+        replace: bool = False,
+        push: bool = True,
+    ) -> pygit2.Oid | None:
         """Create, commit and push a new environment folder to GitLab.
 
         Args:
@@ -376,11 +378,16 @@ def create_environment(
             commit_message: the commit message
             target_tree: pygit2.Tree object with the environment folder you
             want to update as its root
+            replace: if true, it will replace any existing environment at the
+            specified location
+            push: whether or not to push the new environment to the GitLab repo
         """
+        if not replace and self.get(Path(new_env.path), new_env.name):
+            raise FileExistsError()
         root_tree = self.repo.head.peel(pygit2.Tree)
 
         # Create new file
-        contents = self.generate_yaml_contents(env)
+        contents = self.generate_yaml_contents(new_env)
         file_oid = self.repo.create_blob(contents.encode())
 
         # Put new file into new env folder
@@ -396,8 +403,8 @@ def create_environment(
         # Expand tree to include the whole repo
         full_path = (
             Path(self.environments_root)
-            / env.path
-            / env.name
+            / new_env.path
+            / new_env.name
             / self.environments_file
         )
         full_tree = self.build_tree(
@@ -419,19 +426,23 @@ def create_environment(
                         {new_file.path} instead of {full_path}"
                 )
 
-        # Commit and push
-        self.commit(self.repo, full_tree, commit_message)
-        self.push(self.repo)
+        if push:
+            # Commit and push
+            self.commit(self.repo, full_tree, commit_message)
+            self.push(self.repo)
+        else:
+            return full_tree
 
     def update_environment(
-        self, new_env, current_name: str, current_path: str, commit_message: str
-    ):
+        self, current_name: str, current_path: str, new_env, commit_message: str
+    ) -> None:
         """Update an existing environment folder in GitLab.
 
         Args:
-            new_env: an updated Environment object
             current_name: the current name of the environment
             current_path: the current path of the environment
+            new_env: an updated Environment object
+            commit_message: the commit_message
         """
         if new_env.name == current_name and new_env.path == current_path:
             # Update environment in the same location
@@ -442,22 +453,28 @@ def update_environment(
                 new_env,
                 commit_message,
                 target_tree,
+                replace=True,
             )
         else:
             # Update environment in a new location
-            raise KeyError("not matching name or path")
-            # self.delete_environment()
-            # self.create_environment()
+            tree_oid = self.create_environment(new_env, "create new environment", push=False)
+            self.delete_environment(current_name, current_path, commit_message, tree_oid=tree_oid)
 
-    def delete_environment(self, name, path, commit_message):
+    def delete_environment(self, name: str, path: str, commit_message: str, tree_oid: Optional[pygit2.Oid]=None) -> None:
         """Delete an environment folder in GitLab.
         
         Args:
             name: the name of the environment
             path: the path of the environment
+            commit_message: the commit message
+            tree_oid: a Pygit2.Oid object representing a tree. If None, 
+            a tree will be created from the artifacts repo.
         """
         # Get repository tree
-        root_tree = self.repo.head.peel(pygit2.Tree)
+        if not tree_oid:
+            root_tree = self.repo.head.peel(pygit2.Tree)
+        else:
+            root_tree = self.repo.get(tree_oid)
         # Find environment in the tree
         full_path = Path(self.environments_root) / path 
         target_tree = root_tree[full_path]
diff --git a/softpack_core/schemas/environment.py b/softpack_core/schemas/environment.py
index 2dbceb8..0cd021e 100644
--- a/softpack_core/schemas/environment.py
+++ b/softpack_core/schemas/environment.py
@@ -196,10 +196,12 @@ def update(
 
             try:
                 cls.artifacts.update_environment(
-                    new_env, current_name, current_path, "update existing environment"
+                    current_name, current_path, new_env, "update existing environment"
                 )
             except RuntimeError as e:
                 return InvalidInputError(message=str(e))
+            except FileExistsError:
+                return EnvironmentAlreadyExistsError(message="An environment of this name already exists in this location", path=env.path, name=env.name)
 
             return UpdateEnvironmentSuccess(message="Successfully updated environment", environment=new_env)
 
@@ -297,6 +299,7 @@ class EnvironmentAlreadyExistsError(Error):
     "UpdateResponse", [UpdateEnvironmentSuccess,
                        InvalidInputError,
                        EnvironmentNotFoundError,
+                       EnvironmentAlreadyExistsError,
                         ]
 )
 

From 58a474b0b94dac6f2495a7d80b25694545282bc2 Mon Sep 17 00:00:00 2001
From: Samuel Ogg <56794292+SamO135@users.noreply.github.com>
Date: Mon, 7 Aug 2023 17:09:06 +0100
Subject: [PATCH 016/129] add file upload mutation for softpack-builder
 Incorporated the use of this mutation into the create environment mutation

---
 softpack_core/artifacts.py           | 104 +++++++++++++++------------
 softpack_core/schemas/environment.py |  88 +++++++++++++----------
 2 files changed, 109 insertions(+), 83 deletions(-)

diff --git a/softpack_core/artifacts.py b/softpack_core/artifacts.py
index d29945c..e6a9498 100644
--- a/softpack_core/artifacts.py
+++ b/softpack_core/artifacts.py
@@ -363,56 +363,44 @@ def generate_yaml_contents(self, env) -> str:
         contents = f"description: {env.description}\npackages:\n{packages}\n"
         return contents
 
-    def create_environment(
-        self,
-        new_env,
-        commit_message: str,
-        target_tree: Optional[pygit2.Tree] = None,
-        replace: bool = False,
-        push: bool = True,
-    ) -> pygit2.Oid | None:
-        """Create, commit and push a new environment folder to GitLab.
-
+    def create_file(self, folder_path: Path, file_name: str, contents: str, new_folder: bool=False, replace: bool=False) -> pygit2.Oid:
+        """Create a file in the artifacts repo.
+        
         Args:
-            env: an Environment object
-            commit_message: the commit message
-            target_tree: pygit2.Tree object with the environment folder you
-            want to update as its root
-            replace: if true, it will replace any existing environment at the
-            specified location
-            push: whether or not to push the new environment to the GitLab repo
+            folder_path: the path to the folder the file will be placed in
+            file_name: the name of the file
+            contents: the contents of the file
+            new_folder: if True, create the file's parent folder as well
+            replace: if True, replace any existing file with the same name in 
+            the specified location
+            
+        Returns:
+            the OID of the new tree structure of the repository
         """
-        if not replace and self.get(Path(new_env.path), new_env.name):
+        if not replace and self.get(Path(folder_path), file_name):
             raise FileExistsError()
+
         root_tree = self.repo.head.peel(pygit2.Tree)
+        full_path = Path(self.environments_root) / folder_path
 
-        # Create new file
-        contents = self.generate_yaml_contents(new_env)
+        # Create file
         file_oid = self.repo.create_blob(contents.encode())
 
-        # Put new file into new env folder
-        if target_tree:
-            new_treebuilder = self.repo.TreeBuilder(target_tree)
-        else:
+        # Put file in folder
+        if new_folder:
             new_treebuilder = self.repo.TreeBuilder()
-        new_treebuilder.insert(
-            self.environments_file, file_oid, pygit2.GIT_FILEMODE_BLOB
-        )
+        else:
+            folder = root_tree[full_path]
+            new_treebuilder = self.repo.TreeBuilder(folder)
+        new_treebuilder.insert(file_name, file_oid, pygit2.GIT_FILEMODE_BLOB)
         new_tree = new_treebuilder.write()
 
-        # Expand tree to include the whole repo
-        full_path = (
-            Path(self.environments_root)
-            / new_env.path
-            / new_env.name
-            / self.environments_file
-        )
-        full_tree = self.build_tree(
-            self.repo, root_tree, new_tree, full_path.parent
-        )
+        # Expand to include the whole repo
+        full_tree = self.build_tree(self.repo, root_tree, new_tree, full_path)
 
-        new_tree = self.repo.get(full_tree)
         # Check for errors in the new tree
+        new_tree = self.repo.get(full_tree)
+        path = Path(self.environments_root) / folder_path / file_name
         diff = self.repo.diff(new_tree, root_tree)
         if len(diff) > 1:
             raise RuntimeError("Too many changes to the repo")
@@ -420,18 +408,42 @@ def create_environment(
             raise RuntimeError("No changes made to the environment")
         elif len(diff) == 1:
             new_file = diff[0].delta.new_file
-            if new_file.path != str(full_path):
+            if new_file.path != str(path):
                 raise RuntimeError(
                     f"Attempted to add new file added to incorrect path: \
-                        {new_file.path} instead of {full_path}"
+                        {new_file.path} instead of {path}"
                 )
 
-        if push:
-            # Commit and push
-            self.commit(self.repo, full_tree, commit_message)
-            self.push(self.repo)
-        else:
-            return full_tree
+        return full_tree
+
+
+    def create_environment(
+        self,
+        new_env,
+        commit_message: str,
+        target_tree: Optional[pygit2.Tree] = None,
+        replace: bool = False,
+        push: bool = True,
+    ) -> pygit2.Oid | None:
+        """Create, commit and push a new environment folder to GitLab.
+
+        Args:
+            env: an Environment object
+            commit_message: the commit message
+            target_tree: pygit2.Tree object with the environment folder you
+            want to update as its root
+            replace: if true, it will replace any existing environment at the
+            specified location
+            push: whether or not to push the new environment to the GitLab repo
+        """
+        new_folder_path = Path(new_env.path) / new_env.name
+        file_name = "README.md"
+        tree_oid = self.create_file(new_folder_path, file_name, "lorem ipsum", True)
+
+        # Commit and push
+        self.commit(self.repo, tree_oid, commit_message)
+        self.push(self.repo)
+
 
     def update_environment(
         self, current_name: str, current_path: str, new_env, commit_message: str
diff --git a/softpack_core/schemas/environment.py b/softpack_core/schemas/environment.py
index 0cd021e..c73dac9 100644
--- a/softpack_core/schemas/environment.py
+++ b/softpack_core/schemas/environment.py
@@ -74,8 +74,9 @@ def iter(cls, all: bool = False) -> Iterable["Environment"]:
             # TODO: set username from the environment for now
             # eventually this needs to be the name of the authenticated user
             user = os.environ["USER"]
-        environments = cls.artifacts.iter(user=user)
-        return map(cls.from_artifact, environments)
+        environment_folders = cls.artifacts.iter(user=user)
+        environment_objects = map(cls.from_artifact, environment_folders)
+        return filter(lambda x: x is not None, environment_objects)
 
     @classmethod
     def from_artifact(cls, obj: Artifacts.Object) -> "Environment":
@@ -87,18 +88,21 @@ def from_artifact(cls, obj: Artifacts.Object) -> "Environment":
         Returns:
             Environment: An Environment object.
         """
-        spec = obj.spec()
-        return Environment(
-            id=obj.oid,
-            name=obj.name,
-            path=obj.path.parent,
-            description=spec.description,
-            packages=map(
-                lambda package: Package(id=package, name=package),
-                spec.packages,
-            ),  # type: ignore [call-arg]
-            state=None,
-        )
+        try:
+            spec = obj.spec()
+            return Environment(
+                id=obj.oid,
+                name=obj.name,
+                path=obj.path.parent,
+                description=spec.description,
+                packages=map(
+                    lambda package: Package(id=package, name=package),
+                    spec.packages,
+                ),  # type: ignore [call-arg]
+                state=None,
+            )
+        except KeyError:
+            return None
 
     @classmethod
     def create(cls, env: EnvironmentInput):
@@ -113,14 +117,27 @@ def create(cls, env: EnvironmentInput):
         # Check if any field has been left empty
         if any(len(value) == 0 for value in vars(env).values()):
             return InvalidInputError(message="all fields must be filled in")
+        
         # Check if a valid path has been provided
         user = os.environ["USER"]
         if env.path not in ["groups/hgi", f"users/{user}"]:
             return InvalidInputError(message="Invalid path")
+        
         # Check if an env with same name already exists at given path
         if cls.artifacts.get(Path(env.path), env.name):
             return EnvironmentAlreadyExistsError(message="An environment of this name already exists in this location", path=env.path, name=env.name)
 
+        # Create folder with readme
+        new_folder_path = Path(env.path) / env.name
+        file_name = "README.md"
+        try:
+            tree_oid = cls.artifacts.create_file(new_folder_path, file_name, "lorem ipsum", True)
+            cls.artifacts.commit(cls.artifacts.repo, tree_oid, "create empty environment")
+            cls.artifacts.push(cls.artifacts.repo)
+        except RuntimeError as e:
+            return InvalidInputError(message=str(e))
+
+        # Send build request
         response = httpx.post(
             "http://0.0.0.0:7080/environments/build",
             json={
@@ -132,24 +149,8 @@ def create(cls, env: EnvironmentInput):
             },
         ).json()
         print(f"Create: {response}")
-        new_env = Environment(
-            id=uuid.uuid4().hex,
-            name=env.name,
-            path=env.path,
-            description=env.description,
-            packages=list(map(lambda pkg: pkg.to_package(), env.packages)),
-            state=response['state']['type'],
-        )  # type: ignore [call-arg]
-
-        try:
-            cls.artifacts.create_environment(
-                new_env,
-                "create new environment",
-            )
-        except RuntimeError as e:
-            return InvalidInputError(message=str(e))
         
-        return CreateEnvironmentSuccess(message="Successfully scheduled environment creation", environment=new_env)
+        return CreateEnvironmentSuccess(message="Successfully scheduled environment creation")
 
     @classmethod
     def update(
@@ -203,7 +204,7 @@ def update(
             except FileExistsError:
                 return EnvironmentAlreadyExistsError(message="An environment of this name already exists in this location", path=env.path, name=env.name)
 
-            return UpdateEnvironmentSuccess(message="Successfully updated environment", environment=new_env)
+            return UpdateEnvironmentSuccess(message="Successfully updated environment")
 
         return EnvironmentNotFoundError(message="Unable to find an environment of this name in this location", path=current_path, name=current_name)
 
@@ -225,8 +226,23 @@ def delete(cls, name: str, path: str):
         return EnvironmentNotFoundError(message="An environment with that name was not found", path=path, name=name)
 
     @classmethod
-    async def upload_file(cls, file: Upload):
-        return (await file.read()).decode("utf-8")  # type: ignore
+    async def create_artifact(cls, file: Upload, folder_path: str, file_name: str):
+        """Add a file to the Artifacts repo.
+        
+        Args:
+            file: the file to add to the repo
+            folder_path: the path to the folder that the file will be added to
+            file_name: the name of the file
+        """
+        try:
+            contents = (await file.read()).decode()
+            tree_oid = cls.artifacts.create_file(Path(folder_path), file_name, contents, replace=True)
+            cls.artifacts.commit(cls.artifacts.repo, tree_oid, "create artifact")
+            cls.artifacts.push(cls.artifacts.repo)
+            return "created artifact"
+        except Exception as e:
+            return f"something went wrong when creating the artifact: {e}"
+
 
 
 # Interfaces
@@ -249,14 +265,12 @@ class CreateEnvironmentSuccess(Success):
     """Environment successfully scheduled."""
 
     message: str
-    environment: Environment
 
 @strawberry.type
 class UpdateEnvironmentSuccess(Success):
     """Environment successfully updated."""
 
     message: str
-    environment: Environment
 
 @strawberry.type
 class DeleteEnvironmentSuccess(Success):
@@ -326,4 +340,4 @@ class Mutation:
         createEnvironment: CreateResponse = Environment.create  # type: ignore
         updateEnvironment: UpdateResponse = Environment.update  # type: ignore
         deleteEnvironment: DeleteResponse = Environment.delete  # type: ignore
-        upload_file: str = Environment.upload_file  # type: ignore
+        createArtifact: str = Environment.create_artifact  # type: ignore

From b682c9e7824b3043aed305a3782d6dbc50391096 Mon Sep 17 00:00:00 2001
From: Samuel Ogg <56794292+SamO135@users.noreply.github.com>
Date: Tue, 8 Aug 2023 15:25:19 +0100
Subject: [PATCH 017/129] format code with tox

---
 softpack_core/artifacts.py           |  51 ++++++++----
 softpack_core/schemas/environment.py | 120 +++++++++++++++++++--------
 2 files changed, 121 insertions(+), 50 deletions(-)

diff --git a/softpack_core/artifacts.py b/softpack_core/artifacts.py
index e6a9498..58d7e46 100644
--- a/softpack_core/artifacts.py
+++ b/softpack_core/artifacts.py
@@ -363,17 +363,24 @@ def generate_yaml_contents(self, env) -> str:
         contents = f"description: {env.description}\npackages:\n{packages}\n"
         return contents
 
-    def create_file(self, folder_path: Path, file_name: str, contents: str, new_folder: bool=False, replace: bool=False) -> pygit2.Oid:
+    def create_file(
+        self,
+        folder_path: Path,
+        file_name: str,
+        contents: str,
+        new_folder: bool = False,
+        replace: bool = False,
+    ) -> pygit2.Oid:
         """Create a file in the artifacts repo.
-        
+
         Args:
             folder_path: the path to the folder the file will be placed in
             file_name: the name of the file
             contents: the contents of the file
             new_folder: if True, create the file's parent folder as well
-            replace: if True, replace any existing file with the same name in 
+            replace: if True, replace any existing file with the same name in
             the specified location
-            
+
         Returns:
             the OID of the new tree structure of the repository
         """
@@ -416,7 +423,6 @@ def create_file(self, folder_path: Path, file_name: str, contents: str, new_fold
 
         return full_tree
 
-
     def create_environment(
         self,
         new_env,
@@ -428,7 +434,7 @@ def create_environment(
         """Create, commit and push a new environment folder to GitLab.
 
         Args:
-            env: an Environment object
+            new_env: an Environment object
             commit_message: the commit message
             target_tree: pygit2.Tree object with the environment folder you
             want to update as its root
@@ -438,15 +444,20 @@ def create_environment(
         """
         new_folder_path = Path(new_env.path) / new_env.name
         file_name = "README.md"
-        tree_oid = self.create_file(new_folder_path, file_name, "lorem ipsum", True)
+        tree_oid = self.create_file(
+            new_folder_path, file_name, "lorem ipsum", True
+        )
 
         # Commit and push
         self.commit(self.repo, tree_oid, commit_message)
         self.push(self.repo)
 
-
     def update_environment(
-        self, current_name: str, current_path: str, new_env, commit_message: str
+        self,
+        current_name: str,
+        current_path: str,
+        new_env,
+        commit_message: str,
     ) -> None:
         """Update an existing environment folder in GitLab.
 
@@ -469,17 +480,27 @@ def update_environment(
             )
         else:
             # Update environment in a new location
-            tree_oid = self.create_environment(new_env, "create new environment", push=False)
-            self.delete_environment(current_name, current_path, commit_message, tree_oid=tree_oid)
+            tree_oid = self.create_environment(
+                new_env, "create new environment", push=False
+            )
+            self.delete_environment(
+                current_name, current_path, commit_message, tree_oid=tree_oid
+            )
 
-    def delete_environment(self, name: str, path: str, commit_message: str, tree_oid: Optional[pygit2.Oid]=None) -> None:
+    def delete_environment(
+        self,
+        name: str,
+        path: str,
+        commit_message: str,
+        tree_oid: Optional[pygit2.Oid] = None,
+    ) -> None:
         """Delete an environment folder in GitLab.
-        
+
         Args:
             name: the name of the environment
             path: the path of the environment
             commit_message: the commit message
-            tree_oid: a Pygit2.Oid object representing a tree. If None, 
+            tree_oid: a Pygit2.Oid object representing a tree. If None,
             a tree will be created from the artifacts repo.
         """
         # Get repository tree
@@ -488,7 +509,7 @@ def delete_environment(self, name: str, path: str, commit_message: str, tree_oid
         else:
             root_tree = self.repo.get(tree_oid)
         # Find environment in the tree
-        full_path = Path(self.environments_root) / path 
+        full_path = Path(self.environments_root) / path
         target_tree = root_tree[full_path]
         # Remove the environment
         tree_builder = self.repo.TreeBuilder(target_tree)
diff --git a/softpack_core/schemas/environment.py b/softpack_core/schemas/environment.py
index c73dac9..80f7f0f 100644
--- a/softpack_core/schemas/environment.py
+++ b/softpack_core/schemas/environment.py
@@ -32,12 +32,12 @@ class PackageInput(Package):
 
     id: Optional[str] = None
 
-    def to_package(self):
+    def to_package(self) -> Package:
         """Create a Package object from a PackageInput object.
 
         Return: a Package object
         """
-        return Package(**self.__dict__)
+        return Package(**vars(self))
 
 
 @strawberry.input
@@ -117,22 +117,30 @@ def create(cls, env: EnvironmentInput):
         # Check if any field has been left empty
         if any(len(value) == 0 for value in vars(env).values()):
             return InvalidInputError(message="all fields must be filled in")
-        
+
         # Check if a valid path has been provided
         user = os.environ["USER"]
         if env.path not in ["groups/hgi", f"users/{user}"]:
             return InvalidInputError(message="Invalid path")
-        
+
         # Check if an env with same name already exists at given path
         if cls.artifacts.get(Path(env.path), env.name):
-            return EnvironmentAlreadyExistsError(message="An environment of this name already exists in this location", path=env.path, name=env.name)
+            return EnvironmentAlreadyExistsError(
+                message="This name is already used in this location",
+                path=env.path,
+                name=env.name,
+            )
 
         # Create folder with readme
         new_folder_path = Path(env.path) / env.name
         file_name = "README.md"
         try:
-            tree_oid = cls.artifacts.create_file(new_folder_path, file_name, "lorem ipsum", True)
-            cls.artifacts.commit(cls.artifacts.repo, tree_oid, "create empty environment")
+            tree_oid = cls.artifacts.create_file(
+                new_folder_path, file_name, "lorem ipsum", True
+            )
+            cls.artifacts.commit(
+                cls.artifacts.repo, tree_oid, "create empty environment"
+            )
             cls.artifacts.push(cls.artifacts.repo)
         except RuntimeError as e:
             return InvalidInputError(message=str(e))
@@ -149,8 +157,10 @@ def create(cls, env: EnvironmentInput):
             },
         ).json()
         print(f"Create: {response}")
-        
-        return CreateEnvironmentSuccess(message="Successfully scheduled environment creation")
+
+        return CreateEnvironmentSuccess(
+            message="Successfully scheduled environment creation"
+        )
 
     @classmethod
     def update(
@@ -170,7 +180,11 @@ def update(
             A message confirming the success or failure of the operation.
         """
         # Check if any field has been left empty
-        if any(len(value) == 0 for value in vars(env).values()) or current_name == "" or current_path == "":
+        if (
+            any(len(value) == 0 for value in vars(env).values())
+            or current_name == ""
+            or current_path == ""
+        ):
             return InvalidInputError(message="all fields must be filled in")
         # Check if an environment exists at the specified path and name
         if cls.artifacts.get(Path(current_path), current_name):
@@ -197,16 +211,29 @@ def update(
 
             try:
                 cls.artifacts.update_environment(
-                    current_name, current_path, new_env, "update existing environment"
+                    current_name,
+                    current_path,
+                    new_env,
+                    "update existing environment",
                 )
             except RuntimeError as e:
                 return InvalidInputError(message=str(e))
             except FileExistsError:
-                return EnvironmentAlreadyExistsError(message="An environment of this name already exists in this location", path=env.path, name=env.name)
+                return EnvironmentAlreadyExistsError(
+                    message="This name is already used in this location",
+                    path=env.path,
+                    name=env.name,
+                )
 
-            return UpdateEnvironmentSuccess(message="Successfully updated environment")
+            return UpdateEnvironmentSuccess(
+                message="Successfully updated environment"
+            )
 
-        return EnvironmentNotFoundError(message="Unable to find an environment of this name in this location", path=current_path, name=current_name)
+        return EnvironmentNotFoundError(
+            message="No environment with this name found in this location.",
+            path=current_path,
+            name=current_name,
+        )
 
     @classmethod
     def delete(cls, name: str, path: str):
@@ -221,14 +248,22 @@ def delete(cls, name: str, path: str):
         """
         if cls.artifacts.get(Path(path), name):
             cls.artifacts.delete_environment(name, path, "delete environment")
-            return DeleteEnvironmentSuccess(message="Successfully deleted the environment")
+            return DeleteEnvironmentSuccess(
+                message="Successfully deleted the environment"
+            )
 
-        return EnvironmentNotFoundError(message="An environment with that name was not found", path=path, name=name)
+        return EnvironmentNotFoundError(
+            message="No environment with this name found in this location.",
+            path=path,
+            name=name,
+        )
 
     @classmethod
-    async def create_artifact(cls, file: Upload, folder_path: str, file_name: str):
+    async def create_artifact(
+        cls, file: Upload, folder_path: str, file_name: str
+    ):
         """Add a file to the Artifacts repo.
-        
+
         Args:
             file: the file to add to the repo
             folder_path: the path to the folder that the file will be added to
@@ -236,15 +271,18 @@ async def create_artifact(cls, file: Upload, folder_path: str, file_name: str):
         """
         try:
             contents = (await file.read()).decode()
-            tree_oid = cls.artifacts.create_file(Path(folder_path), file_name, contents, replace=True)
-            cls.artifacts.commit(cls.artifacts.repo, tree_oid, "create artifact")
+            tree_oid = cls.artifacts.create_file(
+                Path(folder_path), file_name, contents, replace=True
+            )
+            cls.artifacts.commit(
+                cls.artifacts.repo, tree_oid, "create artifact"
+            )
             cls.artifacts.push(cls.artifacts.repo)
             return "created artifact"
         except Exception as e:
             return f"something went wrong when creating the artifact: {e}"
 
 
-
 # Interfaces
 @strawberry.interface
 class Success:
@@ -252,6 +290,7 @@ class Success:
 
     message: str
 
+
 @strawberry.interface
 class Error:
     """Interface for errors."""
@@ -266,25 +305,29 @@ class CreateEnvironmentSuccess(Success):
 
     message: str
 
+
 @strawberry.type
 class UpdateEnvironmentSuccess(Success):
     """Environment successfully updated."""
 
     message: str
 
+
 @strawberry.type
 class DeleteEnvironmentSuccess(Success):
     """Environment successfully deleted."""
 
     message: str
 
+
 # Error types
 @strawberry.type
 class InvalidInputError(Error):
-    """Invalid input data"""
+    """Invalid input data."""
 
     message: str
 
+
 @strawberry.type
 class EnvironmentNotFoundError(Error):
     """Environment not found."""
@@ -293,34 +336,41 @@ class EnvironmentNotFoundError(Error):
     path: str
     name: str
 
+
 @strawberry.type
 class EnvironmentAlreadyExistsError(Error):
     """Environment name already exists."""
 
-    message:str
+    message: str
     path: str
     name: str
 
 
 CreateResponse = strawberry.union(
-    "CreateResponse", [CreateEnvironmentSuccess,
-                       InvalidInputError,
-                       EnvironmentAlreadyExistsError,
-                        ]
+    "CreateResponse",
+    [
+        CreateEnvironmentSuccess,
+        InvalidInputError,
+        EnvironmentAlreadyExistsError,
+    ],
 )
 
 UpdateResponse = strawberry.union(
-    "UpdateResponse", [UpdateEnvironmentSuccess,
-                       InvalidInputError,
-                       EnvironmentNotFoundError,
-                       EnvironmentAlreadyExistsError,
-                        ]
+    "UpdateResponse",
+    [
+        UpdateEnvironmentSuccess,
+        InvalidInputError,
+        EnvironmentNotFoundError,
+        EnvironmentAlreadyExistsError,
+    ],
 )
 
 DeleteResponse = strawberry.union(
-    "DeleteResponse", [DeleteEnvironmentSuccess,
-                       EnvironmentNotFoundError,
-                       ]
+    "DeleteResponse",
+    [
+        DeleteEnvironmentSuccess,
+        EnvironmentNotFoundError,
+    ],
 )
 
 

From 634e0512bfcad55356e014c3b95b92ba72915f2b Mon Sep 17 00:00:00 2001
From: Samuel Ogg <56794292+SamO135@users.noreply.github.com>
Date: Fri, 11 Aug 2023 10:32:40 +0100
Subject: [PATCH 018/129] rework update method using create_artifacts
 create_artifacts returns oid of new commit

---
 softpack_core/artifacts.py           | 86 ++++------------------------
 softpack_core/schemas/environment.py | 78 ++++++++++++-------------
 2 files changed, 49 insertions(+), 115 deletions(-)

diff --git a/softpack_core/artifacts.py b/softpack_core/artifacts.py
index 58d7e46..571e97f 100644
--- a/softpack_core/artifacts.py
+++ b/softpack_core/artifacts.py
@@ -260,7 +260,7 @@ def get(self, path: Path, name: str) -> Optional[pygit2.Tree]:
 
     def commit(
         self, repo: pygit2.Repository, tree_oid: pygit2.Oid, message: str
-    ) -> pygit2.Commit:
+    ) -> pygit2.Oid:
         """Create and return a commit.
 
         Args:
@@ -278,9 +278,10 @@ def commit(
             self.settings.artifacts.repo.email,
         )
         parents = [repo.lookup_reference(ref).target]
-        return repo.create_commit(
+        commit_oid = repo.create_commit(
             ref, author, committer, message, tree_oid, parents
         )
+        return commit_oid
 
     def error_callback(self, refname: str, message: str) -> None:
         """Push update reference callback.
@@ -369,7 +370,7 @@ def create_file(
         file_name: str,
         contents: str,
         new_folder: bool = False,
-        replace: bool = False,
+        overwrite: bool = False,
     ) -> pygit2.Oid:
         """Create a file in the artifacts repo.
 
@@ -378,17 +379,16 @@ def create_file(
             file_name: the name of the file
             contents: the contents of the file
             new_folder: if True, create the file's parent folder as well
-            replace: if True, replace any existing file with the same name in
-            the specified location
+            overwrite: if True, overwrite the file at the specified path
 
         Returns:
             the OID of the new tree structure of the repository
         """
-        if not replace and self.get(Path(folder_path), file_name):
-            raise FileExistsError()
+        if not overwrite and self.get(Path(folder_path), file_name):
+            raise FileExistsError("File already exists")
 
         root_tree = self.repo.head.peel(pygit2.Tree)
-        full_path = Path(self.environments_root) / folder_path
+        full_path = Path(self.environments_root, folder_path)
 
         # Create file
         file_oid = self.repo.create_blob(contents.encode())
@@ -407,7 +407,7 @@ def create_file(
 
         # Check for errors in the new tree
         new_tree = self.repo.get(full_tree)
-        path = Path(self.environments_root) / folder_path / file_name
+        path = Path(self.environments_root, folder_path, file_name)
         diff = self.repo.diff(new_tree, root_tree)
         if len(diff) > 1:
             raise RuntimeError("Too many changes to the repo")
@@ -418,75 +418,11 @@ def create_file(
             if new_file.path != str(path):
                 raise RuntimeError(
                     f"Attempted to add new file added to incorrect path: \
-                        {new_file.path} instead of {path}"
+                        {new_file.path} instead of {str(path)}"
                 )
 
         return full_tree
 
-    def create_environment(
-        self,
-        new_env,
-        commit_message: str,
-        target_tree: Optional[pygit2.Tree] = None,
-        replace: bool = False,
-        push: bool = True,
-    ) -> pygit2.Oid | None:
-        """Create, commit and push a new environment folder to GitLab.
-
-        Args:
-            new_env: an Environment object
-            commit_message: the commit message
-            target_tree: pygit2.Tree object with the environment folder you
-            want to update as its root
-            replace: if true, it will replace any existing environment at the
-            specified location
-            push: whether or not to push the new environment to the GitLab repo
-        """
-        new_folder_path = Path(new_env.path) / new_env.name
-        file_name = "README.md"
-        tree_oid = self.create_file(
-            new_folder_path, file_name, "lorem ipsum", True
-        )
-
-        # Commit and push
-        self.commit(self.repo, tree_oid, commit_message)
-        self.push(self.repo)
-
-    def update_environment(
-        self,
-        current_name: str,
-        current_path: str,
-        new_env,
-        commit_message: str,
-    ) -> None:
-        """Update an existing environment folder in GitLab.
-
-        Args:
-            current_name: the current name of the environment
-            current_path: the current path of the environment
-            new_env: an updated Environment object
-            commit_message: the commit_message
-        """
-        if new_env.name == current_name and new_env.path == current_path:
-            # Update environment in the same location
-            root_tree = self.repo.head.peel(pygit2.Tree)
-            path = Path(self.environments_root) / current_path / current_name
-            target_tree = root_tree[path]
-            self.create_environment(
-                new_env,
-                commit_message,
-                target_tree,
-                replace=True,
-            )
-        else:
-            # Update environment in a new location
-            tree_oid = self.create_environment(
-                new_env, "create new environment", push=False
-            )
-            self.delete_environment(
-                current_name, current_path, commit_message, tree_oid=tree_oid
-            )
-
     def delete_environment(
         self,
         name: str,
@@ -509,7 +445,7 @@ def delete_environment(
         else:
             root_tree = self.repo.get(tree_oid)
         # Find environment in the tree
-        full_path = Path(self.environments_root) / path
+        full_path = Path(self.environments_root, path)
         target_tree = root_tree[full_path]
         # Remove the environment
         tree_builder = self.repo.TreeBuilder(target_tree)
diff --git a/softpack_core/schemas/environment.py b/softpack_core/schemas/environment.py
index 80f7f0f..85cceba 100644
--- a/softpack_core/schemas/environment.py
+++ b/softpack_core/schemas/environment.py
@@ -5,7 +5,6 @@
 """
 
 import os
-import uuid
 from dataclasses import dataclass
 from pathlib import Path
 from typing import Iterable, Optional
@@ -132,7 +131,7 @@ def create(cls, env: EnvironmentInput):
             )
 
         # Create folder with readme
-        new_folder_path = Path(env.path) / env.name
+        new_folder_path = Path(env.path, env.name)
         file_name = "README.md"
         try:
             tree_oid = cls.artifacts.create_file(
@@ -146,17 +145,16 @@ def create(cls, env: EnvironmentInput):
             return InvalidInputError(message=str(e))
 
         # Send build request
-        response = httpx.post(
+        httpx.post(
             "http://0.0.0.0:7080/environments/build",
             json={
-                "name": env.name,
+                "name": f"{env.path}/{env.name}",
                 "model": {
                     "description": env.description,
                     "packages": [f"{pkg.name}" for pkg in env.packages],
                 },
             },
-        ).json()
-        print(f"Create: {response}")
+        )
 
         return CreateEnvironmentSuccess(
             message="Successfully scheduled environment creation"
@@ -186,45 +184,24 @@ def update(
             or current_path == ""
         ):
             return InvalidInputError(message="all fields must be filled in")
+
+        # Check name and path have not been changed.
+        if env.path != current_path or env.name != current_name:
+            return InvalidInputError(message="cannot change name or path")
+
         # Check if an environment exists at the specified path and name
         if cls.artifacts.get(Path(current_path), current_name):
-            response = httpx.post(
+            httpx.post(
                 "http://0.0.0.0:7080/environments/build",
                 json={
-                    "name": env.name,
+                    "name": f"{env.path}/{env.name}",
                     "model": {
                         "description": env.description,
                         "packages": [pkg.name for pkg in env.packages or []],
                     },
                 },
-            ).json()
-            print(f"Update: {response}")
-
-            new_env = Environment(
-                id=uuid.uuid4().hex,
-                name=env.name,
-                path=env.path,
-                description=env.description,
-                packages=[pkg.to_package() for pkg in env.packages],
-                state=response['state']['type'],
             )
 
-            try:
-                cls.artifacts.update_environment(
-                    current_name,
-                    current_path,
-                    new_env,
-                    "update existing environment",
-                )
-            except RuntimeError as e:
-                return InvalidInputError(message=str(e))
-            except FileExistsError:
-                return EnvironmentAlreadyExistsError(
-                    message="This name is already used in this location",
-                    path=env.path,
-                    name=env.name,
-                )
-
             return UpdateEnvironmentSuccess(
                 message="Successfully updated environment"
             )
@@ -260,7 +237,7 @@ def delete(cls, name: str, path: str):
 
     @classmethod
     async def create_artifact(
-        cls, file: Upload, folder_path: str, file_name: str
+        cls, file: Upload, folder_path: str, file_name: str, overwrite: bool
     ):
         """Add a file to the Artifacts repo.
 
@@ -268,19 +245,21 @@ async def create_artifact(
             file: the file to add to the repo
             folder_path: the path to the folder that the file will be added to
             file_name: the name of the file
+            overwrite: if True, overwrite the file at the specified path
         """
         try:
             contents = (await file.read()).decode()
             tree_oid = cls.artifacts.create_file(
-                Path(folder_path), file_name, contents, replace=True
+                Path(folder_path), file_name, contents, overwrite=overwrite
             )
-            cls.artifacts.commit(
+            commit_oid = cls.artifacts.commit(
                 cls.artifacts.repo, tree_oid, "create artifact"
             )
             cls.artifacts.push(cls.artifacts.repo)
-            return "created artifact"
+            return str(commit_oid)
+
         except Exception as e:
-            return f"something went wrong when creating the artifact: {e}"
+            return InvalidInputError(message=str(e))
 
 
 # Interfaces
@@ -320,6 +299,14 @@ class DeleteEnvironmentSuccess(Success):
     message: str
 
 
+@strawberry.type
+class CreateArtifactSuccess(Success):
+    """Artifact successfully created."""
+
+    message: str
+    commit_oid: str
+
+
 # Error types
 @strawberry.type
 class InvalidInputError(Error):
@@ -346,6 +333,7 @@ class EnvironmentAlreadyExistsError(Error):
     name: str
 
 
+# Unions
 CreateResponse = strawberry.union(
     "CreateResponse",
     [
@@ -373,6 +361,14 @@ class EnvironmentAlreadyExistsError(Error):
     ],
 )
 
+CreateArtifactResponse = strawberry.union(
+    "CreateArtifactResponse",
+    [
+        CreateArtifactSuccess,
+        InvalidInputError,
+    ],
+)
+
 
 class EnvironmentSchema(BaseSchema):
     """Environment schema."""
@@ -390,4 +386,6 @@ class Mutation:
         createEnvironment: CreateResponse = Environment.create  # type: ignore
         updateEnvironment: UpdateResponse = Environment.update  # type: ignore
         deleteEnvironment: DeleteResponse = Environment.delete  # type: ignore
-        createArtifact: str = Environment.create_artifact  # type: ignore
+        createArtifact: CreateArtifactResponse = (
+            Environment.create_artifact
+        )  # type: ignore

From 6bfc9d94882f8023292c70100aca5a58e8643527 Mon Sep 17 00:00:00 2001
From: Samuel Ogg <56794292+SamO135@users.noreply.github.com>
Date: Fri, 11 Aug 2023 15:27:52 +0100
Subject: [PATCH 019/129] rename file upload method also remove overwrite
 option

---
 softpack_core/schemas/environment.py | 28 +++++++++++++++-------------
 1 file changed, 15 insertions(+), 13 deletions(-)

diff --git a/softpack_core/schemas/environment.py b/softpack_core/schemas/environment.py
index 85cceba..156e18f 100644
--- a/softpack_core/schemas/environment.py
+++ b/softpack_core/schemas/environment.py
@@ -138,7 +138,7 @@ def create(cls, env: EnvironmentInput):
                 new_folder_path, file_name, "lorem ipsum", True
             )
             cls.artifacts.commit(
-                cls.artifacts.repo, tree_oid, "create empty environment"
+                cls.artifacts.repo, tree_oid, "create environment folder"
             )
             cls.artifacts.push(cls.artifacts.repo)
         except RuntimeError as e:
@@ -236,8 +236,8 @@ def delete(cls, name: str, path: str):
         )
 
     @classmethod
-    async def create_artifact(
-        cls, file: Upload, folder_path: str, file_name: str, overwrite: bool
+    async def write_artifact(
+        cls, file: Upload, folder_path: str, file_name: str
     ):
         """Add a file to the Artifacts repo.
 
@@ -245,18 +245,20 @@ async def create_artifact(
             file: the file to add to the repo
             folder_path: the path to the folder that the file will be added to
             file_name: the name of the file
-            overwrite: if True, overwrite the file at the specified path
         """
         try:
             contents = (await file.read()).decode()
             tree_oid = cls.artifacts.create_file(
-                Path(folder_path), file_name, contents, overwrite=overwrite
+                Path(folder_path), file_name, contents, overwrite=True
             )
             commit_oid = cls.artifacts.commit(
-                cls.artifacts.repo, tree_oid, "create artifact"
+                cls.artifacts.repo, tree_oid, "write artifact"
             )
             cls.artifacts.push(cls.artifacts.repo)
-            return str(commit_oid)
+            return WriteArtifactSuccess(
+                message="Successfully written artifact",
+                commit_oid=str(commit_oid),
+            )
 
         except Exception as e:
             return InvalidInputError(message=str(e))
@@ -300,7 +302,7 @@ class DeleteEnvironmentSuccess(Success):
 
 
 @strawberry.type
-class CreateArtifactSuccess(Success):
+class WriteArtifactSuccess(Success):
     """Artifact successfully created."""
 
     message: str
@@ -361,10 +363,10 @@ class EnvironmentAlreadyExistsError(Error):
     ],
 )
 
-CreateArtifactResponse = strawberry.union(
-    "CreateArtifactResponse",
+WriteArtifactResponse = strawberry.union(
+    "WriteArtifactResponse",
     [
-        CreateArtifactSuccess,
+        WriteArtifactSuccess,
         InvalidInputError,
     ],
 )
@@ -386,6 +388,6 @@ class Mutation:
         createEnvironment: CreateResponse = Environment.create  # type: ignore
         updateEnvironment: UpdateResponse = Environment.update  # type: ignore
         deleteEnvironment: DeleteResponse = Environment.delete  # type: ignore
-        createArtifact: CreateArtifactResponse = (
-            Environment.create_artifact
+        writeArtifact: WriteArtifactResponse = (
+            Environment.write_artifact
         )  # type: ignore

From 26224f2f8a988af11e791839421234cf057ccc2a Mon Sep 17 00:00:00 2001
From: Samuel Ogg <56794292+SamO135@users.noreply.github.com>
Date: Fri, 11 Aug 2023 16:53:24 +0100
Subject: [PATCH 020/129] remove unused method

---
 softpack_core/artifacts.py | 14 --------------
 1 file changed, 14 deletions(-)

diff --git a/softpack_core/artifacts.py b/softpack_core/artifacts.py
index 571e97f..e4da836 100644
--- a/softpack_core/artifacts.py
+++ b/softpack_core/artifacts.py
@@ -350,20 +350,6 @@ def build_tree(
             path = path.parent
         return new_tree
 
-    def generate_yaml_contents(self, env) -> str:
-        """Generate the softpack.yml file contents.
-
-        Args:
-            env: an Environment object
-        """
-        packages = [
-            f"- {pkg.name}@{pkg.version}" if pkg.version else f"- {pkg.name}"
-            for pkg in env.packages
-        ]
-        packages = "\n".join(packages)
-        contents = f"description: {env.description}\npackages:\n{packages}\n"
-        return contents
-
     def create_file(
         self,
         folder_path: Path,

From 15fc9a3728b2123af57a88544bd5ab92055e60d5 Mon Sep 17 00:00:00 2001
From: Samuel Ogg <56794292+SamO135@users.noreply.github.com>
Date: Mon, 14 Aug 2023 10:41:46 +0100
Subject: [PATCH 021/129] mock testing

---
 tests/test_pygit.py | 21 +++++++++++++++++++++
 1 file changed, 21 insertions(+)
 create mode 100644 tests/test_pygit.py

diff --git a/tests/test_pygit.py b/tests/test_pygit.py
new file mode 100644
index 0000000..6223a50
--- /dev/null
+++ b/tests/test_pygit.py
@@ -0,0 +1,21 @@
+"""Copyright (c) 2023 Genome Research Ltd.
+
+This source code is licensed under the MIT license found in the
+LICENSE file in the root directory of this source tree.
+"""
+
+import pytest_mock
+from softpack_core.artifacts import Artifacts
+
+def test_commit(mocker):
+    # mocker.patch(
+    #     'softpack_core.artifacts.Artifacts.commit',
+    #     return_value="a1b2c3d4e5f6a7b8c9d0e1f2a3b4c5d6e7f8a9b0",
+    # )
+
+    expected = "a1b2c3d4e5f6a7b8c9d0e1f2a3b4c5d6e7f8a9b0"
+    artifacts = Artifacts()
+    tree_oid = "5f6e7d8c9b0a1f2e3d4c5b6a7e8d9c0b1a2f3e4"
+    actual = artifacts.commit(artifacts.repo, tree_oid, "pytest commit")
+    print(actual)
+    assert expected == actual
\ No newline at end of file

From 5aa013137516a494cf1a5eed66353306a12cecd4 Mon Sep 17 00:00:00 2001
From: Samuel Ogg <56794292+SamO135@users.noreply.github.com>
Date: Mon, 14 Aug 2023 12:18:19 +0100
Subject: [PATCH 022/129] pytest for committing to repo

---
 tests/test_pygit.py | 39 +++++++++++++++++++++++++++++----------
 1 file changed, 29 insertions(+), 10 deletions(-)

diff --git a/tests/test_pygit.py b/tests/test_pygit.py
index 6223a50..337a0ac 100644
--- a/tests/test_pygit.py
+++ b/tests/test_pygit.py
@@ -5,17 +5,36 @@
 """
 
 import pytest_mock
+import pygit2
+from pygit2 import Signature
+import tempfile
 from softpack_core.artifacts import Artifacts
 
-def test_commit(mocker):
-    # mocker.patch(
-    #     'softpack_core.artifacts.Artifacts.commit',
-    #     return_value="a1b2c3d4e5f6a7b8c9d0e1f2a3b4c5d6e7f8a9b0",
-    # )
 
-    expected = "a1b2c3d4e5f6a7b8c9d0e1f2a3b4c5d6e7f8a9b0"
+def test_commit():
+    path = tempfile.TemporaryDirectory(ignore_cleanup_errors=True).name
+    repo = pygit2.init_repository(path)
+    open(f"{path}/initial_file.txt", "w").close()
+
+    index = repo.index
+    index.add_all()
+    index.write()
+    ref = "HEAD"
+    author = Signature('Alice Author', 'alice@authors.tld')
+    committer = Signature('Cecil Committer', 'cecil@committers.tld')
+    message = "Initial commit"
+    tree = index.write_tree()
+    parents = []
+    old_commit_oid = repo.create_commit(ref, author, committer, message, tree, parents)
+
+    file_oid = repo.create_blob("test")
+    tree = repo.head.peel(pygit2.Tree)
+    tree_builder = repo.TreeBuilder(tree)
+    tree_builder.insert("new_file.txt", file_oid, pygit2.GIT_FILEMODE_BLOB)
+    new_tree = tree_builder.write()
+    
     artifacts = Artifacts()
-    tree_oid = "5f6e7d8c9b0a1f2e3d4c5b6a7e8d9c0b1a2f3e4"
-    actual = artifacts.commit(artifacts.repo, tree_oid, "pytest commit")
-    print(actual)
-    assert expected == actual
\ No newline at end of file
+    new_commit_oid = artifacts.commit(repo, new_tree, "commit new file")
+
+    assert old_commit_oid != new_commit_oid
+    assert new_commit_oid == repo.head.peel(pygit2.Commit).oid
\ No newline at end of file

From 2e7fd0e450874fce5aafefb7d94be504af22ea60 Mon Sep 17 00:00:00 2001
From: Samuel Ogg <56794292+SamO135@users.noreply.github.com>
Date: Mon, 14 Aug 2023 13:36:32 +0100
Subject: [PATCH 023/129] clean up temp directory in commit test

---
 tests/test_pygit.py | 8 ++++++--
 1 file changed, 6 insertions(+), 2 deletions(-)

diff --git a/tests/test_pygit.py b/tests/test_pygit.py
index 337a0ac..17eaa8e 100644
--- a/tests/test_pygit.py
+++ b/tests/test_pygit.py
@@ -12,7 +12,9 @@
 
 
 def test_commit():
-    path = tempfile.TemporaryDirectory(ignore_cleanup_errors=True).name
+    temp_dir = tempfile.TemporaryDirectory(ignore_cleanup_errors=True)
+    path = temp_dir.name
+    print(path)
     repo = pygit2.init_repository(path)
     open(f"{path}/initial_file.txt", "w").close()
 
@@ -35,6 +37,8 @@ def test_commit():
     
     artifacts = Artifacts()
     new_commit_oid = artifacts.commit(repo, new_tree, "commit new file")
+    repo_head = repo.head.peel(pygit2.Commit).oid
+    temp_dir.cleanup()
 
     assert old_commit_oid != new_commit_oid
-    assert new_commit_oid == repo.head.peel(pygit2.Commit).oid
\ No newline at end of file
+    assert new_commit_oid == repo_head
\ No newline at end of file

From c268d53b26fbefbbbf5bf0a158057418ddfd35dd Mon Sep 17 00:00:00 2001
From: Samuel Ogg <56794292+SamO135@users.noreply.github.com>
Date: Mon, 14 Aug 2023 14:34:28 +0100
Subject: [PATCH 024/129] pytest for pushing to repo

---
 tests/test_pygit.py | 24 ++++++++++++++++++++----
 1 file changed, 20 insertions(+), 4 deletions(-)

diff --git a/tests/test_pygit.py b/tests/test_pygit.py
index 17eaa8e..a2e6bec 100644
--- a/tests/test_pygit.py
+++ b/tests/test_pygit.py
@@ -4,20 +4,21 @@
 LICENSE file in the root directory of this source tree.
 """
 
+import pytest
 import pytest_mock
 import pygit2
 from pygit2 import Signature
 import tempfile
 from softpack_core.artifacts import Artifacts
 
-
-def test_commit():
+@pytest.fixture
+def new_repo():
     temp_dir = tempfile.TemporaryDirectory(ignore_cleanup_errors=True)
     path = temp_dir.name
     print(path)
     repo = pygit2.init_repository(path)
-    open(f"{path}/initial_file.txt", "w").close()
 
+    open(f"{path}/initial_file.txt", "w").close()
     index = repo.index
     index.add_all()
     index.write()
@@ -29,6 +30,13 @@ def test_commit():
     parents = []
     old_commit_oid = repo.create_commit(ref, author, committer, message, tree, parents)
 
+    return (repo, temp_dir, old_commit_oid)
+
+
+def test_commit(new_repo):
+    repo = new_repo[0]
+    temp_dir = new_repo[1]
+    old_commit_oid = new_repo[2]
     file_oid = repo.create_blob("test")
     tree = repo.head.peel(pygit2.Tree)
     tree_builder = repo.TreeBuilder(tree)
@@ -41,4 +49,12 @@ def test_commit():
     temp_dir.cleanup()
 
     assert old_commit_oid != new_commit_oid
-    assert new_commit_oid == repo_head
\ No newline at end of file
+    assert new_commit_oid == repo_head
+
+def test_push(mocker):
+    artifacts = Artifacts()
+
+    push_mock = mocker.patch('pygit2.Remote.push')
+
+    artifacts.push(artifacts.repo)
+    push_mock.assert_called_once()

From caa43d8b9cc18cadaa447cc54ccf5393b7bc5372 Mon Sep 17 00:00:00 2001
From: Samuel Ogg <56794292+SamO135@users.noreply.github.com>
Date: Mon, 14 Aug 2023 16:40:18 +0100
Subject: [PATCH 025/129] add create file test

---
 tests/test_pygit.py | 47 ++++++++++++++++++++++++++++++++++++++++++++-
 1 file changed, 46 insertions(+), 1 deletion(-)

diff --git a/tests/test_pygit.py b/tests/test_pygit.py
index a2e6bec..df0d485 100644
--- a/tests/test_pygit.py
+++ b/tests/test_pygit.py
@@ -4,8 +4,11 @@
 LICENSE file in the root directory of this source tree.
 """
 
+import os
 import pytest
+import shutil
 import pytest_mock
+from pathlib import Path
 import pygit2
 from pygit2 import Signature
 import tempfile
@@ -15,7 +18,6 @@
 def new_repo():
     temp_dir = tempfile.TemporaryDirectory(ignore_cleanup_errors=True)
     path = temp_dir.name
-    print(path)
     repo = pygit2.init_repository(path)
 
     open(f"{path}/initial_file.txt", "w").close()
@@ -58,3 +60,46 @@ def test_push(mocker):
 
     artifacts.push(artifacts.repo)
     push_mock.assert_called_once()
+
+
+
+def test_create_file():
+    artifacts = Artifacts()
+    with tempfile.TemporaryDirectory(ignore_cleanup_errors=True) as temp_dir:
+        shutil.copytree(artifacts.repo.path, temp_dir, dirs_exist_ok=True)
+        artifacts.repo = pygit2.Repository(temp_dir)
+        tree = artifacts.repo.head.peel(pygit2.Tree)
+
+        user_envs_tree = tree[artifacts.environments_root]["users"][os.environ["USER"]]
+        new_test_env = "test_create_file_env"
+        assert new_test_env not in [obj.name for obj in user_envs_tree]
+
+        folder_path = Path("users", os.environ["USER"], new_test_env)
+        oid = artifacts.create_file(str(folder_path), "file.txt", "lorem ipsum", True, False)
+
+        new_tree = artifacts.repo.get(oid)
+        user_envs_tree = new_tree[artifacts.environments_root]["users"][os.environ["USER"]]
+        assert new_test_env in [obj.name for obj in user_envs_tree]
+        assert "file.txt" in [obj.name for obj in user_envs_tree[new_test_env]]
+
+        artifacts.commit(artifacts.repo, oid, "commit file")
+
+        with pytest.raises(RuntimeError) as exc_info:   
+            artifacts.create_file(str(folder_path), "second_file.txt", "lorem ipsum", True, False)
+        assert exc_info.value.args[0] == 'Too many changes to the repo'
+
+        oid = artifacts.create_file(str(folder_path), "second_file.txt", "lorem ipsum", False, False)
+        new_tree = artifacts.repo.get(oid)
+        user_envs_tree = new_tree[artifacts.environments_root]["users"][os.environ["USER"]]
+        assert "second_file.txt" in [obj.name for obj in user_envs_tree[new_test_env]]
+
+        with pytest.raises(FileExistsError) as exc_info:   
+            artifacts.create_file(str(folder_path), "file.txt", "lorem ipsum", False, False)
+        assert exc_info.value.args[0] == 'File already exists'
+        
+        oid = artifacts.create_file(str(folder_path), "file.txt", "override", False, True)
+        new_tree = artifacts.repo.get(oid)
+        user_envs_tree = new_tree[artifacts.environments_root]["users"][os.environ["USER"]]
+        assert "file.txt" in [obj.name for obj in user_envs_tree[new_test_env]]
+        assert user_envs_tree[new_test_env]["file.txt"].data.decode() == "override"
+

From 12297b0b5da0cd1fb1e38be1d7447ee603ffbded Mon Sep 17 00:00:00 2001
From: Samuel Ogg <56794292+SamO135@users.noreply.github.com>
Date: Mon, 14 Aug 2023 17:23:27 +0100
Subject: [PATCH 026/129] add delete environment test

---
 softpack_core/artifacts.py           | 13 ++++++-------
 softpack_core/schemas/environment.py |  6 +++++-
 tests/test_pygit.py                  | 28 ++++++++++++++++++++++++++++
 3 files changed, 39 insertions(+), 8 deletions(-)

diff --git a/softpack_core/artifacts.py b/softpack_core/artifacts.py
index e4da836..a92f4c6 100644
--- a/softpack_core/artifacts.py
+++ b/softpack_core/artifacts.py
@@ -200,7 +200,7 @@ def tree(self, path: str) -> pygit2.Tree:
         Returns:
             Tree: A Tree object
         """
-        return self.repo.lookup_reference(self.reference).peel().tree[path]
+        return self.repo.head.peel(pygit2.Tree)[path]
 
     def environments(self, path: Path) -> Iterable:
         """Return a list of environments in the repo.
@@ -413,9 +413,8 @@ def delete_environment(
         self,
         name: str,
         path: str,
-        commit_message: str,
         tree_oid: Optional[pygit2.Oid] = None,
-    ) -> None:
+    ) -> pygit2.Oid:
         """Delete an environment folder in GitLab.
 
         Args:
@@ -424,6 +423,9 @@ def delete_environment(
             commit_message: the commit message
             tree_oid: a Pygit2.Oid object representing a tree. If None,
             a tree will be created from the artifacts repo.
+
+        Returns:
+            the OID of the new tree structure of the repository
         """
         # Get repository tree
         if not tree_oid:
@@ -437,8 +439,5 @@ def delete_environment(
         tree_builder = self.repo.TreeBuilder(target_tree)
         tree_builder.remove(name)
         new_tree = tree_builder.write()
-        full_tree = self.build_tree(self.repo, root_tree, new_tree, full_path)
 
-        # Commit and push
-        self.commit(self.repo, full_tree, commit_message)
-        self.push(self.repo)
+        return self.build_tree(self.repo, root_tree, new_tree, full_path)
diff --git a/softpack_core/schemas/environment.py b/softpack_core/schemas/environment.py
index 156e18f..f37a9f7 100644
--- a/softpack_core/schemas/environment.py
+++ b/softpack_core/schemas/environment.py
@@ -224,7 +224,11 @@ def delete(cls, name: str, path: str):
             A message confirming the success or failure of the operation.
         """
         if cls.artifacts.get(Path(path), name):
-            cls.artifacts.delete_environment(name, path, "delete environment")
+            tree_oid = cls.artifacts.delete_environment(name, path)
+            cls.artifacts.commit(
+                cls.artifacts.repo, tree_oid, "delete environment"
+            )
+            cls.artifacts.push(cls.artifacts.repo)
             return DeleteEnvironmentSuccess(
                 message="Successfully deleted the environment"
             )
diff --git a/tests/test_pygit.py b/tests/test_pygit.py
index df0d485..ee2f0c6 100644
--- a/tests/test_pygit.py
+++ b/tests/test_pygit.py
@@ -103,3 +103,31 @@ def test_create_file():
         assert "file.txt" in [obj.name for obj in user_envs_tree[new_test_env]]
         assert user_envs_tree[new_test_env]["file.txt"].data.decode() == "override"
 
+def test_delete_environment():
+    artifacts = Artifacts()
+    with tempfile.TemporaryDirectory(ignore_cleanup_errors=True) as temp_dir:
+        shutil.copytree(artifacts.repo.path, temp_dir, dirs_exist_ok=True)
+        artifacts.repo = pygit2.Repository(temp_dir)
+
+        new_test_env = "test_create_file_env"
+        folder_path = Path("users", os.environ["USER"], new_test_env)
+        oid = artifacts.create_file(str(folder_path), "file.txt", "lorem ipsum", True, False)
+        artifacts.commit(artifacts.repo, oid, "commit file")
+
+        new_tree = artifacts.repo.get(oid)
+        user_envs_tree = new_tree[artifacts.environments_root]["users"][os.environ["USER"]]
+        assert new_test_env in [obj.name for obj in user_envs_tree]
+
+        oid = artifacts.delete_environment(new_test_env, Path("users", os.environ["USER"]), oid)
+
+        artifacts.commit(artifacts.repo, oid, "commit file")
+
+        new_tree = artifacts.repo.get(oid)
+        user_envs_tree = new_tree[artifacts.environments_root]["users"][os.environ["USER"]]
+        assert new_test_env not in [obj.name for obj in user_envs_tree]
+
+
+
+# refactor test stuff above that is repeated a lot
+
+# consider if build_tree is needed by replacing with something simpler

From d043767b6b20a2b634603cc3361849a45c7d1c91 Mon Sep 17 00:00:00 2001
From: Samuel Ogg <56794292+SamO135@users.noreply.github.com>
Date: Tue, 15 Aug 2023 10:58:52 +0100
Subject: [PATCH 027/129] removed code duplication in tests

---
 softpack_core/artifacts.py |  6 +++--
 tests/test_pygit.py        | 48 +++++++++++++++++++-------------------
 2 files changed, 28 insertions(+), 26 deletions(-)

diff --git a/softpack_core/artifacts.py b/softpack_core/artifacts.py
index a92f4c6..7b1e918 100644
--- a/softpack_core/artifacts.py
+++ b/softpack_core/artifacts.py
@@ -22,6 +22,8 @@ class Artifacts:
 
     environments_root = "environments"
     environments_file = "softpack.yml"
+    users_folder_name = "users"
+    groups_folder_name = "groups"
 
     @dataclass
     class Object:
@@ -134,7 +136,7 @@ def user_folder(self, user: Optional[str] = None) -> Path:
         Returns:
             Path: A user folder.
         """
-        return self.environments_folder("users", user)
+        return self.environments_folder(self.users_folder_name, user)
 
     def group_folder(self, group: Optional[str] = None) -> Path:
         """Get the group folder for a given group.
@@ -145,7 +147,7 @@ def group_folder(self, group: Optional[str] = None) -> Path:
         Returns:
             Path: A group folder.
         """
-        return self.environments_folder("groups", group)
+        return self.environments_folder(self.groups_folder_name, group)
 
     def environments_folder(self, *args: Optional[str]) -> Path:
         """Get the folder under the environments folder.
diff --git a/tests/test_pygit.py b/tests/test_pygit.py
index ee2f0c6..dc7a6ec 100644
--- a/tests/test_pygit.py
+++ b/tests/test_pygit.py
@@ -61,24 +61,29 @@ def test_push(mocker):
     artifacts.push(artifacts.repo)
     push_mock.assert_called_once()
 
+def get_user_envs_tree(artifacts, oid) -> pygit2.Tree:
+    new_tree = artifacts.repo.get(oid)
+    return new_tree[artifacts.user_folder(os.environ["USER"])]
 
+def copy_of_repo(artifacts):
+    temp_dir = tempfile.TemporaryDirectory(ignore_cleanup_errors=True)
+    shutil.copytree(artifacts.repo.path, temp_dir.name, dirs_exist_ok=True)
+    artifacts.repo = pygit2.Repository(temp_dir.name)
+    return temp_dir
+
+def get_user_path_without_environments(artifacts) -> Path:
+    return Path(*(artifacts.user_folder(os.environ["USER"]).parts[1:]))
 
 def test_create_file():
     artifacts = Artifacts()
-    with tempfile.TemporaryDirectory(ignore_cleanup_errors=True) as temp_dir:
-        shutil.copytree(artifacts.repo.path, temp_dir, dirs_exist_ok=True)
-        artifacts.repo = pygit2.Repository(temp_dir)
-        tree = artifacts.repo.head.peel(pygit2.Tree)
-
-        user_envs_tree = tree[artifacts.environments_root]["users"][os.environ["USER"]]
+    with copy_of_repo(artifacts):
         new_test_env = "test_create_file_env"
-        assert new_test_env not in [obj.name for obj in user_envs_tree]
+        assert new_test_env not in [obj.name for obj in artifacts.iter_user(os.environ["USER"])]
 
-        folder_path = Path("users", os.environ["USER"], new_test_env)
+        folder_path = Path(get_user_path_without_environments(artifacts), new_test_env)
         oid = artifacts.create_file(str(folder_path), "file.txt", "lorem ipsum", True, False)
 
-        new_tree = artifacts.repo.get(oid)
-        user_envs_tree = new_tree[artifacts.environments_root]["users"][os.environ["USER"]]
+        user_envs_tree = get_user_envs_tree(artifacts, oid)
         assert new_test_env in [obj.name for obj in user_envs_tree]
         assert "file.txt" in [obj.name for obj in user_envs_tree[new_test_env]]
 
@@ -89,8 +94,8 @@ def test_create_file():
         assert exc_info.value.args[0] == 'Too many changes to the repo'
 
         oid = artifacts.create_file(str(folder_path), "second_file.txt", "lorem ipsum", False, False)
-        new_tree = artifacts.repo.get(oid)
-        user_envs_tree = new_tree[artifacts.environments_root]["users"][os.environ["USER"]]
+       
+        user_envs_tree = get_user_envs_tree(artifacts, oid)
         assert "second_file.txt" in [obj.name for obj in user_envs_tree[new_test_env]]
 
         with pytest.raises(FileExistsError) as exc_info:   
@@ -98,32 +103,27 @@ def test_create_file():
         assert exc_info.value.args[0] == 'File already exists'
         
         oid = artifacts.create_file(str(folder_path), "file.txt", "override", False, True)
-        new_tree = artifacts.repo.get(oid)
-        user_envs_tree = new_tree[artifacts.environments_root]["users"][os.environ["USER"]]
+
+        user_envs_tree = get_user_envs_tree(artifacts, oid)
         assert "file.txt" in [obj.name for obj in user_envs_tree[new_test_env]]
         assert user_envs_tree[new_test_env]["file.txt"].data.decode() == "override"
 
 def test_delete_environment():
     artifacts = Artifacts()
-    with tempfile.TemporaryDirectory(ignore_cleanup_errors=True) as temp_dir:
-        shutil.copytree(artifacts.repo.path, temp_dir, dirs_exist_ok=True)
-        artifacts.repo = pygit2.Repository(temp_dir)
-
+    with copy_of_repo(artifacts):
         new_test_env = "test_create_file_env"
-        folder_path = Path("users", os.environ["USER"], new_test_env)
+        folder_path = Path(get_user_path_without_environments(artifacts), new_test_env)
         oid = artifacts.create_file(str(folder_path), "file.txt", "lorem ipsum", True, False)
         artifacts.commit(artifacts.repo, oid, "commit file")
 
-        new_tree = artifacts.repo.get(oid)
-        user_envs_tree = new_tree[artifacts.environments_root]["users"][os.environ["USER"]]
+        user_envs_tree = get_user_envs_tree(artifacts, oid)
         assert new_test_env in [obj.name for obj in user_envs_tree]
 
-        oid = artifacts.delete_environment(new_test_env, Path("users", os.environ["USER"]), oid)
+        oid = artifacts.delete_environment(new_test_env, get_user_path_without_environments(artifacts), oid)
 
         artifacts.commit(artifacts.repo, oid, "commit file")
 
-        new_tree = artifacts.repo.get(oid)
-        user_envs_tree = new_tree[artifacts.environments_root]["users"][os.environ["USER"]]
+        user_envs_tree = get_user_envs_tree(artifacts, oid)
         assert new_test_env not in [obj.name for obj in user_envs_tree]
 
 

From d37febea2000da96a7c60a2941b446449e25708e Mon Sep 17 00:00:00 2001
From: Samuel Ogg <56794292+SamO135@users.noreply.github.com>
Date: Tue, 15 Aug 2023 12:37:21 +0100
Subject: [PATCH 028/129] reformatted with tox fixed a couple errors tox found
 also

---
 poetry.lock                          | 218 ++++++++++++++++++++++++--
 pyproject.toml                       |   1 +
 softpack_core/schemas/environment.py | 226 +++++++++++++--------------
 tests/test_pygit.py                  | 112 ++++++++-----
 4 files changed, 393 insertions(+), 164 deletions(-)

diff --git a/poetry.lock b/poetry.lock
index d9493d8..a6f7f07 100644
--- a/poetry.lock
+++ b/poetry.lock
@@ -1,9 +1,10 @@
-# This file is automatically @generated by Poetry 1.5.1 and should not be changed by hand.
+# This file is automatically @generated by Poetry 1.4.2 and should not be changed by hand.
 
 [[package]]
 name = "aiosqlite"
 version = "0.19.0"
 description = "asyncio bridge to the standard sqlite3 module"
+category = "main"
 optional = false
 python-versions = ">=3.7"
 files = [
@@ -19,6 +20,7 @@ docs = ["sphinx (==6.1.3)", "sphinx-mdinclude (==0.5.3)"]
 name = "alembic"
 version = "1.11.0"
 description = "A database migration tool for SQLAlchemy."
+category = "main"
 optional = false
 python-versions = ">=3.7"
 files = [
@@ -38,6 +40,7 @@ tz = ["python-dateutil"]
 name = "altgraph"
 version = "0.17.3"
 description = "Python graph (network) package"
+category = "main"
 optional = false
 python-versions = "*"
 files = [
@@ -49,6 +52,7 @@ files = [
 name = "anyio"
 version = "3.6.2"
 description = "High level compatibility layer for multiple asynchronous event loop implementations"
+category = "main"
 optional = false
 python-versions = ">=3.6.2"
 files = [
@@ -69,6 +73,7 @@ trio = ["trio (>=0.16,<0.22)"]
 name = "apprise"
 version = "1.4.0"
 description = "Push Notifications that work with just about every platform!"
+category = "main"
 optional = false
 python-versions = ">=3.6"
 files = [
@@ -88,6 +93,7 @@ requests-oauthlib = "*"
 name = "archspec"
 version = "0.2.1"
 description = "A library to query system architecture"
+category = "main"
 optional = false
 python-versions = ">=3.6, !=2.7.*, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*, !=3.5.*"
 files = [
@@ -99,6 +105,7 @@ files = [
 name = "asgi-lifespan"
 version = "2.1.0"
 description = "Programmatic startup/shutdown of ASGI apps."
+category = "main"
 optional = false
 python-versions = ">=3.7"
 files = [
@@ -113,6 +120,7 @@ sniffio = "*"
 name = "asyncpg"
 version = "0.27.0"
 description = "An asyncio PostgreSQL driver"
+category = "main"
 optional = false
 python-versions = ">=3.7.0"
 files = [
@@ -163,6 +171,7 @@ test = ["flake8 (>=5.0.4,<5.1.0)", "uvloop (>=0.15.3)"]
 name = "attrs"
 version = "23.1.0"
 description = "Classes Without Boilerplate"
+category = "main"
 optional = false
 python-versions = ">=3.7"
 files = [
@@ -181,6 +190,7 @@ tests-no-zope = ["cloudpickle", "hypothesis", "mypy (>=1.1.1)", "pympler", "pyte
 name = "autoflake"
 version = "1.7.8"
 description = "Removes unused imports and unused variables"
+category = "dev"
 optional = false
 python-versions = ">=3.7"
 files = [
@@ -196,6 +206,7 @@ tomli = {version = ">=2.0.1", markers = "python_version < \"3.11\""}
 name = "black"
 version = "23.3.0"
 description = "The uncompromising code formatter."
+category = "dev"
 optional = false
 python-versions = ">=3.7"
 files = [
@@ -245,6 +256,7 @@ uvloop = ["uvloop (>=0.15.2)"]
 name = "bleach"
 version = "6.0.0"
 description = "An easy safelist-based HTML-sanitizing tool."
+category = "dev"
 optional = false
 python-versions = ">=3.7"
 files = [
@@ -263,6 +275,7 @@ css = ["tinycss2 (>=1.1.0,<1.2)"]
 name = "bokeh"
 version = "2.4.3"
 description = "Interactive plots and applications in the browser from Python"
+category = "main"
 optional = false
 python-versions = ">=3.7"
 files = [
@@ -283,6 +296,7 @@ typing-extensions = ">=3.10.0"
 name = "bump2version"
 version = "1.0.1"
 description = "Version-bump your software with a single command!"
+category = "dev"
 optional = false
 python-versions = ">=3.5"
 files = [
@@ -294,6 +308,7 @@ files = [
 name = "cachetools"
 version = "5.3.0"
 description = "Extensible memoizing collections and decorators"
+category = "main"
 optional = false
 python-versions = "~=3.7"
 files = [
@@ -305,6 +320,7 @@ files = [
 name = "certifi"
 version = "2023.5.7"
 description = "Python package for providing Mozilla's CA Bundle."
+category = "main"
 optional = false
 python-versions = ">=3.6"
 files = [
@@ -316,6 +332,7 @@ files = [
 name = "cffi"
 version = "1.15.1"
 description = "Foreign Function Interface for Python calling C code."
+category = "main"
 optional = false
 python-versions = "*"
 files = [
@@ -392,6 +409,7 @@ pycparser = "*"
 name = "cfgv"
 version = "3.3.1"
 description = "Validate configuration and produce human readable error messages."
+category = "dev"
 optional = false
 python-versions = ">=3.6.1"
 files = [
@@ -403,6 +421,7 @@ files = [
 name = "chardet"
 version = "5.1.0"
 description = "Universal encoding detector for Python 3"
+category = "dev"
 optional = false
 python-versions = ">=3.7"
 files = [
@@ -414,6 +433,7 @@ files = [
 name = "charset-normalizer"
 version = "3.1.0"
 description = "The Real First Universal Charset Detector. Open, modern and actively maintained alternative to Chardet."
+category = "main"
 optional = false
 python-versions = ">=3.7.0"
 files = [
@@ -498,6 +518,7 @@ files = [
 name = "click"
 version = "8.1.3"
 description = "Composable command line interface toolkit"
+category = "main"
 optional = false
 python-versions = ">=3.7"
 files = [
@@ -512,6 +533,7 @@ colorama = {version = "*", markers = "platform_system == \"Windows\""}
 name = "cloudpickle"
 version = "2.2.1"
 description = "Extended pickling support for Python objects"
+category = "main"
 optional = false
 python-versions = ">=3.6"
 files = [
@@ -523,6 +545,7 @@ files = [
 name = "colorama"
 version = "0.4.6"
 description = "Cross-platform colored terminal text."
+category = "main"
 optional = false
 python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,!=3.6.*,>=2.7"
 files = [
@@ -534,6 +557,7 @@ files = [
 name = "coolname"
 version = "2.2.0"
 description = "Random name and slug generator"
+category = "main"
 optional = false
 python-versions = "*"
 files = [
@@ -545,6 +569,7 @@ files = [
 name = "coverage"
 version = "7.2.5"
 description = "Code coverage measurement for Python"
+category = "dev"
 optional = false
 python-versions = ">=3.7"
 files = [
@@ -611,6 +636,7 @@ toml = ["tomli"]
 name = "croniter"
 version = "1.3.14"
 description = "croniter provides iteration for datetime object with cron like format"
+category = "main"
 optional = false
 python-versions = ">=2.6, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*"
 files = [
@@ -625,6 +651,7 @@ python-dateutil = "*"
 name = "cryptography"
 version = "40.0.2"
 description = "cryptography is a package which provides cryptographic recipes and primitives to Python developers."
+category = "main"
 optional = false
 python-versions = ">=3.6"
 files = [
@@ -666,6 +693,7 @@ tox = ["tox"]
 name = "dask"
 version = "2023.3.1"
 description = "Parallel PyData with Task Scheduling"
+category = "main"
 optional = false
 python-versions = ">=3.8"
 files = [
@@ -694,6 +722,7 @@ test = ["pandas[test]", "pre-commit", "pytest", "pytest-rerunfailures", "pytest-
 name = "dateparser"
 version = "1.1.8"
 description = "Date parsing library designed to parse dates from HTML pages"
+category = "main"
 optional = false
 python-versions = ">=3.7"
 files = [
@@ -716,6 +745,7 @@ langdetect = ["langdetect"]
 name = "decopatch"
 version = "1.4.10"
 description = "Create decorators easily in python."
+category = "dev"
 optional = false
 python-versions = "*"
 files = [
@@ -730,6 +760,7 @@ makefun = ">=1.5.0"
 name = "distlib"
 version = "0.3.6"
 description = "Distribution utilities"
+category = "dev"
 optional = false
 python-versions = "*"
 files = [
@@ -741,6 +772,7 @@ files = [
 name = "distributed"
 version = "2023.3.1"
 description = "Distributed scheduler for Dask"
+category = "main"
 optional = false
 python-versions = ">=3.8"
 files = [
@@ -769,6 +801,7 @@ zict = ">=2.1.0"
 name = "docker"
 version = "6.1.2"
 description = "A Python library for the Docker Engine API."
+category = "main"
 optional = false
 python-versions = ">=3.7"
 files = [
@@ -790,6 +823,7 @@ ssh = ["paramiko (>=2.4.3)"]
 name = "docutils"
 version = "0.20"
 description = "Docutils -- Python Documentation Utilities"
+category = "dev"
 optional = false
 python-versions = ">=3.7"
 files = [
@@ -801,6 +835,7 @@ files = [
 name = "exceptiongroup"
 version = "1.1.1"
 description = "Backport of PEP 654 (exception groups)"
+category = "main"
 optional = false
 python-versions = ">=3.7"
 files = [
@@ -815,6 +850,7 @@ test = ["pytest (>=6)"]
 name = "fastapi"
 version = "0.94.1"
 description = "FastAPI framework, high performance, easy to learn, fast to code, ready for production"
+category = "main"
 optional = false
 python-versions = ">=3.7"
 files = [
@@ -836,6 +872,7 @@ test = ["anyio[trio] (>=3.2.1,<4.0.0)", "black (==23.1.0)", "coverage[toml] (>=6
 name = "filelock"
 version = "3.12.0"
 description = "A platform independent file lock."
+category = "dev"
 optional = false
 python-versions = ">=3.7"
 files = [
@@ -851,6 +888,7 @@ testing = ["covdefaults (>=2.3)", "coverage (>=7.2.3)", "diff-cover (>=7.5)", "p
 name = "flake8"
 version = "5.0.4"
 description = "the modular source code checker: pep8 pyflakes and co"
+category = "dev"
 optional = false
 python-versions = ">=3.6.1"
 files = [
@@ -867,6 +905,7 @@ pyflakes = ">=2.5.0,<2.6.0"
 name = "flake8-docstrings"
 version = "1.7.0"
 description = "Extension for flake8 which uses pydocstyle to check docstrings"
+category = "dev"
 optional = false
 python-versions = ">=3.7"
 files = [
@@ -882,6 +921,7 @@ pydocstyle = ">=2.1"
 name = "fsspec"
 version = "2023.5.0"
 description = "File-system specification"
+category = "main"
 optional = false
 python-versions = ">=3.8"
 files = [
@@ -917,6 +957,7 @@ tqdm = ["tqdm"]
 name = "ghp-import"
 version = "2.1.0"
 description = "Copy your docs directly to the gh-pages branch."
+category = "dev"
 optional = false
 python-versions = "*"
 files = [
@@ -934,6 +975,7 @@ dev = ["flake8", "markdown", "twine", "wheel"]
 name = "google-auth"
 version = "2.18.0"
 description = "Google Authentication Library"
+category = "main"
 optional = false
 python-versions = ">=2.7,!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*"
 files = [
@@ -959,6 +1001,7 @@ requests = ["requests (>=2.20.0,<3.0.0dev)"]
 name = "graphql-core"
 version = "3.2.3"
 description = "GraphQL implementation for Python, a port of GraphQL.js, the JavaScript reference implementation for GraphQL."
+category = "main"
 optional = false
 python-versions = ">=3.6,<4"
 files = [
@@ -970,6 +1013,7 @@ files = [
 name = "greenlet"
 version = "2.0.2"
 description = "Lightweight in-process concurrent programming"
+category = "main"
 optional = false
 python-versions = ">=2.7,!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*"
 files = [
@@ -1043,6 +1087,7 @@ test = ["objgraph", "psutil"]
 name = "griffe"
 version = "0.27.5"
 description = "Signatures for entire Python programs. Extract the structure, the frame, the skeleton of your project, to generate API documentation or find breaking changes in your API."
+category = "main"
 optional = false
 python-versions = ">=3.7"
 files = [
@@ -1057,6 +1102,7 @@ colorama = ">=0.4"
 name = "h11"
 version = "0.14.0"
 description = "A pure-Python, bring-your-own-I/O implementation of HTTP/1.1"
+category = "main"
 optional = false
 python-versions = ">=3.7"
 files = [
@@ -1068,6 +1114,7 @@ files = [
 name = "h2"
 version = "4.1.0"
 description = "HTTP/2 State-Machine based protocol implementation"
+category = "main"
 optional = false
 python-versions = ">=3.6.1"
 files = [
@@ -1083,6 +1130,7 @@ hyperframe = ">=6.0,<7"
 name = "hpack"
 version = "4.0.0"
 description = "Pure-Python HPACK header compression"
+category = "main"
 optional = false
 python-versions = ">=3.6.1"
 files = [
@@ -1094,6 +1142,7 @@ files = [
 name = "httpcore"
 version = "0.16.3"
 description = "A minimal low-level HTTP client."
+category = "main"
 optional = false
 python-versions = ">=3.7"
 files = [
@@ -1105,16 +1154,17 @@ files = [
 anyio = ">=3.0,<5.0"
 certifi = "*"
 h11 = ">=0.13,<0.15"
-sniffio = "==1.*"
+sniffio = ">=1.0.0,<2.0.0"
 
 [package.extras]
 http2 = ["h2 (>=3,<5)"]
-socks = ["socksio (==1.*)"]
+socks = ["socksio (>=1.0.0,<2.0.0)"]
 
 [[package]]
 name = "httpx"
 version = "0.23.3"
 description = "The next generation HTTP client."
+category = "main"
 optional = false
 python-versions = ">=3.7"
 files = [
@@ -1131,14 +1181,15 @@ sniffio = "*"
 
 [package.extras]
 brotli = ["brotli", "brotlicffi"]
-cli = ["click (==8.*)", "pygments (==2.*)", "rich (>=10,<13)"]
+cli = ["click (>=8.0.0,<9.0.0)", "pygments (>=2.0.0,<3.0.0)", "rich (>=10,<13)"]
 http2 = ["h2 (>=3,<5)"]
-socks = ["socksio (==1.*)"]
+socks = ["socksio (>=1.0.0,<2.0.0)"]
 
 [[package]]
 name = "hvac"
 version = "1.1.0"
 description = "HashiCorp Vault API client"
+category = "main"
 optional = false
 python-versions = ">=3.6.2,<4.0.0"
 files = [
@@ -1154,6 +1205,7 @@ requests = ">=2.27.1,<3.0.0"
 name = "hyperframe"
 version = "6.0.1"
 description = "HTTP/2 framing layer for Python"
+category = "main"
 optional = false
 python-versions = ">=3.6.1"
 files = [
@@ -1165,6 +1217,7 @@ files = [
 name = "identify"
 version = "2.5.24"
 description = "File identification library for Python"
+category = "dev"
 optional = false
 python-versions = ">=3.7"
 files = [
@@ -1179,6 +1232,7 @@ license = ["ukkonen"]
 name = "idna"
 version = "3.4"
 description = "Internationalized Domain Names in Applications (IDNA)"
+category = "main"
 optional = false
 python-versions = ">=3.5"
 files = [
@@ -1190,6 +1244,7 @@ files = [
 name = "importlib-metadata"
 version = "6.6.0"
 description = "Read metadata from Python packages"
+category = "main"
 optional = false
 python-versions = ">=3.7"
 files = [
@@ -1209,6 +1264,7 @@ testing = ["flake8 (<5)", "flufl.flake8", "importlib-resources (>=1.3)", "packag
 name = "iniconfig"
 version = "2.0.0"
 description = "brain-dead simple config-ini parsing"
+category = "main"
 optional = false
 python-versions = ">=3.7"
 files = [
@@ -1220,6 +1276,7 @@ files = [
 name = "isort"
 version = "5.12.0"
 description = "A Python utility / library to sort Python imports."
+category = "dev"
 optional = false
 python-versions = ">=3.8.0"
 files = [
@@ -1237,6 +1294,7 @@ requirements-deprecated-finder = ["pip-api", "pipreqs"]
 name = "jaraco-classes"
 version = "3.2.3"
 description = "Utility functions for Python class constructs"
+category = "dev"
 optional = false
 python-versions = ">=3.7"
 files = [
@@ -1255,6 +1313,7 @@ testing = ["flake8 (<5)", "pytest (>=6)", "pytest-black (>=0.3.7)", "pytest-chec
 name = "jeepney"
 version = "0.8.0"
 description = "Low-level, pure Python DBus protocol wrapper."
+category = "dev"
 optional = false
 python-versions = ">=3.7"
 files = [
@@ -1270,6 +1329,7 @@ trio = ["async_generator", "trio"]
 name = "jinja2"
 version = "3.1.2"
 description = "A very fast and expressive template engine."
+category = "main"
 optional = false
 python-versions = ">=3.7"
 files = [
@@ -1287,6 +1347,7 @@ i18n = ["Babel (>=2.7)"]
 name = "jsonpatch"
 version = "1.32"
 description = "Apply JSON-Patches (RFC 6902)"
+category = "main"
 optional = false
 python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*"
 files = [
@@ -1301,6 +1362,7 @@ jsonpointer = ">=1.9"
 name = "jsonpointer"
 version = "2.3"
 description = "Identify specific nodes in a JSON document (RFC 6901)"
+category = "main"
 optional = false
 python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*"
 files = [
@@ -1312,6 +1374,7 @@ files = [
 name = "jsonschema"
 version = "4.17.3"
 description = "An implementation of JSON Schema validation for Python"
+category = "main"
 optional = false
 python-versions = ">=3.7"
 files = [
@@ -1331,6 +1394,7 @@ format-nongpl = ["fqdn", "idna", "isoduration", "jsonpointer (>1.13)", "rfc3339-
 name = "keyring"
 version = "23.13.1"
 description = "Store and access your passwords safely."
+category = "dev"
 optional = false
 python-versions = ">=3.7"
 files = [
@@ -1354,6 +1418,7 @@ testing = ["flake8 (<5)", "pytest (>=6)", "pytest-black (>=0.3.7)", "pytest-chec
 name = "kubernetes"
 version = "26.1.0"
 description = "Kubernetes python client"
+category = "main"
 optional = false
 python-versions = ">=3.6"
 files = [
@@ -1371,7 +1436,7 @@ requests-oauthlib = "*"
 setuptools = ">=21.0.0"
 six = ">=1.9.0"
 urllib3 = ">=1.24.2"
-websocket-client = ">=0.32.0,<0.40.0 || >0.40.0,<0.41.dev0 || >=0.43.dev0"
+websocket-client = ">=0.32.0,<0.40.0 || >0.40.0,<0.41.0 || >=0.43.0"
 
 [package.extras]
 adal = ["adal (>=1.0.2)"]
@@ -1380,6 +1445,7 @@ adal = ["adal (>=1.0.2)"]
 name = "locket"
 version = "1.0.0"
 description = "File-based locks for Python on Linux and Windows"
+category = "main"
 optional = false
 python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*"
 files = [
@@ -1391,6 +1457,7 @@ files = [
 name = "macholib"
 version = "1.16.2"
 description = "Mach-O header analysis and editing"
+category = "main"
 optional = false
 python-versions = "*"
 files = [
@@ -1405,6 +1472,7 @@ altgraph = ">=0.17"
 name = "makefun"
 version = "1.15.1"
 description = "Small library to dynamically create python functions."
+category = "dev"
 optional = false
 python-versions = "*"
 files = [
@@ -1416,6 +1484,7 @@ files = [
 name = "mako"
 version = "1.2.4"
 description = "A super-fast templating language that borrows the best ideas from the existing templating languages."
+category = "main"
 optional = false
 python-versions = ">=3.7"
 files = [
@@ -1435,6 +1504,7 @@ testing = ["pytest"]
 name = "markdown"
 version = "3.3.7"
 description = "Python implementation of Markdown."
+category = "main"
 optional = false
 python-versions = ">=3.6"
 files = [
@@ -1452,6 +1522,7 @@ testing = ["coverage", "pyyaml"]
 name = "markdown-it-py"
 version = "2.2.0"
 description = "Python port of markdown-it. Markdown parsing, done right!"
+category = "main"
 optional = false
 python-versions = ">=3.7"
 files = [
@@ -1476,6 +1547,7 @@ testing = ["coverage", "pytest", "pytest-cov", "pytest-regressions"]
 name = "markupsafe"
 version = "2.1.2"
 description = "Safely add untrusted strings to HTML/XML markup."
+category = "main"
 optional = false
 python-versions = ">=3.7"
 files = [
@@ -1535,6 +1607,7 @@ files = [
 name = "mccabe"
 version = "0.7.0"
 description = "McCabe checker, plugin for flake8"
+category = "dev"
 optional = false
 python-versions = ">=3.6"
 files = [
@@ -1546,6 +1619,7 @@ files = [
 name = "mdurl"
 version = "0.1.2"
 description = "Markdown URL utilities"
+category = "main"
 optional = false
 python-versions = ">=3.7"
 files = [
@@ -1557,6 +1631,7 @@ files = [
 name = "mergedeep"
 version = "1.3.4"
 description = "A deep merge function for 🐍."
+category = "main"
 optional = false
 python-versions = ">=3.6"
 files = [
@@ -1568,6 +1643,7 @@ files = [
 name = "mkdocs"
 version = "1.4.3"
 description = "Project documentation with Markdown."
+category = "dev"
 optional = false
 python-versions = ">=3.7"
 files = [
@@ -1596,6 +1672,7 @@ min-versions = ["babel (==2.9.0)", "click (==7.0)", "colorama (==0.4)", "ghp-imp
 name = "mkdocs-autorefs"
 version = "0.4.1"
 description = "Automatically link across pages in MkDocs."
+category = "dev"
 optional = false
 python-versions = ">=3.7"
 files = [
@@ -1611,6 +1688,7 @@ mkdocs = ">=1.1"
 name = "mkdocs-include-markdown-plugin"
 version = "3.9.1"
 description = "Mkdocs Markdown includer plugin."
+category = "dev"
 optional = false
 python-versions = ">=3.6"
 files = [
@@ -1626,6 +1704,7 @@ test = ["mkdocs (==1.4.0)", "pytest (==7.1.3)", "pytest-cov (==3.0.0)"]
 name = "mkdocs-material"
 version = "9.1.12"
 description = "Documentation that simply works"
+category = "dev"
 optional = false
 python-versions = ">=3.7"
 files = [
@@ -1648,6 +1727,7 @@ requests = ">=2.26"
 name = "mkdocs-material-extensions"
 version = "1.1.1"
 description = "Extension pack for Python Markdown and MkDocs Material."
+category = "dev"
 optional = false
 python-versions = ">=3.7"
 files = [
@@ -1659,6 +1739,7 @@ files = [
 name = "mkdocstrings"
 version = "0.20.0"
 description = "Automatic documentation from sources, for MkDocs."
+category = "dev"
 optional = false
 python-versions = ">=3.7"
 files = [
@@ -1683,6 +1764,7 @@ python-legacy = ["mkdocstrings-python-legacy (>=0.2.1)"]
 name = "mkdocstrings-python"
 version = "0.8.3"
 description = "A Python handler for mkdocstrings."
+category = "dev"
 optional = false
 python-versions = ">=3.7"
 files = [
@@ -1698,6 +1780,7 @@ mkdocstrings = ">=0.19"
 name = "more-itertools"
 version = "9.1.0"
 description = "More routines for operating on iterables, beyond itertools"
+category = "dev"
 optional = false
 python-versions = ">=3.7"
 files = [
@@ -1709,6 +1792,7 @@ files = [
 name = "msgpack"
 version = "1.0.5"
 description = "MessagePack serializer"
+category = "main"
 optional = false
 python-versions = "*"
 files = [
@@ -1781,6 +1865,7 @@ files = [
 name = "mypy"
 version = "1.3.0"
 description = "Optional static typing for Python"
+category = "dev"
 optional = false
 python-versions = ">=3.7"
 files = [
@@ -1827,6 +1912,7 @@ reports = ["lxml"]
 name = "mypy-extensions"
 version = "1.0.0"
 description = "Type system extensions for programs checked with the mypy type checker."
+category = "dev"
 optional = false
 python-versions = ">=3.5"
 files = [
@@ -1838,6 +1924,7 @@ files = [
 name = "nodeenv"
 version = "1.8.0"
 description = "Node.js virtual environment builder"
+category = "dev"
 optional = false
 python-versions = ">=2.7,!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,!=3.6.*"
 files = [
@@ -1852,6 +1939,7 @@ setuptools = "*"
 name = "numpy"
 version = "1.24.3"
 description = "Fundamental package for array computing in Python"
+category = "main"
 optional = false
 python-versions = ">=3.8"
 files = [
@@ -1889,6 +1977,7 @@ files = [
 name = "oauthlib"
 version = "3.2.2"
 description = "A generic, spec-compliant, thorough implementation of the OAuth request-signing logic"
+category = "main"
 optional = false
 python-versions = ">=3.6"
 files = [
@@ -1905,6 +1994,7 @@ signedtoken = ["cryptography (>=3.0.0)", "pyjwt (>=2.0.0,<3)"]
 name = "oras"
 version = "0.1.17"
 description = "OCI Registry as Storage Python SDK"
+category = "main"
 optional = false
 python-versions = "*"
 files = [
@@ -1925,6 +2015,7 @@ tests = ["black", "isort", "mypy", "pyflakes", "pytest (>=4.6.2)", "types-reques
 name = "orjson"
 version = "3.8.12"
 description = "Fast, correct Python JSON library supporting dataclasses, datetimes, and numpy"
+category = "main"
 optional = false
 python-versions = ">=3.7"
 files = [
@@ -1980,6 +2071,7 @@ files = [
 name = "packaging"
 version = "23.1"
 description = "Core utilities for Python packages"
+category = "main"
 optional = false
 python-versions = ">=3.7"
 files = [
@@ -1991,6 +2083,7 @@ files = [
 name = "partd"
 version = "1.4.0"
 description = "Appendable key-value storage"
+category = "main"
 optional = false
 python-versions = ">=3.7"
 files = [
@@ -2009,6 +2102,7 @@ complete = ["blosc", "numpy (>=1.9.0)", "pandas (>=0.19.0)", "pyzmq"]
 name = "pathspec"
 version = "0.11.1"
 description = "Utility library for gitignore style pattern matching of file paths."
+category = "main"
 optional = false
 python-versions = ">=3.7"
 files = [
@@ -2020,6 +2114,7 @@ files = [
 name = "pendulum"
 version = "2.1.2"
 description = "Python datetimes made easy"
+category = "main"
 optional = false
 python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*"
 files = [
@@ -2054,6 +2149,7 @@ pytzdata = ">=2020.1"
 name = "pillow"
 version = "9.5.0"
 description = "Python Imaging Library (Fork)"
+category = "main"
 optional = false
 python-versions = ">=3.7"
 files = [
@@ -2133,6 +2229,7 @@ tests = ["check-manifest", "coverage", "defusedxml", "markdown2", "olefile", "pa
 name = "pip"
 version = "23.1.2"
 description = "The PyPA recommended tool for installing Python packages."
+category = "main"
 optional = false
 python-versions = ">=3.7"
 files = [
@@ -2144,6 +2241,7 @@ files = [
 name = "pkginfo"
 version = "1.9.6"
 description = "Query metadata from sdists / bdists / installed packages."
+category = "dev"
 optional = false
 python-versions = ">=3.6"
 files = [
@@ -2158,6 +2256,7 @@ testing = ["pytest", "pytest-cov"]
 name = "platformdirs"
 version = "3.5.1"
 description = "A small Python package for determining appropriate platform-specific dirs, e.g. a \"user data dir\"."
+category = "dev"
 optional = false
 python-versions = ">=3.7"
 files = [
@@ -2173,6 +2272,7 @@ test = ["appdirs (==1.4.4)", "covdefaults (>=2.3)", "pytest (>=7.3.1)", "pytest-
 name = "pluggy"
 version = "1.0.0"
 description = "plugin and hook calling mechanisms for python"
+category = "main"
 optional = false
 python-versions = ">=3.6"
 files = [
@@ -2188,6 +2288,7 @@ testing = ["pytest", "pytest-benchmark"]
 name = "pre-commit"
 version = "3.3.1"
 description = "A framework for managing and maintaining multi-language pre-commit hooks."
+category = "dev"
 optional = false
 python-versions = ">=3.8"
 files = [
@@ -2206,6 +2307,7 @@ virtualenv = ">=20.10.0"
 name = "prefect"
 version = "2.8.6"
 description = "Workflow orchestration and management."
+category = "main"
 optional = false
 python-versions = ">=3.7"
 files = [
@@ -2260,6 +2362,7 @@ dev = ["autoflake8", "cairosvg", "flake8", "flaky", "ipython", "jinja2", "mike",
 name = "prefect-dask"
 version = "0.2.4"
 description = "Prefect integrations with the Dask execution framework."
+category = "main"
 optional = false
 python-versions = ">=3.7"
 files = [
@@ -2278,6 +2381,7 @@ dev = ["black", "coverage", "flake8", "flaky", "interrogate", "isort", "mkdocs",
 name = "prefect-shell"
 version = "0.1.5"
 description = "Prefect tasks and subflows for interacting with shell commands."
+category = "main"
 optional = false
 python-versions = ">=3.7"
 files = [
@@ -2295,6 +2399,7 @@ dev = ["black", "coverage", "flake8", "interrogate", "isort", "mkdocs", "mkdocs-
 name = "psutil"
 version = "5.9.5"
 description = "Cross-platform lib for process and system monitoring in Python."
+category = "main"
 optional = false
 python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*"
 files = [
@@ -2321,6 +2426,7 @@ test = ["enum34", "ipaddress", "mock", "pywin32", "wmi"]
 name = "pyasn1"
 version = "0.5.0"
 description = "Pure-Python implementation of ASN.1 types and DER/BER/CER codecs (X.208)"
+category = "main"
 optional = false
 python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,>=2.7"
 files = [
@@ -2332,6 +2438,7 @@ files = [
 name = "pyasn1-modules"
 version = "0.3.0"
 description = "A collection of ASN.1-based protocols modules"
+category = "main"
 optional = false
 python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,>=2.7"
 files = [
@@ -2346,6 +2453,7 @@ pyasn1 = ">=0.4.6,<0.6.0"
 name = "pycodestyle"
 version = "2.9.1"
 description = "Python style guide checker"
+category = "dev"
 optional = false
 python-versions = ">=3.6"
 files = [
@@ -2357,6 +2465,7 @@ files = [
 name = "pycparser"
 version = "2.21"
 description = "C parser in Python"
+category = "main"
 optional = false
 python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*"
 files = [
@@ -2368,6 +2477,7 @@ files = [
 name = "pydantic"
 version = "1.10.7"
 description = "Data validation and settings management using python type hints"
+category = "main"
 optional = false
 python-versions = ">=3.7"
 files = [
@@ -2420,6 +2530,7 @@ email = ["email-validator (>=1.0.3)"]
 name = "pydocstyle"
 version = "6.3.0"
 description = "Python docstring style checker"
+category = "dev"
 optional = false
 python-versions = ">=3.6"
 files = [
@@ -2437,6 +2548,7 @@ toml = ["tomli (>=1.2.3)"]
 name = "pyflakes"
 version = "2.5.0"
 description = "passive checker of Python programs"
+category = "dev"
 optional = false
 python-versions = ">=3.6"
 files = [
@@ -2448,6 +2560,7 @@ files = [
 name = "pygit2"
 version = "1.12.1"
 description = "Python bindings for libgit2."
+category = "main"
 optional = false
 python-versions = ">=3.8"
 files = [
@@ -2491,6 +2604,7 @@ cffi = ">=1.9.1"
 name = "pygments"
 version = "2.15.1"
 description = "Pygments is a syntax highlighting package written in Python."
+category = "main"
 optional = false
 python-versions = ">=3.7"
 files = [
@@ -2505,6 +2619,7 @@ plugins = ["importlib-metadata"]
 name = "pyhcl"
 version = "0.4.4"
 description = "HCL configuration parser for python"
+category = "main"
 optional = false
 python-versions = "*"
 files = [
@@ -2515,6 +2630,7 @@ files = [
 name = "pymdown-extensions"
 version = "10.0.1"
 description = "Extension pack for Python Markdown."
+category = "dev"
 optional = false
 python-versions = ">=3.7"
 files = [
@@ -2530,6 +2646,7 @@ pyyaml = "*"
 name = "pyproject-api"
 version = "1.5.1"
 description = "API to interact with the python pyproject.toml based projects"
+category = "dev"
 optional = false
 python-versions = ">=3.7"
 files = [
@@ -2549,6 +2666,7 @@ testing = ["covdefaults (>=2.2.2)", "importlib-metadata (>=6)", "pytest (>=7.2.1
 name = "pyrsistent"
 version = "0.19.3"
 description = "Persistent/Functional/Immutable data structures"
+category = "main"
 optional = false
 python-versions = ">=3.7"
 files = [
@@ -2585,6 +2703,7 @@ files = [
 name = "pytest"
 version = "7.3.1"
 description = "pytest: simple powerful testing with Python"
+category = "main"
 optional = false
 python-versions = ">=3.7"
 files = [
@@ -2607,6 +2726,7 @@ testing = ["argcomplete", "attrs (>=19.2.0)", "hypothesis (>=3.56)", "mock", "no
 name = "pytest-asyncio"
 version = "0.21.0"
 description = "Pytest support for asyncio"
+category = "dev"
 optional = false
 python-versions = ">=3.7"
 files = [
@@ -2625,6 +2745,7 @@ testing = ["coverage (>=6.2)", "flaky (>=3.5.0)", "hypothesis (>=5.7.1)", "mypy
 name = "pytest-cases"
 version = "3.6.14"
 description = "Separate test code from test cases in pytest."
+category = "dev"
 optional = false
 python-versions = "*"
 files = [
@@ -2640,6 +2761,7 @@ makefun = ">=1.9.5"
 name = "pytest-cov"
 version = "4.0.0"
 description = "Pytest plugin for measuring coverage."
+category = "dev"
 optional = false
 python-versions = ">=3.6"
 files = [
@@ -2658,6 +2780,7 @@ testing = ["fields", "hunter", "process-tests", "pytest-xdist", "six", "virtuale
 name = "pytest-httpx"
 version = "0.21.3"
 description = "Send responses to httpx."
+category = "dev"
 optional = false
 python-versions = ">=3.7"
 files = [
@@ -2666,16 +2789,35 @@ files = [
 ]
 
 [package.dependencies]
-httpx = "==0.23.*"
+httpx = ">=0.23.0,<0.24.0"
 pytest = ">=6.0,<8.0"
 
 [package.extras]
-testing = ["pytest-asyncio (==0.20.*)", "pytest-cov (==4.*)"]
+testing = ["pytest-asyncio (>=0.20.0,<0.21.0)", "pytest-cov (>=4.0.0,<5.0.0)"]
+
+[[package]]
+name = "pytest-mock"
+version = "3.11.1"
+description = "Thin-wrapper around the mock package for easier use with pytest"
+category = "main"
+optional = false
+python-versions = ">=3.7"
+files = [
+    {file = "pytest-mock-3.11.1.tar.gz", hash = "sha256:7f6b125602ac6d743e523ae0bfa71e1a697a2f5534064528c6ff84c2f7c2fc7f"},
+    {file = "pytest_mock-3.11.1-py3-none-any.whl", hash = "sha256:21c279fff83d70763b05f8874cc9cfb3fcacd6d354247a976f9529d19f9acf39"},
+]
+
+[package.dependencies]
+pytest = ">=5.0"
+
+[package.extras]
+dev = ["pre-commit", "pytest-asyncio", "tox"]
 
 [[package]]
 name = "python-box"
 version = "7.0.1"
 description = "Advanced Python dictionaries with dot notation access"
+category = "main"
 optional = false
 python-versions = ">=3.7"
 files = [
@@ -2710,6 +2852,7 @@ yaml = ["ruamel.yaml (>=0.17)"]
 name = "python-dateutil"
 version = "2.8.2"
 description = "Extensions to the standard Python datetime module"
+category = "main"
 optional = false
 python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,>=2.7"
 files = [
@@ -2724,6 +2867,7 @@ six = ">=1.5"
 name = "python-ldap"
 version = "3.4.3"
 description = "Python modules for implementing LDAP clients"
+category = "main"
 optional = false
 python-versions = ">=3.6"
 files = [
@@ -2738,6 +2882,7 @@ pyasn1_modules = ">=0.1.5"
 name = "python-slugify"
 version = "8.0.1"
 description = "A Python slugify application that also handles Unicode"
+category = "main"
 optional = false
 python-versions = ">=3.7"
 files = [
@@ -2755,6 +2900,7 @@ unidecode = ["Unidecode (>=1.1.1)"]
 name = "pytz"
 version = "2023.3"
 description = "World timezone definitions, modern and historical"
+category = "main"
 optional = false
 python-versions = "*"
 files = [
@@ -2766,6 +2912,7 @@ files = [
 name = "pytzdata"
 version = "2020.1"
 description = "The Olson timezone database for Python."
+category = "main"
 optional = false
 python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*"
 files = [
@@ -2777,6 +2924,7 @@ files = [
 name = "pywin32"
 version = "306"
 description = "Python for Window Extensions"
+category = "main"
 optional = false
 python-versions = "*"
 files = [
@@ -2800,6 +2948,7 @@ files = [
 name = "pywin32-ctypes"
 version = "0.2.0"
 description = ""
+category = "dev"
 optional = false
 python-versions = "*"
 files = [
@@ -2811,6 +2960,7 @@ files = [
 name = "pyyaml"
 version = "6.0"
 description = "YAML parser and emitter for Python"
+category = "main"
 optional = false
 python-versions = ">=3.6"
 files = [
@@ -2860,6 +3010,7 @@ files = [
 name = "pyyaml-env-tag"
 version = "0.1"
 description = "A custom YAML tag for referencing environment variables in YAML files. "
+category = "dev"
 optional = false
 python-versions = ">=3.6"
 files = [
@@ -2874,6 +3025,7 @@ pyyaml = "*"
 name = "readchar"
 version = "4.0.5"
 description = "Library to easily read single chars and key strokes"
+category = "main"
 optional = false
 python-versions = ">=3.7"
 files = [
@@ -2888,6 +3040,7 @@ setuptools = ">=41.0"
 name = "readme-renderer"
 version = "37.3"
 description = "readme_renderer is a library for rendering \"readme\" descriptions for Warehouse"
+category = "dev"
 optional = false
 python-versions = ">=3.7"
 files = [
@@ -2907,6 +3060,7 @@ md = ["cmarkgfm (>=0.8.0)"]
 name = "regex"
 version = "2023.5.5"
 description = "Alternative regular expression module, to replace re."
+category = "main"
 optional = false
 python-versions = ">=3.6"
 files = [
@@ -3004,6 +3158,7 @@ files = [
 name = "requests"
 version = "2.29.0"
 description = "Python HTTP for Humans."
+category = "main"
 optional = false
 python-versions = ">=3.7"
 files = [
@@ -3025,6 +3180,7 @@ use-chardet-on-py3 = ["chardet (>=3.0.2,<6)"]
 name = "requests-oauthlib"
 version = "1.3.1"
 description = "OAuthlib authentication support for Requests."
+category = "main"
 optional = false
 python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*"
 files = [
@@ -3043,6 +3199,7 @@ rsa = ["oauthlib[signedtoken] (>=3.0.0)"]
 name = "requests-toolbelt"
 version = "1.0.0"
 description = "A utility belt for advanced users of python-requests"
+category = "dev"
 optional = false
 python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*"
 files = [
@@ -3057,6 +3214,7 @@ requests = ">=2.0.1,<3.0.0"
 name = "rfc3986"
 version = "1.5.0"
 description = "Validating URI References per RFC 3986"
+category = "main"
 optional = false
 python-versions = "*"
 files = [
@@ -3074,6 +3232,7 @@ idna2008 = ["idna"]
 name = "rich"
 version = "13.3.5"
 description = "Render rich text, tables, progress bars, syntax highlighting, markdown and more to the terminal"
+category = "main"
 optional = false
 python-versions = ">=3.7.0"
 files = [
@@ -3092,6 +3251,7 @@ jupyter = ["ipywidgets (>=7.5.1,<9)"]
 name = "rsa"
 version = "4.9"
 description = "Pure-Python RSA implementation"
+category = "main"
 optional = false
 python-versions = ">=3.6,<4"
 files = [
@@ -3106,6 +3266,7 @@ pyasn1 = ">=0.1.3"
 name = "ruamel-yaml"
 version = "0.17.26"
 description = "ruamel.yaml is a YAML parser/emitter that supports roundtrip preservation of comments, seq/map flow style, and map key order"
+category = "main"
 optional = false
 python-versions = ">=3"
 files = [
@@ -3124,6 +3285,7 @@ jinja2 = ["ruamel.yaml.jinja2 (>=0.2)"]
 name = "ruamel-yaml-clib"
 version = "0.2.7"
 description = "C version of reader, parser and emitter for ruamel.yaml derived from libyaml"
+category = "main"
 optional = false
 python-versions = ">=3.5"
 files = [
@@ -3169,6 +3331,7 @@ files = [
 name = "secretstorage"
 version = "3.3.3"
 description = "Python bindings to FreeDesktop.org Secret Service API"
+category = "dev"
 optional = false
 python-versions = ">=3.6"
 files = [
@@ -3184,6 +3347,7 @@ jeepney = ">=0.6"
 name = "semver"
 version = "3.0.0"
 description = "Python helper for Semantic Versioning (https://semver.org)"
+category = "main"
 optional = false
 python-versions = ">=3.7"
 files = [
@@ -3195,6 +3359,7 @@ files = [
 name = "setuptools"
 version = "67.7.2"
 description = "Easily download, build, install, upgrade, and uninstall Python packages"
+category = "main"
 optional = false
 python-versions = ">=3.7"
 files = [
@@ -3211,6 +3376,7 @@ testing-integration = ["build[virtualenv]", "filelock (>=3.4.0)", "jaraco.envs (
 name = "singleton-decorator"
 version = "1.0.0"
 description = "A testable singleton decorator"
+category = "main"
 optional = false
 python-versions = "*"
 files = [
@@ -3221,6 +3387,7 @@ files = [
 name = "six"
 version = "1.16.0"
 description = "Python 2 and 3 compatibility utilities"
+category = "main"
 optional = false
 python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*"
 files = [
@@ -3232,6 +3399,7 @@ files = [
 name = "sniffio"
 version = "1.3.0"
 description = "Sniff out which async library your code is running under"
+category = "main"
 optional = false
 python-versions = ">=3.7"
 files = [
@@ -3243,6 +3411,7 @@ files = [
 name = "snowballstemmer"
 version = "2.2.0"
 description = "This package provides 29 stemmers for 28 languages generated from Snowball algorithms."
+category = "dev"
 optional = false
 python-versions = "*"
 files = [
@@ -3254,6 +3423,7 @@ files = [
 name = "sortedcontainers"
 version = "2.4.0"
 description = "Sorted Containers -- Sorted List, Sorted Dict, Sorted Set"
+category = "main"
 optional = false
 python-versions = "*"
 files = [
@@ -3265,6 +3435,7 @@ files = [
 name = "sqlalchemy"
 version = "1.4.45"
 description = "Database Abstraction Library"
+category = "main"
 optional = false
 python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,>=2.7"
 files = [
@@ -3312,7 +3483,7 @@ files = [
 ]
 
 [package.dependencies]
-greenlet = {version = "!=0.4.17", optional = true, markers = "python_version >= \"3\" and (platform_machine == \"win32\" or platform_machine == \"WIN32\" or platform_machine == \"AMD64\" or platform_machine == \"amd64\" or platform_machine == \"x86_64\" or platform_machine == \"ppc64le\" or platform_machine == \"aarch64\" or extra == \"asyncio\")"}
+greenlet = {version = "!=0.4.17", optional = true, markers = "python_version >= \"3\" and platform_machine == \"aarch64\" or python_version >= \"3\" and platform_machine == \"ppc64le\" or python_version >= \"3\" and platform_machine == \"x86_64\" or python_version >= \"3\" and platform_machine == \"amd64\" or python_version >= \"3\" and platform_machine == \"AMD64\" or python_version >= \"3\" and platform_machine == \"win32\" or python_version >= \"3\" and platform_machine == \"WIN32\" or python_version >= \"3\" and extra == \"asyncio\""}
 
 [package.extras]
 aiomysql = ["aiomysql", "greenlet (!=0.4.17)"]
@@ -3339,6 +3510,7 @@ sqlcipher = ["sqlcipher3-binary"]
 name = "starlette"
 version = "0.26.1"
 description = "The little ASGI library that shines."
+category = "main"
 optional = false
 python-versions = ">=3.7"
 files = [
@@ -3357,6 +3529,7 @@ full = ["httpx (>=0.22.0)", "itsdangerous", "jinja2", "python-multipart", "pyyam
 name = "strawberry-graphql"
 version = "0.177.1"
 description = "A library for creating GraphQL APIs"
+category = "main"
 optional = false
 python-versions = ">=3.7,<4.0"
 files = [
@@ -3390,6 +3563,7 @@ starlite = ["starlite (>=1.48.0)"]
 name = "tblib"
 version = "1.7.0"
 description = "Traceback serialization library."
+category = "main"
 optional = false
 python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*"
 files = [
@@ -3401,6 +3575,7 @@ files = [
 name = "text-unidecode"
 version = "1.3"
 description = "The most basic Text::Unidecode port"
+category = "main"
 optional = false
 python-versions = "*"
 files = [
@@ -3412,6 +3587,7 @@ files = [
 name = "toml"
 version = "0.10.2"
 description = "Python Library for Tom's Obvious, Minimal Language"
+category = "main"
 optional = false
 python-versions = ">=2.6, !=3.0.*, !=3.1.*, !=3.2.*"
 files = [
@@ -3423,6 +3599,7 @@ files = [
 name = "tomli"
 version = "2.0.1"
 description = "A lil' TOML parser"
+category = "main"
 optional = false
 python-versions = ">=3.7"
 files = [
@@ -3434,6 +3611,7 @@ files = [
 name = "toolz"
 version = "0.12.0"
 description = "List processing tools and functional utilities"
+category = "main"
 optional = false
 python-versions = ">=3.5"
 files = [
@@ -3445,6 +3623,7 @@ files = [
 name = "tornado"
 version = "6.3.2"
 description = "Tornado is a Python web framework and asynchronous networking library, originally developed at FriendFeed."
+category = "main"
 optional = false
 python-versions = ">= 3.8"
 files = [
@@ -3465,6 +3644,7 @@ files = [
 name = "tox"
 version = "4.5.1"
 description = "tox is a generic virtualenv management and test command line tool"
+category = "dev"
 optional = false
 python-versions = ">=3.7"
 files = [
@@ -3492,6 +3672,7 @@ testing = ["build[virtualenv] (>=0.10)", "covdefaults (>=2.3)", "devpi-process (
 name = "twine"
 version = "4.0.2"
 description = "Collection of utilities for publishing packages on PyPI"
+category = "dev"
 optional = false
 python-versions = ">=3.7"
 files = [
@@ -3514,6 +3695,7 @@ urllib3 = ">=1.26.0"
 name = "typer"
 version = "0.9.0"
 description = "Typer, build great CLIs. Easy to code. Based on Python type hints."
+category = "main"
 optional = false
 python-versions = ">=3.6"
 files = [
@@ -3535,6 +3717,7 @@ test = ["black (>=22.3.0,<23.0.0)", "coverage (>=6.2,<7.0)", "isort (>=5.0.6,<6.
 name = "types-pyyaml"
 version = "6.0.12.9"
 description = "Typing stubs for PyYAML"
+category = "dev"
 optional = false
 python-versions = "*"
 files = [
@@ -3546,6 +3729,7 @@ files = [
 name = "types-requests"
 version = "2.30.0.0"
 description = "Typing stubs for requests"
+category = "dev"
 optional = false
 python-versions = "*"
 files = [
@@ -3560,6 +3744,7 @@ types-urllib3 = "*"
 name = "types-setuptools"
 version = "67.7.0.2"
 description = "Typing stubs for setuptools"
+category = "dev"
 optional = false
 python-versions = "*"
 files = [
@@ -3571,6 +3756,7 @@ files = [
 name = "types-urllib3"
 version = "1.26.25.13"
 description = "Typing stubs for urllib3"
+category = "dev"
 optional = false
 python-versions = "*"
 files = [
@@ -3582,6 +3768,7 @@ files = [
 name = "typing-extensions"
 version = "4.5.0"
 description = "Backported and Experimental Type Hints for Python 3.7+"
+category = "main"
 optional = false
 python-versions = ">=3.7"
 files = [
@@ -3593,6 +3780,7 @@ files = [
 name = "tzdata"
 version = "2023.3"
 description = "Provider of IANA time zone data"
+category = "main"
 optional = false
 python-versions = ">=2"
 files = [
@@ -3604,6 +3792,7 @@ files = [
 name = "tzlocal"
 version = "5.0.1"
 description = "tzinfo object for the local timezone"
+category = "main"
 optional = false
 python-versions = ">=3.7"
 files = [
@@ -3621,6 +3810,7 @@ devenv = ["black", "check-manifest", "flake8", "pyroma", "pytest (>=4.3)", "pyte
 name = "urllib3"
 version = "1.26.15"
 description = "HTTP library with thread-safe connection pooling, file post, and more."
+category = "main"
 optional = false
 python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*, !=3.5.*"
 files = [
@@ -3637,6 +3827,7 @@ socks = ["PySocks (>=1.5.6,!=1.5.7,<2.0)"]
 name = "uvicorn"
 version = "0.22.0"
 description = "The lightning-fast ASGI server."
+category = "main"
 optional = false
 python-versions = ">=3.7"
 files = [
@@ -3655,6 +3846,7 @@ standard = ["colorama (>=0.4)", "httptools (>=0.5.0)", "python-dotenv (>=0.13)",
 name = "virtualenv"
 version = "20.23.0"
 description = "Virtual Python Environment builder"
+category = "dev"
 optional = false
 python-versions = ">=3.7"
 files = [
@@ -3675,6 +3867,7 @@ test = ["covdefaults (>=2.3)", "coverage (>=7.2.3)", "coverage-enable-subprocess
 name = "watchdog"
 version = "3.0.0"
 description = "Filesystem events monitoring"
+category = "dev"
 optional = false
 python-versions = ">=3.7"
 files = [
@@ -3714,6 +3907,7 @@ watchmedo = ["PyYAML (>=3.10)"]
 name = "webencodings"
 version = "0.5.1"
 description = "Character encoding aliases for legacy web content"
+category = "dev"
 optional = false
 python-versions = "*"
 files = [
@@ -3725,6 +3919,7 @@ files = [
 name = "websocket-client"
 version = "1.5.1"
 description = "WebSocket client for Python with low level API options"
+category = "main"
 optional = false
 python-versions = ">=3.7"
 files = [
@@ -3741,6 +3936,7 @@ test = ["websockets"]
 name = "websockets"
 version = "11.0.3"
 description = "An implementation of the WebSocket Protocol (RFC 6455 & 7692)"
+category = "main"
 optional = false
 python-versions = ">=3.7"
 files = [
@@ -3820,6 +4016,7 @@ files = [
 name = "zict"
 version = "3.0.0"
 description = "Mutable mapping tools"
+category = "main"
 optional = false
 python-versions = ">=3.8"
 files = [
@@ -3831,6 +4028,7 @@ files = [
 name = "zipp"
 version = "3.15.0"
 description = "Backport of pathlib-compatible object wrapper for zip files"
+category = "main"
 optional = false
 python-versions = ">=3.7"
 files = [
@@ -3845,4 +4043,4 @@ testing = ["big-O", "flake8 (<5)", "jaraco.functools", "jaraco.itertools", "more
 [metadata]
 lock-version = "2.0"
 python-versions = ">=3.9,<4.0"
-content-hash = "edfd9534063f496bf6c7de432111da487ed9fa24990f181ec1472b1c0fdf0038"
+content-hash = "1464d35bb7dea62d6540a946ccdb1fb7a40acddf5ba9b1355822414ee1462ca1"
diff --git a/pyproject.toml b/pyproject.toml
index 8616ac2..ca77294 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -48,6 +48,7 @@ singleton-decorator = "^1.0.0"
 sqlalchemy = "1.4.45"
 strawberry-graphql = "^0.177.1"
 typer = "^0.9.0"
+pytest-mock = "^3.11.1"
 
 [tool.poetry.group.dev]
 optional = true
diff --git a/softpack_core/schemas/environment.py b/softpack_core/schemas/environment.py
index f37a9f7..b04388e 100644
--- a/softpack_core/schemas/environment.py
+++ b/softpack_core/schemas/environment.py
@@ -18,6 +18,114 @@
 from softpack_core.spack import Spack
 
 
+# Interfaces
+@strawberry.interface
+class Success:
+    """Interface for successful results."""
+
+    message: str
+
+
+@strawberry.interface
+class Error:
+    """Interface for errors."""
+
+    message: str
+
+
+# Success types
+@strawberry.type
+class CreateEnvironmentSuccess(Success):
+    """Environment successfully scheduled."""
+
+    message: str
+
+
+@strawberry.type
+class UpdateEnvironmentSuccess(Success):
+    """Environment successfully updated."""
+
+    message: str
+
+
+@strawberry.type
+class DeleteEnvironmentSuccess(Success):
+    """Environment successfully deleted."""
+
+    message: str
+
+
+@strawberry.type
+class WriteArtifactSuccess(Success):
+    """Artifact successfully created."""
+
+    message: str
+    commit_oid: str
+
+
+# Error types
+@strawberry.type
+class InvalidInputError(Error):
+    """Invalid input data."""
+
+    message: str
+
+
+@strawberry.type
+class EnvironmentNotFoundError(Error):
+    """Environment not found."""
+
+    message: str
+    path: str
+    name: str
+
+
+@strawberry.type
+class EnvironmentAlreadyExistsError(Error):
+    """Environment name already exists."""
+
+    message: str
+    path: str
+    name: str
+
+
+# Unions
+CreateResponse = strawberry.union(
+    "CreateResponse",
+    [
+        CreateEnvironmentSuccess,
+        InvalidInputError,
+        EnvironmentAlreadyExistsError,
+    ],
+)
+
+UpdateResponse = strawberry.union(
+    "UpdateResponse",
+    [
+        UpdateEnvironmentSuccess,
+        InvalidInputError,
+        EnvironmentNotFoundError,
+        EnvironmentAlreadyExistsError,
+    ],
+)
+
+DeleteResponse = strawberry.union(
+    "DeleteResponse",
+    [
+        DeleteEnvironmentSuccess,
+        EnvironmentNotFoundError,
+    ],
+)
+
+WriteArtifactResponse = strawberry.union(
+    "WriteArtifactResponse",
+    [
+        WriteArtifactSuccess,
+        InvalidInputError,
+    ],
+)
+
+
 @strawberry.type
 class Package(Spack.PackageBase):
     """A Strawberry model representing a package."""
@@ -78,7 +186,7 @@ def iter(cls, all: bool = False) -> Iterable["Environment"]:
         return filter(lambda x: x is not None, environment_objects)
 
     @classmethod
-    def from_artifact(cls, obj: Artifacts.Object) -> "Environment":
+    def from_artifact(cls, obj: Artifacts.Object) -> Optional["Environment"]:
         """Create an Environment object from an artifact.
 
         Args:
@@ -104,7 +212,7 @@ def from_artifact(cls, obj: Artifacts.Object) -> "Environment":
             return None
 
     @classmethod
-    def create(cls, env: EnvironmentInput):
+    def create(cls, env: EnvironmentInput) -> CreateResponse:
         """Create an Environment.
 
         Args:
@@ -166,7 +274,7 @@ def update(
         env: EnvironmentInput,
         current_path: str,
         current_name: str,
-    ):
+    ) -> UpdateResponse:
         """Update an Environment.
 
         Args:
@@ -213,7 +321,7 @@ def update(
         )
 
     @classmethod
-    def delete(cls, name: str, path: str):
+    def delete(cls, name: str, path: str) -> DeleteResponse:
         """Delete an Environment.
 
         Args:
@@ -242,7 +350,7 @@ def delete(cls, name: str, path: str):
     @classmethod
     async def write_artifact(
         cls, file: Upload, folder_path: str, file_name: str
-    ):
+    ) -> WriteArtifactResponse:
         """Add a file to the Artifacts repo.
 
         Args:
@@ -268,114 +376,6 @@ async def write_artifact(
             return InvalidInputError(message=str(e))
 
 
-# Interfaces
-@strawberry.interface
-class Success:
-    """Interface for successful results."""
-
-    message: str
-
-
-@strawberry.interface
-class Error:
-    """Interface for errors."""
-
-    message: str
-
-
-# Success types
-@strawberry.type
-class CreateEnvironmentSuccess(Success):
-    """Environment successfully scheduled."""
-
-    message: str
-
-
-@strawberry.type
-class UpdateEnvironmentSuccess(Success):
-    """Environment successfully updated."""
-
-    message: str
-
-
-@strawberry.type
-class DeleteEnvironmentSuccess(Success):
-    """Environment successfully deleted."""
-
-    message: str
-
-
-@strawberry.type
-class WriteArtifactSuccess(Success):
-    """Artifact successfully created."""
-
-    message: str
-    commit_oid: str
-
-
-# Error types
-@strawberry.type
-class InvalidInputError(Error):
-    """Invalid input data."""
-
-    message: str
-
-
-@strawberry.type
-class EnvironmentNotFoundError(Error):
-    """Environment not found."""
-
-    message: str
-    path: str
-    name: str
-
-
-@strawberry.type
-class EnvironmentAlreadyExistsError(Error):
-    """Environment name already exists."""
-
-    message: str
-    path: str
-    name: str
-
-
-# Unions
-CreateResponse = strawberry.union(
-    "CreateResponse",
-    [
-        CreateEnvironmentSuccess,
-        InvalidInputError,
-        EnvironmentAlreadyExistsError,
-    ],
-)
-
-UpdateResponse = strawberry.union(
-    "UpdateResponse",
-    [
-        UpdateEnvironmentSuccess,
-        InvalidInputError,
-        EnvironmentNotFoundError,
-        EnvironmentAlreadyExistsError,
-    ],
-)
-
-DeleteResponse = strawberry.union(
-    "DeleteResponse",
-    [
-        DeleteEnvironmentSuccess,
-        EnvironmentNotFoundError,
-    ],
-)
-
-WriteArtifactResponse = strawberry.union(
-    "WriteArtifactResponse",
-    [
-        WriteArtifactSuccess,
-        InvalidInputError,
-    ],
-)
-
-
 class EnvironmentSchema(BaseSchema):
     """Environment schema."""
 
diff --git a/tests/test_pygit.py b/tests/test_pygit.py
index dc7a6ec..dbe0431 100644
--- a/tests/test_pygit.py
+++ b/tests/test_pygit.py
@@ -5,18 +5,20 @@
 """
 
 import os
-import pytest
 import shutil
-import pytest_mock
+import tempfile
 from pathlib import Path
+
 import pygit2
+import pytest
 from pygit2 import Signature
-import tempfile
+
 from softpack_core.artifacts import Artifacts
 
+
 @pytest.fixture
 def new_repo():
-    temp_dir = tempfile.TemporaryDirectory(ignore_cleanup_errors=True)
+    temp_dir = tempfile.TemporaryDirectory()
     path = temp_dir.name
     repo = pygit2.init_repository(path)
 
@@ -30,28 +32,30 @@ def new_repo():
     message = "Initial commit"
     tree = index.write_tree()
     parents = []
-    old_commit_oid = repo.create_commit(ref, author, committer, message, tree, parents)
+    old_commit_oid = repo.create_commit(
+        ref, author, committer, message, tree, parents
+    )
 
     return (repo, temp_dir, old_commit_oid)
 
 
 def test_commit(new_repo):
     repo = new_repo[0]
-    temp_dir = new_repo[1]
     old_commit_oid = new_repo[2]
-    file_oid = repo.create_blob("test")
-    tree = repo.head.peel(pygit2.Tree)
-    tree_builder = repo.TreeBuilder(tree)
-    tree_builder.insert("new_file.txt", file_oid, pygit2.GIT_FILEMODE_BLOB)
-    new_tree = tree_builder.write()
-    
-    artifacts = Artifacts()
-    new_commit_oid = artifacts.commit(repo, new_tree, "commit new file")
-    repo_head = repo.head.peel(pygit2.Commit).oid
-    temp_dir.cleanup()
+    with new_repo[1]:
+        file_oid = repo.create_blob("test")
+        tree = repo.head.peel(pygit2.Tree)
+        tree_builder = repo.TreeBuilder(tree)
+        tree_builder.insert("new_file.txt", file_oid, pygit2.GIT_FILEMODE_BLOB)
+        new_tree = tree_builder.write()
+
+        artifacts = Artifacts()
+        new_commit_oid = artifacts.commit(repo, new_tree, "commit new file")
+        repo_head = repo.head.peel(pygit2.Commit).oid
+
+        assert old_commit_oid != new_commit_oid
+        assert new_commit_oid == repo_head
 
-    assert old_commit_oid != new_commit_oid
-    assert new_commit_oid == repo_head
 
 def test_push(mocker):
     artifacts = Artifacts()
@@ -61,65 +65,94 @@ def test_push(mocker):
     artifacts.push(artifacts.repo)
     push_mock.assert_called_once()
 
+
 def get_user_envs_tree(artifacts, oid) -> pygit2.Tree:
     new_tree = artifacts.repo.get(oid)
     return new_tree[artifacts.user_folder(os.environ["USER"])]
 
+
 def copy_of_repo(artifacts):
-    temp_dir = tempfile.TemporaryDirectory(ignore_cleanup_errors=True)
+    temp_dir = tempfile.TemporaryDirectory()
     shutil.copytree(artifacts.repo.path, temp_dir.name, dirs_exist_ok=True)
     artifacts.repo = pygit2.Repository(temp_dir.name)
     return temp_dir
 
+
 def get_user_path_without_environments(artifacts) -> Path:
     return Path(*(artifacts.user_folder(os.environ["USER"]).parts[1:]))
 
+
 def test_create_file():
     artifacts = Artifacts()
     with copy_of_repo(artifacts):
         new_test_env = "test_create_file_env"
-        assert new_test_env not in [obj.name for obj in artifacts.iter_user(os.environ["USER"])]
+        assert new_test_env not in [
+            obj.name for obj in artifacts.iter_user(os.environ["USER"])
+        ]
 
-        folder_path = Path(get_user_path_without_environments(artifacts), new_test_env)
-        oid = artifacts.create_file(str(folder_path), "file.txt", "lorem ipsum", True, False)
+        fname = "file.txt"
+
+        folder_path = Path(
+            get_user_path_without_environments(artifacts), new_test_env
+        )
+        oid = artifacts.create_file(
+            str(folder_path), fname, "lorem ipsum", True, False
+        )
 
         user_envs_tree = get_user_envs_tree(artifacts, oid)
         assert new_test_env in [obj.name for obj in user_envs_tree]
-        assert "file.txt" in [obj.name for obj in user_envs_tree[new_test_env]]
+        assert fname in [obj.name for obj in user_envs_tree[new_test_env]]
 
         artifacts.commit(artifacts.repo, oid, "commit file")
 
-        with pytest.raises(RuntimeError) as exc_info:   
-            artifacts.create_file(str(folder_path), "second_file.txt", "lorem ipsum", True, False)
+        with pytest.raises(RuntimeError) as exc_info:
+            artifacts.create_file(
+                str(folder_path), "second_file.txt", "lorem ipsum", True, False
+            )
         assert exc_info.value.args[0] == 'Too many changes to the repo'
 
-        oid = artifacts.create_file(str(folder_path), "second_file.txt", "lorem ipsum", False, False)
-       
-        user_envs_tree = get_user_envs_tree(artifacts, oid)
-        assert "second_file.txt" in [obj.name for obj in user_envs_tree[new_test_env]]
+        oid = artifacts.create_file(
+            str(folder_path), "second_file.txt", "lorem ipsum", False, False
+        )
 
-        with pytest.raises(FileExistsError) as exc_info:   
-            artifacts.create_file(str(folder_path), "file.txt", "lorem ipsum", False, False)
+        user_envs_tree = get_user_envs_tree(artifacts, oid)
+        assert "second_file.txt" in [
+            obj.name for obj in user_envs_tree[new_test_env]
+        ]
+
+        with pytest.raises(FileExistsError) as exc_info:
+            artifacts.create_file(
+                str(folder_path), fname, "lorem ipsum", False, False
+            )
         assert exc_info.value.args[0] == 'File already exists'
-        
-        oid = artifacts.create_file(str(folder_path), "file.txt", "override", False, True)
+
+        oid = artifacts.create_file(
+            str(folder_path), fname, "override", False, True
+        )
 
         user_envs_tree = get_user_envs_tree(artifacts, oid)
-        assert "file.txt" in [obj.name for obj in user_envs_tree[new_test_env]]
-        assert user_envs_tree[new_test_env]["file.txt"].data.decode() == "override"
+        assert fname in [obj.name for obj in user_envs_tree[new_test_env]]
+        assert user_envs_tree[new_test_env][fname].data.decode() == "override"
+
 
 def test_delete_environment():
     artifacts = Artifacts()
     with copy_of_repo(artifacts):
         new_test_env = "test_create_file_env"
-        folder_path = Path(get_user_path_without_environments(artifacts), new_test_env)
-        oid = artifacts.create_file(str(folder_path), "file.txt", "lorem ipsum", True, False)
+        folder_path = Path(
+            get_user_path_without_environments(artifacts), new_test_env
+        )
+        oid = artifacts.create_file(
+            str(folder_path), "file.txt", "lorem ipsum", True, False
+        )
         artifacts.commit(artifacts.repo, oid, "commit file")
 
         user_envs_tree = get_user_envs_tree(artifacts, oid)
         assert new_test_env in [obj.name for obj in user_envs_tree]
 
-        oid = artifacts.delete_environment(new_test_env, get_user_path_without_environments(artifacts), oid)
+        oid = artifacts.delete_environment(
+            new_test_env, get_user_path_without_environments(artifacts), oid
+        )
 
         artifacts.commit(artifacts.repo, oid, "commit file")
 
@@ -127,7 +160,4 @@ def test_delete_environment():
         assert new_test_env not in [obj.name for obj in user_envs_tree]
 
 
-
-# refactor test stuff above that is repeated a lot
-
 # consider if build_tree is needed by replacing with something simpler

From 52567c319718b884b70682c26c8c93289f73431e Mon Sep 17 00:00:00 2001
From: Samuel Ogg <56794292+SamO135@users.noreply.github.com>
Date: Tue, 15 Aug 2023 16:33:55 +0100
Subject: [PATCH 029/129] format and improve code coverage

---
 softpack_core/artifacts.py | 49 ++++++----------------
 tests/test_pygit.py        | 83 +++++++++++++++++++++++++++++++++++++-
 2 files changed, 94 insertions(+), 38 deletions(-)

diff --git a/softpack_core/artifacts.py b/softpack_core/artifacts.py
index 7b1e918..cc52830 100644
--- a/softpack_core/artifacts.py
+++ b/softpack_core/artifacts.py
@@ -227,23 +227,17 @@ def iter(self, user: Optional[str] = None) -> Iterable:
         Returns:
             Iterator: an iterator
         """
-        try:
-            if user:
-                folders = list(
-                    itertools.chain(
-                        [self.user_folder(user)],
-                        map(self.group_folder, self.ldap.groups(user) or []),
-                    )
+        if user:
+            folders = list(
+                itertools.chain(
+                    [self.user_folder(user)],
+                    map(self.group_folder, self.ldap.groups(user) or []),
                 )
-            else:
-                folders = self.iter_user() + self.iter_group()
-
-            return itertools.chain.from_iterable(
-                map(self.environments, folders)
             )
+        else:
+            folders = self.iter_user() + self.iter_group()
 
-        except KeyError:
-            return iter(())
+        return itertools.chain.from_iterable(map(self.environments, folders))
 
     def get(self, path: Path, name: str) -> Optional[pygit2.Tree]:
         """Return the environment at the specified name and path.
@@ -285,19 +279,6 @@ def commit(
         )
         return commit_oid
 
-    def error_callback(self, refname: str, message: str) -> None:
-        """Push update reference callback.
-
-        Args:
-            refname: the name of the reference (on the remote)
-            message: rejection message from the remote. If None, the update was
-            accepted
-        """
-        if message is not None:
-            print(
-                f"An error occurred during push to ref '{refname}': {message}"
-            )
-
     def push(self, repo: pygit2.Repository) -> None:
         """Push all commits to a repository.
 
@@ -314,7 +295,6 @@ def push(self, repo: pygit2.Repository) -> None:
         except Exception as e:
             print(e)
         callbacks = pygit2.RemoteCallbacks(credentials=credentials)
-        callbacks.push_update_reference = self.error_callback
         remote.push([repo.head.name], callbacks=callbacks)
 
     def build_tree(
@@ -405,7 +385,7 @@ def create_file(
             new_file = diff[0].delta.new_file
             if new_file.path != str(path):
                 raise RuntimeError(
-                    f"Attempted to add new file added to incorrect path: \
+                    f"New file added to incorrect path: \
                         {new_file.path} instead of {str(path)}"
                 )
 
@@ -415,7 +395,6 @@ def delete_environment(
         self,
         name: str,
         path: str,
-        tree_oid: Optional[pygit2.Oid] = None,
     ) -> pygit2.Oid:
         """Delete an environment folder in GitLab.
 
@@ -423,17 +402,15 @@ def delete_environment(
             name: the name of the environment
             path: the path of the environment
             commit_message: the commit message
-            tree_oid: a Pygit2.Oid object representing a tree. If None,
-            a tree will be created from the artifacts repo.
 
         Returns:
             the OID of the new tree structure of the repository
         """
+        if len(Path(path).parts) != 2:
+            raise ValueError("Not a valid environment path")
+
         # Get repository tree
-        if not tree_oid:
-            root_tree = self.repo.head.peel(pygit2.Tree)
-        else:
-            root_tree = self.repo.get(tree_oid)
+        root_tree = self.repo.head.peel(pygit2.Tree)
         # Find environment in the tree
         full_path = Path(self.environments_root, path)
         target_tree = root_tree[full_path]
diff --git a/tests/test_pygit.py b/tests/test_pygit.py
index dbe0431..0b49811 100644
--- a/tests/test_pygit.py
+++ b/tests/test_pygit.py
@@ -39,6 +39,17 @@ def new_repo():
     return (repo, temp_dir, old_commit_oid)
 
 
+def test_clone():
+    artifacts = Artifacts()
+    path = artifacts.repo.path
+
+    shutil.rmtree(path)
+    assert os.path.isdir(path) is False
+
+    artifacts = Artifacts()
+    assert os.path.isdir(path) is True
+
+
 def test_commit(new_repo):
     repo = new_repo[0]
     old_commit_oid = new_repo[2]
@@ -105,6 +116,12 @@ def test_create_file():
 
         artifacts.commit(artifacts.repo, oid, "commit file")
 
+        with pytest.raises(RuntimeError) as exc_info:
+            oid = artifacts.create_file(
+                str(folder_path), fname, "lorem ipsum", False, True
+            )
+        assert exc_info.value.args[0] == 'No changes made to the environment'
+
         with pytest.raises(RuntimeError) as exc_info:
             artifacts.create_file(
                 str(folder_path), "second_file.txt", "lorem ipsum", True, False
@@ -151,7 +168,7 @@ def test_delete_environment():
         assert new_test_env in [obj.name for obj in user_envs_tree]
 
         oid = artifacts.delete_environment(
-            new_test_env, get_user_path_without_environments(artifacts), oid
+            new_test_env, get_user_path_without_environments(artifacts)
         )
 
         artifacts.commit(artifacts.repo, oid, "commit file")
@@ -159,5 +176,67 @@ def test_delete_environment():
         user_envs_tree = get_user_envs_tree(artifacts, oid)
         assert new_test_env not in [obj.name for obj in user_envs_tree]
 
+        with pytest.raises(ValueError) as exc_info:
+            artifacts.delete_environment(
+                os.environ["USER"], artifacts.users_folder_name
+            )
+        assert exc_info.value.args[0] == 'Not a valid environment path'
+
+        with pytest.raises(KeyError) as exc_info:
+            artifacts.delete_environment(new_test_env, "foo/bar")
+        assert exc_info
+
+
+def count_user_and_group_envs(artifacts, envs) -> (int, int):
+    num_user_envs = 0
+    num_group_envs = 0
+
+    for env in envs:
+        if str(env.path).startswith(artifacts.users_folder_name):
+            num_user_envs += 1
+        elif str(env.path).startswith(artifacts.groups_folder_name):
+            num_group_envs += 1
 
-# consider if build_tree is needed by replacing with something simpler
+    return num_user_envs, num_group_envs
+
+
+def test_iter():
+    artifacts = Artifacts()
+    user = os.environ["USER"]
+    envs = artifacts.iter(user)
+
+    user_found = False
+    only_this_user = True
+    num_user_envs = 0
+    num_group_envs = 0
+
+    for env in envs:
+        if str(env.path).startswith(artifacts.users_folder_name):
+            num_user_envs += 1
+            if str(env.path).startswith(
+                f"{artifacts.users_folder_name}/{user}"
+            ):
+                user_found = True
+            else:
+                only_this_user = False
+        elif str(env.path).startswith(artifacts.groups_folder_name):
+            num_group_envs += 1
+
+    assert user_found is True
+    assert only_this_user is True
+    assert num_group_envs > 0
+
+    envs = artifacts.iter()
+    no_user_num_user_envs, no_user_num_group_envs = count_user_and_group_envs(
+        artifacts, envs
+    )
+    assert no_user_num_user_envs > num_user_envs
+    assert no_user_num_group_envs > num_group_envs
+
+    envs = artifacts.iter("!@£$%")
+    (
+        bad_user_num_user_envs,
+        bad_user_num_group_envs,
+    ) = count_user_and_group_envs(artifacts, envs)
+    assert bad_user_num_user_envs == 0
+    assert bad_user_num_group_envs == 0

From 2f83f3d1928d4559fb255ff7249bc711da2d4b47 Mon Sep 17 00:00:00 2001
From: Samuel Ogg <56794292+SamO135@users.noreply.github.com>
Date: Tue, 15 Aug 2023 17:36:11 +0100
Subject: [PATCH 030/129] add create environment test

---
 tests/test_environment.py | 39 +++++++++++++++++++++++++++++++++++++++
 1 file changed, 39 insertions(+)
 create mode 100644 tests/test_environment.py

diff --git a/tests/test_environment.py b/tests/test_environment.py
new file mode 100644
index 0000000..592991f
--- /dev/null
+++ b/tests/test_environment.py
@@ -0,0 +1,39 @@
+"""Copyright (c) 2023 Genome Research Ltd.
+
+This source code is licensed under the MIT license found in the
+LICENSE file in the root directory of this source tree.
+"""
+
+from softpack_core.artifacts import Artifacts
+from softpack_core.schemas.environment import (
+    CreateEnvironmentSuccess,
+    Environment,
+    EnvironmentInput,
+    PackageInput,
+)
+
+from .test_pygit import copy_of_repo, get_user_path_without_environments
+
+
+def test_create(mocker):
+    artifacts = Artifacts()
+    environment = Environment(
+        id="", name="", path="", description="", packages=[], state=None
+    )
+    with copy_of_repo(artifacts) as temp_dir:
+        print(temp_dir)
+        environment.artifacts = artifacts
+        push_mock = mocker.patch('pygit2.Remote.push')
+        post_mock = mocker.patch('httpx.post')
+        env_input = EnvironmentInput(
+            name="environment_test",
+            path=str(get_user_path_without_environments(artifacts)),
+            description="description",
+            packages=[PackageInput(name="pkg_test")],
+        )
+        result = environment.create(env_input)
+        print(result)
+
+        assert isinstance(result, CreateEnvironmentSuccess)
+        push_mock.assert_called_once()
+        post_mock.assert_called_once()

From d8228356b113b5418a8c8e62c7ee09ce9f77e076 Mon Sep 17 00:00:00 2001
From: Samuel Ogg <56794292+SamO135@users.noreply.github.com>
Date: Wed, 16 Aug 2023 10:32:32 +0100
Subject: [PATCH 031/129] finish create environment test

---
 softpack_core/schemas/environment.py |  6 ------
 tests/test_environment.py            | 28 +++++++++++++++++++++++++---
 2 files changed, 25 insertions(+), 9 deletions(-)

diff --git a/softpack_core/schemas/environment.py b/softpack_core/schemas/environment.py
index b04388e..88dc240 100644
--- a/softpack_core/schemas/environment.py
+++ b/softpack_core/schemas/environment.py
@@ -169,7 +169,6 @@ class Environment:
     state: Optional[str]
     artifacts = Artifacts()
 
-    @classmethod
     def iter(cls, all: bool = False) -> Iterable["Environment"]:
         """Get an iterator over Environment objects.
 
@@ -185,7 +184,6 @@ def iter(cls, all: bool = False) -> Iterable["Environment"]:
         environment_objects = map(cls.from_artifact, environment_folders)
         return filter(lambda x: x is not None, environment_objects)
 
-    @classmethod
     def from_artifact(cls, obj: Artifacts.Object) -> Optional["Environment"]:
         """Create an Environment object from an artifact.
 
@@ -211,7 +209,6 @@ def from_artifact(cls, obj: Artifacts.Object) -> Optional["Environment"]:
         except KeyError:
             return None
 
-    @classmethod
     def create(cls, env: EnvironmentInput) -> CreateResponse:
         """Create an Environment.
 
@@ -268,7 +265,6 @@ def create(cls, env: EnvironmentInput) -> CreateResponse:
             message="Successfully scheduled environment creation"
         )
 
-    @classmethod
     def update(
         cls,
         env: EnvironmentInput,
@@ -320,7 +316,6 @@ def update(
             name=current_name,
         )
 
-    @classmethod
     def delete(cls, name: str, path: str) -> DeleteResponse:
         """Delete an Environment.
 
@@ -347,7 +342,6 @@ def delete(cls, name: str, path: str) -> DeleteResponse:
             name=name,
         )
 
-    @classmethod
     async def write_artifact(
         cls, file: Upload, folder_path: str, file_name: str
     ) -> WriteArtifactResponse:
diff --git a/tests/test_environment.py b/tests/test_environment.py
index 592991f..8be3b9f 100644
--- a/tests/test_environment.py
+++ b/tests/test_environment.py
@@ -10,6 +10,8 @@
     Environment,
     EnvironmentInput,
     PackageInput,
+    InvalidInputError,
+    EnvironmentAlreadyExistsError,
 )
 
 from .test_pygit import copy_of_repo, get_user_path_without_environments
@@ -21,19 +23,39 @@ def test_create(mocker):
         id="", name="", path="", description="", packages=[], state=None
     )
     with copy_of_repo(artifacts) as temp_dir:
-        print(temp_dir)
         environment.artifacts = artifacts
         push_mock = mocker.patch('pygit2.Remote.push')
         post_mock = mocker.patch('httpx.post')
+        env_name = "environment_test"
         env_input = EnvironmentInput(
-            name="environment_test",
+            name=env_name,
             path=str(get_user_path_without_environments(artifacts)),
             description="description",
             packages=[PackageInput(name="pkg_test")],
         )
         result = environment.create(env_input)
-        print(result)
 
         assert isinstance(result, CreateEnvironmentSuccess)
         push_mock.assert_called_once()
         post_mock.assert_called_once()
+
+        post_mock.assert_called_with("http://0.0.0.0:7080/environments/build", json={
+                "name": f"{env_input.path}/{env_input.name}",
+                "model": {
+                    "description": env_input.description,
+                    "packages": [f"{pkg.name}" for pkg in env_input.packages],
+                },
+            })
+        
+        result = environment.create(env_input)
+        assert isinstance(result, EnvironmentAlreadyExistsError)
+        
+        env_input.name = ""
+        result = environment.create(env_input)
+        assert isinstance(result, InvalidInputError)
+
+        env_input.name = env_name
+        env_input.path = "invalid/path"
+        result = environment.create(env_input)
+        assert isinstance(result, InvalidInputError)
+

From 2e1b01e8865ce54bfd43d7bee45b98395a9ad903 Mon Sep 17 00:00:00 2001
From: Altaf Ali <7231912+altaf-ali@users.noreply.github.com>
Date: Wed, 16 Aug 2023 12:07:01 +0100
Subject: [PATCH 032/129] =?UTF-8?q?=F0=9F=93=9D=20docs(CODEOWNERS):=20add?=
 =?UTF-8?q?=20CODEOWNERS=20file=20to=20specify=20code=20owners=20for=20the?=
 =?UTF-8?q?=20repository?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

---
 .github/CODEOWNERS | 1 +
 1 file changed, 1 insertion(+)
 create mode 100644 .github/CODEOWNERS

diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS
new file mode 100644
index 0000000..fe19a31
--- /dev/null
+++ b/.github/CODEOWNERS
@@ -0,0 +1 @@
+* @altaf-ali

From 2663bc7bbb608451ae31c6aa850f930e64ebb514 Mon Sep 17 00:00:00 2001
From: Samuel Ogg <56794292+SamO135@users.noreply.github.com>
Date: Wed, 16 Aug 2023 17:13:03 +0100
Subject: [PATCH 033/129] add tests for update

---
 tests/test_environment.py | 27 ++++++++++++++++++++++++---
 1 file changed, 24 insertions(+), 3 deletions(-)

diff --git a/tests/test_environment.py b/tests/test_environment.py
index 8be3b9f..123b5f8 100644
--- a/tests/test_environment.py
+++ b/tests/test_environment.py
@@ -12,17 +12,19 @@
     PackageInput,
     InvalidInputError,
     EnvironmentAlreadyExistsError,
+    UpdateEnvironmentSuccess,
+    EnvironmentNotFoundError,
 )
 
 from .test_pygit import copy_of_repo, get_user_path_without_environments
 
 
-def test_create(mocker):
+def test_create_and_update(mocker):
     artifacts = Artifacts()
     environment = Environment(
         id="", name="", path="", description="", packages=[], state=None
     )
-    with copy_of_repo(artifacts) as temp_dir:
+    with copy_of_repo(artifacts):
         environment.artifacts = artifacts
         push_mock = mocker.patch('pygit2.Remote.push')
         post_mock = mocker.patch('httpx.post')
@@ -33,8 +35,9 @@ def test_create(mocker):
             description="description",
             packages=[PackageInput(name="pkg_test")],
         )
-        result = environment.create(env_input)
 
+        # Test create
+        result = environment.create(env_input)
         assert isinstance(result, CreateEnvironmentSuccess)
         push_mock.assert_called_once()
         post_mock.assert_called_once()
@@ -59,3 +62,21 @@ def test_create(mocker):
         result = environment.create(env_input)
         assert isinstance(result, InvalidInputError)
 
+        # Test update
+        env_input.path = str(get_user_path_without_environments(artifacts))
+        env_input.description = "updated description"
+        result = environment.update(env_input, env_input.path, env_input.name)
+        assert isinstance(result, UpdateEnvironmentSuccess)
+
+        result = environment.update(env_input, "invalid/path", "invalid_name")
+        assert isinstance(result, InvalidInputError)
+
+        env_input.name = ""
+        result = environment.update(env_input, "invalid/path", "invalid_name")
+        assert isinstance(result, InvalidInputError)
+
+        env_input.name = "invalid_name"
+        env_input.path = "invalid/path"
+        result = environment.update(env_input, "invalid/path", "invalid_name")
+        assert isinstance(result, EnvironmentNotFoundError)
+

From 9adca648766115d1fc876c52c91e5964a942e6f5 Mon Sep 17 00:00:00 2001
From: Samuel Ogg <56794292+SamO135@users.noreply.github.com>
Date: Thu, 17 Aug 2023 15:43:35 +0100
Subject: [PATCH 034/129] add classmethod decorators back

---
 softpack_core/schemas/environment.py |  6 ++++++
 tests/test_environment.py            | 25 +++++++++++++++----------
 tests/test_pygit.py                  |  8 ++++----
 3 files changed, 25 insertions(+), 14 deletions(-)

diff --git a/softpack_core/schemas/environment.py b/softpack_core/schemas/environment.py
index 88dc240..b04388e 100644
--- a/softpack_core/schemas/environment.py
+++ b/softpack_core/schemas/environment.py
@@ -169,6 +169,7 @@ class Environment:
     state: Optional[str]
     artifacts = Artifacts()
 
+    @classmethod
     def iter(cls, all: bool = False) -> Iterable["Environment"]:
         """Get an iterator over Environment objects.
 
@@ -184,6 +185,7 @@ def iter(cls, all: bool = False) -> Iterable["Environment"]:
         environment_objects = map(cls.from_artifact, environment_folders)
         return filter(lambda x: x is not None, environment_objects)
 
+    @classmethod
     def from_artifact(cls, obj: Artifacts.Object) -> Optional["Environment"]:
         """Create an Environment object from an artifact.
 
@@ -209,6 +211,7 @@ def from_artifact(cls, obj: Artifacts.Object) -> Optional["Environment"]:
         except KeyError:
             return None
 
+    @classmethod
     def create(cls, env: EnvironmentInput) -> CreateResponse:
         """Create an Environment.
 
@@ -265,6 +268,7 @@ def create(cls, env: EnvironmentInput) -> CreateResponse:
             message="Successfully scheduled environment creation"
         )
 
+    @classmethod
     def update(
         cls,
         env: EnvironmentInput,
@@ -316,6 +320,7 @@ def update(
             name=current_name,
         )
 
+    @classmethod
     def delete(cls, name: str, path: str) -> DeleteResponse:
         """Delete an Environment.
 
@@ -342,6 +347,7 @@ def delete(cls, name: str, path: str) -> DeleteResponse:
             name=name,
         )
 
+    @classmethod
     async def write_artifact(
         cls, file: Upload, folder_path: str, file_name: str
     ) -> WriteArtifactResponse:
diff --git a/tests/test_environment.py b/tests/test_environment.py
index 123b5f8..038a7f5 100644
--- a/tests/test_environment.py
+++ b/tests/test_environment.py
@@ -4,16 +4,18 @@
 LICENSE file in the root directory of this source tree.
 """
 
+import pygit2
+
 from softpack_core.artifacts import Artifacts
 from softpack_core.schemas.environment import (
     CreateEnvironmentSuccess,
     Environment,
+    EnvironmentAlreadyExistsError,
     EnvironmentInput,
-    PackageInput,
+    EnvironmentNotFoundError,
     InvalidInputError,
-    EnvironmentAlreadyExistsError,
+    PackageInput,
     UpdateEnvironmentSuccess,
-    EnvironmentNotFoundError,
 )
 
 from .test_pygit import copy_of_repo, get_user_path_without_environments
@@ -24,8 +26,9 @@ def test_create_and_update(mocker):
     environment = Environment(
         id="", name="", path="", description="", packages=[], state=None
     )
-    with copy_of_repo(artifacts):
-        environment.artifacts = artifacts
+    with copy_of_repo(artifacts) as temp_dir:
+        # repo needs to be modified via environment obj for change to persist
+        environment.artifacts.repo = pygit2.Repository(temp_dir)
         push_mock = mocker.patch('pygit2.Remote.push')
         post_mock = mocker.patch('httpx.post')
         env_name = "environment_test"
@@ -42,17 +45,20 @@ def test_create_and_update(mocker):
         push_mock.assert_called_once()
         post_mock.assert_called_once()
 
-        post_mock.assert_called_with("http://0.0.0.0:7080/environments/build", json={
+        post_mock.assert_called_with(
+            "http://0.0.0.0:7080/environments/build",
+            json={
                 "name": f"{env_input.path}/{env_input.name}",
                 "model": {
                     "description": env_input.description,
                     "packages": [f"{pkg.name}" for pkg in env_input.packages],
                 },
-            })
-        
+            },
+        )
+
         result = environment.create(env_input)
         assert isinstance(result, EnvironmentAlreadyExistsError)
-        
+
         env_input.name = ""
         result = environment.create(env_input)
         assert isinstance(result, InvalidInputError)
@@ -79,4 +85,3 @@ def test_create_and_update(mocker):
         env_input.path = "invalid/path"
         result = environment.update(env_input, "invalid/path", "invalid_name")
         assert isinstance(result, EnvironmentNotFoundError)
-
diff --git a/tests/test_pygit.py b/tests/test_pygit.py
index 0b49811..e47281c 100644
--- a/tests/test_pygit.py
+++ b/tests/test_pygit.py
@@ -85,7 +85,6 @@ def get_user_envs_tree(artifacts, oid) -> pygit2.Tree:
 def copy_of_repo(artifacts):
     temp_dir = tempfile.TemporaryDirectory()
     shutil.copytree(artifacts.repo.path, temp_dir.name, dirs_exist_ok=True)
-    artifacts.repo = pygit2.Repository(temp_dir.name)
     return temp_dir
 
 
@@ -95,7 +94,8 @@ def get_user_path_without_environments(artifacts) -> Path:
 
 def test_create_file():
     artifacts = Artifacts()
-    with copy_of_repo(artifacts):
+    with copy_of_repo(artifacts) as temp_dir:
+        artifacts.repo = pygit2.Repository(temp_dir)
         new_test_env = "test_create_file_env"
         assert new_test_env not in [
             obj.name for obj in artifacts.iter_user(os.environ["USER"])
@@ -230,8 +230,8 @@ def test_iter():
     no_user_num_user_envs, no_user_num_group_envs = count_user_and_group_envs(
         artifacts, envs
     )
-    assert no_user_num_user_envs > num_user_envs
-    assert no_user_num_group_envs > num_group_envs
+    assert no_user_num_user_envs >= num_user_envs
+    assert no_user_num_group_envs >= num_group_envs
 
     envs = artifacts.iter("!@£$%")
     (

From d436d6a45fb31977859c482a037f7707c85203cc Mon Sep 17 00:00:00 2001
From: Samuel Ogg <56794292+SamO135@users.noreply.github.com>
Date: Thu, 17 Aug 2023 16:37:10 +0100
Subject: [PATCH 035/129] split create and update tests by adding a fixture

---
 tests/test_environment.py | 103 +++++++++++++++++++++++---------------
 tests/test_pygit.py       |  14 +++---
 2 files changed, 69 insertions(+), 48 deletions(-)

diff --git a/tests/test_environment.py b/tests/test_environment.py
index 038a7f5..90445f5 100644
--- a/tests/test_environment.py
+++ b/tests/test_environment.py
@@ -5,6 +5,7 @@
 """
 
 import pygit2
+import pytest
 
 from softpack_core.artifacts import Artifacts
 from softpack_core.schemas.environment import (
@@ -21,7 +22,8 @@
 from .test_pygit import copy_of_repo, get_user_path_without_environments
 
 
-def test_create_and_update(mocker):
+@pytest.fixture
+def temp_git_repo(mocker):
     artifacts = Artifacts()
     environment = Environment(
         id="", name="", path="", description="", packages=[], state=None
@@ -29,8 +31,8 @@ def test_create_and_update(mocker):
     with copy_of_repo(artifacts) as temp_dir:
         # repo needs to be modified via environment obj for change to persist
         environment.artifacts.repo = pygit2.Repository(temp_dir)
-        push_mock = mocker.patch('pygit2.Remote.push')
-        post_mock = mocker.patch('httpx.post')
+        mocker.patch('pygit2.Remote.push')
+        mocker.patch('httpx.post')
         env_name = "environment_test"
         env_input = EnvironmentInput(
             name=env_name,
@@ -39,49 +41,68 @@ def test_create_and_update(mocker):
             packages=[PackageInput(name="pkg_test")],
         )
 
-        # Test create
-        result = environment.create(env_input)
-        assert isinstance(result, CreateEnvironmentSuccess)
-        push_mock.assert_called_once()
-        post_mock.assert_called_once()
-
-        post_mock.assert_called_with(
-            "http://0.0.0.0:7080/environments/build",
-            json={
-                "name": f"{env_input.path}/{env_input.name}",
-                "model": {
-                    "description": env_input.description,
-                    "packages": [f"{pkg.name}" for pkg in env_input.packages],
-                },
+        environment.create(env_input)
+
+        yield environment, artifacts, env_input, env_name
+
+
+def test_create(mocker, temp_git_repo) -> None:
+    # Setup
+    environment, artifacts, env_input, env_name = temp_git_repo
+    push_mock = mocker.patch('pygit2.Remote.push')
+    post_mock = mocker.patch('httpx.post')
+
+    # Tests
+    env_input.name = "test_create_new"
+    result = environment.create(env_input)
+    assert isinstance(result, CreateEnvironmentSuccess)
+    push_mock.assert_called_once()
+    post_mock.assert_called_once()
+
+    post_mock.assert_called_with(
+        "http://0.0.0.0:7080/environments/build",
+        json={
+            "name": f"{env_input.path}/{env_input.name}",
+            "model": {
+                "description": env_input.description,
+                "packages": [f"{pkg.name}" for pkg in env_input.packages],
             },
-        )
+        },
+    )
+
+    result = environment.create(env_input)
+    assert isinstance(result, EnvironmentAlreadyExistsError)
+
+    env_input.name = ""
+    result = environment.create(env_input)
+    assert isinstance(result, InvalidInputError)
 
-        result = environment.create(env_input)
-        assert isinstance(result, EnvironmentAlreadyExistsError)
+    env_input.name = env_name
+    env_input.path = "invalid/path"
+    result = environment.create(env_input)
+    assert isinstance(result, InvalidInputError)
 
-        env_input.name = ""
-        result = environment.create(env_input)
-        assert isinstance(result, InvalidInputError)
 
-        env_input.name = env_name
-        env_input.path = "invalid/path"
-        result = environment.create(env_input)
-        assert isinstance(result, InvalidInputError)
+def test_update(mocker, temp_git_repo) -> None:
+    # Setup
+    environment, artifacts, env_input, env_name = temp_git_repo
+    post_mock = mocker.patch('httpx.post')
 
-        # Test update
-        env_input.path = str(get_user_path_without_environments(artifacts))
-        env_input.description = "updated description"
-        result = environment.update(env_input, env_input.path, env_input.name)
-        assert isinstance(result, UpdateEnvironmentSuccess)
+    # Tests
+    env_input.path = str(get_user_path_without_environments(artifacts))
+    env_input.description = "updated description"
+    result = environment.update(env_input, env_input.path, env_input.name)
+    assert isinstance(result, UpdateEnvironmentSuccess)
+    post_mock.assert_called_once()
 
-        result = environment.update(env_input, "invalid/path", "invalid_name")
-        assert isinstance(result, InvalidInputError)
+    result = environment.update(env_input, "invalid/path", "invalid_name")
+    assert isinstance(result, InvalidInputError)
 
-        env_input.name = ""
-        result = environment.update(env_input, "invalid/path", "invalid_name")
-        assert isinstance(result, InvalidInputError)
+    env_input.name = ""
+    result = environment.update(env_input, "invalid/path", "invalid_name")
+    assert isinstance(result, InvalidInputError)
 
-        env_input.name = "invalid_name"
-        env_input.path = "invalid/path"
-        result = environment.update(env_input, "invalid/path", "invalid_name")
-        assert isinstance(result, EnvironmentNotFoundError)
+    env_input.name = "invalid_name"
+    env_input.path = "invalid/path"
+    result = environment.update(env_input, "invalid/path", "invalid_name")
+    assert isinstance(result, EnvironmentNotFoundError)
diff --git a/tests/test_pygit.py b/tests/test_pygit.py
index e47281c..dc90b42 100644
--- a/tests/test_pygit.py
+++ b/tests/test_pygit.py
@@ -39,7 +39,7 @@ def new_repo():
     return (repo, temp_dir, old_commit_oid)
 
 
-def test_clone():
+def test_clone() -> None:
     artifacts = Artifacts()
     path = artifacts.repo.path
 
@@ -50,7 +50,7 @@ def test_clone():
     assert os.path.isdir(path) is True
 
 
-def test_commit(new_repo):
+def test_commit(new_repo) -> None:
     repo = new_repo[0]
     old_commit_oid = new_repo[2]
     with new_repo[1]:
@@ -68,7 +68,7 @@ def test_commit(new_repo):
         assert new_commit_oid == repo_head
 
 
-def test_push(mocker):
+def test_push(mocker) -> None:
     artifacts = Artifacts()
 
     push_mock = mocker.patch('pygit2.Remote.push')
@@ -82,7 +82,7 @@ def get_user_envs_tree(artifacts, oid) -> pygit2.Tree:
     return new_tree[artifacts.user_folder(os.environ["USER"])]
 
 
-def copy_of_repo(artifacts):
+def copy_of_repo(artifacts) -> tempfile.TemporaryDirectory:
     temp_dir = tempfile.TemporaryDirectory()
     shutil.copytree(artifacts.repo.path, temp_dir.name, dirs_exist_ok=True)
     return temp_dir
@@ -92,7 +92,7 @@ def get_user_path_without_environments(artifacts) -> Path:
     return Path(*(artifacts.user_folder(os.environ["USER"]).parts[1:]))
 
 
-def test_create_file():
+def test_create_file() -> None:
     artifacts = Artifacts()
     with copy_of_repo(artifacts) as temp_dir:
         artifacts.repo = pygit2.Repository(temp_dir)
@@ -152,7 +152,7 @@ def test_create_file():
         assert user_envs_tree[new_test_env][fname].data.decode() == "override"
 
 
-def test_delete_environment():
+def test_delete_environment() -> None:
     artifacts = Artifacts()
     with copy_of_repo(artifacts):
         new_test_env = "test_create_file_env"
@@ -200,7 +200,7 @@ def count_user_and_group_envs(artifacts, envs) -> (int, int):
     return num_user_envs, num_group_envs
 
 
-def test_iter():
+def test_iter() -> None:
     artifacts = Artifacts()
     user = os.environ["USER"]
     envs = artifacts.iter(user)

From 2e1b95673019aec22df43a820ecb67b01c096c2d Mon Sep 17 00:00:00 2001
From: Samuel Ogg <56794292+SamO135@users.noreply.github.com>
Date: Fri, 18 Aug 2023 11:36:48 +0100
Subject: [PATCH 036/129] add tests for delete

---
 tests/test_environment.py | 45 +++++++++++++++++++++++++++++----------
 1 file changed, 34 insertions(+), 11 deletions(-)

diff --git a/tests/test_environment.py b/tests/test_environment.py
index 90445f5..8539816 100644
--- a/tests/test_environment.py
+++ b/tests/test_environment.py
@@ -15,8 +15,9 @@
     EnvironmentInput,
     EnvironmentNotFoundError,
     InvalidInputError,
-    PackageInput,
+    Package,
     UpdateEnvironmentSuccess,
+    DeleteEnvironmentSuccess,
 )
 
 from .test_pygit import copy_of_repo, get_user_path_without_environments
@@ -26,29 +27,33 @@
 def temp_git_repo(mocker):
     artifacts = Artifacts()
     environment = Environment(
-        id="", name="", path="", description="", packages=[], state=None
+        id="",
+        name="environment_test",
+        path=str(get_user_path_without_environments(artifacts)),
+        description="description",
+        packages=[Package(id="", name="pkg_test")],
+        state=None,
     )
     with copy_of_repo(artifacts) as temp_dir:
         # repo needs to be modified via environment obj for change to persist
         environment.artifacts.repo = pygit2.Repository(temp_dir)
         mocker.patch('pygit2.Remote.push')
         mocker.patch('httpx.post')
-        env_name = "environment_test"
         env_input = EnvironmentInput(
-            name=env_name,
-            path=str(get_user_path_without_environments(artifacts)),
-            description="description",
-            packages=[PackageInput(name="pkg_test")],
+            name=environment.name,
+            path=environment.path,
+            description=environment.description,
+            packages=environment.packages,
         )
 
         environment.create(env_input)
 
-        yield environment, artifacts, env_input, env_name
+        yield environment, artifacts, env_input
 
 
 def test_create(mocker, temp_git_repo) -> None:
     # Setup
-    environment, artifacts, env_input, env_name = temp_git_repo
+    environment, artifacts, env_input = temp_git_repo
     push_mock = mocker.patch('pygit2.Remote.push')
     post_mock = mocker.patch('httpx.post')
 
@@ -77,7 +82,7 @@ def test_create(mocker, temp_git_repo) -> None:
     result = environment.create(env_input)
     assert isinstance(result, InvalidInputError)
 
-    env_input.name = env_name
+    env_input.name = environment.name
     env_input.path = "invalid/path"
     result = environment.create(env_input)
     assert isinstance(result, InvalidInputError)
@@ -85,7 +90,7 @@ def test_create(mocker, temp_git_repo) -> None:
 
 def test_update(mocker, temp_git_repo) -> None:
     # Setup
-    environment, artifacts, env_input, env_name = temp_git_repo
+    environment, artifacts, env_input = temp_git_repo
     post_mock = mocker.patch('httpx.post')
 
     # Tests
@@ -106,3 +111,21 @@ def test_update(mocker, temp_git_repo) -> None:
     env_input.path = "invalid/path"
     result = environment.update(env_input, "invalid/path", "invalid_name")
     assert isinstance(result, EnvironmentNotFoundError)
+
+
+def test_delete(mocker, temp_git_repo) -> None:
+    # Setup
+    environment, artifacts, env_input = temp_git_repo
+    push_mock = mocker.patch('pygit2.Remote.push')
+    post_mock = mocker.patch('httpx.post')
+
+    # Test
+    result = environment.delete(env_input.name, env_input.path)
+    assert isinstance(result, DeleteEnvironmentSuccess)
+
+    env_input.name = "invalid_name"
+    env_input.path = "invalid/path"
+    result = environment.delete(env_input.name, env_input.path)
+    assert isinstance(result, EnvironmentNotFoundError)
+
+

From 52595d7bd427a2a3c9362910df91795adca10495 Mon Sep 17 00:00:00 2001
From: Samuel Ogg <56794292+SamO135@users.noreply.github.com>
Date: Fri, 18 Aug 2023 15:12:26 +0100
Subject: [PATCH 037/129] add test for write_artifact

---
 poetry.lock               | 10 +++++-----
 pyproject.toml            |  1 +
 tests/test_environment.py | 36 +++++++++++++++++++++++++++++++++---
 3 files changed, 39 insertions(+), 8 deletions(-)

diff --git a/poetry.lock b/poetry.lock
index a6f7f07..2c72d5d 100644
--- a/poetry.lock
+++ b/poetry.lock
@@ -2724,14 +2724,14 @@ testing = ["argcomplete", "attrs (>=19.2.0)", "hypothesis (>=3.56)", "mock", "no
 
 [[package]]
 name = "pytest-asyncio"
-version = "0.21.0"
+version = "0.21.1"
 description = "Pytest support for asyncio"
-category = "dev"
+category = "main"
 optional = false
 python-versions = ">=3.7"
 files = [
-    {file = "pytest-asyncio-0.21.0.tar.gz", hash = "sha256:2b38a496aef56f56b0e87557ec313e11e1ab9276fc3863f6a7be0f1d0e415e1b"},
-    {file = "pytest_asyncio-0.21.0-py3-none-any.whl", hash = "sha256:f2b3366b7cd501a4056858bd39349d5af19742aed2d81660b7998b6341c7eb9c"},
+    {file = "pytest-asyncio-0.21.1.tar.gz", hash = "sha256:40a7eae6dded22c7b604986855ea48400ab15b069ae38116e8c01238e9eeb64d"},
+    {file = "pytest_asyncio-0.21.1-py3-none-any.whl", hash = "sha256:8666c1c8ac02631d7c51ba282e0c69a8a452b211ffedf2599099845da5c5c37b"},
 ]
 
 [package.dependencies]
@@ -4043,4 +4043,4 @@ testing = ["big-O", "flake8 (<5)", "jaraco.functools", "jaraco.itertools", "more
 [metadata]
 lock-version = "2.0"
 python-versions = ">=3.9,<4.0"
-content-hash = "1464d35bb7dea62d6540a946ccdb1fb7a40acddf5ba9b1355822414ee1462ca1"
+content-hash = "d9352aa50d84b36bfeae85c3339f95cd4a6a6d0d2bb6cfb60b85f03a2398392a"
diff --git a/pyproject.toml b/pyproject.toml
index ca77294..f1299b0 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -49,6 +49,7 @@ sqlalchemy = "1.4.45"
 strawberry-graphql = "^0.177.1"
 typer = "^0.9.0"
 pytest-mock = "^3.11.1"
+pytest-asyncio = "^0.21.1"
 
 [tool.poetry.group.dev]
 optional = true
diff --git a/tests/test_environment.py b/tests/test_environment.py
index 8539816..f22adf8 100644
--- a/tests/test_environment.py
+++ b/tests/test_environment.py
@@ -6,10 +6,12 @@
 
 import pygit2
 import pytest
+from starlette.datastructures import UploadFile
 
 from softpack_core.artifacts import Artifacts
 from softpack_core.schemas.environment import (
     CreateEnvironmentSuccess,
+    DeleteEnvironmentSuccess,
     Environment,
     EnvironmentAlreadyExistsError,
     EnvironmentInput,
@@ -17,7 +19,7 @@
     InvalidInputError,
     Package,
     UpdateEnvironmentSuccess,
-    DeleteEnvironmentSuccess,
+    WriteArtifactSuccess,
 )
 
 from .test_pygit import copy_of_repo, get_user_path_without_environments
@@ -116,8 +118,8 @@ def test_update(mocker, temp_git_repo) -> None:
 def test_delete(mocker, temp_git_repo) -> None:
     # Setup
     environment, artifacts, env_input = temp_git_repo
-    push_mock = mocker.patch('pygit2.Remote.push')
-    post_mock = mocker.patch('httpx.post')
+    mocker.patch('pygit2.Remote.push')
+    mocker.patch('httpx.post')
 
     # Test
     result = environment.delete(env_input.name, env_input.path)
@@ -129,3 +131,31 @@ def test_delete(mocker, temp_git_repo) -> None:
     assert isinstance(result, EnvironmentNotFoundError)
 
 
+@pytest.mark.asyncio
+async def test_write_artifact(mocker, temp_git_repo):
+    # Setup
+    environment, artifacts, env_input = temp_git_repo
+    push_mock = mocker.patch('pygit2.Remote.push')
+    mocker.patch('httpx.post')
+
+    # Mock the file upload
+    upload = mocker.Mock(spec=UploadFile)
+    upload.filename = "example.txt"
+    upload.content_type = "text/plain"
+    upload.read.return_value = b"mock data"
+
+    # Test
+    result = await environment.write_artifact(
+        file=upload,
+        folder_path=f"{env_input.path}/{env_input.name}",
+        file_name=upload.filename,
+    )
+    assert isinstance(result, WriteArtifactSuccess)
+    push_mock.assert_called_once()
+
+    result = await environment.write_artifact(
+        file=upload,
+        folder_path="invalid/env/path",
+        file_name=upload.filename,
+    )
+    assert isinstance(result, InvalidInputError)

From af7b497656e65cadd4cd2b6e923bc168ab5262d3 Mon Sep 17 00:00:00 2001
From: Samuel Ogg <56794292+SamO135@users.noreply.github.com>
Date: Fri, 18 Aug 2023 16:42:23 +0100
Subject: [PATCH 038/129] add new fixture to help with tests this fixture has
 not been integrated with any tests yet.

---
 tests/test_pygit.py | 54 +++++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 54 insertions(+)

diff --git a/tests/test_pygit.py b/tests/test_pygit.py
index dc90b42..393ec62 100644
--- a/tests/test_pygit.py
+++ b/tests/test_pygit.py
@@ -16,6 +16,60 @@
 from softpack_core.artifacts import Artifacts
 
 
+# This is a fixture that sets up a new repo in a temporary directory to act
+# as dummy data/repo for tests, instead of creating a copy of the real repo
+# and accessing user folders with os.environ["USERS"].
+@pytest.fixture
+def new_test_repo():
+    # Create new temp folder and repo
+    temp_dir = tempfile.TemporaryDirectory()
+    dir_path = temp_dir.name
+    repo = pygit2.init_repository(dir_path)
+
+    # Create directory structure
+    users_folder = "users"
+    groups_folder = "groups"
+    test_user = "test_user"
+    test_group = "test_group"
+    test_env = "test_environment"
+    user_env_path = Path(
+        dir_path, "environments", users_folder, test_user, test_env
+    )
+    group_env_path = Path(
+        dir_path, "environments", groups_folder, test_group, test_env
+    )
+    os.makedirs(user_env_path)
+    os.makedirs(group_env_path)
+    open(f"{user_env_path}/initial_file.txt", "w").close()
+    open(f"{group_env_path}/initial_file.txt", "w").close()
+
+    # Make initial commit
+    index = repo.index
+    index.add_all()
+    index.write()
+    ref = "HEAD"
+    author = Signature('Alice Author', 'alice@authors.tld')
+    committer = Signature('Cecil Committer', 'cecil@committers.tld')
+    message = "Initial commit"
+    tree = index.write_tree()
+    parents = []
+    initial_commit_oid = repo.create_commit(
+        ref, author, committer, message, tree, parents
+    )
+
+    repo_dict = {
+        "repo": repo,
+        "temp_dir": temp_dir,
+        "initial_commit_oid": initial_commit_oid,
+        "users_folder": users_folder,
+        "groups_folder": groups_folder,
+        "test_user": test_user,
+        "test_group": test_group,
+        "test_environment": test_env,
+    }
+    return repo_dict
+
+
 @pytest.fixture
 def new_repo():
     temp_dir = tempfile.TemporaryDirectory()

From 38ffac564ae43ba082b188d6d267bb80601451ee Mon Sep 17 00:00:00 2001
From: Sendu Bala <sb10@sanger.ac.uk>
Date: Tue, 22 Aug 2023 13:09:40 +0100
Subject: [PATCH 039/129] Start to refactor integration tests to subdir.

---
 tests/integration/conftest.py       | 55 +++++++++++++++++++++++++++++
 tests/integration/test_artifacts.py | 14 ++++++++
 2 files changed, 69 insertions(+)
 create mode 100644 tests/integration/conftest.py
 create mode 100644 tests/integration/test_artifacts.py

diff --git a/tests/integration/conftest.py b/tests/integration/conftest.py
new file mode 100644
index 0000000..42bde87
--- /dev/null
+++ b/tests/integration/conftest.py
@@ -0,0 +1,55 @@
+"""Copyright (c) 2023 Genome Research Ltd.
+
+This source code is licensed under the MIT license found in the
+LICENSE file in the root directory of this source tree.
+"""
+
+import os
+from pathlib import Path
+import pytest
+import tempfile
+
+from softpack_core.artifacts import Artifacts, app
+
+
+@pytest.fixture(scope="package", autouse=True)
+def safe_testing_artifacts():
+    repo_url = os.getenv("SOFTPACK_TEST_ARTIFACTS_REPO_URL")
+    repo_user = os.getenv("SOFTPACK_TEST_ARTIFACTS_REPO_USER")
+    repo_token = os.getenv("SOFTPACK_TEST_ARTIFACTS_REPO_TOKEN")
+    if repo_url is None or repo_user is None or repo_token is None:
+        pytest.skip(("SOFTPACK_TEST_ARTIFACTS_REPO_URL, _USER and _TOKEN "
+                     "env vars are all required for these tests"))
+
+    user = repo_user.split('@', 1)[0]
+    app.settings.artifacts.repo.url = repo_url
+    app.settings.artifacts.repo.username = repo_user
+    app.settings.artifacts.repo.author = user
+    app.settings.artifacts.repo.email = repo_user
+    app.settings.artifacts.repo.writer = repo_token
+    temp_dir = tempfile.TemporaryDirectory()
+    app.settings.artifacts.path = Path(temp_dir.name)
+
+    artifacts = Artifacts()
+
+    try:
+        user_branch = artifacts.repo.branches.remote[f"origin/{user}"]
+    except Exception as e:
+        pytest.fail(f"There must be a branch named after your username. [{e}]")
+
+    commit_ref = artifacts.repo.lookup_reference(user_branch.name)
+    artifacts.repo.set_head(commit_ref.target)
+
+    dict = {
+        "artifacts": artifacts,
+        # "repo": repo,
+        "repo_url": repo_url,
+        "temp_dir": temp_dir,
+        # "initial_commit_oid": initial_commit_oid,
+        # "users_folder": users_folder,
+        # "groups_folder": groups_folder,
+        # "test_user": test_user,
+        # "test_group": test_group,
+        # "test_environment": test_env,
+    }
+    return dict
diff --git a/tests/integration/test_artifacts.py b/tests/integration/test_artifacts.py
new file mode 100644
index 0000000..0879b39
--- /dev/null
+++ b/tests/integration/test_artifacts.py
@@ -0,0 +1,14 @@
+"""Copyright (c) 2023 Genome Research Ltd.
+
+This source code is licensed under the MIT license found in the
+LICENSE file in the root directory of this source tree.
+"""
+
+import pytest
+
+
+def test_commit() -> None:
+    # print(new_safe_test_repo["repo_url"])
+    # with safe_testing_artifacts["temp_dir"]:
+	# print(safe_testing_artifacts["temp_dir"])
+	assert True
\ No newline at end of file

From 1ae5740564644aaaddf06272fc476479b3ac515c Mon Sep 17 00:00:00 2001
From: Sendu Bala <sb10@sanger.ac.uk>
Date: Wed, 23 Aug 2023 09:20:20 +0100
Subject: [PATCH 040/129] Refactor some replicated code and allow branch
 changing and checkout non-bare.

---
 softpack_core/artifacts.py     | 46 +++++++++++++++-------------------
 softpack_core/config/models.py |  1 +
 2 files changed, 21 insertions(+), 26 deletions(-)

diff --git a/softpack_core/artifacts.py b/softpack_core/artifacts.py
index cc52830..45992bf 100644
--- a/softpack_core/artifacts.py
+++ b/softpack_core/artifacts.py
@@ -11,7 +11,6 @@
 
 import pygit2
 from box import Box
-from pygit2 import Signature
 
 from .app import app
 from .ldapapi import LDAP
@@ -24,6 +23,9 @@ class Artifacts:
     environments_file = "softpack.yml"
     users_folder_name = "users"
     groups_folder_name = "groups"
+    head_name = ""
+    credentials_callback = None
+    signature = None
 
     @dataclass
     class Object:
@@ -107,7 +109,12 @@ def __init__(self) -> None:
         except Exception as e:
             print(e)
 
-        callbacks = pygit2.RemoteCallbacks(credentials=credentials)
+        self.credentials_callback = pygit2.RemoteCallbacks(
+            credentials=credentials)
+
+        branch = self.settings.artifacts.repo.branch
+        if branch is None:
+            branch = "main"
 
         if path.is_dir():
             self.repo = pygit2.Repository(path)
@@ -115,18 +122,18 @@ def __init__(self) -> None:
             self.repo = pygit2.clone_repository(
                 self.settings.artifacts.repo.url,
                 path=path,
-                callbacks=callbacks,
-                bare=True,
+                callbacks=self.credentials_callback,
+                bare=False,
+                checkout_branch=branch,
             )
 
-        self.reference = "/".join(
-            [
-                "refs/remotes",
-                self.repo.remotes[0].name,
-                self.repo.head.shorthand,
-            ]
+        self.signature = pygit2.Signature(
+            self.settings.artifacts.repo.author,
+            self.settings.artifacts.repo.email,
         )
 
+        self.head_name = self.repo.head.name
+
     def user_folder(self, user: Optional[str] = None) -> Path:
         """Get the user folder for a given user.
 
@@ -268,14 +275,10 @@ def commit(
         Returns:
             pygit2.Commit: the commit oid
         """
-        ref = repo.head.name
-        author = committer = Signature(
-            self.settings.artifacts.repo.author,
-            self.settings.artifacts.repo.email,
-        )
+        ref = self.head_name
         parents = [repo.lookup_reference(ref).target]
         commit_oid = repo.create_commit(
-            ref, author, committer, message, tree_oid, parents
+            ref, self.signature, self.signature, message, tree_oid, parents
         )
         return commit_oid
 
@@ -286,16 +289,7 @@ def push(self, repo: pygit2.Repository) -> None:
             repo: the repository to push to
         """
         remote = self.repo.remotes[0]
-        credentials = None
-        try:
-            credentials = pygit2.UserPass(
-                self.settings.artifacts.repo.username,
-                self.settings.artifacts.repo.writer,
-            )
-        except Exception as e:
-            print(e)
-        callbacks = pygit2.RemoteCallbacks(credentials=credentials)
-        remote.push([repo.head.name], callbacks=callbacks)
+        remote.push([self.head_name], callbacks=self.credentials_callback)
 
     def build_tree(
         self,
diff --git a/softpack_core/config/models.py b/softpack_core/config/models.py
index 89bdbb3..d054fef 100644
--- a/softpack_core/config/models.py
+++ b/softpack_core/config/models.py
@@ -43,6 +43,7 @@ class Repo(BaseModel):
         email: str
         reader: Optional[str]
         writer: Optional[str]
+        branch: Optional[str]
 
     path: Path
     repo: Repo

From 54aa3cd2c50f8c6cfc5e29f81dad62fdfe80c11d Mon Sep 17 00:00:00 2001
From: Sendu Bala <sb10@sanger.ac.uk>
Date: Wed, 23 Aug 2023 09:21:45 +0100
Subject: [PATCH 041/129] Broken attempt at removing existing envs folder.

---
 tests/integration/conftest.py | 87 +++++++++++++++++++++++++++++++++--
 1 file changed, 83 insertions(+), 4 deletions(-)

diff --git a/tests/integration/conftest.py b/tests/integration/conftest.py
index 42bde87..90cf627 100644
--- a/tests/integration/conftest.py
+++ b/tests/integration/conftest.py
@@ -11,9 +11,11 @@
 
 from softpack_core.artifacts import Artifacts, app
 
+import pygit2
+
 
 @pytest.fixture(scope="package", autouse=True)
-def safe_testing_artifacts():
+def testable_artifacts():
     repo_url = os.getenv("SOFTPACK_TEST_ARTIFACTS_REPO_URL")
     repo_user = os.getenv("SOFTPACK_TEST_ARTIFACTS_REPO_USER")
     repo_token = os.getenv("SOFTPACK_TEST_ARTIFACTS_REPO_TOKEN")
@@ -29,6 +31,7 @@ def safe_testing_artifacts():
     app.settings.artifacts.repo.writer = repo_token
     temp_dir = tempfile.TemporaryDirectory()
     app.settings.artifacts.path = Path(temp_dir.name)
+    app.settings.artifacts.repo.branch = user
 
     artifacts = Artifacts()
 
@@ -37,15 +40,24 @@ def safe_testing_artifacts():
     except Exception as e:
         pytest.fail(f"There must be a branch named after your username. [{e}]")
 
-    commit_ref = artifacts.repo.lookup_reference(user_branch.name)
-    artifacts.repo.set_head(commit_ref.target)
+    artifacts.repo.set_head(user_branch.name)
+    artifacts.head_name = user_branch.name
+
+    tree = artifacts.repo.head.peel(pygit2.Tree)
+    print([(e.name, e) for e in tree])
+
+    oid = reset_test_repo(artifacts)
+
+    tree = artifacts.repo.head.peel(pygit2.Tree)
+    print([(e.name, e) for e in tree])
 
     dict = {
         "artifacts": artifacts,
+        "user_branch": user_branch,
         # "repo": repo,
         "repo_url": repo_url,
         "temp_dir": temp_dir,
-        # "initial_commit_oid": initial_commit_oid,
+        "initial_commit_oid": oid,
         # "users_folder": users_folder,
         # "groups_folder": groups_folder,
         # "test_user": test_user,
@@ -53,3 +65,70 @@ def safe_testing_artifacts():
         # "test_environment": test_env,
     }
     return dict
+
+
+def reset_test_repo(artifacts: Artifacts) -> pygit2.Oid:
+    tree = artifacts.repo.head.peel(pygit2.Tree)
+    dir_path = app.settings.artifacts.path
+
+    if artifacts.environments_root in tree:
+        exitcode = os.system(
+            f"cd {dir_path} && git rm -r {artifacts.environments_root} && git commit -m 'remove environments'")
+        if exitcode != 0:
+            pytest.fail("failed to remove environments")
+
+        # exitcode = os.system(
+        #     f"cd {dir_path} && git commit -m 'remove environements' && git push")
+        # print(exitcode)
+        # tb = artifacts.repo.TreeBuilder(tree)
+
+        # sub_tb = artifacts.repo.TreeBuilder(tree[artifacts.environments_root])
+        # for obj in tree[artifacts.environments_root]:
+        #     sub_tb.remove(obj.name)
+
+        # tb.insert(artifacts.environments_root,
+        #           sub_tb.write(), pygit2.GIT_FILEMODE_TREE)
+        # tb.remove(artifacts.environments_root)
+        # oid = tb.write()
+
+        # ref = artifacts.head_name
+        # parents = [artifacts.repo.lookup_reference(ref).target]
+        # artifacts.repo.create_commit(
+        #     ref, artifacts.signature, artifacts.signature, "rm environments", oid, parents
+        # )
+
+        # remote = artifacts.repo.remotes[0]
+        # remote.push([artifacts.head_name],
+        #             callbacks=artifacts.credentials_callback)
+        print("removed environments")
+    else:
+        print("repo started empty")
+
+    # Create directory structure
+    users_folder = "users"
+    groups_folder = "groups"
+    test_user = "test_user"
+    test_group = "test_group"
+    test_env = "test_environment"
+    user_env_path = Path(
+        dir_path, "environments", users_folder, test_user, test_env
+    )
+    group_env_path = Path(
+        dir_path, "environments", groups_folder, test_group, test_env
+    )
+    os.makedirs(user_env_path)
+    os.makedirs(group_env_path)
+    open(f"{user_env_path}/initial_file.txt", "w").close()
+    open(f"{group_env_path}/initial_file.txt", "w").close()
+
+    # Commit
+    index = artifacts.repo.index
+    index.add_all()
+    index.write()
+    ref = artifacts.head_name  # "HEAD"
+    parents = [artifacts.repo.lookup_reference(ref).target]
+    message = "Add test environments"
+    tree = index.write_tree()
+    return artifacts.repo.create_commit(
+        ref, artifacts.signature, artifacts.signature, message, tree, parents
+    )

From 666631cdd8df3d4b97cd9363364cbe7ed43641d0 Mon Sep 17 00:00:00 2001
From: Sendu Bala <sb10@sanger.ac.uk>
Date: Wed, 23 Aug 2023 13:15:31 +0100
Subject: [PATCH 042/129] Working testable_artifacts test fixture.

---
 tests/integration/conftest.py       | 120 +++++++++++-----------------
 tests/integration/test_artifacts.py |  17 ++--
 2 files changed, 57 insertions(+), 80 deletions(-)

diff --git a/tests/integration/conftest.py b/tests/integration/conftest.py
index 90cf627..c45661c 100644
--- a/tests/integration/conftest.py
+++ b/tests/integration/conftest.py
@@ -6,16 +6,19 @@
 
 import os
 from pathlib import Path
+import pygit2
 import pytest
 import tempfile
+import shutil
 
 from softpack_core.artifacts import Artifacts, app
 
-import pygit2
+artifacts_dict = dict[str, str | pygit2.Oid | Path
+                      | Artifacts | tempfile.TemporaryDirectory[str]]
 
 
 @pytest.fixture(scope="package", autouse=True)
-def testable_artifacts():
+def testable_artifacts() -> artifacts_dict:
     repo_url = os.getenv("SOFTPACK_TEST_ARTIFACTS_REPO_URL")
     repo_user = os.getenv("SOFTPACK_TEST_ARTIFACTS_REPO_USER")
     repo_token = os.getenv("SOFTPACK_TEST_ARTIFACTS_REPO_TOKEN")
@@ -34,77 +37,42 @@ def testable_artifacts():
     app.settings.artifacts.repo.branch = user
 
     artifacts = Artifacts()
+    dict = reset_test_repo(artifacts)
+    dict["temp_dir"] = temp_dir
+    dict["artifacts"] = artifacts
 
-    try:
-        user_branch = artifacts.repo.branches.remote[f"origin/{user}"]
-    except Exception as e:
-        pytest.fail(f"There must be a branch named after your username. [{e}]")
-
-    artifacts.repo.set_head(user_branch.name)
-    artifacts.head_name = user_branch.name
+    return dict
 
-    tree = artifacts.repo.head.peel(pygit2.Tree)
-    print([(e.name, e) for e in tree])
 
-    oid = reset_test_repo(artifacts)
+def reset_test_repo(artifacts: Artifacts) -> artifacts_dict:
+    delete_environments_folder_from_test_repo(artifacts)
 
-    tree = artifacts.repo.head.peel(pygit2.Tree)
-    print([(e.name, e) for e in tree])
-
-    dict = {
-        "artifacts": artifacts,
-        "user_branch": user_branch,
-        # "repo": repo,
-        "repo_url": repo_url,
-        "temp_dir": temp_dir,
-        "initial_commit_oid": oid,
-        # "users_folder": users_folder,
-        # "groups_folder": groups_folder,
-        # "test_user": test_user,
-        # "test_group": test_group,
-        # "test_environment": test_env,
-    }
-    return dict
+    return create_initial_test_repo_state(artifacts)
 
 
-def reset_test_repo(artifacts: Artifacts) -> pygit2.Oid:
+def delete_environments_folder_from_test_repo(artifacts: Artifacts):
     tree = artifacts.repo.head.peel(pygit2.Tree)
-    dir_path = app.settings.artifacts.path
 
     if artifacts.environments_root in tree:
-        exitcode = os.system(
-            f"cd {dir_path} && git rm -r {artifacts.environments_root} && git commit -m 'remove environments'")
-        if exitcode != 0:
-            pytest.fail("failed to remove environments")
-
-        # exitcode = os.system(
-        #     f"cd {dir_path} && git commit -m 'remove environements' && git push")
-        # print(exitcode)
-        # tb = artifacts.repo.TreeBuilder(tree)
-
-        # sub_tb = artifacts.repo.TreeBuilder(tree[artifacts.environments_root])
-        # for obj in tree[artifacts.environments_root]:
-        #     sub_tb.remove(obj.name)
-
-        # tb.insert(artifacts.environments_root,
-        #           sub_tb.write(), pygit2.GIT_FILEMODE_TREE)
-        # tb.remove(artifacts.environments_root)
-        # oid = tb.write()
-
-        # ref = artifacts.head_name
-        # parents = [artifacts.repo.lookup_reference(ref).target]
-        # artifacts.repo.create_commit(
-        #     ref, artifacts.signature, artifacts.signature, "rm environments", oid, parents
-        # )
-
-        # remote = artifacts.repo.remotes[0]
-        # remote.push([artifacts.head_name],
-        #             callbacks=artifacts.credentials_callback)
-        print("removed environments")
-    else:
-        print("repo started empty")
-
-    # Create directory structure
+        shutil.rmtree(Path(app.settings.artifacts.path,
+                      artifacts.environments_root))
+        commit_local_file_changes(artifacts, "delete environments")
+
+
+def commit_local_file_changes(artifacts: Artifacts, msg: str) -> pygit2.Oid:
+    index = artifacts.repo.index
+    index.add_all()
+    index.write()
+    ref = artifacts.head_name
+    parents = [artifacts.repo.lookup_reference(ref).target]
+    oid = index.write_tree()
+    return artifacts.repo.create_commit(
+        ref, artifacts.signature, artifacts.signature, msg, oid, parents
+    )
+
+
+def create_initial_test_repo_state(artifacts: Artifacts) -> artifacts_dict:
+    dir_path = app.settings.artifacts.path
     users_folder = "users"
     groups_folder = "groups"
     test_user = "test_user"
@@ -121,14 +89,16 @@ def reset_test_repo(artifacts: Artifacts) -> pygit2.Oid:
     open(f"{user_env_path}/initial_file.txt", "w").close()
     open(f"{group_env_path}/initial_file.txt", "w").close()
 
-    # Commit
-    index = artifacts.repo.index
-    index.add_all()
-    index.write()
-    ref = artifacts.head_name  # "HEAD"
-    parents = [artifacts.repo.lookup_reference(ref).target]
-    message = "Add test environments"
-    tree = index.write_tree()
-    return artifacts.repo.create_commit(
-        ref, artifacts.signature, artifacts.signature, message, tree, parents
-    )
+    oid = commit_local_file_changes(artifacts, "Add test environments")
+
+    dict: artifacts_dict = {
+        "initial_commit_oid": oid,
+        "users_folder": users_folder,
+        "groups_folder": groups_folder,
+        "test_user": test_user,
+        "test_group": test_group,
+        "test_environment": test_env,
+        "user_env_path": user_env_path,
+        "group_env_path": group_env_path,
+    }
+    return dict
diff --git a/tests/integration/test_artifacts.py b/tests/integration/test_artifacts.py
index 0879b39..a60a0e9 100644
--- a/tests/integration/test_artifacts.py
+++ b/tests/integration/test_artifacts.py
@@ -4,11 +4,18 @@
 LICENSE file in the root directory of this source tree.
 """
 
+import pygit2
 import pytest
 
+from softpack_core.artifacts import Artifacts
 
-def test_commit() -> None:
-    # print(new_safe_test_repo["repo_url"])
-    # with safe_testing_artifacts["temp_dir"]:
-	# print(safe_testing_artifacts["temp_dir"])
-	assert True
\ No newline at end of file
+
+def test_commit(testable_artifacts) -> None:
+    artifacts = testable_artifacts["artifacts"]
+    tree = artifacts.repo.head.peel(pygit2.Tree)
+    print([(e.name, e) for e in tree])
+    print([(e.name, e) for e in tree[artifacts.environments_root]])
+    print([(e.name, e) for e in tree[artifacts.environments_root]["users"]])
+    print(testable_artifacts["user_env_path"])
+
+    assert True

From a9a8e5efc4abc0203d4d5080588984a96d399639 Mon Sep 17 00:00:00 2001
From: Sendu Bala <sb10@sanger.ac.uk>
Date: Wed, 23 Aug 2023 14:22:00 +0100
Subject: [PATCH 043/129] temp (?) hack disable ldap to avoid no-config
 warnings.

---
 softpack_core/ldapapi.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/softpack_core/ldapapi.py b/softpack_core/ldapapi.py
index 899face..dff5d48 100644
--- a/softpack_core/ldapapi.py
+++ b/softpack_core/ldapapi.py
@@ -28,6 +28,7 @@ def initialize(self) -> None:
         Returns:
             None.
         """
+        return
         try:
             self.ldap = ldap.initialize(self.settings.server)
             self.group_regex = re.compile(self.settings.group.pattern)

From c8068d0fe44e94bca6c9e717ed6ce547d4edc49c Mon Sep 17 00:00:00 2001
From: Sendu Bala <sb10@sanger.ac.uk>
Date: Wed, 23 Aug 2023 14:36:56 +0100
Subject: [PATCH 044/129] Refactor fixture to only change settings, to allow
 for independent tests of clone and commit.

---
 tests/integration/conftest.py       | 23 ++++++++-------
 tests/integration/test_artifacts.py | 43 +++++++++++++++++++++++------
 2 files changed, 47 insertions(+), 19 deletions(-)

diff --git a/tests/integration/conftest.py b/tests/integration/conftest.py
index c45661c..5270ab3 100644
--- a/tests/integration/conftest.py
+++ b/tests/integration/conftest.py
@@ -18,7 +18,7 @@
 
 
 @pytest.fixture(scope="package", autouse=True)
-def testable_artifacts() -> artifacts_dict:
+def testable_artifacts_setup():
     repo_url = os.getenv("SOFTPACK_TEST_ARTIFACTS_REPO_URL")
     repo_user = os.getenv("SOFTPACK_TEST_ARTIFACTS_REPO_USER")
     repo_token = os.getenv("SOFTPACK_TEST_ARTIFACTS_REPO_TOKEN")
@@ -32,9 +32,12 @@ def testable_artifacts() -> artifacts_dict:
     app.settings.artifacts.repo.author = user
     app.settings.artifacts.repo.email = repo_user
     app.settings.artifacts.repo.writer = repo_token
+    app.settings.artifacts.repo.branch = user
+
+
+def new_test_artifacts() -> artifacts_dict:
     temp_dir = tempfile.TemporaryDirectory()
     app.settings.artifacts.path = Path(temp_dir.name)
-    app.settings.artifacts.repo.branch = user
 
     artifacts = Artifacts()
     dict = reset_test_repo(artifacts)
@@ -73,32 +76,32 @@ def commit_local_file_changes(artifacts: Artifacts, msg: str) -> pygit2.Oid:
 
 def create_initial_test_repo_state(artifacts: Artifacts) -> artifacts_dict:
     dir_path = app.settings.artifacts.path
-    users_folder = "users"
-    groups_folder = "groups"
     test_user = "test_user"
     test_group = "test_group"
     test_env = "test_environment"
     user_env_path = Path(
-        dir_path, "environments", users_folder, test_user, test_env
+        dir_path, "environments", artifacts.users_folder_name, test_user,
+        test_env
     )
     group_env_path = Path(
-        dir_path, "environments", groups_folder, test_group, test_env
+        dir_path, "environments", artifacts.groups_folder_name, test_group,
+        test_env
     )
     os.makedirs(user_env_path)
     os.makedirs(group_env_path)
-    open(f"{user_env_path}/initial_file.txt", "w").close()
-    open(f"{group_env_path}/initial_file.txt", "w").close()
+    file_basename = "file.txt"
+    open(Path(user_env_path, file_basename), "w").close()
+    open(Path(group_env_path, file_basename), "w").close()
 
     oid = commit_local_file_changes(artifacts, "Add test environments")
 
     dict: artifacts_dict = {
         "initial_commit_oid": oid,
-        "users_folder": users_folder,
-        "groups_folder": groups_folder,
         "test_user": test_user,
         "test_group": test_group,
         "test_environment": test_env,
         "user_env_path": user_env_path,
         "group_env_path": group_env_path,
+        "basename": file_basename,
     }
     return dict
diff --git a/tests/integration/test_artifacts.py b/tests/integration/test_artifacts.py
index a60a0e9..83c6bac 100644
--- a/tests/integration/test_artifacts.py
+++ b/tests/integration/test_artifacts.py
@@ -4,18 +4,43 @@
 LICENSE file in the root directory of this source tree.
 """
 
+import os
+from pathlib import Path
 import pygit2
 import pytest
+import shutil
 
-from softpack_core.artifacts import Artifacts
+from softpack_core.artifacts import Artifacts, app
+from tests.integration.conftest import new_test_artifacts
 
 
-def test_commit(testable_artifacts) -> None:
-    artifacts = testable_artifacts["artifacts"]
-    tree = artifacts.repo.head.peel(pygit2.Tree)
-    print([(e.name, e) for e in tree])
-    print([(e.name, e) for e in tree[artifacts.environments_root]])
-    print([(e.name, e) for e in tree[artifacts.environments_root]["users"]])
-    print(testable_artifacts["user_env_path"])
+def test_clone() -> None:
+    ad = new_test_artifacts()
+    artifacts = ad["artifacts"]
+    path = artifacts.repo.path
+    assert path.startswith(ad["temp_dir"].name)
 
-    assert True
+    shutil.rmtree(ad["temp_dir"].name)
+    assert os.path.isdir(path) is False
+
+    artifacts = Artifacts()
+    assert os.path.isdir(path) is True
+
+
+def test_commit() -> None:
+    ad = new_test_artifacts()
+    artifacts = ad["artifacts"]
+    repo = artifacts.repo
+    old_commit_oid = ad["initial_commit_oid"]
+
+    file_oid = repo.create_blob("test")
+    tree = repo.head.peel(pygit2.Tree)
+    tree_builder = repo.TreeBuilder(tree)
+    tree_builder.insert("new_file.txt", file_oid, pygit2.GIT_FILEMODE_BLOB)
+    new_tree = tree_builder.write()
+
+    new_commit_oid = artifacts.commit(repo, new_tree, "commit new file")
+    repo_head = repo.head.peel(pygit2.Commit).oid
+
+    assert old_commit_oid != new_commit_oid
+    assert new_commit_oid == repo_head

From 84fe41e8fa5c7cdedcc4199e1c533b94c17b5e29 Mon Sep 17 00:00:00 2001
From: Sendu Bala <sb10@sanger.ac.uk>
Date: Wed, 23 Aug 2023 15:40:25 +0100
Subject: [PATCH 045/129] Test commit and real push; change signatures to not
 take repo.

---
 softpack_core/artifacts.py           | 12 +++-----
 softpack_core/schemas/environment.py |  6 ++--
 tests/integration/test_artifacts.py  | 42 ++++++++++++++++++++--------
 tests/test_pygit.py                  |  2 +-
 4 files changed, 39 insertions(+), 23 deletions(-)

diff --git a/softpack_core/artifacts.py b/softpack_core/artifacts.py
index 45992bf..c515d68 100644
--- a/softpack_core/artifacts.py
+++ b/softpack_core/artifacts.py
@@ -262,12 +262,11 @@ def get(self, path: Path, name: str) -> Optional[pygit2.Tree]:
             return None
 
     def commit(
-        self, repo: pygit2.Repository, tree_oid: pygit2.Oid, message: str
+        self, tree_oid: pygit2.Oid, message: str
     ) -> pygit2.Oid:
         """Create and return a commit.
 
         Args:
-            repo: the repository to commit to
             tree_oid: the oid of the tree object that will be committed. The
             tree this refers to will replace the entire contents of the repo.
             message: the commit message
@@ -276,17 +275,14 @@ def commit(
             pygit2.Commit: the commit oid
         """
         ref = self.head_name
-        parents = [repo.lookup_reference(ref).target]
-        commit_oid = repo.create_commit(
+        parents = [self.repo.lookup_reference(ref).target]
+        commit_oid = self.repo.create_commit(
             ref, self.signature, self.signature, message, tree_oid, parents
         )
         return commit_oid
 
-    def push(self, repo: pygit2.Repository) -> None:
+    def push(self) -> None:
         """Push all commits to a repository.
-
-        Args:
-            repo: the repository to push to
         """
         remote = self.repo.remotes[0]
         remote.push([self.head_name], callbacks=self.credentials_callback)
diff --git a/softpack_core/schemas/environment.py b/softpack_core/schemas/environment.py
index b04388e..65f1dfe 100644
--- a/softpack_core/schemas/environment.py
+++ b/softpack_core/schemas/environment.py
@@ -248,7 +248,7 @@ def create(cls, env: EnvironmentInput) -> CreateResponse:
             cls.artifacts.commit(
                 cls.artifacts.repo, tree_oid, "create environment folder"
             )
-            cls.artifacts.push(cls.artifacts.repo)
+            cls.artifacts.push()
         except RuntimeError as e:
             return InvalidInputError(message=str(e))
 
@@ -336,7 +336,7 @@ def delete(cls, name: str, path: str) -> DeleteResponse:
             cls.artifacts.commit(
                 cls.artifacts.repo, tree_oid, "delete environment"
             )
-            cls.artifacts.push(cls.artifacts.repo)
+            cls.artifacts.push()
             return DeleteEnvironmentSuccess(
                 message="Successfully deleted the environment"
             )
@@ -366,7 +366,7 @@ async def write_artifact(
             commit_oid = cls.artifacts.commit(
                 cls.artifacts.repo, tree_oid, "write artifact"
             )
-            cls.artifacts.push(cls.artifacts.repo)
+            cls.artifacts.push()
             return WriteArtifactSuccess(
                 message="Successfully written artifact",
                 commit_oid=str(commit_oid),
diff --git a/tests/integration/test_artifacts.py b/tests/integration/test_artifacts.py
index 83c6bac..4018f7f 100644
--- a/tests/integration/test_artifacts.py
+++ b/tests/integration/test_artifacts.py
@@ -9,6 +9,7 @@
 import pygit2
 import pytest
 import shutil
+import tempfile
 
 from softpack_core.artifacts import Artifacts, app
 from tests.integration.conftest import new_test_artifacts
@@ -16,7 +17,7 @@
 
 def test_clone() -> None:
     ad = new_test_artifacts()
-    artifacts = ad["artifacts"]
+    artifacts: Artifacts = ad["artifacts"]
     path = artifacts.repo.path
     assert path.startswith(ad["temp_dir"].name)
 
@@ -27,20 +28,39 @@ def test_clone() -> None:
     assert os.path.isdir(path) is True
 
 
-def test_commit() -> None:
+def test_commit_and_push() -> None:
     ad = new_test_artifacts()
-    artifacts = ad["artifacts"]
-    repo = artifacts.repo
+    artifacts: Artifacts = ad["artifacts"]
     old_commit_oid = ad["initial_commit_oid"]
 
-    file_oid = repo.create_blob("test")
-    tree = repo.head.peel(pygit2.Tree)
-    tree_builder = repo.TreeBuilder(tree)
-    tree_builder.insert("new_file.txt", file_oid, pygit2.GIT_FILEMODE_BLOB)
-    new_tree = tree_builder.write()
+    tree = artifacts.repo.head.peel(pygit2.Tree)
 
-    new_commit_oid = artifacts.commit(repo, new_tree, "commit new file")
-    repo_head = repo.head.peel(pygit2.Commit).oid
+    new_file_name = "new_file.txt"
+    path = Path(ad["temp_dir"].name, artifacts.environments_root,
+                artifacts.users_folder_name, ad["test_user"],
+                ad["test_environment"], new_file_name)
+
+    open(path, "w").close()
+
+    index = artifacts.repo.index
+    index.add_all()
+    index.write()
+    new_tree = index.write_tree()
+
+    new_commit_oid = artifacts.commit(new_tree, "commit new file")
+    repo_head = artifacts.repo.head.peel(pygit2.Commit).oid
 
     assert old_commit_oid != new_commit_oid
     assert new_commit_oid == repo_head
+
+    artifacts.push()
+
+    temp_dir = tempfile.TemporaryDirectory()
+    app.settings.artifacts.path = Path(temp_dir.name)
+    artifacts = Artifacts()
+
+    path = Path(temp_dir.name, artifacts.environments_root,
+                artifacts.users_folder_name, ad["test_user"],
+                ad["test_environment"], new_file_name)
+
+    assert os.path.isfile(path)
diff --git a/tests/test_pygit.py b/tests/test_pygit.py
index 393ec62..509db4c 100644
--- a/tests/test_pygit.py
+++ b/tests/test_pygit.py
@@ -127,7 +127,7 @@ def test_push(mocker) -> None:
 
     push_mock = mocker.patch('pygit2.Remote.push')
 
-    artifacts.push(artifacts.repo)
+    artifacts.push()
     push_mock.assert_called_once()
 
 

From ccd0a50bd4a5a066f0ea03e987642eaaef5c7296 Mon Sep 17 00:00:00 2001
From: Sendu Bala <sb10@sanger.ac.uk>
Date: Wed, 23 Aug 2023 15:41:11 +0100
Subject: [PATCH 046/129] Test commit and real push; change signatures to not
 take repo.

---
 softpack_core/schemas/environment.py | 12 +++---------
 1 file changed, 3 insertions(+), 9 deletions(-)

diff --git a/softpack_core/schemas/environment.py b/softpack_core/schemas/environment.py
index 65f1dfe..544525b 100644
--- a/softpack_core/schemas/environment.py
+++ b/softpack_core/schemas/environment.py
@@ -245,9 +245,7 @@ def create(cls, env: EnvironmentInput) -> CreateResponse:
             tree_oid = cls.artifacts.create_file(
                 new_folder_path, file_name, "lorem ipsum", True
             )
-            cls.artifacts.commit(
-                cls.artifacts.repo, tree_oid, "create environment folder"
-            )
+            cls.artifacts.commit(tree_oid, "create environment folder")
             cls.artifacts.push()
         except RuntimeError as e:
             return InvalidInputError(message=str(e))
@@ -333,9 +331,7 @@ def delete(cls, name: str, path: str) -> DeleteResponse:
         """
         if cls.artifacts.get(Path(path), name):
             tree_oid = cls.artifacts.delete_environment(name, path)
-            cls.artifacts.commit(
-                cls.artifacts.repo, tree_oid, "delete environment"
-            )
+            cls.artifacts.commit(tree_oid, "delete environment")
             cls.artifacts.push()
             return DeleteEnvironmentSuccess(
                 message="Successfully deleted the environment"
@@ -363,9 +359,7 @@ async def write_artifact(
             tree_oid = cls.artifacts.create_file(
                 Path(folder_path), file_name, contents, overwrite=True
             )
-            commit_oid = cls.artifacts.commit(
-                cls.artifacts.repo, tree_oid, "write artifact"
-            )
+            commit_oid = cls.artifacts.commit(tree_oid, "write artifact")
             cls.artifacts.push()
             return WriteArtifactSuccess(
                 message="Successfully written artifact",

From 4dda585c19221a0836b88ab76b1605cda03caa27 Mon Sep 17 00:00:00 2001
From: Sendu Bala <sb10@sanger.ac.uk>
Date: Wed, 23 Aug 2023 16:08:27 +0100
Subject: [PATCH 047/129] Test create_file.

---
 tests/integration/test_artifacts.py | 98 +++++++++++++++++++++++++++--
 1 file changed, 94 insertions(+), 4 deletions(-)

diff --git a/tests/integration/test_artifacts.py b/tests/integration/test_artifacts.py
index 4018f7f..e2e7d45 100644
--- a/tests/integration/test_artifacts.py
+++ b/tests/integration/test_artifacts.py
@@ -55,12 +55,102 @@ def test_commit_and_push() -> None:
 
     artifacts.push()
 
+    path = Path(artifacts.users_folder_name, ad["test_user"],
+                ad["test_environment"], new_file_name)
+
+    assert file_was_pushed(path)
+
+
+def file_was_pushed(*paths_without_environment: str) -> bool:
     temp_dir = tempfile.TemporaryDirectory()
     app.settings.artifacts.path = Path(temp_dir.name)
     artifacts = Artifacts()
 
-    path = Path(temp_dir.name, artifacts.environments_root,
-                artifacts.users_folder_name, ad["test_user"],
-                ad["test_environment"], new_file_name)
+    for path_without_environment in paths_without_environment:
+        path = Path(temp_dir.name, artifacts.environments_root,
+                    path_without_environment)
+        if not os.path.isfile(path):
+            return False
+
+    return True
+
+
+def test_create_file() -> None:
+    ad = new_test_artifacts()
+    artifacts: Artifacts = ad["artifacts"]
+    user = ad["test_user"]
+
+    new_test_env = "test_create_file_env"
+    assert new_test_env not in [
+        obj.name for obj in artifacts.iter_user(user)
+    ]
+
+    folder_path = Path(
+        get_user_path_without_environments(
+            artifacts, user), new_test_env
+    )
+    basename = "create_file.txt"
+
+    oid = artifacts.create_file(
+        str(folder_path), basename, "lorem ipsum", True, False
+    )
+
+    user_envs_tree = get_user_envs_tree(artifacts, user, oid)
+    assert new_test_env in [obj.name for obj in user_envs_tree]
+    assert basename in [obj.name for obj in user_envs_tree[new_test_env]]
+
+    artifacts.commit(oid, "create file")
+
+    with pytest.raises(RuntimeError) as exc_info:
+        artifacts.create_file(
+            str(folder_path), basename, "lorem ipsum", False, True
+        )
+    assert exc_info.value.args[0] == 'No changes made to the environment'
+
+    basename2 = "create_file2.txt"
+    with pytest.raises(RuntimeError) as exc_info:
+        artifacts.create_file(
+            str(folder_path), basename2, "lorem ipsum", True, False
+        )
+    assert exc_info.value.args[0] == 'Too many changes to the repo'
+
+    oid = artifacts.create_file(
+        str(folder_path), basename2, "lorem ipsum", False, False
+    )
+
+    artifacts.commit(oid, "create file2")
+
+    user_envs_tree = get_user_envs_tree(artifacts, user, oid)
+    assert basename2 in [
+        obj.name for obj in user_envs_tree[new_test_env]
+    ]
+
+    with pytest.raises(FileExistsError) as exc_info:
+        artifacts.create_file(
+            str(folder_path), basename, "lorem ipsum", False, False
+        )
+    assert exc_info.value.args[0] == 'File already exists'
+
+    oid = artifacts.create_file(
+        str(folder_path), basename, "override", False, True
+    )
+
+    artifacts.commit(oid, "update created file")
+
+    user_envs_tree = get_user_envs_tree(artifacts, user, oid)
+    assert basename in [obj.name for obj in user_envs_tree[new_test_env]]
+    assert user_envs_tree[new_test_env][basename].data.decode() == "override"
+
+    artifacts.push()
+
+    assert file_was_pushed(Path(folder_path, basename),
+                           Path(folder_path, basename2))
+
+
+def get_user_path_without_environments(artifacts: Artifacts, user: str) -> Path:
+    return Path(*(artifacts.user_folder(user).parts[1:]))
+
 
-    assert os.path.isfile(path)
+def get_user_envs_tree(artifacts: Artifacts, user: str, oid: pygit2.Oid) -> pygit2.Tree:
+    new_tree = artifacts.repo.get(oid)
+    return new_tree[artifacts.user_folder(user)]

From 56bd743bc97ac406082c9f809c9f39c6e3a61333 Mon Sep 17 00:00:00 2001
From: Sendu Bala <sb10@sanger.ac.uk>
Date: Wed, 23 Aug 2023 16:30:42 +0100
Subject: [PATCH 048/129] Test delete_environment.

---
 tests/integration/test_artifacts.py | 30 +++++++++++++++++++++++++++++
 1 file changed, 30 insertions(+)

diff --git a/tests/integration/test_artifacts.py b/tests/integration/test_artifacts.py
index e2e7d45..a84a0e6 100644
--- a/tests/integration/test_artifacts.py
+++ b/tests/integration/test_artifacts.py
@@ -154,3 +154,33 @@ def get_user_path_without_environments(artifacts: Artifacts, user: str) -> Path:
 def get_user_envs_tree(artifacts: Artifacts, user: str, oid: pygit2.Oid) -> pygit2.Tree:
     new_tree = artifacts.repo.get(oid)
     return new_tree[artifacts.user_folder(user)]
+
+
+def test_delete_environment() -> None:
+    ad = new_test_artifacts()
+    artifacts: Artifacts = ad["artifacts"]
+    user = ad["test_user"]
+    env_for_deleting = ad["test_environment"]
+
+    user_envs_tree = get_user_envs_tree(
+        artifacts, user, artifacts.repo.head.peel(pygit2.Tree).oid)
+    assert env_for_deleting in [obj.name for obj in user_envs_tree]
+
+    oid = artifacts.delete_environment(
+        env_for_deleting, get_user_path_without_environments(artifacts, user)
+    )
+
+    artifacts.commit(oid, "delete new env")
+
+    user_envs_tree = get_user_envs_tree(artifacts, user, oid)
+    assert env_for_deleting not in [obj.name for obj in user_envs_tree]
+
+    with pytest.raises(ValueError) as exc_info:
+        artifacts.delete_environment(
+            user, artifacts.users_folder_name
+        )
+    assert exc_info.value.args[0] == 'Not a valid environment path'
+
+    with pytest.raises(KeyError) as exc_info:
+        artifacts.delete_environment(env_for_deleting, "foo/bar")
+    assert exc_info

From 884a21755d6be9133c057dda6a7925d76cc68925 Mon Sep 17 00:00:00 2001
From: Sendu Bala <sb10@sanger.ac.uk>
Date: Wed, 23 Aug 2023 17:15:12 +0100
Subject: [PATCH 049/129] Refactor iter stuff to not be user-specific; test new
 iter.

---
 softpack_core/artifacts.py           | 37 ++++++++--------------------
 softpack_core/schemas/environment.py | 11 +++------
 tests/integration/test_artifacts.py  | 33 ++++++++++++++++++++++---
 3 files changed, 43 insertions(+), 38 deletions(-)

diff --git a/softpack_core/artifacts.py b/softpack_core/artifacts.py
index c515d68..b0269cf 100644
--- a/softpack_core/artifacts.py
+++ b/softpack_core/artifacts.py
@@ -167,27 +167,21 @@ def environments_folder(self, *args: Optional[str]) -> Path:
         """
         return Path(self.environments_root, *filter(None, list(args)))
 
-    def iter_user(self, user: Optional[str] = None) -> list[pygit2.Tree]:
-        """Iterate environments for a given user.
-
-        Args:
-            user: A username or None.
+    def iter_users(self) -> list[pygit2.Tree]:
+        """Iterate environments for all users.
 
         Returns:
             list[pygit2.Tree]: List of environments
         """
-        return self.iter_environments(self.user_folder(user))
+        return self.iter_environments(self.environments_folder(self.users_folder_name))
 
-    def iter_group(self, group: Optional[str] = None) -> list[pygit2.Tree]:
-        """Iterate environments for a given group.
-
-        Args:
-            group: A group name or None.
+    def iter_groups(self) -> list[pygit2.Tree]:
+        """Iterate environments for all groups.
 
         Returns:
             list[pygit2.Tree]: List of environments
         """
-        return self.iter_environments(self.group_folder(group))
+        return self.iter_environments(self.environments_folder(self.groups_folder_name))
 
     def iter_environments(self, path: Path) -> list[pygit2.Tree]:
         """Iterate environments under a path.
@@ -212,7 +206,7 @@ def tree(self, path: str) -> pygit2.Tree:
         return self.repo.head.peel(pygit2.Tree)[path]
 
     def environments(self, path: Path) -> Iterable:
-        """Return a list of environments in the repo.
+        """Return a list of environments in the repo under the given path.
 
         Args:
             path: a searchable path within the repo
@@ -225,24 +219,13 @@ def environments(self, path: Path) -> Iterable:
         except KeyError:
             return iter(())
 
-    def iter(self, user: Optional[str] = None) -> Iterable:
-        """Return an iterator for the specified user.
-
-        Args:
-            user: a username
+    def iter(self) -> Iterable:
+        """Return an iterator over all environments.
 
         Returns:
             Iterator: an iterator
         """
-        if user:
-            folders = list(
-                itertools.chain(
-                    [self.user_folder(user)],
-                    map(self.group_folder, self.ldap.groups(user) or []),
-                )
-            )
-        else:
-            folders = self.iter_user() + self.iter_group()
+        folders = self.iter_users() + self.iter_groups()
 
         return itertools.chain.from_iterable(map(self.environments, folders))
 
diff --git a/softpack_core/schemas/environment.py b/softpack_core/schemas/environment.py
index 544525b..89e855b 100644
--- a/softpack_core/schemas/environment.py
+++ b/softpack_core/schemas/environment.py
@@ -170,18 +170,13 @@ class Environment:
     artifacts = Artifacts()
 
     @classmethod
-    def iter(cls, all: bool = False) -> Iterable["Environment"]:
-        """Get an iterator over Environment objects.
+    def iter(cls) -> Iterable["Environment"]:
+        """Get an iterator over all Environment objects.
 
         Returns:
             Iterable[Environment]: An iterator of Environment objects.
         """
-        user = None
-        if not user:
-            # TODO: set username from the environment for now
-            # eventually this needs to be the name of the authenticated user
-            user = os.environ["USER"]
-        environment_folders = cls.artifacts.iter(user=user)
+        environment_folders = cls.artifacts.iter()
         environment_objects = map(cls.from_artifact, environment_folders)
         return filter(lambda x: x is not None, environment_objects)
 
diff --git a/tests/integration/test_artifacts.py b/tests/integration/test_artifacts.py
index a84a0e6..e09e2ff 100644
--- a/tests/integration/test_artifacts.py
+++ b/tests/integration/test_artifacts.py
@@ -81,9 +81,10 @@ def test_create_file() -> None:
     user = ad["test_user"]
 
     new_test_env = "test_create_file_env"
-    assert new_test_env not in [
-        obj.name for obj in artifacts.iter_user(user)
-    ]
+
+    user_envs_tree = get_user_envs_tree(
+        artifacts, user, artifacts.repo.head.peel(pygit2.Tree).oid)
+    assert new_test_env not in [obj.name for obj in user_envs_tree]
 
     folder_path = Path(
         get_user_path_without_environments(
@@ -184,3 +185,29 @@ def test_delete_environment() -> None:
     with pytest.raises(KeyError) as exc_info:
         artifacts.delete_environment(env_for_deleting, "foo/bar")
     assert exc_info
+
+
+def test_iter() -> None:
+    ad = new_test_artifacts()
+    artifacts: Artifacts = ad["artifacts"]
+    user = ad["test_user"]
+
+    user_found = False
+    num_user_envs = 0
+    num_group_envs = 0
+
+    envs = artifacts.iter()
+
+    for env in envs:
+        if str(env.path).startswith(artifacts.users_folder_name):
+            num_user_envs += 1
+            if str(env.path).startswith(
+                f"{artifacts.users_folder_name}/{user}"
+            ):
+                user_found = True
+        elif str(env.path).startswith(artifacts.groups_folder_name):
+            num_group_envs += 1
+
+    assert user_found is True
+    assert num_user_envs == 1
+    assert num_group_envs == 1

From 82022fee0207775e60d345272b04b4d0ba0d6a50 Mon Sep 17 00:00:00 2001
From: Sendu Bala <sb10@sanger.ac.uk>
Date: Wed, 23 Aug 2023 17:20:12 +0100
Subject: [PATCH 050/129] Move unit tests to subfolder.

---
 tests/__init__.py                    | 1 -
 tests/unit/__init__.py               | 1 +
 tests/{ => unit}/conftest.py         | 0
 tests/{ => unit}/test_app.py         | 0
 tests/{ => unit}/test_environment.py | 0
 tests/{ => unit}/test_graphql.py     | 0
 tests/{ => unit}/test_main.py        | 0
 tests/{ => unit}/test_pygit.py       | 0
 tests/{ => unit}/test_service.py     | 0
 9 files changed, 1 insertion(+), 1 deletion(-)
 delete mode 100644 tests/__init__.py
 create mode 100644 tests/unit/__init__.py
 rename tests/{ => unit}/conftest.py (100%)
 rename tests/{ => unit}/test_app.py (100%)
 rename tests/{ => unit}/test_environment.py (100%)
 rename tests/{ => unit}/test_graphql.py (100%)
 rename tests/{ => unit}/test_main.py (100%)
 rename tests/{ => unit}/test_pygit.py (100%)
 rename tests/{ => unit}/test_service.py (100%)

diff --git a/tests/__init__.py b/tests/__init__.py
deleted file mode 100644
index f4b2d43..0000000
--- a/tests/__init__.py
+++ /dev/null
@@ -1 +0,0 @@
-"""Unit test package for softpack-core."""
diff --git a/tests/unit/__init__.py b/tests/unit/__init__.py
new file mode 100644
index 0000000..1521582
--- /dev/null
+++ b/tests/unit/__init__.py
@@ -0,0 +1 @@
+"""Unit tests for softpack-core."""
diff --git a/tests/conftest.py b/tests/unit/conftest.py
similarity index 100%
rename from tests/conftest.py
rename to tests/unit/conftest.py
diff --git a/tests/test_app.py b/tests/unit/test_app.py
similarity index 100%
rename from tests/test_app.py
rename to tests/unit/test_app.py
diff --git a/tests/test_environment.py b/tests/unit/test_environment.py
similarity index 100%
rename from tests/test_environment.py
rename to tests/unit/test_environment.py
diff --git a/tests/test_graphql.py b/tests/unit/test_graphql.py
similarity index 100%
rename from tests/test_graphql.py
rename to tests/unit/test_graphql.py
diff --git a/tests/test_main.py b/tests/unit/test_main.py
similarity index 100%
rename from tests/test_main.py
rename to tests/unit/test_main.py
diff --git a/tests/test_pygit.py b/tests/unit/test_pygit.py
similarity index 100%
rename from tests/test_pygit.py
rename to tests/unit/test_pygit.py
diff --git a/tests/test_service.py b/tests/unit/test_service.py
similarity index 100%
rename from tests/test_service.py
rename to tests/unit/test_service.py

From 2d3da1f74ea81d9c826d3a4b39f70567ffd81e68 Mon Sep 17 00:00:00 2001
From: Sendu Bala <sb10@sanger.ac.uk>
Date: Wed, 23 Aug 2023 17:23:12 +0100
Subject: [PATCH 051/129] Add init py.

---
 tests/integration/__init__.py | 1 +
 1 file changed, 1 insertion(+)
 create mode 100644 tests/integration/__init__.py

diff --git a/tests/integration/__init__.py b/tests/integration/__init__.py
new file mode 100644
index 0000000..4a69d90
--- /dev/null
+++ b/tests/integration/__init__.py
@@ -0,0 +1 @@
+"""Integration tests for softpack-core."""

From 71bbdb439605913028603918a96584423612a9bf Mon Sep 17 00:00:00 2001
From: Sendu Bala <sb10@sanger.ac.uk>
Date: Wed, 23 Aug 2023 17:28:10 +0100
Subject: [PATCH 052/129] Rename test_pygit to test_artifacts, and comment out
 all tests for now, prior to conversion to unit tests.

---
 tests/unit/test_artifacts.py | 219 ++++++++++++++++++++++++++
 tests/unit/test_pygit.py     | 296 -----------------------------------
 2 files changed, 219 insertions(+), 296 deletions(-)
 create mode 100644 tests/unit/test_artifacts.py
 delete mode 100644 tests/unit/test_pygit.py

diff --git a/tests/unit/test_artifacts.py b/tests/unit/test_artifacts.py
new file mode 100644
index 0000000..912e9af
--- /dev/null
+++ b/tests/unit/test_artifacts.py
@@ -0,0 +1,219 @@
+"""Copyright (c) 2023 Genome Research Ltd.
+
+This source code is licensed under the MIT license found in the
+LICENSE file in the root directory of this source tree.
+"""
+
+import os
+import shutil
+import tempfile
+from pathlib import Path
+
+import pygit2
+import pytest
+from pygit2 import Signature
+
+from softpack_core.artifacts import Artifacts
+
+
+# def test_clone() -> None:
+#     artifacts = Artifacts()
+#     path = artifacts.repo.path
+
+#     shutil.rmtree(path)
+#     assert os.path.isdir(path) is False
+
+#     artifacts = Artifacts()
+#     assert os.path.isdir(path) is True
+
+
+# def test_commit(new_repo) -> None:
+#     repo = new_repo[0]
+#     old_commit_oid = new_repo[2]
+#     with new_repo[1]:
+#         file_oid = repo.create_blob("test")
+#         tree = repo.head.peel(pygit2.Tree)
+#         tree_builder = repo.TreeBuilder(tree)
+#         tree_builder.insert("new_file.txt", file_oid, pygit2.GIT_FILEMODE_BLOB)
+#         new_tree = tree_builder.write()
+
+#         artifacts = Artifacts()
+#         new_commit_oid = artifacts.commit(repo, new_tree, "commit new file")
+#         repo_head = repo.head.peel(pygit2.Commit).oid
+
+#         assert old_commit_oid != new_commit_oid
+#         assert new_commit_oid == repo_head
+
+
+# def test_push(mocker) -> None:
+#     artifacts = Artifacts()
+
+#     push_mock = mocker.patch('pygit2.Remote.push')
+
+#     artifacts.push()
+#     push_mock.assert_called_once()
+
+
+# def get_user_envs_tree(artifacts, oid) -> pygit2.Tree:
+#     new_tree = artifacts.repo.get(oid)
+#     return new_tree[artifacts.user_folder(os.environ["USER"])]
+
+
+# def copy_of_repo(artifacts) -> tempfile.TemporaryDirectory:
+#     temp_dir = tempfile.TemporaryDirectory()
+#     shutil.copytree(artifacts.repo.path, temp_dir.name, dirs_exist_ok=True)
+#     return temp_dir
+
+
+# def get_user_path_without_environments(artifacts) -> Path:
+#     return Path(*(artifacts.user_folder(os.environ["USER"]).parts[1:]))
+
+
+# def test_create_file() -> None:
+#     artifacts = Artifacts()
+#     with copy_of_repo(artifacts) as temp_dir:
+#         artifacts.repo = pygit2.Repository(temp_dir)
+#         new_test_env = "test_create_file_env"
+#         assert new_test_env not in [
+#             obj.name for obj in artifacts.iter_user(os.environ["USER"])
+#         ]
+
+#         fname = "file.txt"
+
+#         folder_path = Path(
+#             get_user_path_without_environments(artifacts), new_test_env
+#         )
+#         oid = artifacts.create_file(
+#             str(folder_path), fname, "lorem ipsum", True, False
+#         )
+
+#         user_envs_tree = get_user_envs_tree(artifacts, oid)
+#         assert new_test_env in [obj.name for obj in user_envs_tree]
+#         assert fname in [obj.name for obj in user_envs_tree[new_test_env]]
+
+#         artifacts.commit(artifacts.repo, oid, "commit file")
+
+#         with pytest.raises(RuntimeError) as exc_info:
+#             oid = artifacts.create_file(
+#                 str(folder_path), fname, "lorem ipsum", False, True
+#             )
+#         assert exc_info.value.args[0] == 'No changes made to the environment'
+
+#         with pytest.raises(RuntimeError) as exc_info:
+#             artifacts.create_file(
+#                 str(folder_path), "second_file.txt", "lorem ipsum", True, False
+#             )
+#         assert exc_info.value.args[0] == 'Too many changes to the repo'
+
+#         oid = artifacts.create_file(
+#             str(folder_path), "second_file.txt", "lorem ipsum", False, False
+#         )
+
+#         user_envs_tree = get_user_envs_tree(artifacts, oid)
+#         assert "second_file.txt" in [
+#             obj.name for obj in user_envs_tree[new_test_env]
+#         ]
+
+#         with pytest.raises(FileExistsError) as exc_info:
+#             artifacts.create_file(
+#                 str(folder_path), fname, "lorem ipsum", False, False
+#             )
+#         assert exc_info.value.args[0] == 'File already exists'
+
+#         oid = artifacts.create_file(
+#             str(folder_path), fname, "override", False, True
+#         )
+
+#         user_envs_tree = get_user_envs_tree(artifacts, oid)
+#         assert fname in [obj.name for obj in user_envs_tree[new_test_env]]
+#         assert user_envs_tree[new_test_env][fname].data.decode() == "override"
+
+
+# def test_delete_environment() -> None:
+#     artifacts = Artifacts()
+#     with copy_of_repo(artifacts):
+#         new_test_env = "test_create_file_env"
+#         folder_path = Path(
+#             get_user_path_without_environments(artifacts), new_test_env
+#         )
+#         oid = artifacts.create_file(
+#             str(folder_path), "file.txt", "lorem ipsum", True, False
+#         )
+#         artifacts.commit(artifacts.repo, oid, "commit file")
+
+#         user_envs_tree = get_user_envs_tree(artifacts, oid)
+#         assert new_test_env in [obj.name for obj in user_envs_tree]
+
+#         oid = artifacts.delete_environment(
+#             new_test_env, get_user_path_without_environments(artifacts)
+#         )
+
+#         artifacts.commit(artifacts.repo, oid, "commit file")
+
+#         user_envs_tree = get_user_envs_tree(artifacts, oid)
+#         assert new_test_env not in [obj.name for obj in user_envs_tree]
+
+#         with pytest.raises(ValueError) as exc_info:
+#             artifacts.delete_environment(
+#                 os.environ["USER"], artifacts.users_folder_name
+#             )
+#         assert exc_info.value.args[0] == 'Not a valid environment path'
+
+#         with pytest.raises(KeyError) as exc_info:
+#             artifacts.delete_environment(new_test_env, "foo/bar")
+#         assert exc_info
+
+
+# def count_user_and_group_envs(artifacts, envs) -> (int, int):
+#     num_user_envs = 0
+#     num_group_envs = 0
+
+#     for env in envs:
+#         if str(env.path).startswith(artifacts.users_folder_name):
+#             num_user_envs += 1
+#         elif str(env.path).startswith(artifacts.groups_folder_name):
+#             num_group_envs += 1
+
+#     return num_user_envs, num_group_envs
+
+
+# def test_iter() -> None:
+#     artifacts = Artifacts()
+#     user = os.environ["USER"]
+#     envs = artifacts.iter(user)
+
+#     user_found = False
+#     only_this_user = True
+#     num_user_envs = 0
+#     num_group_envs = 0
+
+#     for env in envs:
+#         if str(env.path).startswith(artifacts.users_folder_name):
+#             num_user_envs += 1
+#             if str(env.path).startswith(
+#                 f"{artifacts.users_folder_name}/{user}"
+#             ):
+#                 user_found = True
+#             else:
+#                 only_this_user = False
+#         elif str(env.path).startswith(artifacts.groups_folder_name):
+#             num_group_envs += 1
+
+#     assert user_found is True
+#     assert only_this_user is True
+#     assert num_group_envs > 0
+
+#     envs = artifacts.iter()
+#     no_user_num_user_envs, no_user_num_group_envs = count_user_and_group_envs(
+#         artifacts, envs
+#     )
+#     assert no_user_num_user_envs >= num_user_envs
+#     assert no_user_num_group_envs >= num_group_envs
+
+#     envs = artifacts.iter("!@£$%")
+#     (
+#         bad_user_num_user_envs,
+#         bad_user_num_group_envs,
+#     ) = count_user_and_group_envs(artifacts, envs)
+#     assert bad_user_num_user_envs == 0
+#     assert bad_user_num_group_envs == 0
diff --git a/tests/unit/test_pygit.py b/tests/unit/test_pygit.py
deleted file mode 100644
index 509db4c..0000000
--- a/tests/unit/test_pygit.py
+++ /dev/null
@@ -1,296 +0,0 @@
-"""Copyright (c) 2023 Genome Research Ltd.
-
-This source code is licensed under the MIT license found in the
-LICENSE file in the root directory of this source tree.
-"""
-
-import os
-import shutil
-import tempfile
-from pathlib import Path
-
-import pygit2
-import pytest
-from pygit2 import Signature
-
-from softpack_core.artifacts import Artifacts
-
-
-# This is a fixture that sets up a new repo in a temporary directory to act
-# as dummy data/repo for tests, instead of creating a copy of the real repo
-# and accessing user folders with os.environ["USERS"].
-@pytest.fixture
-def new_test_repo():
-    # Create new temp folder and repo
-    temp_dir = tempfile.TemporaryDirectory()
-    dir_path = temp_dir.name
-    repo = pygit2.init_repository(dir_path)
-
-    # Create directory structure
-    users_folder = "users"
-    groups_folder = "groups"
-    test_user = "test_user"
-    test_group = "test_group"
-    test_env = "test_environment"
-    user_env_path = Path(
-        dir_path, "environments", users_folder, test_user, test_env
-    )
-    group_env_path = Path(
-        dir_path, "environments", groups_folder, test_group, test_env
-    )
-    os.makedirs(user_env_path)
-    os.makedirs(group_env_path)
-    open(f"{user_env_path}/initial_file.txt", "w").close()
-    open(f"{group_env_path}/initial_file.txt", "w").close()
-
-    # Make initial commit
-    index = repo.index
-    index.add_all()
-    index.write()
-    ref = "HEAD"
-    author = Signature('Alice Author', 'alice@authors.tld')
-    committer = Signature('Cecil Committer', 'cecil@committers.tld')
-    message = "Initial commit"
-    tree = index.write_tree()
-    parents = []
-    initial_commit_oid = repo.create_commit(
-        ref, author, committer, message, tree, parents
-    )
-
-    repo_dict = {
-        "repo": repo,
-        "temp_dir": temp_dir,
-        "initial_commit_oid": initial_commit_oid,
-        "users_folder": users_folder,
-        "groups_folder": groups_folder,
-        "test_user": test_user,
-        "test_group": test_group,
-        "test_environment": test_env,
-    }
-    return repo_dict
-
-
-@pytest.fixture
-def new_repo():
-    temp_dir = tempfile.TemporaryDirectory()
-    path = temp_dir.name
-    repo = pygit2.init_repository(path)
-
-    open(f"{path}/initial_file.txt", "w").close()
-    index = repo.index
-    index.add_all()
-    index.write()
-    ref = "HEAD"
-    author = Signature('Alice Author', 'alice@authors.tld')
-    committer = Signature('Cecil Committer', 'cecil@committers.tld')
-    message = "Initial commit"
-    tree = index.write_tree()
-    parents = []
-    old_commit_oid = repo.create_commit(
-        ref, author, committer, message, tree, parents
-    )
-
-    return (repo, temp_dir, old_commit_oid)
-
-
-def test_clone() -> None:
-    artifacts = Artifacts()
-    path = artifacts.repo.path
-
-    shutil.rmtree(path)
-    assert os.path.isdir(path) is False
-
-    artifacts = Artifacts()
-    assert os.path.isdir(path) is True
-
-
-def test_commit(new_repo) -> None:
-    repo = new_repo[0]
-    old_commit_oid = new_repo[2]
-    with new_repo[1]:
-        file_oid = repo.create_blob("test")
-        tree = repo.head.peel(pygit2.Tree)
-        tree_builder = repo.TreeBuilder(tree)
-        tree_builder.insert("new_file.txt", file_oid, pygit2.GIT_FILEMODE_BLOB)
-        new_tree = tree_builder.write()
-
-        artifacts = Artifacts()
-        new_commit_oid = artifacts.commit(repo, new_tree, "commit new file")
-        repo_head = repo.head.peel(pygit2.Commit).oid
-
-        assert old_commit_oid != new_commit_oid
-        assert new_commit_oid == repo_head
-
-
-def test_push(mocker) -> None:
-    artifacts = Artifacts()
-
-    push_mock = mocker.patch('pygit2.Remote.push')
-
-    artifacts.push()
-    push_mock.assert_called_once()
-
-
-def get_user_envs_tree(artifacts, oid) -> pygit2.Tree:
-    new_tree = artifacts.repo.get(oid)
-    return new_tree[artifacts.user_folder(os.environ["USER"])]
-
-
-def copy_of_repo(artifacts) -> tempfile.TemporaryDirectory:
-    temp_dir = tempfile.TemporaryDirectory()
-    shutil.copytree(artifacts.repo.path, temp_dir.name, dirs_exist_ok=True)
-    return temp_dir
-
-
-def get_user_path_without_environments(artifacts) -> Path:
-    return Path(*(artifacts.user_folder(os.environ["USER"]).parts[1:]))
-
-
-def test_create_file() -> None:
-    artifacts = Artifacts()
-    with copy_of_repo(artifacts) as temp_dir:
-        artifacts.repo = pygit2.Repository(temp_dir)
-        new_test_env = "test_create_file_env"
-        assert new_test_env not in [
-            obj.name for obj in artifacts.iter_user(os.environ["USER"])
-        ]
-
-        fname = "file.txt"
-
-        folder_path = Path(
-            get_user_path_without_environments(artifacts), new_test_env
-        )
-        oid = artifacts.create_file(
-            str(folder_path), fname, "lorem ipsum", True, False
-        )
-
-        user_envs_tree = get_user_envs_tree(artifacts, oid)
-        assert new_test_env in [obj.name for obj in user_envs_tree]
-        assert fname in [obj.name for obj in user_envs_tree[new_test_env]]
-
-        artifacts.commit(artifacts.repo, oid, "commit file")
-
-        with pytest.raises(RuntimeError) as exc_info:
-            oid = artifacts.create_file(
-                str(folder_path), fname, "lorem ipsum", False, True
-            )
-        assert exc_info.value.args[0] == 'No changes made to the environment'
-
-        with pytest.raises(RuntimeError) as exc_info:
-            artifacts.create_file(
-                str(folder_path), "second_file.txt", "lorem ipsum", True, False
-            )
-        assert exc_info.value.args[0] == 'Too many changes to the repo'
-
-        oid = artifacts.create_file(
-            str(folder_path), "second_file.txt", "lorem ipsum", False, False
-        )
-
-        user_envs_tree = get_user_envs_tree(artifacts, oid)
-        assert "second_file.txt" in [
-            obj.name for obj in user_envs_tree[new_test_env]
-        ]
-
-        with pytest.raises(FileExistsError) as exc_info:
-            artifacts.create_file(
-                str(folder_path), fname, "lorem ipsum", False, False
-            )
-        assert exc_info.value.args[0] == 'File already exists'
-
-        oid = artifacts.create_file(
-            str(folder_path), fname, "override", False, True
-        )
-
-        user_envs_tree = get_user_envs_tree(artifacts, oid)
-        assert fname in [obj.name for obj in user_envs_tree[new_test_env]]
-        assert user_envs_tree[new_test_env][fname].data.decode() == "override"
-
-
-def test_delete_environment() -> None:
-    artifacts = Artifacts()
-    with copy_of_repo(artifacts):
-        new_test_env = "test_create_file_env"
-        folder_path = Path(
-            get_user_path_without_environments(artifacts), new_test_env
-        )
-        oid = artifacts.create_file(
-            str(folder_path), "file.txt", "lorem ipsum", True, False
-        )
-        artifacts.commit(artifacts.repo, oid, "commit file")
-
-        user_envs_tree = get_user_envs_tree(artifacts, oid)
-        assert new_test_env in [obj.name for obj in user_envs_tree]
-
-        oid = artifacts.delete_environment(
-            new_test_env, get_user_path_without_environments(artifacts)
-        )
-
-        artifacts.commit(artifacts.repo, oid, "commit file")
-
-        user_envs_tree = get_user_envs_tree(artifacts, oid)
-        assert new_test_env not in [obj.name for obj in user_envs_tree]
-
-        with pytest.raises(ValueError) as exc_info:
-            artifacts.delete_environment(
-                os.environ["USER"], artifacts.users_folder_name
-            )
-        assert exc_info.value.args[0] == 'Not a valid environment path'
-
-        with pytest.raises(KeyError) as exc_info:
-            artifacts.delete_environment(new_test_env, "foo/bar")
-        assert exc_info
-
-
-def count_user_and_group_envs(artifacts, envs) -> (int, int):
-    num_user_envs = 0
-    num_group_envs = 0
-
-    for env in envs:
-        if str(env.path).startswith(artifacts.users_folder_name):
-            num_user_envs += 1
-        elif str(env.path).startswith(artifacts.groups_folder_name):
-            num_group_envs += 1
-
-    return num_user_envs, num_group_envs
-
-
-def test_iter() -> None:
-    artifacts = Artifacts()
-    user = os.environ["USER"]
-    envs = artifacts.iter(user)
-
-    user_found = False
-    only_this_user = True
-    num_user_envs = 0
-    num_group_envs = 0
-
-    for env in envs:
-        if str(env.path).startswith(artifacts.users_folder_name):
-            num_user_envs += 1
-            if str(env.path).startswith(
-                f"{artifacts.users_folder_name}/{user}"
-            ):
-                user_found = True
-            else:
-                only_this_user = False
-        elif str(env.path).startswith(artifacts.groups_folder_name):
-            num_group_envs += 1
-
-    assert user_found is True
-    assert only_this_user is True
-    assert num_group_envs > 0
-
-    envs = artifacts.iter()
-    no_user_num_user_envs, no_user_num_group_envs = count_user_and_group_envs(
-        artifacts, envs
-    )
-    assert no_user_num_user_envs >= num_user_envs
-    assert no_user_num_group_envs >= num_group_envs
-
-    envs = artifacts.iter("!@£$%")
-    (
-        bad_user_num_user_envs,
-        bad_user_num_group_envs,
-    ) = count_user_and_group_envs(artifacts, envs)
-    assert bad_user_num_user_envs == 0
-    assert bad_user_num_group_envs == 0

From 51e9ce634889a91e08a66b6eddfcd3ced549b4e2 Mon Sep 17 00:00:00 2001
From: Sendu Bala <sb10@sanger.ac.uk>
Date: Thu, 24 Aug 2023 09:14:37 +0100
Subject: [PATCH 053/129] Delint.

---
 tests/integration/test_artifacts.py | 16 +++++++---------
 1 file changed, 7 insertions(+), 9 deletions(-)

diff --git a/tests/integration/test_artifacts.py b/tests/integration/test_artifacts.py
index e09e2ff..586bf79 100644
--- a/tests/integration/test_artifacts.py
+++ b/tests/integration/test_artifacts.py
@@ -33,8 +33,6 @@ def test_commit_and_push() -> None:
     artifacts: Artifacts = ad["artifacts"]
     old_commit_oid = ad["initial_commit_oid"]
 
-    tree = artifacts.repo.head.peel(pygit2.Tree)
-
     new_file_name = "new_file.txt"
     path = Path(ad["temp_dir"].name, artifacts.environments_root,
                 artifacts.users_folder_name, ad["test_user"],
@@ -61,7 +59,7 @@ def test_commit_and_push() -> None:
     assert file_was_pushed(path)
 
 
-def file_was_pushed(*paths_without_environment: str) -> bool:
+def file_was_pushed(*paths_without_environment: str | Path) -> bool:
     temp_dir = tempfile.TemporaryDirectory()
     app.settings.artifacts.path = Path(temp_dir.name)
     artifacts = Artifacts()
@@ -93,7 +91,7 @@ def test_create_file() -> None:
     basename = "create_file.txt"
 
     oid = artifacts.create_file(
-        str(folder_path), basename, "lorem ipsum", True, False
+        folder_path, basename, "lorem ipsum", True, False
     )
 
     user_envs_tree = get_user_envs_tree(artifacts, user, oid)
@@ -104,19 +102,19 @@ def test_create_file() -> None:
 
     with pytest.raises(RuntimeError) as exc_info:
         artifacts.create_file(
-            str(folder_path), basename, "lorem ipsum", False, True
+            folder_path, basename, "lorem ipsum", False, True
         )
     assert exc_info.value.args[0] == 'No changes made to the environment'
 
     basename2 = "create_file2.txt"
     with pytest.raises(RuntimeError) as exc_info:
         artifacts.create_file(
-            str(folder_path), basename2, "lorem ipsum", True, False
+            folder_path, basename2, "lorem ipsum", True, False
         )
     assert exc_info.value.args[0] == 'Too many changes to the repo'
 
     oid = artifacts.create_file(
-        str(folder_path), basename2, "lorem ipsum", False, False
+        folder_path, basename2, "lorem ipsum", False, False
     )
 
     artifacts.commit(oid, "create file2")
@@ -128,12 +126,12 @@ def test_create_file() -> None:
 
     with pytest.raises(FileExistsError) as exc_info:
         artifacts.create_file(
-            str(folder_path), basename, "lorem ipsum", False, False
+            folder_path, basename, "lorem ipsum", False, False
         )
     assert exc_info.value.args[0] == 'File already exists'
 
     oid = artifacts.create_file(
-        str(folder_path), basename, "override", False, True
+        folder_path, basename, "override", False, True
     )
 
     artifacts.commit(oid, "update created file")

From 9dc18163394bd3b9fdeacf307a40885448237712 Mon Sep 17 00:00:00 2001
From: Sendu Bala <sb10@sanger.ac.uk>
Date: Thu, 24 Aug 2023 10:03:10 +0100
Subject: [PATCH 054/129] Test environment.create().

---
 softpack_core/schemas/environment.py  |  9 ++-
 tests/integration/conftest.py         |  4 ++
 tests/integration/test_artifacts.py   |  8 +--
 tests/integration/test_environment.py | 86 +++++++++++++++++++++++++++
 4 files changed, 99 insertions(+), 8 deletions(-)
 create mode 100644 tests/integration/test_environment.py

diff --git a/softpack_core/schemas/environment.py b/softpack_core/schemas/environment.py
index 89e855b..6d11ea5 100644
--- a/softpack_core/schemas/environment.py
+++ b/softpack_core/schemas/environment.py
@@ -220,9 +220,12 @@ def create(cls, env: EnvironmentInput) -> CreateResponse:
         if any(len(value) == 0 for value in vars(env).values()):
             return InvalidInputError(message="all fields must be filled in")
 
-        # Check if a valid path has been provided
-        user = os.environ["USER"]
-        if env.path not in ["groups/hgi", f"users/{user}"]:
+        # Check if a valid path has been provided. TODO: improve this to check
+        # that they can only create stuff in their own users folder, or in
+        # group folders of unix groups they belong to.
+        valid_dirs = [cls.artifacts.users_folder_name,
+                      cls.artifacts.groups_folder_name]
+        if not any(env.path.startswith(dir) for dir in valid_dirs):
             return InvalidInputError(message="Invalid path")
 
         # Check if an env with same name already exists at given path
diff --git a/tests/integration/conftest.py b/tests/integration/conftest.py
index 5270ab3..33e9cc9 100644
--- a/tests/integration/conftest.py
+++ b/tests/integration/conftest.py
@@ -105,3 +105,7 @@ def create_initial_test_repo_state(artifacts: Artifacts) -> artifacts_dict:
         "basename": file_basename,
     }
     return dict
+
+
+def get_user_path_without_environments(artifacts: Artifacts, user: str) -> Path:
+    return Path(*(artifacts.user_folder(user).parts[1:]))
diff --git a/tests/integration/test_artifacts.py b/tests/integration/test_artifacts.py
index 586bf79..50ede05 100644
--- a/tests/integration/test_artifacts.py
+++ b/tests/integration/test_artifacts.py
@@ -12,7 +12,9 @@
 import tempfile
 
 from softpack_core.artifacts import Artifacts, app
-from tests.integration.conftest import new_test_artifacts
+
+from tests.integration.conftest import (new_test_artifacts,
+                                        get_user_path_without_environments)
 
 
 def test_clone() -> None:
@@ -146,10 +148,6 @@ def test_create_file() -> None:
                            Path(folder_path, basename2))
 
 
-def get_user_path_without_environments(artifacts: Artifacts, user: str) -> Path:
-    return Path(*(artifacts.user_folder(user).parts[1:]))
-
-
 def get_user_envs_tree(artifacts: Artifacts, user: str, oid: pygit2.Oid) -> pygit2.Tree:
     new_tree = artifacts.repo.get(oid)
     return new_tree[artifacts.user_folder(user)]
diff --git a/tests/integration/test_environment.py b/tests/integration/test_environment.py
new file mode 100644
index 0000000..2232952
--- /dev/null
+++ b/tests/integration/test_environment.py
@@ -0,0 +1,86 @@
+"""Copyright (c) 2023 Genome Research Ltd.
+
+This source code is licensed under the MIT license found in the
+LICENSE file in the root directory of this source tree.
+"""
+
+import pytest
+
+from softpack_core.artifacts import Artifacts
+from softpack_core.schemas.environment import (
+    CreateEnvironmentSuccess,
+    DeleteEnvironmentSuccess,
+    Environment,
+    EnvironmentAlreadyExistsError,
+    EnvironmentInput,
+    EnvironmentNotFoundError,
+    InvalidInputError,
+    Package,
+    UpdateEnvironmentSuccess,
+    WriteArtifactSuccess,
+)
+
+from tests.integration.conftest import (new_test_artifacts,
+                                        get_user_path_without_environments)
+
+from softpack_core.artifacts import Artifacts, app
+
+
+@pytest.fixture
+def testable_environment(mocker):
+    ad = new_test_artifacts()
+    artifacts: Artifacts = ad["artifacts"]
+    user = ad["test_user"]
+
+    mocker.patch.object(Environment, 'artifacts', new=artifacts)
+
+    environment = Environment(
+        id="",
+        name="test_env_create",
+        path=str(get_user_path_without_environments(artifacts, user)),
+        description="description",
+        packages=[Package(id="", name="pkg_test")],
+        state=None,
+    )
+
+    env_input = EnvironmentInput(
+        name=environment.name,
+        path=environment.path,
+        description=environment.description,
+        packages=environment.packages,
+    )
+
+    yield artifacts, environment, env_input
+
+
+def test_create(mocker, testable_environment) -> None:
+    _, environment, env_input = testable_environment
+    post_mock = mocker.patch('httpx.post')
+
+    result = environment.create(env_input)
+    assert isinstance(result, CreateEnvironmentSuccess)
+    # push_mock.assert_called_once()
+
+    # TODO: don't mock this; actually have a real builder service to test with?
+    post_mock.assert_called_once_with(
+        "http://0.0.0.0:7080/environments/build",
+        json={
+            "name": f"{env_input.path}/{env_input.name}",
+            "model": {
+                "description": env_input.description,
+                "packages": [f"{pkg.name}" for pkg in env_input.packages],
+            },
+        },
+    )
+
+    result = environment.create(env_input)
+    assert isinstance(result, EnvironmentAlreadyExistsError)
+
+    env_input.name = ""
+    result = environment.create(env_input)
+    assert isinstance(result, InvalidInputError)
+
+    env_input.name = environment.name
+    env_input.path = "invalid/path"
+    result = environment.create(env_input)
+    assert isinstance(result, InvalidInputError)

From de1e243d5d504c4c68d11bd277a0ba05aab10cac Mon Sep 17 00:00:00 2001
From: Sendu Bala <sb10@sanger.ac.uk>
Date: Thu, 24 Aug 2023 10:12:18 +0100
Subject: [PATCH 055/129] Test the push worked during create.

---
 softpack_core/schemas/environment.py  | 4 ++--
 tests/integration/test_environment.py | 8 ++++++--
 2 files changed, 8 insertions(+), 4 deletions(-)

diff --git a/softpack_core/schemas/environment.py b/softpack_core/schemas/environment.py
index 6d11ea5..f1836d1 100644
--- a/softpack_core/schemas/environment.py
+++ b/softpack_core/schemas/environment.py
@@ -238,10 +238,10 @@ def create(cls, env: EnvironmentInput) -> CreateResponse:
 
         # Create folder with readme
         new_folder_path = Path(env.path, env.name)
-        file_name = "README.md"
+        file_name = ".created"
         try:
             tree_oid = cls.artifacts.create_file(
-                new_folder_path, file_name, "lorem ipsum", True
+                new_folder_path, file_name, "", True
             )
             cls.artifacts.commit(tree_oid, "create environment folder")
             cls.artifacts.push()
diff --git a/tests/integration/test_environment.py b/tests/integration/test_environment.py
index 2232952..e8ef959 100644
--- a/tests/integration/test_environment.py
+++ b/tests/integration/test_environment.py
@@ -4,6 +4,7 @@
 LICENSE file in the root directory of this source tree.
 """
 
+from pathlib import Path
 import pytest
 
 from softpack_core.artifacts import Artifacts
@@ -22,8 +23,9 @@
 
 from tests.integration.conftest import (new_test_artifacts,
                                         get_user_path_without_environments)
+from .test_artifacts import file_was_pushed
 
-from softpack_core.artifacts import Artifacts, app
+from softpack_core.artifacts import Artifacts
 
 
 @pytest.fixture
@@ -59,7 +61,9 @@ def test_create(mocker, testable_environment) -> None:
 
     result = environment.create(env_input)
     assert isinstance(result, CreateEnvironmentSuccess)
-    # push_mock.assert_called_once()
+
+    path = Path(env_input.path, env_input.name, ".created")
+    assert file_was_pushed(path)
 
     # TODO: don't mock this; actually have a real builder service to test with?
     post_mock.assert_called_once_with(

From 0a502e4d898e1e6dea2c9b99967889b75486b077 Mon Sep 17 00:00:00 2001
From: Sendu Bala <sb10@sanger.ac.uk>
Date: Thu, 24 Aug 2023 10:29:12 +0100
Subject: [PATCH 056/129] Test create.update().

---
 softpack_core/schemas/environment.py  |  3 +-
 tests/integration/test_environment.py | 45 +++++++++++++++++++++++----
 2 files changed, 41 insertions(+), 7 deletions(-)

diff --git a/softpack_core/schemas/environment.py b/softpack_core/schemas/environment.py
index f1836d1..388785d 100644
--- a/softpack_core/schemas/environment.py
+++ b/softpack_core/schemas/environment.py
@@ -291,7 +291,8 @@ def update(
 
         # Check name and path have not been changed.
         if env.path != current_path or env.name != current_name:
-            return InvalidInputError(message="cannot change name or path")
+            return InvalidInputError(message=("change of name or path not "
+                                              "currently supported"))
 
         # Check if an environment exists at the specified path and name
         if cls.artifacts.get(Path(current_path), current_name):
diff --git a/tests/integration/test_environment.py b/tests/integration/test_environment.py
index e8ef959..32c5cdb 100644
--- a/tests/integration/test_environment.py
+++ b/tests/integration/test_environment.py
@@ -65,8 +65,26 @@ def test_create(mocker, testable_environment) -> None:
     path = Path(env_input.path, env_input.name, ".created")
     assert file_was_pushed(path)
 
+    post_mock.assert_called_once()
+    builder_called_correctly(post_mock, env_input)
+
+    result = environment.create(env_input)
+    assert isinstance(result, EnvironmentAlreadyExistsError)
+
+    env_input.name = ""
+    result = environment.create(env_input)
+    assert isinstance(result, InvalidInputError)
+
+    env_input.name = environment.name
+    env_input.path = "invalid/path"
+    result = environment.create(env_input)
+    assert isinstance(result, InvalidInputError)
+
+
+def builder_called_correctly(post_mock, env_input: EnvironmentInput) -> None:
     # TODO: don't mock this; actually have a real builder service to test with?
-    post_mock.assert_called_once_with(
+    # Also need to not hard-code the url here.
+    post_mock.assert_called_with(
         "http://0.0.0.0:7080/environments/build",
         json={
             "name": f"{env_input.path}/{env_input.name}",
@@ -77,14 +95,29 @@ def test_create(mocker, testable_environment) -> None:
         },
     )
 
+
+def test_update(mocker, testable_environment) -> None:
+    _, environment, env_input = testable_environment
+    post_mock = mocker.patch('httpx.post')
+
     result = environment.create(env_input)
-    assert isinstance(result, EnvironmentAlreadyExistsError)
+    assert isinstance(result, CreateEnvironmentSuccess)
+    post_mock.assert_called_once()
+
+    env_input.description = "updated description"
+    result = environment.update(env_input, env_input.path, env_input.name)
+    assert isinstance(result, UpdateEnvironmentSuccess)
+
+    builder_called_correctly(post_mock, env_input)
+
+    result = environment.update(env_input, "invalid/path", "invalid_name")
+    assert isinstance(result, InvalidInputError)
 
     env_input.name = ""
-    result = environment.create(env_input)
+    result = environment.update(env_input, env_input.path, env_input.name)
     assert isinstance(result, InvalidInputError)
 
-    env_input.name = environment.name
+    env_input.name = "invalid_name"
     env_input.path = "invalid/path"
-    result = environment.create(env_input)
-    assert isinstance(result, InvalidInputError)
+    result = environment.update(env_input, "invalid/path", "invalid_name")
+    assert isinstance(result, EnvironmentNotFoundError)

From 31d0ed5b9f6c0e7cb8546c2ff47a3cf1d7d1b5d9 Mon Sep 17 00:00:00 2001
From: Sendu Bala <sb10@sanger.ac.uk>
Date: Thu, 24 Aug 2023 10:51:59 +0100
Subject: [PATCH 057/129] Test environment.delete().

---
 tests/integration/conftest.py         | 14 ++++++++++++++
 tests/integration/test_artifacts.py   | 17 ++---------------
 tests/integration/test_environment.py | 24 ++++++++++++++++++++++--
 3 files changed, 38 insertions(+), 17 deletions(-)

diff --git a/tests/integration/conftest.py b/tests/integration/conftest.py
index 33e9cc9..556d7f1 100644
--- a/tests/integration/conftest.py
+++ b/tests/integration/conftest.py
@@ -109,3 +109,17 @@ def create_initial_test_repo_state(artifacts: Artifacts) -> artifacts_dict:
 
 def get_user_path_without_environments(artifacts: Artifacts, user: str) -> Path:
     return Path(*(artifacts.user_folder(user).parts[1:]))
+
+
+def file_was_pushed(*paths_without_environment: str | Path) -> bool:
+    temp_dir = tempfile.TemporaryDirectory()
+    app.settings.artifacts.path = Path(temp_dir.name)
+    artifacts = Artifacts()
+
+    for path_without_environment in paths_without_environment:
+        path = Path(temp_dir.name, artifacts.environments_root,
+                    path_without_environment)
+        if not os.path.isfile(path):
+            return False
+
+    return True
diff --git a/tests/integration/test_artifacts.py b/tests/integration/test_artifacts.py
index 50ede05..570e011 100644
--- a/tests/integration/test_artifacts.py
+++ b/tests/integration/test_artifacts.py
@@ -14,7 +14,8 @@
 from softpack_core.artifacts import Artifacts, app
 
 from tests.integration.conftest import (new_test_artifacts,
-                                        get_user_path_without_environments)
+                                        get_user_path_without_environments,
+                                        file_was_pushed)
 
 
 def test_clone() -> None:
@@ -61,20 +62,6 @@ def test_commit_and_push() -> None:
     assert file_was_pushed(path)
 
 
-def file_was_pushed(*paths_without_environment: str | Path) -> bool:
-    temp_dir = tempfile.TemporaryDirectory()
-    app.settings.artifacts.path = Path(temp_dir.name)
-    artifacts = Artifacts()
-
-    for path_without_environment in paths_without_environment:
-        path = Path(temp_dir.name, artifacts.environments_root,
-                    path_without_environment)
-        if not os.path.isfile(path):
-            return False
-
-    return True
-
-
 def test_create_file() -> None:
     ad = new_test_artifacts()
     artifacts: Artifacts = ad["artifacts"]
diff --git a/tests/integration/test_environment.py b/tests/integration/test_environment.py
index 32c5cdb..be348aa 100644
--- a/tests/integration/test_environment.py
+++ b/tests/integration/test_environment.py
@@ -22,8 +22,8 @@
 )
 
 from tests.integration.conftest import (new_test_artifacts,
-                                        get_user_path_without_environments)
-from .test_artifacts import file_was_pushed
+                                        get_user_path_without_environments,
+                                        file_was_pushed)
 
 from softpack_core.artifacts import Artifacts
 
@@ -121,3 +121,23 @@ def test_update(mocker, testable_environment) -> None:
     env_input.path = "invalid/path"
     result = environment.update(env_input, "invalid/path", "invalid_name")
     assert isinstance(result, EnvironmentNotFoundError)
+
+
+def test_delete(mocker, testable_environment) -> None:
+    _, environment, env_input = testable_environment
+
+    result = environment.delete(env_input.name, env_input.path)
+    assert isinstance(result, EnvironmentNotFoundError)
+
+    post_mock = mocker.patch('httpx.post')
+    result = environment.create(env_input)
+    assert isinstance(result, CreateEnvironmentSuccess)
+    post_mock.assert_called_once()
+
+    path = Path(env_input.path, env_input.name, ".created")
+    assert file_was_pushed(path)
+
+    result = environment.delete(env_input.name, env_input.path)
+    assert isinstance(result, DeleteEnvironmentSuccess)
+
+    assert not file_was_pushed(path)

From f681109f76d50e46d7f32ba48f1857d8395a4c2e Mon Sep 17 00:00:00 2001
From: Sendu Bala <sb10@sanger.ac.uk>
Date: Thu, 24 Aug 2023 11:04:45 +0100
Subject: [PATCH 058/129] Test write_artifact.

---
 tests/integration/test_environment.py | 48 ++++++++++++++++++++++++---
 1 file changed, 44 insertions(+), 4 deletions(-)

diff --git a/tests/integration/test_environment.py b/tests/integration/test_environment.py
index be348aa..a728acd 100644
--- a/tests/integration/test_environment.py
+++ b/tests/integration/test_environment.py
@@ -6,6 +6,7 @@
 
 from pathlib import Path
 import pytest
+from starlette.datastructures import UploadFile
 
 from softpack_core.artifacts import Artifacts
 from softpack_core.schemas.environment import (
@@ -52,11 +53,11 @@ def testable_environment(mocker):
         packages=environment.packages,
     )
 
-    yield artifacts, environment, env_input
+    yield environment, env_input
 
 
 def test_create(mocker, testable_environment) -> None:
-    _, environment, env_input = testable_environment
+    environment, env_input = testable_environment
     post_mock = mocker.patch('httpx.post')
 
     result = environment.create(env_input)
@@ -97,7 +98,7 @@ def builder_called_correctly(post_mock, env_input: EnvironmentInput) -> None:
 
 
 def test_update(mocker, testable_environment) -> None:
-    _, environment, env_input = testable_environment
+    environment, env_input = testable_environment
     post_mock = mocker.patch('httpx.post')
 
     result = environment.create(env_input)
@@ -124,7 +125,7 @@ def test_update(mocker, testable_environment) -> None:
 
 
 def test_delete(mocker, testable_environment) -> None:
-    _, environment, env_input = testable_environment
+    environment, env_input = testable_environment
 
     result = environment.delete(env_input.name, env_input.path)
     assert isinstance(result, EnvironmentNotFoundError)
@@ -141,3 +142,42 @@ def test_delete(mocker, testable_environment) -> None:
     assert isinstance(result, DeleteEnvironmentSuccess)
 
     assert not file_was_pushed(path)
+
+
+@pytest.mark.asyncio
+async def test_write_artifact(mocker, testable_environment):
+    environment, env_input = testable_environment
+
+    upload = mocker.Mock(spec=UploadFile)
+    upload.filename = "example.txt"
+    upload.content_type = "text/plain"
+    upload.read.return_value = b"mock data"
+
+    result = await environment.write_artifact(
+        file=upload,
+        folder_path=f"{env_input.path}/{env_input.name}",
+        file_name=upload.filename,
+    )
+    assert isinstance(result, InvalidInputError)
+
+    post_mock = mocker.patch('httpx.post')
+    result = environment.create(env_input)
+    assert isinstance(result, CreateEnvironmentSuccess)
+    post_mock.assert_called_once()
+
+    result = await environment.write_artifact(
+        file=upload,
+        folder_path=f"{env_input.path}/{env_input.name}",
+        file_name=upload.filename,
+    )
+    assert isinstance(result, WriteArtifactSuccess)
+
+    path = Path(env_input.path, env_input.name, upload.filename)
+    assert file_was_pushed(path)
+
+    result = await environment.write_artifact(
+        file=upload,
+        folder_path="invalid/env/path",
+        file_name=upload.filename,
+    )
+    assert isinstance(result, InvalidInputError)

From c999436d0664a86fb541fde1fb2b51d721539192 Mon Sep 17 00:00:00 2001
From: Sendu Bala <sb10@sanger.ac.uk>
Date: Thu, 24 Aug 2023 11:18:48 +0100
Subject: [PATCH 059/129] Comment out tests for now until replaced with fully
 mocked unit tests.

---
 tests/unit/test_environment.py | 274 ++++++++++++++++-----------------
 1 file changed, 137 insertions(+), 137 deletions(-)

diff --git a/tests/unit/test_environment.py b/tests/unit/test_environment.py
index f22adf8..9909ff4 100644
--- a/tests/unit/test_environment.py
+++ b/tests/unit/test_environment.py
@@ -22,140 +22,140 @@
     WriteArtifactSuccess,
 )
 
-from .test_pygit import copy_of_repo, get_user_path_without_environments
-
-
-@pytest.fixture
-def temp_git_repo(mocker):
-    artifacts = Artifacts()
-    environment = Environment(
-        id="",
-        name="environment_test",
-        path=str(get_user_path_without_environments(artifacts)),
-        description="description",
-        packages=[Package(id="", name="pkg_test")],
-        state=None,
-    )
-    with copy_of_repo(artifacts) as temp_dir:
-        # repo needs to be modified via environment obj for change to persist
-        environment.artifacts.repo = pygit2.Repository(temp_dir)
-        mocker.patch('pygit2.Remote.push')
-        mocker.patch('httpx.post')
-        env_input = EnvironmentInput(
-            name=environment.name,
-            path=environment.path,
-            description=environment.description,
-            packages=environment.packages,
-        )
-
-        environment.create(env_input)
-
-        yield environment, artifacts, env_input
-
-
-def test_create(mocker, temp_git_repo) -> None:
-    # Setup
-    environment, artifacts, env_input = temp_git_repo
-    push_mock = mocker.patch('pygit2.Remote.push')
-    post_mock = mocker.patch('httpx.post')
-
-    # Tests
-    env_input.name = "test_create_new"
-    result = environment.create(env_input)
-    assert isinstance(result, CreateEnvironmentSuccess)
-    push_mock.assert_called_once()
-    post_mock.assert_called_once()
-
-    post_mock.assert_called_with(
-        "http://0.0.0.0:7080/environments/build",
-        json={
-            "name": f"{env_input.path}/{env_input.name}",
-            "model": {
-                "description": env_input.description,
-                "packages": [f"{pkg.name}" for pkg in env_input.packages],
-            },
-        },
-    )
-
-    result = environment.create(env_input)
-    assert isinstance(result, EnvironmentAlreadyExistsError)
-
-    env_input.name = ""
-    result = environment.create(env_input)
-    assert isinstance(result, InvalidInputError)
-
-    env_input.name = environment.name
-    env_input.path = "invalid/path"
-    result = environment.create(env_input)
-    assert isinstance(result, InvalidInputError)
-
-
-def test_update(mocker, temp_git_repo) -> None:
-    # Setup
-    environment, artifacts, env_input = temp_git_repo
-    post_mock = mocker.patch('httpx.post')
-
-    # Tests
-    env_input.path = str(get_user_path_without_environments(artifacts))
-    env_input.description = "updated description"
-    result = environment.update(env_input, env_input.path, env_input.name)
-    assert isinstance(result, UpdateEnvironmentSuccess)
-    post_mock.assert_called_once()
-
-    result = environment.update(env_input, "invalid/path", "invalid_name")
-    assert isinstance(result, InvalidInputError)
-
-    env_input.name = ""
-    result = environment.update(env_input, "invalid/path", "invalid_name")
-    assert isinstance(result, InvalidInputError)
-
-    env_input.name = "invalid_name"
-    env_input.path = "invalid/path"
-    result = environment.update(env_input, "invalid/path", "invalid_name")
-    assert isinstance(result, EnvironmentNotFoundError)
-
-
-def test_delete(mocker, temp_git_repo) -> None:
-    # Setup
-    environment, artifacts, env_input = temp_git_repo
-    mocker.patch('pygit2.Remote.push')
-    mocker.patch('httpx.post')
-
-    # Test
-    result = environment.delete(env_input.name, env_input.path)
-    assert isinstance(result, DeleteEnvironmentSuccess)
-
-    env_input.name = "invalid_name"
-    env_input.path = "invalid/path"
-    result = environment.delete(env_input.name, env_input.path)
-    assert isinstance(result, EnvironmentNotFoundError)
-
-
-@pytest.mark.asyncio
-async def test_write_artifact(mocker, temp_git_repo):
-    # Setup
-    environment, artifacts, env_input = temp_git_repo
-    push_mock = mocker.patch('pygit2.Remote.push')
-    mocker.patch('httpx.post')
-
-    # Mock the file upload
-    upload = mocker.Mock(spec=UploadFile)
-    upload.filename = "example.txt"
-    upload.content_type = "text/plain"
-    upload.read.return_value = b"mock data"
-
-    # Test
-    result = await environment.write_artifact(
-        file=upload,
-        folder_path=f"{env_input.path}/{env_input.name}",
-        file_name=upload.filename,
-    )
-    assert isinstance(result, WriteArtifactSuccess)
-    push_mock.assert_called_once()
-
-    result = await environment.write_artifact(
-        file=upload,
-        folder_path="invalid/env/path",
-        file_name=upload.filename,
-    )
-    assert isinstance(result, InvalidInputError)
+# from .test_pygit import copy_of_repo, get_user_path_without_environments
+
+
+# @pytest.fixture
+# def temp_git_repo(mocker):
+#     artifacts = Artifacts()
+#     environment = Environment(
+#         id="",
+#         name="environment_test",
+#         path=str(get_user_path_without_environments(artifacts)),
+#         description="description",
+#         packages=[Package(id="", name="pkg_test")],
+#         state=None,
+#     )
+#     with copy_of_repo(artifacts) as temp_dir:
+#         # repo needs to be modified via environment obj for change to persist
+#         environment.artifacts.repo = pygit2.Repository(temp_dir)
+#         mocker.patch('pygit2.Remote.push')
+#         mocker.patch('httpx.post')
+#         env_input = EnvironmentInput(
+#             name=environment.name,
+#             path=environment.path,
+#             description=environment.description,
+#             packages=environment.packages,
+#         )
+
+#         environment.create(env_input)
+
+#         yield environment, artifacts, env_input
+
+
+# def test_create(mocker, temp_git_repo) -> None:
+#     # Setup
+#     environment, artifacts, env_input = temp_git_repo
+#     push_mock = mocker.patch('pygit2.Remote.push')
+#     post_mock = mocker.patch('httpx.post')
+
+#     # Tests
+#     env_input.name = "test_create_new"
+#     result = environment.create(env_input)
+#     assert isinstance(result, CreateEnvironmentSuccess)
+#     push_mock.assert_called_once()
+#     post_mock.assert_called_once()
+
+#     post_mock.assert_called_with(
+#         "http://0.0.0.0:7080/environments/build",
+#         json={
+#             "name": f"{env_input.path}/{env_input.name}",
+#             "model": {
+#                 "description": env_input.description,
+#                 "packages": [f"{pkg.name}" for pkg in env_input.packages],
+#             },
+#         },
+#     )
+
+#     result = environment.create(env_input)
+#     assert isinstance(result, EnvironmentAlreadyExistsError)
+
+#     env_input.name = ""
+#     result = environment.create(env_input)
+#     assert isinstance(result, InvalidInputError)
+
+#     env_input.name = environment.name
+#     env_input.path = "invalid/path"
+#     result = environment.create(env_input)
+#     assert isinstance(result, InvalidInputError)
+
+
+# def test_update(mocker, temp_git_repo) -> None:
+#     # Setup
+#     environment, artifacts, env_input = temp_git_repo
+#     post_mock = mocker.patch('httpx.post')
+
+#     # Tests
+#     env_input.path = str(get_user_path_without_environments(artifacts))
+#     env_input.description = "updated description"
+#     result = environment.update(env_input, env_input.path, env_input.name)
+#     assert isinstance(result, UpdateEnvironmentSuccess)
+#     post_mock.assert_called_once()
+
+#     result = environment.update(env_input, "invalid/path", "invalid_name")
+#     assert isinstance(result, InvalidInputError)
+
+#     env_input.name = ""
+#     result = environment.update(env_input, "invalid/path", "invalid_name")
+#     assert isinstance(result, InvalidInputError)
+
+#     env_input.name = "invalid_name"
+#     env_input.path = "invalid/path"
+#     result = environment.update(env_input, "invalid/path", "invalid_name")
+#     assert isinstance(result, EnvironmentNotFoundError)
+
+
+# def test_delete(mocker, temp_git_repo) -> None:
+#     # Setup
+#     environment, artifacts, env_input = temp_git_repo
+#     mocker.patch('pygit2.Remote.push')
+#     mocker.patch('httpx.post')
+
+#     # Test
+#     result = environment.delete(env_input.name, env_input.path)
+#     assert isinstance(result, DeleteEnvironmentSuccess)
+
+#     env_input.name = "invalid_name"
+#     env_input.path = "invalid/path"
+#     result = environment.delete(env_input.name, env_input.path)
+#     assert isinstance(result, EnvironmentNotFoundError)
+
+
+# @pytest.mark.asyncio
+# async def test_write_artifact(mocker, temp_git_repo):
+#     # Setup
+#     environment, artifacts, env_input = temp_git_repo
+#     push_mock = mocker.patch('pygit2.Remote.push')
+#     mocker.patch('httpx.post')
+
+#     # Mock the file upload
+#     upload = mocker.Mock(spec=UploadFile)
+#     upload.filename = "example.txt"
+#     upload.content_type = "text/plain"
+#     upload.read.return_value = b"mock data"
+
+#     # Test
+#     result = await environment.write_artifact(
+#         file=upload,
+#         folder_path=f"{env_input.path}/{env_input.name}",
+#         file_name=upload.filename,
+#     )
+#     assert isinstance(result, WriteArtifactSuccess)
+#     push_mock.assert_called_once()
+
+#     result = await environment.write_artifact(
+#         file=upload,
+#         folder_path="invalid/env/path",
+#         file_name=upload.filename,
+#     )
+#     assert isinstance(result, InvalidInputError)

From fcbc2c2adb62d1a03b02114b7491debd7064cc68 Mon Sep 17 00:00:00 2001
From: Sendu Bala <sb10@sanger.ac.uk>
Date: Thu, 24 Aug 2023 11:34:54 +0100
Subject: [PATCH 060/129] Update with additional testing notes.

---
 README.md | 25 +++++++++++++++++++++++++
 1 file changed, 25 insertions(+)

diff --git a/README.md b/README.md
index f59b25a..2b4fc7c 100644
--- a/README.md
+++ b/README.md
@@ -26,6 +26,15 @@ SoftPack Core - GraphQL backend service
 
 ## Installation
 
+### External dependencies
+
+SoftPack Core relies on Spack. Install that first:
+
+``` console
+$ git clone -c feature.manyFiles=true --depth 1 https://github.com/spack/spack.git
+$ source spack/share/spack/setup-env.sh
+```
+
 ### Stable release
 
 To install SoftPack Core, run this command in your
@@ -90,6 +99,22 @@ Run tests with [Tox][]
 poetry run tox
 ```
 
+To run integration tests, you need a git repository set up with token access and
+a branch named after your git repo username. Then set these environment
+variables:
+
+```
+export SOFTPACK_TEST_ARTIFACTS_REPO_URL='https://[...]artifacts.git'
+export SOFTPACK_TEST_ARTIFACTS_REPO_USER='username@domain'
+export SOFTPACK_TEST_ARTIFACTS_REPO_TOKEN='token'
+```
+
+To run an individual test:
+
+``` console
+poetry run pytest tests/integration/artifacts.py::test_commit -sv
+```
+
 Run [MkDocs] server to view documentation:
 
 ``` console

From 562dd24e356257005dbe5f8611e2bfa870719b31 Mon Sep 17 00:00:00 2001
From: Sendu Bala <sb10@sanger.ac.uk>
Date: Thu, 24 Aug 2023 12:56:02 +0100
Subject: [PATCH 061/129] Test environment.iter().

---
 softpack_core/schemas/environment.py  |  2 +-
 tests/integration/test_environment.py | 38 +++++++++++++++++++++++++++
 2 files changed, 39 insertions(+), 1 deletion(-)

diff --git a/softpack_core/schemas/environment.py b/softpack_core/schemas/environment.py
index 388785d..0c23323 100644
--- a/softpack_core/schemas/environment.py
+++ b/softpack_core/schemas/environment.py
@@ -236,7 +236,7 @@ def create(cls, env: EnvironmentInput) -> CreateResponse:
                 name=env.name,
             )
 
-        # Create folder with readme
+        # Create folder with place-holder file
         new_folder_path = Path(env.path, env.name)
         file_name = ".created"
         try:
diff --git a/tests/integration/test_environment.py b/tests/integration/test_environment.py
index a728acd..0c60a1c 100644
--- a/tests/integration/test_environment.py
+++ b/tests/integration/test_environment.py
@@ -181,3 +181,41 @@ async def test_write_artifact(mocker, testable_environment):
         file_name=upload.filename,
     )
     assert isinstance(result, InvalidInputError)
+
+
+@pytest.mark.asyncio
+async def test_iter(mocker, testable_environment):
+    environment, env_input = testable_environment
+
+    envs_filter = environment.iter()
+    count = 0
+    for env in envs_filter:
+        count += 1
+
+    assert count == 0
+
+    post_mock = mocker.patch('httpx.post')
+    result = environment.create(env_input)
+    assert isinstance(result, CreateEnvironmentSuccess)
+    post_mock.assert_called_once()
+
+    upload = mocker.Mock(spec=UploadFile)
+    upload.filename = Artifacts.environments_file
+    upload.content_type = "text/plain"
+    upload.read.return_value = b"description: test env\npackages:\n- zlib\n"
+
+    result = await environment.write_artifact(
+        file=upload,
+        folder_path=f"{env_input.path}/{env_input.name}",
+        file_name=upload.filename,
+    )
+    assert isinstance(result, WriteArtifactSuccess)
+
+    envs_filter = environment.iter()
+    count = 0
+    for env in envs_filter:
+        assert env.name == env_input.name
+        assert any(p.name == "zlib" for p in env.packages)
+        count += 1
+
+    assert count == 1

From 865e5909a1ee1963448b532acc0728c7270dcd80 Mon Sep 17 00:00:00 2001
From: Sendu Bala <sb10@sanger.ac.uk>
Date: Thu, 24 Aug 2023 13:44:00 +0100
Subject: [PATCH 062/129] Remove commented-out test files for now, since they
 had too-long lines.

---
 tests/unit/test_artifacts.py   | 219 ---------------------------------
 tests/unit/test_environment.py | 161 ------------------------
 2 files changed, 380 deletions(-)
 delete mode 100644 tests/unit/test_artifacts.py
 delete mode 100644 tests/unit/test_environment.py

diff --git a/tests/unit/test_artifacts.py b/tests/unit/test_artifacts.py
deleted file mode 100644
index 912e9af..0000000
--- a/tests/unit/test_artifacts.py
+++ /dev/null
@@ -1,219 +0,0 @@
-"""Copyright (c) 2023 Genome Research Ltd.
-
-This source code is licensed under the MIT license found in the
-LICENSE file in the root directory of this source tree.
-"""
-
-import os
-import shutil
-import tempfile
-from pathlib import Path
-
-import pygit2
-import pytest
-from pygit2 import Signature
-
-from softpack_core.artifacts import Artifacts
-
-
-# def test_clone() -> None:
-#     artifacts = Artifacts()
-#     path = artifacts.repo.path
-
-#     shutil.rmtree(path)
-#     assert os.path.isdir(path) is False
-
-#     artifacts = Artifacts()
-#     assert os.path.isdir(path) is True
-
-
-# def test_commit(new_repo) -> None:
-#     repo = new_repo[0]
-#     old_commit_oid = new_repo[2]
-#     with new_repo[1]:
-#         file_oid = repo.create_blob("test")
-#         tree = repo.head.peel(pygit2.Tree)
-#         tree_builder = repo.TreeBuilder(tree)
-#         tree_builder.insert("new_file.txt", file_oid, pygit2.GIT_FILEMODE_BLOB)
-#         new_tree = tree_builder.write()
-
-#         artifacts = Artifacts()
-#         new_commit_oid = artifacts.commit(repo, new_tree, "commit new file")
-#         repo_head = repo.head.peel(pygit2.Commit).oid
-
-#         assert old_commit_oid != new_commit_oid
-#         assert new_commit_oid == repo_head
-
-
-# def test_push(mocker) -> None:
-#     artifacts = Artifacts()
-
-#     push_mock = mocker.patch('pygit2.Remote.push')
-
-#     artifacts.push()
-#     push_mock.assert_called_once()
-
-
-# def get_user_envs_tree(artifacts, oid) -> pygit2.Tree:
-#     new_tree = artifacts.repo.get(oid)
-#     return new_tree[artifacts.user_folder(os.environ["USER"])]
-
-
-# def copy_of_repo(artifacts) -> tempfile.TemporaryDirectory:
-#     temp_dir = tempfile.TemporaryDirectory()
-#     shutil.copytree(artifacts.repo.path, temp_dir.name, dirs_exist_ok=True)
-#     return temp_dir
-
-
-# def get_user_path_without_environments(artifacts) -> Path:
-#     return Path(*(artifacts.user_folder(os.environ["USER"]).parts[1:]))
-
-
-# def test_create_file() -> None:
-#     artifacts = Artifacts()
-#     with copy_of_repo(artifacts) as temp_dir:
-#         artifacts.repo = pygit2.Repository(temp_dir)
-#         new_test_env = "test_create_file_env"
-#         assert new_test_env not in [
-#             obj.name for obj in artifacts.iter_user(os.environ["USER"])
-#         ]
-
-#         fname = "file.txt"
-
-#         folder_path = Path(
-#             get_user_path_without_environments(artifacts), new_test_env
-#         )
-#         oid = artifacts.create_file(
-#             str(folder_path), fname, "lorem ipsum", True, False
-#         )
-
-#         user_envs_tree = get_user_envs_tree(artifacts, oid)
-#         assert new_test_env in [obj.name for obj in user_envs_tree]
-#         assert fname in [obj.name for obj in user_envs_tree[new_test_env]]
-
-#         artifacts.commit(artifacts.repo, oid, "commit file")
-
-#         with pytest.raises(RuntimeError) as exc_info:
-#             oid = artifacts.create_file(
-#                 str(folder_path), fname, "lorem ipsum", False, True
-#             )
-#         assert exc_info.value.args[0] == 'No changes made to the environment'
-
-#         with pytest.raises(RuntimeError) as exc_info:
-#             artifacts.create_file(
-#                 str(folder_path), "second_file.txt", "lorem ipsum", True, False
-#             )
-#         assert exc_info.value.args[0] == 'Too many changes to the repo'
-
-#         oid = artifacts.create_file(
-#             str(folder_path), "second_file.txt", "lorem ipsum", False, False
-#         )
-
-#         user_envs_tree = get_user_envs_tree(artifacts, oid)
-#         assert "second_file.txt" in [
-#             obj.name for obj in user_envs_tree[new_test_env]
-#         ]
-
-#         with pytest.raises(FileExistsError) as exc_info:
-#             artifacts.create_file(
-#                 str(folder_path), fname, "lorem ipsum", False, False
-#             )
-#         assert exc_info.value.args[0] == 'File already exists'
-
-#         oid = artifacts.create_file(
-#             str(folder_path), fname, "override", False, True
-#         )
-
-#         user_envs_tree = get_user_envs_tree(artifacts, oid)
-#         assert fname in [obj.name for obj in user_envs_tree[new_test_env]]
-#         assert user_envs_tree[new_test_env][fname].data.decode() == "override"
-
-
-# def test_delete_environment() -> None:
-#     artifacts = Artifacts()
-#     with copy_of_repo(artifacts):
-#         new_test_env = "test_create_file_env"
-#         folder_path = Path(
-#             get_user_path_without_environments(artifacts), new_test_env
-#         )
-#         oid = artifacts.create_file(
-#             str(folder_path), "file.txt", "lorem ipsum", True, False
-#         )
-#         artifacts.commit(artifacts.repo, oid, "commit file")
-
-#         user_envs_tree = get_user_envs_tree(artifacts, oid)
-#         assert new_test_env in [obj.name for obj in user_envs_tree]
-
-#         oid = artifacts.delete_environment(
-#             new_test_env, get_user_path_without_environments(artifacts)
-#         )
-
-#         artifacts.commit(artifacts.repo, oid, "commit file")
-
-#         user_envs_tree = get_user_envs_tree(artifacts, oid)
-#         assert new_test_env not in [obj.name for obj in user_envs_tree]
-
-#         with pytest.raises(ValueError) as exc_info:
-#             artifacts.delete_environment(
-#                 os.environ["USER"], artifacts.users_folder_name
-#             )
-#         assert exc_info.value.args[0] == 'Not a valid environment path'
-
-#         with pytest.raises(KeyError) as exc_info:
-#             artifacts.delete_environment(new_test_env, "foo/bar")
-#         assert exc_info
-
-
-# def count_user_and_group_envs(artifacts, envs) -> (int, int):
-#     num_user_envs = 0
-#     num_group_envs = 0
-
-#     for env in envs:
-#         if str(env.path).startswith(artifacts.users_folder_name):
-#             num_user_envs += 1
-#         elif str(env.path).startswith(artifacts.groups_folder_name):
-#             num_group_envs += 1
-
-#     return num_user_envs, num_group_envs
-
-
-# def test_iter() -> None:
-#     artifacts = Artifacts()
-#     user = os.environ["USER"]
-#     envs = artifacts.iter(user)
-
-#     user_found = False
-#     only_this_user = True
-#     num_user_envs = 0
-#     num_group_envs = 0
-
-#     for env in envs:
-#         if str(env.path).startswith(artifacts.users_folder_name):
-#             num_user_envs += 1
-#             if str(env.path).startswith(
-#                 f"{artifacts.users_folder_name}/{user}"
-#             ):
-#                 user_found = True
-#             else:
-#                 only_this_user = False
-#         elif str(env.path).startswith(artifacts.groups_folder_name):
-#             num_group_envs += 1
-
-#     assert user_found is True
-#     assert only_this_user is True
-#     assert num_group_envs > 0
-
-#     envs = artifacts.iter()
-#     no_user_num_user_envs, no_user_num_group_envs = count_user_and_group_envs(
-#         artifacts, envs
-#     )
-#     assert no_user_num_user_envs >= num_user_envs
-#     assert no_user_num_group_envs >= num_group_envs
-
-#     envs = artifacts.iter("!@£$%")
-#     (
-#         bad_user_num_user_envs,
-#         bad_user_num_group_envs,
-#     ) = count_user_and_group_envs(artifacts, envs)
-#     assert bad_user_num_user_envs == 0
-#     assert bad_user_num_group_envs == 0
diff --git a/tests/unit/test_environment.py b/tests/unit/test_environment.py
deleted file mode 100644
index 9909ff4..0000000
--- a/tests/unit/test_environment.py
+++ /dev/null
@@ -1,161 +0,0 @@
-"""Copyright (c) 2023 Genome Research Ltd.
-
-This source code is licensed under the MIT license found in the
-LICENSE file in the root directory of this source tree.
-"""
-
-import pygit2
-import pytest
-from starlette.datastructures import UploadFile
-
-from softpack_core.artifacts import Artifacts
-from softpack_core.schemas.environment import (
-    CreateEnvironmentSuccess,
-    DeleteEnvironmentSuccess,
-    Environment,
-    EnvironmentAlreadyExistsError,
-    EnvironmentInput,
-    EnvironmentNotFoundError,
-    InvalidInputError,
-    Package,
-    UpdateEnvironmentSuccess,
-    WriteArtifactSuccess,
-)
-
-# from .test_pygit import copy_of_repo, get_user_path_without_environments
-
-
-# @pytest.fixture
-# def temp_git_repo(mocker):
-#     artifacts = Artifacts()
-#     environment = Environment(
-#         id="",
-#         name="environment_test",
-#         path=str(get_user_path_without_environments(artifacts)),
-#         description="description",
-#         packages=[Package(id="", name="pkg_test")],
-#         state=None,
-#     )
-#     with copy_of_repo(artifacts) as temp_dir:
-#         # repo needs to be modified via environment obj for change to persist
-#         environment.artifacts.repo = pygit2.Repository(temp_dir)
-#         mocker.patch('pygit2.Remote.push')
-#         mocker.patch('httpx.post')
-#         env_input = EnvironmentInput(
-#             name=environment.name,
-#             path=environment.path,
-#             description=environment.description,
-#             packages=environment.packages,
-#         )
-
-#         environment.create(env_input)
-
-#         yield environment, artifacts, env_input
-
-
-# def test_create(mocker, temp_git_repo) -> None:
-#     # Setup
-#     environment, artifacts, env_input = temp_git_repo
-#     push_mock = mocker.patch('pygit2.Remote.push')
-#     post_mock = mocker.patch('httpx.post')
-
-#     # Tests
-#     env_input.name = "test_create_new"
-#     result = environment.create(env_input)
-#     assert isinstance(result, CreateEnvironmentSuccess)
-#     push_mock.assert_called_once()
-#     post_mock.assert_called_once()
-
-#     post_mock.assert_called_with(
-#         "http://0.0.0.0:7080/environments/build",
-#         json={
-#             "name": f"{env_input.path}/{env_input.name}",
-#             "model": {
-#                 "description": env_input.description,
-#                 "packages": [f"{pkg.name}" for pkg in env_input.packages],
-#             },
-#         },
-#     )
-
-#     result = environment.create(env_input)
-#     assert isinstance(result, EnvironmentAlreadyExistsError)
-
-#     env_input.name = ""
-#     result = environment.create(env_input)
-#     assert isinstance(result, InvalidInputError)
-
-#     env_input.name = environment.name
-#     env_input.path = "invalid/path"
-#     result = environment.create(env_input)
-#     assert isinstance(result, InvalidInputError)
-
-
-# def test_update(mocker, temp_git_repo) -> None:
-#     # Setup
-#     environment, artifacts, env_input = temp_git_repo
-#     post_mock = mocker.patch('httpx.post')
-
-#     # Tests
-#     env_input.path = str(get_user_path_without_environments(artifacts))
-#     env_input.description = "updated description"
-#     result = environment.update(env_input, env_input.path, env_input.name)
-#     assert isinstance(result, UpdateEnvironmentSuccess)
-#     post_mock.assert_called_once()
-
-#     result = environment.update(env_input, "invalid/path", "invalid_name")
-#     assert isinstance(result, InvalidInputError)
-
-#     env_input.name = ""
-#     result = environment.update(env_input, "invalid/path", "invalid_name")
-#     assert isinstance(result, InvalidInputError)
-
-#     env_input.name = "invalid_name"
-#     env_input.path = "invalid/path"
-#     result = environment.update(env_input, "invalid/path", "invalid_name")
-#     assert isinstance(result, EnvironmentNotFoundError)
-
-
-# def test_delete(mocker, temp_git_repo) -> None:
-#     # Setup
-#     environment, artifacts, env_input = temp_git_repo
-#     mocker.patch('pygit2.Remote.push')
-#     mocker.patch('httpx.post')
-
-#     # Test
-#     result = environment.delete(env_input.name, env_input.path)
-#     assert isinstance(result, DeleteEnvironmentSuccess)
-
-#     env_input.name = "invalid_name"
-#     env_input.path = "invalid/path"
-#     result = environment.delete(env_input.name, env_input.path)
-#     assert isinstance(result, EnvironmentNotFoundError)
-
-
-# @pytest.mark.asyncio
-# async def test_write_artifact(mocker, temp_git_repo):
-#     # Setup
-#     environment, artifacts, env_input = temp_git_repo
-#     push_mock = mocker.patch('pygit2.Remote.push')
-#     mocker.patch('httpx.post')
-
-#     # Mock the file upload
-#     upload = mocker.Mock(spec=UploadFile)
-#     upload.filename = "example.txt"
-#     upload.content_type = "text/plain"
-#     upload.read.return_value = b"mock data"
-
-#     # Test
-#     result = await environment.write_artifact(
-#         file=upload,
-#         folder_path=f"{env_input.path}/{env_input.name}",
-#         file_name=upload.filename,
-#     )
-#     assert isinstance(result, WriteArtifactSuccess)
-#     push_mock.assert_called_once()
-
-#     result = await environment.write_artifact(
-#         file=upload,
-#         folder_path="invalid/env/path",
-#         file_name=upload.filename,
-#     )
-#     assert isinstance(result, InvalidInputError)

From 9e7a96ad8f31ab1d59baecabeb5679195785921f Mon Sep 17 00:00:00 2001
From: Sendu Bala <sb10@sanger.ac.uk>
Date: Thu, 24 Aug 2023 13:44:30 +0100
Subject: [PATCH 063/129] Reformat.

---
 softpack_core/artifacts.py            | 18 ++++----
 softpack_core/schemas/environment.py  | 12 ++---
 tests/integration/conftest.py         | 48 +++++++++++++-------
 tests/integration/test_artifacts.py   | 64 +++++++++++++++------------
 tests/integration/test_environment.py | 12 ++---
 5 files changed, 92 insertions(+), 62 deletions(-)

diff --git a/softpack_core/artifacts.py b/softpack_core/artifacts.py
index b0269cf..c04dad5 100644
--- a/softpack_core/artifacts.py
+++ b/softpack_core/artifacts.py
@@ -110,7 +110,8 @@ def __init__(self) -> None:
             print(e)
 
         self.credentials_callback = pygit2.RemoteCallbacks(
-            credentials=credentials)
+            credentials=credentials
+        )
 
         branch = self.settings.artifacts.repo.branch
         if branch is None:
@@ -173,7 +174,9 @@ def iter_users(self) -> list[pygit2.Tree]:
         Returns:
             list[pygit2.Tree]: List of environments
         """
-        return self.iter_environments(self.environments_folder(self.users_folder_name))
+        return self.iter_environments(
+            self.environments_folder(self.users_folder_name)
+        )
 
     def iter_groups(self) -> list[pygit2.Tree]:
         """Iterate environments for all groups.
@@ -181,7 +184,9 @@ def iter_groups(self) -> list[pygit2.Tree]:
         Returns:
             list[pygit2.Tree]: List of environments
         """
-        return self.iter_environments(self.environments_folder(self.groups_folder_name))
+        return self.iter_environments(
+            self.environments_folder(self.groups_folder_name)
+        )
 
     def iter_environments(self, path: Path) -> list[pygit2.Tree]:
         """Iterate environments under a path.
@@ -244,9 +249,7 @@ def get(self, path: Path, name: str) -> Optional[pygit2.Tree]:
         except KeyError:
             return None
 
-    def commit(
-        self, tree_oid: pygit2.Oid, message: str
-    ) -> pygit2.Oid:
+    def commit(self, tree_oid: pygit2.Oid, message: str) -> pygit2.Oid:
         """Create and return a commit.
 
         Args:
@@ -265,8 +268,7 @@ def commit(
         return commit_oid
 
     def push(self) -> None:
-        """Push all commits to a repository.
-        """
+        """Push all commits to a repository."""
         remote = self.repo.remotes[0]
         remote.push([self.head_name], callbacks=self.credentials_callback)
 
diff --git a/softpack_core/schemas/environment.py b/softpack_core/schemas/environment.py
index 0c23323..2ca86f8 100644
--- a/softpack_core/schemas/environment.py
+++ b/softpack_core/schemas/environment.py
@@ -4,7 +4,6 @@
 LICENSE file in the root directory of this source tree.
 """
 
-import os
 from dataclasses import dataclass
 from pathlib import Path
 from typing import Iterable, Optional
@@ -223,8 +222,10 @@ def create(cls, env: EnvironmentInput) -> CreateResponse:
         # Check if a valid path has been provided. TODO: improve this to check
         # that they can only create stuff in their own users folder, or in
         # group folders of unix groups they belong to.
-        valid_dirs = [cls.artifacts.users_folder_name,
-                      cls.artifacts.groups_folder_name]
+        valid_dirs = [
+            cls.artifacts.users_folder_name,
+            cls.artifacts.groups_folder_name,
+        ]
         if not any(env.path.startswith(dir) for dir in valid_dirs):
             return InvalidInputError(message="Invalid path")
 
@@ -291,8 +292,9 @@ def update(
 
         # Check name and path have not been changed.
         if env.path != current_path or env.name != current_name:
-            return InvalidInputError(message=("change of name or path not "
-                                              "currently supported"))
+            return InvalidInputError(
+                message=("change of name or path not " "currently supported")
+            )
 
         # Check if an environment exists at the specified path and name
         if cls.artifacts.get(Path(current_path), current_name):
diff --git a/tests/integration/conftest.py b/tests/integration/conftest.py
index 556d7f1..404285b 100644
--- a/tests/integration/conftest.py
+++ b/tests/integration/conftest.py
@@ -5,16 +5,18 @@
 """
 
 import os
+import shutil
+import tempfile
 from pathlib import Path
+
 import pygit2
 import pytest
-import tempfile
-import shutil
 
 from softpack_core.artifacts import Artifacts, app
 
-artifacts_dict = dict[str, str | pygit2.Oid | Path
-                      | Artifacts | tempfile.TemporaryDirectory[str]]
+artifacts_dict = dict[
+    str, str | pygit2.Oid | Path | Artifacts | tempfile.TemporaryDirectory[str]
+]
 
 
 @pytest.fixture(scope="package", autouse=True)
@@ -23,8 +25,12 @@ def testable_artifacts_setup():
     repo_user = os.getenv("SOFTPACK_TEST_ARTIFACTS_REPO_USER")
     repo_token = os.getenv("SOFTPACK_TEST_ARTIFACTS_REPO_TOKEN")
     if repo_url is None or repo_user is None or repo_token is None:
-        pytest.skip(("SOFTPACK_TEST_ARTIFACTS_REPO_URL, _USER and _TOKEN "
-                     "env vars are all required for these tests"))
+        pytest.skip(
+            (
+                "SOFTPACK_TEST_ARTIFACTS_REPO_URL, _USER and _TOKEN "
+                "env vars are all required for these tests"
+            )
+        )
 
     user = repo_user.split('@', 1)[0]
     app.settings.artifacts.repo.url = repo_url
@@ -57,8 +63,9 @@ def delete_environments_folder_from_test_repo(artifacts: Artifacts):
     tree = artifacts.repo.head.peel(pygit2.Tree)
 
     if artifacts.environments_root in tree:
-        shutil.rmtree(Path(app.settings.artifacts.path,
-                      artifacts.environments_root))
+        shutil.rmtree(
+            Path(app.settings.artifacts.path, artifacts.environments_root)
+        )
         commit_local_file_changes(artifacts, "delete environments")
 
 
@@ -80,12 +87,18 @@ def create_initial_test_repo_state(artifacts: Artifacts) -> artifacts_dict:
     test_group = "test_group"
     test_env = "test_environment"
     user_env_path = Path(
-        dir_path, "environments", artifacts.users_folder_name, test_user,
-        test_env
+        dir_path,
+        "environments",
+        artifacts.users_folder_name,
+        test_user,
+        test_env,
     )
     group_env_path = Path(
-        dir_path, "environments", artifacts.groups_folder_name, test_group,
-        test_env
+        dir_path,
+        "environments",
+        artifacts.groups_folder_name,
+        test_group,
+        test_env,
     )
     os.makedirs(user_env_path)
     os.makedirs(group_env_path)
@@ -107,7 +120,9 @@ def create_initial_test_repo_state(artifacts: Artifacts) -> artifacts_dict:
     return dict
 
 
-def get_user_path_without_environments(artifacts: Artifacts, user: str) -> Path:
+def get_user_path_without_environments(
+    artifacts: Artifacts, user: str
+) -> Path:
     return Path(*(artifacts.user_folder(user).parts[1:]))
 
 
@@ -117,8 +132,11 @@ def file_was_pushed(*paths_without_environment: str | Path) -> bool:
     artifacts = Artifacts()
 
     for path_without_environment in paths_without_environment:
-        path = Path(temp_dir.name, artifacts.environments_root,
-                    path_without_environment)
+        path = Path(
+            temp_dir.name,
+            artifacts.environments_root,
+            path_without_environment,
+        )
         if not os.path.isfile(path):
             return False
 
diff --git a/tests/integration/test_artifacts.py b/tests/integration/test_artifacts.py
index 570e011..43db67b 100644
--- a/tests/integration/test_artifacts.py
+++ b/tests/integration/test_artifacts.py
@@ -5,17 +5,18 @@
 """
 
 import os
+import shutil
 from pathlib import Path
+
 import pygit2
 import pytest
-import shutil
-import tempfile
 
-from softpack_core.artifacts import Artifacts, app
-
-from tests.integration.conftest import (new_test_artifacts,
-                                        get_user_path_without_environments,
-                                        file_was_pushed)
+from softpack_core.artifacts import Artifacts
+from tests.integration.conftest import (
+    file_was_pushed,
+    get_user_path_without_environments,
+    new_test_artifacts,
+)
 
 
 def test_clone() -> None:
@@ -37,9 +38,14 @@ def test_commit_and_push() -> None:
     old_commit_oid = ad["initial_commit_oid"]
 
     new_file_name = "new_file.txt"
-    path = Path(ad["temp_dir"].name, artifacts.environments_root,
-                artifacts.users_folder_name, ad["test_user"],
-                ad["test_environment"], new_file_name)
+    path = Path(
+        ad["temp_dir"].name,
+        artifacts.environments_root,
+        artifacts.users_folder_name,
+        ad["test_user"],
+        ad["test_environment"],
+        new_file_name,
+    )
 
     open(path, "w").close()
 
@@ -56,8 +62,12 @@ def test_commit_and_push() -> None:
 
     artifacts.push()
 
-    path = Path(artifacts.users_folder_name, ad["test_user"],
-                ad["test_environment"], new_file_name)
+    path = Path(
+        artifacts.users_folder_name,
+        ad["test_user"],
+        ad["test_environment"],
+        new_file_name,
+    )
 
     assert file_was_pushed(path)
 
@@ -70,12 +80,12 @@ def test_create_file() -> None:
     new_test_env = "test_create_file_env"
 
     user_envs_tree = get_user_envs_tree(
-        artifacts, user, artifacts.repo.head.peel(pygit2.Tree).oid)
+        artifacts, user, artifacts.repo.head.peel(pygit2.Tree).oid
+    )
     assert new_test_env not in [obj.name for obj in user_envs_tree]
 
     folder_path = Path(
-        get_user_path_without_environments(
-            artifacts, user), new_test_env
+        get_user_path_without_environments(artifacts, user), new_test_env
     )
     basename = "create_file.txt"
 
@@ -109,9 +119,7 @@ def test_create_file() -> None:
     artifacts.commit(oid, "create file2")
 
     user_envs_tree = get_user_envs_tree(artifacts, user, oid)
-    assert basename2 in [
-        obj.name for obj in user_envs_tree[new_test_env]
-    ]
+    assert basename2 in [obj.name for obj in user_envs_tree[new_test_env]]
 
     with pytest.raises(FileExistsError) as exc_info:
         artifacts.create_file(
@@ -119,9 +127,7 @@ def test_create_file() -> None:
         )
     assert exc_info.value.args[0] == 'File already exists'
 
-    oid = artifacts.create_file(
-        folder_path, basename, "override", False, True
-    )
+    oid = artifacts.create_file(folder_path, basename, "override", False, True)
 
     artifacts.commit(oid, "update created file")
 
@@ -131,11 +137,14 @@ def test_create_file() -> None:
 
     artifacts.push()
 
-    assert file_was_pushed(Path(folder_path, basename),
-                           Path(folder_path, basename2))
+    assert file_was_pushed(
+        Path(folder_path, basename), Path(folder_path, basename2)
+    )
 
 
-def get_user_envs_tree(artifacts: Artifacts, user: str, oid: pygit2.Oid) -> pygit2.Tree:
+def get_user_envs_tree(
+    artifacts: Artifacts, user: str, oid: pygit2.Oid
+) -> pygit2.Tree:
     new_tree = artifacts.repo.get(oid)
     return new_tree[artifacts.user_folder(user)]
 
@@ -147,7 +156,8 @@ def test_delete_environment() -> None:
     env_for_deleting = ad["test_environment"]
 
     user_envs_tree = get_user_envs_tree(
-        artifacts, user, artifacts.repo.head.peel(pygit2.Tree).oid)
+        artifacts, user, artifacts.repo.head.peel(pygit2.Tree).oid
+    )
     assert env_for_deleting in [obj.name for obj in user_envs_tree]
 
     oid = artifacts.delete_environment(
@@ -160,9 +170,7 @@ def test_delete_environment() -> None:
     assert env_for_deleting not in [obj.name for obj in user_envs_tree]
 
     with pytest.raises(ValueError) as exc_info:
-        artifacts.delete_environment(
-            user, artifacts.users_folder_name
-        )
+        artifacts.delete_environment(user, artifacts.users_folder_name)
     assert exc_info.value.args[0] == 'Not a valid environment path'
 
     with pytest.raises(KeyError) as exc_info:
diff --git a/tests/integration/test_environment.py b/tests/integration/test_environment.py
index 0c60a1c..939a841 100644
--- a/tests/integration/test_environment.py
+++ b/tests/integration/test_environment.py
@@ -5,6 +5,7 @@
 """
 
 from pathlib import Path
+
 import pytest
 from starlette.datastructures import UploadFile
 
@@ -21,12 +22,11 @@
     UpdateEnvironmentSuccess,
     WriteArtifactSuccess,
 )
-
-from tests.integration.conftest import (new_test_artifacts,
-                                        get_user_path_without_environments,
-                                        file_was_pushed)
-
-from softpack_core.artifacts import Artifacts
+from tests.integration.conftest import (
+    file_was_pushed,
+    get_user_path_without_environments,
+    new_test_artifacts,
+)
 
 
 @pytest.fixture

From 21cc7a681f7bc8076d98c9699087de57479611e4 Mon Sep 17 00:00:00 2001
From: Sendu Bala <sb10@sanger.ac.uk>
Date: Tue, 29 Aug 2023 13:16:25 +0100
Subject: [PATCH 064/129] Move functions from conftest.py to utils.py.

---
 tests/integration/conftest.py         | 106 --------------------------
 tests/integration/test_artifacts.py   |   2 +-
 tests/integration/test_environment.py |   2 +-
 3 files changed, 2 insertions(+), 108 deletions(-)

diff --git a/tests/integration/conftest.py b/tests/integration/conftest.py
index 404285b..7164255 100644
--- a/tests/integration/conftest.py
+++ b/tests/integration/conftest.py
@@ -14,10 +14,6 @@
 
 from softpack_core.artifacts import Artifacts, app
 
-artifacts_dict = dict[
-    str, str | pygit2.Oid | Path | Artifacts | tempfile.TemporaryDirectory[str]
-]
-
 
 @pytest.fixture(scope="package", autouse=True)
 def testable_artifacts_setup():
@@ -39,105 +35,3 @@ def testable_artifacts_setup():
     app.settings.artifacts.repo.email = repo_user
     app.settings.artifacts.repo.writer = repo_token
     app.settings.artifacts.repo.branch = user
-
-
-def new_test_artifacts() -> artifacts_dict:
-    temp_dir = tempfile.TemporaryDirectory()
-    app.settings.artifacts.path = Path(temp_dir.name)
-
-    artifacts = Artifacts()
-    dict = reset_test_repo(artifacts)
-    dict["temp_dir"] = temp_dir
-    dict["artifacts"] = artifacts
-
-    return dict
-
-
-def reset_test_repo(artifacts: Artifacts) -> artifacts_dict:
-    delete_environments_folder_from_test_repo(artifacts)
-
-    return create_initial_test_repo_state(artifacts)
-
-
-def delete_environments_folder_from_test_repo(artifacts: Artifacts):
-    tree = artifacts.repo.head.peel(pygit2.Tree)
-
-    if artifacts.environments_root in tree:
-        shutil.rmtree(
-            Path(app.settings.artifacts.path, artifacts.environments_root)
-        )
-        commit_local_file_changes(artifacts, "delete environments")
-
-
-def commit_local_file_changes(artifacts: Artifacts, msg: str) -> pygit2.Oid:
-    index = artifacts.repo.index
-    index.add_all()
-    index.write()
-    ref = artifacts.head_name
-    parents = [artifacts.repo.lookup_reference(ref).target]
-    oid = index.write_tree()
-    return artifacts.repo.create_commit(
-        ref, artifacts.signature, artifacts.signature, msg, oid, parents
-    )
-
-
-def create_initial_test_repo_state(artifacts: Artifacts) -> artifacts_dict:
-    dir_path = app.settings.artifacts.path
-    test_user = "test_user"
-    test_group = "test_group"
-    test_env = "test_environment"
-    user_env_path = Path(
-        dir_path,
-        "environments",
-        artifacts.users_folder_name,
-        test_user,
-        test_env,
-    )
-    group_env_path = Path(
-        dir_path,
-        "environments",
-        artifacts.groups_folder_name,
-        test_group,
-        test_env,
-    )
-    os.makedirs(user_env_path)
-    os.makedirs(group_env_path)
-    file_basename = "file.txt"
-    open(Path(user_env_path, file_basename), "w").close()
-    open(Path(group_env_path, file_basename), "w").close()
-
-    oid = commit_local_file_changes(artifacts, "Add test environments")
-
-    dict: artifacts_dict = {
-        "initial_commit_oid": oid,
-        "test_user": test_user,
-        "test_group": test_group,
-        "test_environment": test_env,
-        "user_env_path": user_env_path,
-        "group_env_path": group_env_path,
-        "basename": file_basename,
-    }
-    return dict
-
-
-def get_user_path_without_environments(
-    artifacts: Artifacts, user: str
-) -> Path:
-    return Path(*(artifacts.user_folder(user).parts[1:]))
-
-
-def file_was_pushed(*paths_without_environment: str | Path) -> bool:
-    temp_dir = tempfile.TemporaryDirectory()
-    app.settings.artifacts.path = Path(temp_dir.name)
-    artifacts = Artifacts()
-
-    for path_without_environment in paths_without_environment:
-        path = Path(
-            temp_dir.name,
-            artifacts.environments_root,
-            path_without_environment,
-        )
-        if not os.path.isfile(path):
-            return False
-
-    return True
diff --git a/tests/integration/test_artifacts.py b/tests/integration/test_artifacts.py
index 43db67b..ca8e430 100644
--- a/tests/integration/test_artifacts.py
+++ b/tests/integration/test_artifacts.py
@@ -12,7 +12,7 @@
 import pytest
 
 from softpack_core.artifacts import Artifacts
-from tests.integration.conftest import (
+from tests.integration.utils import (
     file_was_pushed,
     get_user_path_without_environments,
     new_test_artifacts,
diff --git a/tests/integration/test_environment.py b/tests/integration/test_environment.py
index 939a841..4f92570 100644
--- a/tests/integration/test_environment.py
+++ b/tests/integration/test_environment.py
@@ -22,7 +22,7 @@
     UpdateEnvironmentSuccess,
     WriteArtifactSuccess,
 )
-from tests.integration.conftest import (
+from tests.integration.utils import (
     file_was_pushed,
     get_user_path_without_environments,
     new_test_artifacts,

From 8a419f769b21dee893187339cb02414da7a41b72 Mon Sep 17 00:00:00 2001
From: Sendu Bala <sb10@sanger.ac.uk>
Date: Tue, 29 Aug 2023 14:44:43 +0100
Subject: [PATCH 065/129] Move functions from conftest.py to utils.py.

---
 tests/integration/conftest.py         |   6 ++
 tests/integration/test_environment.py |  29 +++---
 tests/integration/utils.py            | 121 ++++++++++++++++++++++++++
 3 files changed, 139 insertions(+), 17 deletions(-)
 create mode 100644 tests/integration/utils.py

diff --git a/tests/integration/conftest.py b/tests/integration/conftest.py
index 7164255..6985faf 100644
--- a/tests/integration/conftest.py
+++ b/tests/integration/conftest.py
@@ -35,3 +35,9 @@ def testable_artifacts_setup():
     app.settings.artifacts.repo.email = repo_user
     app.settings.artifacts.repo.writer = repo_token
     app.settings.artifacts.repo.branch = user
+
+
+@pytest.fixture(autouse=True)
+def patch_post(mocker):
+    post_mock = mocker.patch('httpx.post')
+    return post_mock
diff --git a/tests/integration/test_environment.py b/tests/integration/test_environment.py
index 4f92570..c8f2251 100644
--- a/tests/integration/test_environment.py
+++ b/tests/integration/test_environment.py
@@ -56,9 +56,8 @@ def testable_environment(mocker):
     yield environment, env_input
 
 
-def test_create(mocker, testable_environment) -> None:
+def test_create(patch_post, testable_environment) -> None:
     environment, env_input = testable_environment
-    post_mock = mocker.patch('httpx.post')
 
     result = environment.create(env_input)
     assert isinstance(result, CreateEnvironmentSuccess)
@@ -66,8 +65,8 @@ def test_create(mocker, testable_environment) -> None:
     path = Path(env_input.path, env_input.name, ".created")
     assert file_was_pushed(path)
 
-    post_mock.assert_called_once()
-    builder_called_correctly(post_mock, env_input)
+    patch_post.assert_called_once()
+    builder_called_correctly(patch_post, env_input)
 
     result = environment.create(env_input)
     assert isinstance(result, EnvironmentAlreadyExistsError)
@@ -97,19 +96,18 @@ def builder_called_correctly(post_mock, env_input: EnvironmentInput) -> None:
     )
 
 
-def test_update(mocker, testable_environment) -> None:
+def test_update(patch_post, testable_environment) -> None:
     environment, env_input = testable_environment
-    post_mock = mocker.patch('httpx.post')
 
     result = environment.create(env_input)
     assert isinstance(result, CreateEnvironmentSuccess)
-    post_mock.assert_called_once()
+    patch_post.assert_called_once()
 
     env_input.description = "updated description"
     result = environment.update(env_input, env_input.path, env_input.name)
     assert isinstance(result, UpdateEnvironmentSuccess)
 
-    builder_called_correctly(post_mock, env_input)
+    builder_called_correctly(patch_post, env_input)
 
     result = environment.update(env_input, "invalid/path", "invalid_name")
     assert isinstance(result, InvalidInputError)
@@ -124,16 +122,15 @@ def test_update(mocker, testable_environment) -> None:
     assert isinstance(result, EnvironmentNotFoundError)
 
 
-def test_delete(mocker, testable_environment) -> None:
+def test_delete(patch_post, testable_environment) -> None:
     environment, env_input = testable_environment
 
     result = environment.delete(env_input.name, env_input.path)
     assert isinstance(result, EnvironmentNotFoundError)
 
-    post_mock = mocker.patch('httpx.post')
     result = environment.create(env_input)
     assert isinstance(result, CreateEnvironmentSuccess)
-    post_mock.assert_called_once()
+    patch_post.assert_called_once()
 
     path = Path(env_input.path, env_input.name, ".created")
     assert file_was_pushed(path)
@@ -145,7 +142,7 @@ def test_delete(mocker, testable_environment) -> None:
 
 
 @pytest.mark.asyncio
-async def test_write_artifact(mocker, testable_environment):
+async def test_write_artifact(mocker, patch_post, testable_environment):
     environment, env_input = testable_environment
 
     upload = mocker.Mock(spec=UploadFile)
@@ -160,10 +157,9 @@ async def test_write_artifact(mocker, testable_environment):
     )
     assert isinstance(result, InvalidInputError)
 
-    post_mock = mocker.patch('httpx.post')
     result = environment.create(env_input)
     assert isinstance(result, CreateEnvironmentSuccess)
-    post_mock.assert_called_once()
+    patch_post.assert_called_once()
 
     result = await environment.write_artifact(
         file=upload,
@@ -184,7 +180,7 @@ async def test_write_artifact(mocker, testable_environment):
 
 
 @pytest.mark.asyncio
-async def test_iter(mocker, testable_environment):
+async def test_iter(mocker, patch_post, testable_environment):
     environment, env_input = testable_environment
 
     envs_filter = environment.iter()
@@ -194,10 +190,9 @@ async def test_iter(mocker, testable_environment):
 
     assert count == 0
 
-    post_mock = mocker.patch('httpx.post')
     result = environment.create(env_input)
     assert isinstance(result, CreateEnvironmentSuccess)
-    post_mock.assert_called_once()
+    patch_post.assert_called_once()
 
     upload = mocker.Mock(spec=UploadFile)
     upload.filename = Artifacts.environments_file
diff --git a/tests/integration/utils.py b/tests/integration/utils.py
new file mode 100644
index 0000000..43429b2
--- /dev/null
+++ b/tests/integration/utils.py
@@ -0,0 +1,121 @@
+"""Copyright (c) 2023 Genome Research Ltd.
+
+This source code is licensed under the MIT license found in the
+LICENSE file in the root directory of this source tree.
+"""
+
+import os
+import shutil
+import tempfile
+from pathlib import Path
+
+import pygit2
+import pytest
+
+from softpack_core.artifacts import Artifacts, app
+
+artifacts_dict = dict[
+    str, str | pygit2.Oid | Path | Artifacts | tempfile.TemporaryDirectory[str]
+]
+
+
+def new_test_artifacts() -> artifacts_dict:
+    temp_dir = tempfile.TemporaryDirectory()
+    app.settings.artifacts.path = Path(temp_dir.name)
+
+    artifacts = Artifacts()
+    dict = reset_test_repo(artifacts)
+    dict["temp_dir"] = temp_dir
+    dict["artifacts"] = artifacts
+
+    return dict
+
+
+def reset_test_repo(artifacts: Artifacts) -> artifacts_dict:
+    delete_environments_folder_from_test_repo(artifacts)
+
+    return create_initial_test_repo_state(artifacts)
+
+
+def delete_environments_folder_from_test_repo(artifacts: Artifacts):
+    tree = artifacts.repo.head.peel(pygit2.Tree)
+
+    if artifacts.environments_root in tree:
+        shutil.rmtree(
+            Path(app.settings.artifacts.path, artifacts.environments_root)
+        )
+        commit_local_file_changes(artifacts, "delete environments")
+
+
+def commit_local_file_changes(artifacts: Artifacts, msg: str) -> pygit2.Oid:
+    index = artifacts.repo.index
+    index.add_all()
+    index.write()
+    ref = artifacts.head_name
+    parents = [artifacts.repo.lookup_reference(ref).target]
+    oid = index.write_tree()
+    return artifacts.repo.create_commit(
+        ref, artifacts.signature, artifacts.signature, msg, oid, parents
+    )
+
+
+def create_initial_test_repo_state(artifacts: Artifacts) -> artifacts_dict:
+    dir_path = app.settings.artifacts.path
+    test_user = "test_user"
+    test_group = "test_group"
+    test_env = "test_environment"
+    user_env_path = Path(
+        dir_path,
+        "environments",
+        artifacts.users_folder_name,
+        test_user,
+        test_env,
+    )
+    group_env_path = Path(
+        dir_path,
+        "environments",
+        artifacts.groups_folder_name,
+        test_group,
+        test_env,
+    )
+    os.makedirs(user_env_path)
+    os.makedirs(group_env_path)
+    file_basename = "file.txt"
+    open(Path(user_env_path, file_basename), "w").close()
+    open(Path(group_env_path, file_basename), "w").close()
+
+    oid = commit_local_file_changes(artifacts, "Add test environments")
+
+    dict: artifacts_dict = {
+        "initial_commit_oid": oid,
+        "test_user": test_user,
+        "test_group": test_group,
+        "test_environment": test_env,
+        "user_env_path": user_env_path,
+        "group_env_path": group_env_path,
+        "basename": file_basename,
+    }
+    return dict
+
+
+def get_user_path_without_environments(
+    artifacts: Artifacts, user: str
+) -> Path:
+    return Path(*(artifacts.user_folder(user).parts[1:]))
+
+
+def file_was_pushed(*paths_without_environment: str | Path) -> bool:
+    temp_dir = tempfile.TemporaryDirectory()
+    app.settings.artifacts.path = Path(temp_dir.name)
+    artifacts = Artifacts()
+
+    for path_without_environment in paths_without_environment:
+        path = Path(
+            temp_dir.name,
+            artifacts.environments_root,
+            path_without_environment,
+        )
+        if not os.path.isfile(path):
+            return False
+
+    return True

From 6393e1724873b2b6ce25a2dc3503ef138cd586eb Mon Sep 17 00:00:00 2001
From: Sendu Bala <sb10@sanger.ac.uk>
Date: Tue, 29 Aug 2023 15:58:23 +0100
Subject: [PATCH 066/129] Use fixtures for the repeated mocks.

---
 tests/integration/conftest.py         |  8 +++++++-
 tests/integration/test_environment.py | 27 ++++++++++++---------------
 2 files changed, 19 insertions(+), 16 deletions(-)

diff --git a/tests/integration/conftest.py b/tests/integration/conftest.py
index 6985faf..0438149 100644
--- a/tests/integration/conftest.py
+++ b/tests/integration/conftest.py
@@ -11,6 +11,7 @@
 
 import pygit2
 import pytest
+from starlette.datastructures import UploadFile
 
 from softpack_core.artifacts import Artifacts, app
 
@@ -38,6 +39,11 @@ def testable_artifacts_setup():
 
 
 @pytest.fixture(autouse=True)
-def patch_post(mocker):
+def post(mocker):
     post_mock = mocker.patch('httpx.post')
     return post_mock
+
+
+@pytest.fixture()
+def upload(mocker):
+    return mocker.Mock(spec=UploadFile)
diff --git a/tests/integration/test_environment.py b/tests/integration/test_environment.py
index c8f2251..82e06d8 100644
--- a/tests/integration/test_environment.py
+++ b/tests/integration/test_environment.py
@@ -7,7 +7,6 @@
 from pathlib import Path
 
 import pytest
-from starlette.datastructures import UploadFile
 
 from softpack_core.artifacts import Artifacts
 from softpack_core.schemas.environment import (
@@ -56,7 +55,7 @@ def testable_environment(mocker):
     yield environment, env_input
 
 
-def test_create(patch_post, testable_environment) -> None:
+def test_create(post, testable_environment) -> None:
     environment, env_input = testable_environment
 
     result = environment.create(env_input)
@@ -65,8 +64,8 @@ def test_create(patch_post, testable_environment) -> None:
     path = Path(env_input.path, env_input.name, ".created")
     assert file_was_pushed(path)
 
-    patch_post.assert_called_once()
-    builder_called_correctly(patch_post, env_input)
+    post.assert_called_once()
+    builder_called_correctly(post, env_input)
 
     result = environment.create(env_input)
     assert isinstance(result, EnvironmentAlreadyExistsError)
@@ -96,18 +95,18 @@ def builder_called_correctly(post_mock, env_input: EnvironmentInput) -> None:
     )
 
 
-def test_update(patch_post, testable_environment) -> None:
+def test_update(post, testable_environment) -> None:
     environment, env_input = testable_environment
 
     result = environment.create(env_input)
     assert isinstance(result, CreateEnvironmentSuccess)
-    patch_post.assert_called_once()
+    post.assert_called_once()
 
     env_input.description = "updated description"
     result = environment.update(env_input, env_input.path, env_input.name)
     assert isinstance(result, UpdateEnvironmentSuccess)
 
-    builder_called_correctly(patch_post, env_input)
+    builder_called_correctly(post, env_input)
 
     result = environment.update(env_input, "invalid/path", "invalid_name")
     assert isinstance(result, InvalidInputError)
@@ -122,7 +121,7 @@ def test_update(patch_post, testable_environment) -> None:
     assert isinstance(result, EnvironmentNotFoundError)
 
 
-def test_delete(patch_post, testable_environment) -> None:
+def test_delete(post, testable_environment) -> None:
     environment, env_input = testable_environment
 
     result = environment.delete(env_input.name, env_input.path)
@@ -130,7 +129,7 @@ def test_delete(patch_post, testable_environment) -> None:
 
     result = environment.create(env_input)
     assert isinstance(result, CreateEnvironmentSuccess)
-    patch_post.assert_called_once()
+    post.assert_called_once()
 
     path = Path(env_input.path, env_input.name, ".created")
     assert file_was_pushed(path)
@@ -142,10 +141,9 @@ def test_delete(patch_post, testable_environment) -> None:
 
 
 @pytest.mark.asyncio
-async def test_write_artifact(mocker, patch_post, testable_environment):
+async def test_write_artifact(post, testable_environment, upload):
     environment, env_input = testable_environment
 
-    upload = mocker.Mock(spec=UploadFile)
     upload.filename = "example.txt"
     upload.content_type = "text/plain"
     upload.read.return_value = b"mock data"
@@ -159,7 +157,7 @@ async def test_write_artifact(mocker, patch_post, testable_environment):
 
     result = environment.create(env_input)
     assert isinstance(result, CreateEnvironmentSuccess)
-    patch_post.assert_called_once()
+    post.assert_called_once()
 
     result = await environment.write_artifact(
         file=upload,
@@ -180,7 +178,7 @@ async def test_write_artifact(mocker, patch_post, testable_environment):
 
 
 @pytest.mark.asyncio
-async def test_iter(mocker, patch_post, testable_environment):
+async def test_iter(post, testable_environment, upload):
     environment, env_input = testable_environment
 
     envs_filter = environment.iter()
@@ -192,9 +190,8 @@ async def test_iter(mocker, patch_post, testable_environment):
 
     result = environment.create(env_input)
     assert isinstance(result, CreateEnvironmentSuccess)
-    patch_post.assert_called_once()
+    post.assert_called_once()
 
-    upload = mocker.Mock(spec=UploadFile)
     upload.filename = Artifacts.environments_file
     upload.content_type = "text/plain"
     upload.read.return_value = b"description: test env\npackages:\n- zlib\n"

From c5e516a1bd40471a26861faeaa9272952719d1fd Mon Sep 17 00:00:00 2001
From: Sendu Bala <sb10@sanger.ac.uk>
Date: Wed, 30 Aug 2023 13:58:10 +0100
Subject: [PATCH 067/129] Switch from env vars to markers to skip integration
 tests.

---
 README.md                             | 33 ++++++++++++++++----
 tests/integration/conftest.py         | 45 ++++++++++++++++++++-------
 tests/integration/test_artifacts.py   |  3 ++
 tests/integration/test_environment.py |  3 ++
 4 files changed, 66 insertions(+), 18 deletions(-)

diff --git a/README.md b/README.md
index 2b4fc7c..d966b25 100644
--- a/README.md
+++ b/README.md
@@ -100,19 +100,40 @@ poetry run tox
 ```
 
 To run integration tests, you need a git repository set up with token access and
-a branch named after your git repo username. Then set these environment
-variables:
+a branch named after your git repo username (stripped of any @domain if your
+username is an email address).
 
+Make sure the artifacts/repo section of ~/.softpack/core/config.yml is
+configured correctly:
+
+```
+artifacts:
+  repo:
+    url: https://github.com/[your-org]/development-softpack-artifacts.git
+    username: [your-username]
+    author: [your-name]
+    email: [your-email]
+    writer: [your-token]
+```
+
+Then enable the integration tests by suppling --repo to `poetry run pytest`.
+
+To discover all tests and run them (skipping integration tests with no --repo):
+
+``` console
+poetry run pytest tests -sv
 ```
-export SOFTPACK_TEST_ARTIFACTS_REPO_URL='https://[...]artifacts.git'
-export SOFTPACK_TEST_ARTIFACTS_REPO_USER='username@domain'
-export SOFTPACK_TEST_ARTIFACTS_REPO_TOKEN='token'
+
+To run just the integration tests:
+
+``` console
+poetry run pytest tests/integration -sv --repo
 ```
 
 To run an individual test:
 
 ``` console
-poetry run pytest tests/integration/artifacts.py::test_commit -sv
+poetry run pytest tests/integration/test_artifacts.py::test_clone -sv --repo
 ```
 
 Run [MkDocs] server to view documentation:
diff --git a/tests/integration/conftest.py b/tests/integration/conftest.py
index 0438149..65dfd6d 100644
--- a/tests/integration/conftest.py
+++ b/tests/integration/conftest.py
@@ -16,25 +16,46 @@
 from softpack_core.artifacts import Artifacts, app
 
 
+def pytest_addoption(parser):
+    parser.addoption(
+        "--repo", action="store_true", default=False, help=(
+            "run integration tests that alter your real git repo")
+    )
+
+
+def pytest_configure(config):
+    config.addinivalue_line(
+        "markers", "repo: mark test as altering a real git repo")
+
+
+def pytest_collection_modifyitems(config, items):
+    if config.getoption("--repo"):
+        return
+    skip_repo = pytest.mark.skip(reason=("specify --repo to run integration "
+                                         "tests that will alter your "
+                                         "configured git repo"))
+    for item in items:
+        if "repo" in item.keywords:
+            item.add_marker(skip_repo)
+
+
 @pytest.fixture(scope="package", autouse=True)
 def testable_artifacts_setup():
-    repo_url = os.getenv("SOFTPACK_TEST_ARTIFACTS_REPO_URL")
-    repo_user = os.getenv("SOFTPACK_TEST_ARTIFACTS_REPO_USER")
-    repo_token = os.getenv("SOFTPACK_TEST_ARTIFACTS_REPO_TOKEN")
-    if repo_url is None or repo_user is None or repo_token is None:
+    user = app.settings.artifacts.repo.username.split('@', 1)[0]
+    if user is None or user == "main":
+        pytest.skip(
+            (
+                "Your artifacts repo username must be defined in your config."
+            )
+        )
+
+    if app.settings.artifacts.repo.writer is None:
         pytest.skip(
             (
-                "SOFTPACK_TEST_ARTIFACTS_REPO_URL, _USER and _TOKEN "
-                "env vars are all required for these tests"
+                "Your artifacts repo writer must be defined in your config."
             )
         )
 
-    user = repo_user.split('@', 1)[0]
-    app.settings.artifacts.repo.url = repo_url
-    app.settings.artifacts.repo.username = repo_user
-    app.settings.artifacts.repo.author = user
-    app.settings.artifacts.repo.email = repo_user
-    app.settings.artifacts.repo.writer = repo_token
     app.settings.artifacts.repo.branch = user
 
 
diff --git a/tests/integration/test_artifacts.py b/tests/integration/test_artifacts.py
index ca8e430..a99f135 100644
--- a/tests/integration/test_artifacts.py
+++ b/tests/integration/test_artifacts.py
@@ -19,6 +19,9 @@
 )
 
 
+pytestmark = pytest.mark.repo
+
+
 def test_clone() -> None:
     ad = new_test_artifacts()
     artifacts: Artifacts = ad["artifacts"]
diff --git a/tests/integration/test_environment.py b/tests/integration/test_environment.py
index 82e06d8..5aa4608 100644
--- a/tests/integration/test_environment.py
+++ b/tests/integration/test_environment.py
@@ -28,6 +28,9 @@
 )
 
 
+pytestmark = pytest.mark.repo
+
+
 @pytest.fixture
 def testable_environment(mocker):
     ad = new_test_artifacts()

From 5edfa947698c93a234a069720385fb0783ca89eb Mon Sep 17 00:00:00 2001
From: Sendu Bala <sb10@sanger.ac.uk>
Date: Thu, 31 Aug 2023 10:08:34 +0100
Subject: [PATCH 068/129] Broken attempt at switching to bare repo: push not
 working.

---
 softpack_core/artifacts.py |  5 ++--
 tests/integration/utils.py | 52 ++++++++++++++++++++++++++++----------
 2 files changed, 42 insertions(+), 15 deletions(-)

diff --git a/softpack_core/artifacts.py b/softpack_core/artifacts.py
index c04dad5..ca68552 100644
--- a/softpack_core/artifacts.py
+++ b/softpack_core/artifacts.py
@@ -24,6 +24,7 @@ class Artifacts:
     users_folder_name = "users"
     groups_folder_name = "groups"
     head_name = ""
+    head = None
     credentials_callback = None
     signature = None
 
@@ -124,8 +125,7 @@ def __init__(self) -> None:
                 self.settings.artifacts.repo.url,
                 path=path,
                 callbacks=self.credentials_callback,
-                bare=False,
-                checkout_branch=branch,
+                bare=True,
             )
 
         self.signature = pygit2.Signature(
@@ -134,6 +134,7 @@ def __init__(self) -> None:
         )
 
         self.head_name = self.repo.head.name
+        self.head = self.repo.head
 
     def user_folder(self, user: Optional[str] = None) -> Path:
         """Get the user folder for a given user.
diff --git a/tests/integration/utils.py b/tests/integration/utils.py
index 43429b2..4860a24 100644
--- a/tests/integration/utils.py
+++ b/tests/integration/utils.py
@@ -24,6 +24,20 @@ def new_test_artifacts() -> artifacts_dict:
     app.settings.artifacts.path = Path(temp_dir.name)
 
     artifacts = Artifacts()
+
+    branch_name = "origin/" + app.settings.artifacts.repo.branch
+    branch = artifacts.repo.branches.get(branch_name)
+
+    if branch is None:
+        pytest.skip(
+            (
+                "Your artifacts repo must have a branch named after your username."
+            )
+        )
+
+    artifacts.head_name = branch.branch_name
+    artifacts.head = branch
+
     dict = reset_test_repo(artifacts)
     dict["temp_dir"] = temp_dir
     dict["artifacts"] = artifacts
@@ -38,24 +52,36 @@ def reset_test_repo(artifacts: Artifacts) -> artifacts_dict:
 
 
 def delete_environments_folder_from_test_repo(artifacts: Artifacts):
-    tree = artifacts.repo.head.peel(pygit2.Tree)
-
+    tree = artifacts.head.peel(pygit2.Tree)
     if artifacts.environments_root in tree:
-        shutil.rmtree(
-            Path(app.settings.artifacts.path, artifacts.environments_root)
-        )
-        commit_local_file_changes(artifacts, "delete environments")
+        oid = delete_all_files_from_tree(
+            artifacts.repo, tree[artifacts.environments_root])
+        commit_changes(artifacts, oid, "delete environments")
+
+        remote = artifacts.repo.remotes[0]
+        remote.push(["refs/remotes/" + artifacts.head_name],
+                    callbacks=artifacts.credentials_callback)
+
+
+def delete_all_files_from_tree(repo: pygit2.Repository, tree: pygit2.Tree) -> pygit2.Oid:
+    treeBuilder = repo.TreeBuilder(tree)
+    for entry in tree:
+        if entry is pygit2.Tree:
+            delete_all_files_from_tree(entry)
+        else:
+            treeBuilder.remove(entry.name)
+    return treeBuilder.write()
 
 
-def commit_local_file_changes(artifacts: Artifacts, msg: str) -> pygit2.Oid:
-    index = artifacts.repo.index
-    index.add_all()
-    index.write()
+def commit_changes(artifacts: Artifacts, oid: pygit2.Oid, msg: str) -> pygit2.Oid:
     ref = artifacts.head_name
-    parents = [artifacts.repo.lookup_reference(ref).target]
-    oid = index.write_tree()
     return artifacts.repo.create_commit(
-        ref, artifacts.signature, artifacts.signature, msg, oid, parents
+        ref,
+        artifacts.signature,
+        artifacts.signature,
+        msg,
+        oid,
+        [artifacts.head.target]
     )
 
 

From 89323d68d498c1bb3b40f41151431d8c69c85494 Mon Sep 17 00:00:00 2001
From: Sendu Bala <sb10@sanger.ac.uk>
Date: Thu, 31 Aug 2023 11:26:38 +0100
Subject: [PATCH 069/129] Fixed push to branch of bare repo; but delete
 deleting everything.

---
 softpack_core/artifacts.py | 1 +
 tests/integration/utils.py | 9 +++------
 2 files changed, 4 insertions(+), 6 deletions(-)

diff --git a/softpack_core/artifacts.py b/softpack_core/artifacts.py
index ca68552..97235f4 100644
--- a/softpack_core/artifacts.py
+++ b/softpack_core/artifacts.py
@@ -126,6 +126,7 @@ def __init__(self) -> None:
                 path=path,
                 callbacks=self.credentials_callback,
                 bare=True,
+                checkout_branch=branch
             )
 
         self.signature = pygit2.Signature(
diff --git a/tests/integration/utils.py b/tests/integration/utils.py
index 4860a24..816c7fa 100644
--- a/tests/integration/utils.py
+++ b/tests/integration/utils.py
@@ -25,19 +25,16 @@ def new_test_artifacts() -> artifacts_dict:
 
     artifacts = Artifacts()
 
-    branch_name = "origin/" + app.settings.artifacts.repo.branch
+    branch_name = app.settings.artifacts.repo.branch
     branch = artifacts.repo.branches.get(branch_name)
 
-    if branch is None:
+    if branch is None or branch_name == "main":
         pytest.skip(
             (
                 "Your artifacts repo must have a branch named after your username."
             )
         )
 
-    artifacts.head_name = branch.branch_name
-    artifacts.head = branch
-
     dict = reset_test_repo(artifacts)
     dict["temp_dir"] = temp_dir
     dict["artifacts"] = artifacts
@@ -59,7 +56,7 @@ def delete_environments_folder_from_test_repo(artifacts: Artifacts):
         commit_changes(artifacts, oid, "delete environments")
 
         remote = artifacts.repo.remotes[0]
-        remote.push(["refs/remotes/" + artifacts.head_name],
+        remote.push([artifacts.head_name],
                     callbacks=artifacts.credentials_callback)
 
 

From df4cacc7df8eb4ed59501edd8e293eeb147286f0 Mon Sep 17 00:00:00 2001
From: Sendu Bala <sb10@sanger.ac.uk>
Date: Thu, 31 Aug 2023 11:40:50 +0100
Subject: [PATCH 070/129] Fix delete_environments_folder

---
 tests/integration/utils.py | 15 +++------------
 1 file changed, 3 insertions(+), 12 deletions(-)

diff --git a/tests/integration/utils.py b/tests/integration/utils.py
index 816c7fa..70309e4 100644
--- a/tests/integration/utils.py
+++ b/tests/integration/utils.py
@@ -51,8 +51,9 @@ def reset_test_repo(artifacts: Artifacts) -> artifacts_dict:
 def delete_environments_folder_from_test_repo(artifacts: Artifacts):
     tree = artifacts.head.peel(pygit2.Tree)
     if artifacts.environments_root in tree:
-        oid = delete_all_files_from_tree(
-            artifacts.repo, tree[artifacts.environments_root])
+        treeBuilder = artifacts.repo.TreeBuilder(tree)
+        treeBuilder.remove(artifacts.environments_root)
+        oid = treeBuilder.write()
         commit_changes(artifacts, oid, "delete environments")
 
         remote = artifacts.repo.remotes[0]
@@ -60,16 +61,6 @@ def delete_environments_folder_from_test_repo(artifacts: Artifacts):
                     callbacks=artifacts.credentials_callback)
 
 
-def delete_all_files_from_tree(repo: pygit2.Repository, tree: pygit2.Tree) -> pygit2.Oid:
-    treeBuilder = repo.TreeBuilder(tree)
-    for entry in tree:
-        if entry is pygit2.Tree:
-            delete_all_files_from_tree(entry)
-        else:
-            treeBuilder.remove(entry.name)
-    return treeBuilder.write()
-
-
 def commit_changes(artifacts: Artifacts, oid: pygit2.Oid, msg: str) -> pygit2.Oid:
     ref = artifacts.head_name
     return artifacts.repo.create_commit(

From 570eb2b94923e9d5227b7dc98912061d57acdf68 Mon Sep 17 00:00:00 2001
From: Sendu Bala <sb10@sanger.ac.uk>
Date: Thu, 31 Aug 2023 14:26:22 +0100
Subject: [PATCH 071/129] Refactor tests to work with bare repo.

---
 softpack_core/artifacts.py            |  9 +---
 tests/integration/test_artifacts.py   | 31 +++--------
 tests/integration/test_environment.py | 15 ++++--
 tests/integration/utils.py            | 76 ++++++++++++++++++---------
 4 files changed, 71 insertions(+), 60 deletions(-)

diff --git a/softpack_core/artifacts.py b/softpack_core/artifacts.py
index 97235f4..8898aac 100644
--- a/softpack_core/artifacts.py
+++ b/softpack_core/artifacts.py
@@ -23,8 +23,6 @@ class Artifacts:
     environments_file = "softpack.yml"
     users_folder_name = "users"
     groups_folder_name = "groups"
-    head_name = ""
-    head = None
     credentials_callback = None
     signature = None
 
@@ -134,9 +132,6 @@ def __init__(self) -> None:
             self.settings.artifacts.repo.email,
         )
 
-        self.head_name = self.repo.head.name
-        self.head = self.repo.head
-
     def user_folder(self, user: Optional[str] = None) -> Path:
         """Get the user folder for a given user.
 
@@ -262,7 +257,7 @@ def commit(self, tree_oid: pygit2.Oid, message: str) -> pygit2.Oid:
         Returns:
             pygit2.Commit: the commit oid
         """
-        ref = self.head_name
+        ref = self.repo.head.name
         parents = [self.repo.lookup_reference(ref).target]
         commit_oid = self.repo.create_commit(
             ref, self.signature, self.signature, message, tree_oid, parents
@@ -272,7 +267,7 @@ def commit(self, tree_oid: pygit2.Oid, message: str) -> pygit2.Oid:
     def push(self) -> None:
         """Push all commits to a repository."""
         remote = self.repo.remotes[0]
-        remote.push([self.head_name], callbacks=self.credentials_callback)
+        remote.push([self.repo.head.name], callbacks=self.credentials_callback)
 
     def build_tree(
         self,
diff --git a/tests/integration/test_artifacts.py b/tests/integration/test_artifacts.py
index a99f135..bbf23e1 100644
--- a/tests/integration/test_artifacts.py
+++ b/tests/integration/test_artifacts.py
@@ -41,21 +41,11 @@ def test_commit_and_push() -> None:
     old_commit_oid = ad["initial_commit_oid"]
 
     new_file_name = "new_file.txt"
-    path = Path(
-        ad["temp_dir"].name,
-        artifacts.environments_root,
-        artifacts.users_folder_name,
-        ad["test_user"],
-        ad["test_environment"],
-        new_file_name,
-    )
-
-    open(path, "w").close()
+    oid = artifacts.repo.create_blob(b"")
 
-    index = artifacts.repo.index
-    index.add_all()
-    index.write()
-    new_tree = index.write_tree()
+    tb = artifacts.repo.TreeBuilder()
+    tb.insert(new_file_name, oid, pygit2.GIT_FILEMODE_BLOB)
+    new_tree = tb.write()
 
     new_commit_oid = artifacts.commit(new_tree, "commit new file")
     repo_head = artifacts.repo.head.peel(pygit2.Commit).oid
@@ -64,15 +54,7 @@ def test_commit_and_push() -> None:
     assert new_commit_oid == repo_head
 
     artifacts.push()
-
-    path = Path(
-        artifacts.users_folder_name,
-        ad["test_user"],
-        ad["test_environment"],
-        new_file_name,
-    )
-
-    assert file_was_pushed(path)
+    assert file_was_pushed(Path(new_file_name))
 
 
 def test_create_file() -> None:
@@ -141,7 +123,8 @@ def test_create_file() -> None:
     artifacts.push()
 
     assert file_was_pushed(
-        Path(folder_path, basename), Path(folder_path, basename2)
+        Path(artifacts.environments_root, folder_path, basename),
+        Path(artifacts.environments_root, folder_path, basename2)
     )
 
 
diff --git a/tests/integration/test_environment.py b/tests/integration/test_environment.py
index 5aa4608..dbbf41f 100644
--- a/tests/integration/test_environment.py
+++ b/tests/integration/test_environment.py
@@ -30,9 +30,11 @@
 
 pytestmark = pytest.mark.repo
 
+tetype = tuple[Environment, EnvironmentInput]
+
 
 @pytest.fixture
-def testable_environment(mocker):
+def testable_environment(mocker) -> tetype:
     ad = new_test_artifacts()
     artifacts: Artifacts = ad["artifacts"]
     user = ad["test_user"]
@@ -58,13 +60,14 @@ def testable_environment(mocker):
     yield environment, env_input
 
 
-def test_create(post, testable_environment) -> None:
+def test_create(post, testable_environment: tetype) -> None:
     environment, env_input = testable_environment
 
     result = environment.create(env_input)
     assert isinstance(result, CreateEnvironmentSuccess)
 
-    path = Path(env_input.path, env_input.name, ".created")
+    path = Path(environment.artifacts.environments_root, env_input.path,
+                env_input.name, ".created")
     assert file_was_pushed(path)
 
     post.assert_called_once()
@@ -134,7 +137,8 @@ def test_delete(post, testable_environment) -> None:
     assert isinstance(result, CreateEnvironmentSuccess)
     post.assert_called_once()
 
-    path = Path(env_input.path, env_input.name, ".created")
+    path = Path(environment.artifacts.environments_root, env_input.path,
+                env_input.name, ".created")
     assert file_was_pushed(path)
 
     result = environment.delete(env_input.name, env_input.path)
@@ -169,7 +173,8 @@ async def test_write_artifact(post, testable_environment, upload):
     )
     assert isinstance(result, WriteArtifactSuccess)
 
-    path = Path(env_input.path, env_input.name, upload.filename)
+    path = Path(environment.artifacts.environments_root, env_input.path,
+                env_input.name, upload.filename)
     assert file_was_pushed(path)
 
     result = await environment.write_artifact(
diff --git a/tests/integration/utils.py b/tests/integration/utils.py
index 70309e4..fafdbeb 100644
--- a/tests/integration/utils.py
+++ b/tests/integration/utils.py
@@ -31,7 +31,8 @@ def new_test_artifacts() -> artifacts_dict:
     if branch is None or branch_name == "main":
         pytest.skip(
             (
-                "Your artifacts repo must have a branch named after your username."
+                "Your artifacts repo must have a branch named after your "
+                "username."
             )
         )
 
@@ -49,27 +50,23 @@ def reset_test_repo(artifacts: Artifacts) -> artifacts_dict:
 
 
 def delete_environments_folder_from_test_repo(artifacts: Artifacts):
-    tree = artifacts.head.peel(pygit2.Tree)
+    tree = artifacts.repo.head.peel(pygit2.Tree)
     if artifacts.environments_root in tree:
         treeBuilder = artifacts.repo.TreeBuilder(tree)
         treeBuilder.remove(artifacts.environments_root)
         oid = treeBuilder.write()
-        commit_changes(artifacts, oid, "delete environments")
+        commit_test_repo_changes(artifacts, oid, "delete environments")
 
-        remote = artifacts.repo.remotes[0]
-        remote.push([artifacts.head_name],
-                    callbacks=artifacts.credentials_callback)
 
-
-def commit_changes(artifacts: Artifacts, oid: pygit2.Oid, msg: str) -> pygit2.Oid:
-    ref = artifacts.head_name
+def commit_test_repo_changes(artifacts: Artifacts, oid: pygit2.Oid, msg: str) -> pygit2.Oid:
+    ref = artifacts.repo.head.name
     return artifacts.repo.create_commit(
         ref,
         artifacts.signature,
         artifacts.signature,
         msg,
         oid,
-        [artifacts.head.target]
+        [artifacts.repo.lookup_reference(ref).target]
     )
 
 
@@ -92,13 +89,44 @@ def create_initial_test_repo_state(artifacts: Artifacts) -> artifacts_dict:
         test_group,
         test_env,
     )
-    os.makedirs(user_env_path)
-    os.makedirs(group_env_path)
     file_basename = "file.txt"
-    open(Path(user_env_path, file_basename), "w").close()
-    open(Path(group_env_path, file_basename), "w").close()
 
-    oid = commit_local_file_changes(artifacts, "Add test environments")
+    oid = artifacts.repo.create_blob(b"")
+
+    userTestEnv = artifacts.repo.TreeBuilder()
+    userTestEnv.insert(file_basename, oid, pygit2.GIT_FILEMODE_BLOB)
+
+    testUser = artifacts.repo.TreeBuilder()
+    testUser.insert(test_env, userTestEnv.write(), pygit2.GIT_FILEMODE_TREE)
+
+    usersFolder = artifacts.repo.TreeBuilder()
+    usersFolder.insert(test_user, testUser.write(), pygit2.GIT_FILEMODE_TREE)
+
+    oid = artifacts.repo.create_blob(b"")
+
+    userGroupEnv = artifacts.repo.TreeBuilder()
+    userGroupEnv.insert(file_basename, oid, pygit2.GIT_FILEMODE_BLOB)
+
+    testGroup = artifacts.repo.TreeBuilder()
+    testGroup.insert(test_env, userGroupEnv.write(), pygit2.GIT_FILEMODE_TREE)
+
+    groupsFolder = artifacts.repo.TreeBuilder()
+    groupsFolder.insert(test_group, testGroup.write(),
+                        pygit2.GIT_FILEMODE_TREE)
+
+    environments = artifacts.repo.TreeBuilder()
+    environments.insert(artifacts.users_folder_name,
+                        usersFolder.write(), pygit2.GIT_FILEMODE_TREE)
+    environments.insert(artifacts.groups_folder_name,
+                        groupsFolder.write(), pygit2.GIT_FILEMODE_TREE)
+
+    tree = artifacts.repo.head.peel(pygit2.Tree)
+    treeBuilder = artifacts.repo.TreeBuilder(tree)
+    treeBuilder.insert(artifacts.environments_root,
+                       environments.write(), pygit2.GIT_FILEMODE_TREE)
+
+    oid = commit_test_repo_changes(artifacts, treeBuilder.write(),
+                                   "Add test environments")
 
     dict: artifacts_dict = {
         "initial_commit_oid": oid,
@@ -118,18 +146,18 @@ def get_user_path_without_environments(
     return Path(*(artifacts.user_folder(user).parts[1:]))
 
 
-def file_was_pushed(*paths_without_environment: str | Path) -> bool:
+def file_was_pushed(*paths_with_environment: str | Path) -> bool:
     temp_dir = tempfile.TemporaryDirectory()
     app.settings.artifacts.path = Path(temp_dir.name)
     artifacts = Artifacts()
 
-    for path_without_environment in paths_without_environment:
-        path = Path(
-            temp_dir.name,
-            artifacts.environments_root,
-            path_without_environment,
-        )
-        if not os.path.isfile(path):
-            return False
+    for path_with_environment in paths_with_environment:
+        path = Path(path_with_environment)
+
+        current = artifacts.repo.head.peel(pygit2.Tree)
+        for part in path.parts:
+            if part not in current:
+                return False
+            current = current[part]
 
     return True

From 8f9494584c87338911f33aee532ac28eaad37903 Mon Sep 17 00:00:00 2001
From: Sendu Bala <sb10@sanger.ac.uk>
Date: Thu, 31 Aug 2023 16:14:30 +0100
Subject: [PATCH 072/129] Require successful push for attempted changes to be
 visible.

---
 softpack_core/artifacts.py           | 24 +++++++++++++-----------
 softpack_core/schemas/environment.py | 11 +++++------
 tests/integration/test_artifacts.py  | 13 +++++--------
 tests/integration/utils.py           | 15 ++++++++++-----
 4 files changed, 33 insertions(+), 30 deletions(-)

diff --git a/softpack_core/artifacts.py b/softpack_core/artifacts.py
index 8898aac..cb1ef3d 100644
--- a/softpack_core/artifacts.py
+++ b/softpack_core/artifacts.py
@@ -127,6 +127,14 @@ def __init__(self) -> None:
                 checkout_branch=branch
             )
 
+        self.reference = "/".join(
+            [
+                "refs/remotes",
+                self.repo.remotes[0].name,
+                self.repo.head.shorthand,
+            ]
+        )
+
         self.signature = pygit2.Signature(
             self.settings.artifacts.repo.author,
             self.settings.artifacts.repo.email,
@@ -205,7 +213,7 @@ def tree(self, path: str) -> pygit2.Tree:
         Returns:
             Tree: A Tree object
         """
-        return self.repo.head.peel(pygit2.Tree)[path]
+        return self.repo.lookup_reference(self.reference).peel().tree[path]
 
     def environments(self, path: Path) -> Iterable:
         """Return a list of environments in the repo under the given path.
@@ -246,28 +254,22 @@ def get(self, path: Path, name: str) -> Optional[pygit2.Tree]:
         except KeyError:
             return None
 
-    def commit(self, tree_oid: pygit2.Oid, message: str) -> pygit2.Oid:
-        """Create and return a commit.
+    def commit_and_push(self, tree_oid: pygit2.Oid, message: str) -> pygit2.Oid:
+        """Commit and push current changes to the remote repository.
 
         Args:
             tree_oid: the oid of the tree object that will be committed. The
             tree this refers to will replace the entire contents of the repo.
             message: the commit message
-
-        Returns:
-            pygit2.Commit: the commit oid
         """
         ref = self.repo.head.name
         parents = [self.repo.lookup_reference(ref).target]
-        commit_oid = self.repo.create_commit(
+        oid = self.repo.create_commit(
             ref, self.signature, self.signature, message, tree_oid, parents
         )
-        return commit_oid
-
-    def push(self) -> None:
-        """Push all commits to a repository."""
         remote = self.repo.remotes[0]
         remote.push([self.repo.head.name], callbacks=self.credentials_callback)
+        return oid
 
     def build_tree(
         self,
diff --git a/softpack_core/schemas/environment.py b/softpack_core/schemas/environment.py
index 2ca86f8..623d15b 100644
--- a/softpack_core/schemas/environment.py
+++ b/softpack_core/schemas/environment.py
@@ -244,8 +244,8 @@ def create(cls, env: EnvironmentInput) -> CreateResponse:
             tree_oid = cls.artifacts.create_file(
                 new_folder_path, file_name, "", True
             )
-            cls.artifacts.commit(tree_oid, "create environment folder")
-            cls.artifacts.push()
+            cls.artifacts.commit_and_push(
+                tree_oid, "create environment folder")
         except RuntimeError as e:
             return InvalidInputError(message=str(e))
 
@@ -332,8 +332,7 @@ def delete(cls, name: str, path: str) -> DeleteResponse:
         """
         if cls.artifacts.get(Path(path), name):
             tree_oid = cls.artifacts.delete_environment(name, path)
-            cls.artifacts.commit(tree_oid, "delete environment")
-            cls.artifacts.push()
+            cls.artifacts.commit_and_push(tree_oid, "delete environment")
             return DeleteEnvironmentSuccess(
                 message="Successfully deleted the environment"
             )
@@ -360,8 +359,8 @@ async def write_artifact(
             tree_oid = cls.artifacts.create_file(
                 Path(folder_path), file_name, contents, overwrite=True
             )
-            commit_oid = cls.artifacts.commit(tree_oid, "write artifact")
-            cls.artifacts.push()
+            commit_oid = cls.artifacts.commit_and_push(
+                tree_oid, "write artifact")
             return WriteArtifactSuccess(
                 message="Successfully written artifact",
                 commit_oid=str(commit_oid),
diff --git a/tests/integration/test_artifacts.py b/tests/integration/test_artifacts.py
index bbf23e1..0da2e6d 100644
--- a/tests/integration/test_artifacts.py
+++ b/tests/integration/test_artifacts.py
@@ -47,13 +47,12 @@ def test_commit_and_push() -> None:
     tb.insert(new_file_name, oid, pygit2.GIT_FILEMODE_BLOB)
     new_tree = tb.write()
 
-    new_commit_oid = artifacts.commit(new_tree, "commit new file")
+    new_commit_oid = artifacts.commit_and_push(new_tree, "commit new file")
     repo_head = artifacts.repo.head.peel(pygit2.Commit).oid
 
     assert old_commit_oid != new_commit_oid
     assert new_commit_oid == repo_head
 
-    artifacts.push()
     assert file_was_pushed(Path(new_file_name))
 
 
@@ -82,7 +81,7 @@ def test_create_file() -> None:
     assert new_test_env in [obj.name for obj in user_envs_tree]
     assert basename in [obj.name for obj in user_envs_tree[new_test_env]]
 
-    artifacts.commit(oid, "create file")
+    artifacts.commit_and_push(oid, "create file")
 
     with pytest.raises(RuntimeError) as exc_info:
         artifacts.create_file(
@@ -101,7 +100,7 @@ def test_create_file() -> None:
         folder_path, basename2, "lorem ipsum", False, False
     )
 
-    artifacts.commit(oid, "create file2")
+    artifacts.commit_and_push(oid, "create file2")
 
     user_envs_tree = get_user_envs_tree(artifacts, user, oid)
     assert basename2 in [obj.name for obj in user_envs_tree[new_test_env]]
@@ -114,14 +113,12 @@ def test_create_file() -> None:
 
     oid = artifacts.create_file(folder_path, basename, "override", False, True)
 
-    artifacts.commit(oid, "update created file")
+    artifacts.commit_and_push(oid, "update created file")
 
     user_envs_tree = get_user_envs_tree(artifacts, user, oid)
     assert basename in [obj.name for obj in user_envs_tree[new_test_env]]
     assert user_envs_tree[new_test_env][basename].data.decode() == "override"
 
-    artifacts.push()
-
     assert file_was_pushed(
         Path(artifacts.environments_root, folder_path, basename),
         Path(artifacts.environments_root, folder_path, basename2)
@@ -150,7 +147,7 @@ def test_delete_environment() -> None:
         env_for_deleting, get_user_path_without_environments(artifacts, user)
     )
 
-    artifacts.commit(oid, "delete new env")
+    artifacts.commit_and_push(oid, "delete new env")
 
     user_envs_tree = get_user_envs_tree(artifacts, user, oid)
     assert env_for_deleting not in [obj.name for obj in user_envs_tree]
diff --git a/tests/integration/utils.py b/tests/integration/utils.py
index fafdbeb..e7c59ad 100644
--- a/tests/integration/utils.py
+++ b/tests/integration/utils.py
@@ -55,12 +55,13 @@ def delete_environments_folder_from_test_repo(artifacts: Artifacts):
         treeBuilder = artifacts.repo.TreeBuilder(tree)
         treeBuilder.remove(artifacts.environments_root)
         oid = treeBuilder.write()
-        commit_test_repo_changes(artifacts, oid, "delete environments")
+        commit_and_push_test_repo_changes(
+            artifacts, oid, "delete environments")
 
 
-def commit_test_repo_changes(artifacts: Artifacts, oid: pygit2.Oid, msg: str) -> pygit2.Oid:
+def commit_and_push_test_repo_changes(artifacts: Artifacts, oid: pygit2.Oid, msg: str) -> pygit2.Oid:
     ref = artifacts.repo.head.name
-    return artifacts.repo.create_commit(
+    oid = artifacts.repo.create_commit(
         ref,
         artifacts.signature,
         artifacts.signature,
@@ -68,6 +69,10 @@ def commit_test_repo_changes(artifacts: Artifacts, oid: pygit2.Oid, msg: str) ->
         oid,
         [artifacts.repo.lookup_reference(ref).target]
     )
+    remote = artifacts.repo.remotes[0]
+    remote.push([artifacts.repo.head.name],
+                callbacks=artifacts.credentials_callback)
+    return oid
 
 
 def create_initial_test_repo_state(artifacts: Artifacts) -> artifacts_dict:
@@ -125,8 +130,8 @@ def create_initial_test_repo_state(artifacts: Artifacts) -> artifacts_dict:
     treeBuilder.insert(artifacts.environments_root,
                        environments.write(), pygit2.GIT_FILEMODE_TREE)
 
-    oid = commit_test_repo_changes(artifacts, treeBuilder.write(),
-                                   "Add test environments")
+    oid = commit_and_push_test_repo_changes(artifacts, treeBuilder.write(),
+                                            "Add test environments")
 
     dict: artifacts_dict = {
         "initial_commit_oid": oid,

From ea18ddb3e9e499d8f3b423ba830ff6bf839fa372 Mon Sep 17 00:00:00 2001
From: Sendu Bala <sb10@sanger.ac.uk>
Date: Thu, 31 Aug 2023 16:59:00 +0100
Subject: [PATCH 073/129] Document that this does not work on empty repo.

---
 README.md | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/README.md b/README.md
index d966b25..204c93b 100644
--- a/README.md
+++ b/README.md
@@ -35,6 +35,10 @@ $ git clone -c feature.manyFiles=true --depth 1 https://github.com/spack/spack.g
 $ source spack/share/spack/setup-env.sh
 ```
 
+To start the service, you will also need to configure a git repository to store
+artifacts. That respository must have at least 1 file in
+environments/users/<subdir> and another file in environments/groups/<subdir>.
+
 ### Stable release
 
 To install SoftPack Core, run this command in your

From 469078d2b5eaf53cd9cf9eb8e467aac4dcaf7e20 Mon Sep 17 00:00:00 2001
From: Sendu Bala <sb10@sanger.ac.uk>
Date: Thu, 31 Aug 2023 16:59:40 +0100
Subject: [PATCH 074/129] Fix test_commit_and_push() to only affect
 environments folder.

---
 tests/integration/test_artifacts.py | 15 +++++++++++++--
 1 file changed, 13 insertions(+), 2 deletions(-)

diff --git a/tests/integration/test_artifacts.py b/tests/integration/test_artifacts.py
index 0da2e6d..73ea8cf 100644
--- a/tests/integration/test_artifacts.py
+++ b/tests/integration/test_artifacts.py
@@ -16,6 +16,7 @@
     file_was_pushed,
     get_user_path_without_environments,
     new_test_artifacts,
+    delete_environments_folder_from_test_repo,
 )
 
 
@@ -34,6 +35,11 @@ def test_clone() -> None:
     artifacts = Artifacts()
     assert os.path.isdir(path) is True
 
+    # add test where we make a change to the repo in a different clone dir,
+    # then call Artifacts() in an existing clone dir, and we should see the
+    # change: ie. implement pull on init.
+    # And then possibly something to test pull on every iter?
+
 
 def test_commit_and_push() -> None:
     ad = new_test_artifacts()
@@ -43,8 +49,13 @@ def test_commit_and_push() -> None:
     new_file_name = "new_file.txt"
     oid = artifacts.repo.create_blob(b"")
 
-    tb = artifacts.repo.TreeBuilder()
+    root = artifacts.repo.head.peel(pygit2.Tree)
+    tree = root[artifacts.environments_root]
+    tb = artifacts.repo.TreeBuilder(tree)
     tb.insert(new_file_name, oid, pygit2.GIT_FILEMODE_BLOB)
+    oid = tb.write()
+    tb = artifacts.repo.TreeBuilder(root)
+    tb.insert(artifacts.environments_root, oid, pygit2.GIT_FILEMODE_TREE)
     new_tree = tb.write()
 
     new_commit_oid = artifacts.commit_and_push(new_tree, "commit new file")
@@ -53,7 +64,7 @@ def test_commit_and_push() -> None:
     assert old_commit_oid != new_commit_oid
     assert new_commit_oid == repo_head
 
-    assert file_was_pushed(Path(new_file_name))
+    assert file_was_pushed(Path(artifacts.environments_root, new_file_name))
 
 
 def test_create_file() -> None:

From cbe4f7c63746c60f6c4ccd80578394bcc5bf35b8 Mon Sep 17 00:00:00 2001
From: Sendu Bala <sb10@sanger.ac.uk>
Date: Fri, 1 Sep 2023 10:28:45 +0100
Subject: [PATCH 075/129] Get latest repo during startup.

---
 softpack_core/artifacts.py          | 19 +++++++------
 tests/integration/test_artifacts.py | 43 ++++++++++++++++++-----------
 tests/integration/utils.py          | 17 ++++++++----
 3 files changed, 49 insertions(+), 30 deletions(-)

diff --git a/softpack_core/artifacts.py b/softpack_core/artifacts.py
index cb1ef3d..db04638 100644
--- a/softpack_core/artifacts.py
+++ b/softpack_core/artifacts.py
@@ -8,6 +8,7 @@
 from dataclasses import dataclass
 from pathlib import Path
 from typing import Iterable, Iterator, Optional
+import shutil
 
 import pygit2
 from box import Box
@@ -117,15 +118,15 @@ def __init__(self) -> None:
             branch = "main"
 
         if path.is_dir():
-            self.repo = pygit2.Repository(path)
-        else:
-            self.repo = pygit2.clone_repository(
-                self.settings.artifacts.repo.url,
-                path=path,
-                callbacks=self.credentials_callback,
-                bare=True,
-                checkout_branch=branch
-            )
+            shutil.rmtree(path)
+
+        self.repo = pygit2.clone_repository(
+            self.settings.artifacts.repo.url,
+            path=path,
+            callbacks=self.credentials_callback,
+            bare=True,
+            checkout_branch=branch
+        )
 
         self.reference = "/".join(
             [
diff --git a/tests/integration/test_artifacts.py b/tests/integration/test_artifacts.py
index 73ea8cf..96e347d 100644
--- a/tests/integration/test_artifacts.py
+++ b/tests/integration/test_artifacts.py
@@ -11,12 +11,13 @@
 import pygit2
 import pytest
 
-from softpack_core.artifacts import Artifacts
+from softpack_core.artifacts import Artifacts, app
 from tests.integration.utils import (
     file_was_pushed,
     get_user_path_without_environments,
     new_test_artifacts,
-    delete_environments_folder_from_test_repo,
+    commit_and_push_test_repo_changes,
+    file_in_repo,
 )
 
 
@@ -35,10 +36,17 @@ def test_clone() -> None:
     artifacts = Artifacts()
     assert os.path.isdir(path) is True
 
-    # add test where we make a change to the repo in a different clone dir,
-    # then call Artifacts() in an existing clone dir, and we should see the
-    # change: ie. implement pull on init.
-    # And then possibly something to test pull on every iter?
+    orig_repo_path = app.settings.artifacts.path
+    ad_for_changing = new_test_artifacts()
+    artifacts_for_changing: Artifacts = ad_for_changing["artifacts"]
+
+    oid, file_path = add_test_file_to_repo(artifacts_for_changing)
+    commit_and_push_test_repo_changes(artifacts_for_changing, oid, "add file")
+
+    app.settings.artifacts.path = orig_repo_path
+    artifacts = Artifacts()
+
+    assert file_in_repo(artifacts, file_path)
 
 
 def test_commit_and_push() -> None:
@@ -46,9 +54,20 @@ def test_commit_and_push() -> None:
     artifacts: Artifacts = ad["artifacts"]
     old_commit_oid = ad["initial_commit_oid"]
 
+    new_tree, file_path = add_test_file_to_repo(artifacts)
+
+    new_commit_oid = artifacts.commit_and_push(new_tree, "commit new file")
+    repo_head = artifacts.repo.head.peel(pygit2.Commit).oid
+
+    assert old_commit_oid != new_commit_oid
+    assert new_commit_oid == repo_head
+
+    assert file_was_pushed(file_path)
+
+
+def add_test_file_to_repo(artifacts: Artifacts) -> tuple[pygit2.Oid, Path]:
     new_file_name = "new_file.txt"
     oid = artifacts.repo.create_blob(b"")
-
     root = artifacts.repo.head.peel(pygit2.Tree)
     tree = root[artifacts.environments_root]
     tb = artifacts.repo.TreeBuilder(tree)
@@ -56,15 +75,7 @@ def test_commit_and_push() -> None:
     oid = tb.write()
     tb = artifacts.repo.TreeBuilder(root)
     tb.insert(artifacts.environments_root, oid, pygit2.GIT_FILEMODE_TREE)
-    new_tree = tb.write()
-
-    new_commit_oid = artifacts.commit_and_push(new_tree, "commit new file")
-    repo_head = artifacts.repo.head.peel(pygit2.Commit).oid
-
-    assert old_commit_oid != new_commit_oid
-    assert new_commit_oid == repo_head
-
-    assert file_was_pushed(Path(artifacts.environments_root, new_file_name))
+    return tb.write(), Path(artifacts.environments_root, new_file_name)
 
 
 def test_create_file() -> None:
diff --git a/tests/integration/utils.py b/tests/integration/utils.py
index e7c59ad..1e7dc74 100644
--- a/tests/integration/utils.py
+++ b/tests/integration/utils.py
@@ -159,10 +159,17 @@ def file_was_pushed(*paths_with_environment: str | Path) -> bool:
     for path_with_environment in paths_with_environment:
         path = Path(path_with_environment)
 
-        current = artifacts.repo.head.peel(pygit2.Tree)
-        for part in path.parts:
-            if part not in current:
-                return False
-            current = current[part]
+        if not file_in_repo(artifacts, path):
+            return False
+
+    return True
+
+
+def file_in_repo(artifacts: Artifacts, path: Path) -> bool:
+    current = artifacts.repo.head.peel(pygit2.Tree)
+    for part in path.parts:
+        if part not in current:
+            return False
+        current = current[part]
 
     return True

From 3997c3c74f8f8d86993bec9415bf43d3bdc59ca6 Mon Sep 17 00:00:00 2001
From: Sendu Bala <sb10@sanger.ac.uk>
Date: Fri, 1 Sep 2023 10:32:06 +0100
Subject: [PATCH 076/129] Reformat.

---
 softpack_core/artifacts.py            |  8 +++--
 softpack_core/schemas/environment.py  |  6 ++--
 tests/integration/conftest.py         | 34 +++++++++----------
 tests/integration/test_artifacts.py   |  7 ++--
 tests/integration/test_environment.py | 25 ++++++++++----
 tests/integration/utils.py            | 47 +++++++++++++++++----------
 6 files changed, 76 insertions(+), 51 deletions(-)

diff --git a/softpack_core/artifacts.py b/softpack_core/artifacts.py
index db04638..4c4f31b 100644
--- a/softpack_core/artifacts.py
+++ b/softpack_core/artifacts.py
@@ -5,10 +5,10 @@
 """
 
 import itertools
+import shutil
 from dataclasses import dataclass
 from pathlib import Path
 from typing import Iterable, Iterator, Optional
-import shutil
 
 import pygit2
 from box import Box
@@ -125,7 +125,7 @@ def __init__(self) -> None:
             path=path,
             callbacks=self.credentials_callback,
             bare=True,
-            checkout_branch=branch
+            checkout_branch=branch,
         )
 
         self.reference = "/".join(
@@ -255,7 +255,9 @@ def get(self, path: Path, name: str) -> Optional[pygit2.Tree]:
         except KeyError:
             return None
 
-    def commit_and_push(self, tree_oid: pygit2.Oid, message: str) -> pygit2.Oid:
+    def commit_and_push(
+        self, tree_oid: pygit2.Oid, message: str
+    ) -> pygit2.Oid:
         """Commit and push current changes to the remote repository.
 
         Args:
diff --git a/softpack_core/schemas/environment.py b/softpack_core/schemas/environment.py
index 623d15b..eaada15 100644
--- a/softpack_core/schemas/environment.py
+++ b/softpack_core/schemas/environment.py
@@ -245,7 +245,8 @@ def create(cls, env: EnvironmentInput) -> CreateResponse:
                 new_folder_path, file_name, "", True
             )
             cls.artifacts.commit_and_push(
-                tree_oid, "create environment folder")
+                tree_oid, "create environment folder"
+            )
         except RuntimeError as e:
             return InvalidInputError(message=str(e))
 
@@ -360,7 +361,8 @@ async def write_artifact(
                 Path(folder_path), file_name, contents, overwrite=True
             )
             commit_oid = cls.artifacts.commit_and_push(
-                tree_oid, "write artifact")
+                tree_oid, "write artifact"
+            )
             return WriteArtifactSuccess(
                 message="Successfully written artifact",
                 commit_oid=str(commit_oid),
diff --git a/tests/integration/conftest.py b/tests/integration/conftest.py
index 65dfd6d..090e70b 100644
--- a/tests/integration/conftest.py
+++ b/tests/integration/conftest.py
@@ -4,36 +4,38 @@
 LICENSE file in the root directory of this source tree.
 """
 
-import os
-import shutil
-import tempfile
-from pathlib import Path
 
-import pygit2
 import pytest
 from starlette.datastructures import UploadFile
 
-from softpack_core.artifacts import Artifacts, app
+from softpack_core.artifacts import app
 
 
 def pytest_addoption(parser):
     parser.addoption(
-        "--repo", action="store_true", default=False, help=(
-            "run integration tests that alter your real git repo")
+        "--repo",
+        action="store_true",
+        default=False,
+        help=("run integration tests that alter your real git repo"),
     )
 
 
 def pytest_configure(config):
     config.addinivalue_line(
-        "markers", "repo: mark test as altering a real git repo")
+        "markers", "repo: mark test as altering a real git repo"
+    )
 
 
 def pytest_collection_modifyitems(config, items):
     if config.getoption("--repo"):
         return
-    skip_repo = pytest.mark.skip(reason=("specify --repo to run integration "
-                                         "tests that will alter your "
-                                         "configured git repo"))
+    skip_repo = pytest.mark.skip(
+        reason=(
+            "specify --repo to run integration "
+            "tests that will alter your "
+            "configured git repo"
+        )
+    )
     for item in items:
         if "repo" in item.keywords:
             item.add_marker(skip_repo)
@@ -44,16 +46,12 @@ def testable_artifacts_setup():
     user = app.settings.artifacts.repo.username.split('@', 1)[0]
     if user is None or user == "main":
         pytest.skip(
-            (
-                "Your artifacts repo username must be defined in your config."
-            )
+            ("Your artifacts repo username must be defined in your config.")
         )
 
     if app.settings.artifacts.repo.writer is None:
         pytest.skip(
-            (
-                "Your artifacts repo writer must be defined in your config."
-            )
+            ("Your artifacts repo writer must be defined in your config.")
         )
 
     app.settings.artifacts.repo.branch = user
diff --git a/tests/integration/test_artifacts.py b/tests/integration/test_artifacts.py
index 96e347d..bc346b7 100644
--- a/tests/integration/test_artifacts.py
+++ b/tests/integration/test_artifacts.py
@@ -13,14 +13,13 @@
 
 from softpack_core.artifacts import Artifacts, app
 from tests.integration.utils import (
+    commit_and_push_test_repo_changes,
+    file_in_repo,
     file_was_pushed,
     get_user_path_without_environments,
     new_test_artifacts,
-    commit_and_push_test_repo_changes,
-    file_in_repo,
 )
 
-
 pytestmark = pytest.mark.repo
 
 
@@ -143,7 +142,7 @@ def test_create_file() -> None:
 
     assert file_was_pushed(
         Path(artifacts.environments_root, folder_path, basename),
-        Path(artifacts.environments_root, folder_path, basename2)
+        Path(artifacts.environments_root, folder_path, basename2),
     )
 
 
diff --git a/tests/integration/test_environment.py b/tests/integration/test_environment.py
index dbbf41f..ad964fa 100644
--- a/tests/integration/test_environment.py
+++ b/tests/integration/test_environment.py
@@ -27,7 +27,6 @@
     new_test_artifacts,
 )
 
-
 pytestmark = pytest.mark.repo
 
 tetype = tuple[Environment, EnvironmentInput]
@@ -66,8 +65,12 @@ def test_create(post, testable_environment: tetype) -> None:
     result = environment.create(env_input)
     assert isinstance(result, CreateEnvironmentSuccess)
 
-    path = Path(environment.artifacts.environments_root, env_input.path,
-                env_input.name, ".created")
+    path = Path(
+        environment.artifacts.environments_root,
+        env_input.path,
+        env_input.name,
+        ".created",
+    )
     assert file_was_pushed(path)
 
     post.assert_called_once()
@@ -137,8 +140,12 @@ def test_delete(post, testable_environment) -> None:
     assert isinstance(result, CreateEnvironmentSuccess)
     post.assert_called_once()
 
-    path = Path(environment.artifacts.environments_root, env_input.path,
-                env_input.name, ".created")
+    path = Path(
+        environment.artifacts.environments_root,
+        env_input.path,
+        env_input.name,
+        ".created",
+    )
     assert file_was_pushed(path)
 
     result = environment.delete(env_input.name, env_input.path)
@@ -173,8 +180,12 @@ async def test_write_artifact(post, testable_environment, upload):
     )
     assert isinstance(result, WriteArtifactSuccess)
 
-    path = Path(environment.artifacts.environments_root, env_input.path,
-                env_input.name, upload.filename)
+    path = Path(
+        environment.artifacts.environments_root,
+        env_input.path,
+        env_input.name,
+        upload.filename,
+    )
     assert file_was_pushed(path)
 
     result = await environment.write_artifact(
diff --git a/tests/integration/utils.py b/tests/integration/utils.py
index 1e7dc74..9ee065a 100644
--- a/tests/integration/utils.py
+++ b/tests/integration/utils.py
@@ -4,8 +4,6 @@
 LICENSE file in the root directory of this source tree.
 """
 
-import os
-import shutil
 import tempfile
 from pathlib import Path
 
@@ -56,10 +54,13 @@ def delete_environments_folder_from_test_repo(artifacts: Artifacts):
         treeBuilder.remove(artifacts.environments_root)
         oid = treeBuilder.write()
         commit_and_push_test_repo_changes(
-            artifacts, oid, "delete environments")
+            artifacts, oid, "delete environments"
+        )
 
 
-def commit_and_push_test_repo_changes(artifacts: Artifacts, oid: pygit2.Oid, msg: str) -> pygit2.Oid:
+def commit_and_push_test_repo_changes(
+    artifacts: Artifacts, oid: pygit2.Oid, msg: str
+) -> pygit2.Oid:
     ref = artifacts.repo.head.name
     oid = artifacts.repo.create_commit(
         ref,
@@ -67,11 +68,12 @@ def commit_and_push_test_repo_changes(artifacts: Artifacts, oid: pygit2.Oid, msg
         artifacts.signature,
         msg,
         oid,
-        [artifacts.repo.lookup_reference(ref).target]
+        [artifacts.repo.lookup_reference(ref).target],
     )
     remote = artifacts.repo.remotes[0]
-    remote.push([artifacts.repo.head.name],
-                callbacks=artifacts.credentials_callback)
+    remote.push(
+        [artifacts.repo.head.name], callbacks=artifacts.credentials_callback
+    )
     return oid
 
 
@@ -116,22 +118,33 @@ def create_initial_test_repo_state(artifacts: Artifacts) -> artifacts_dict:
     testGroup.insert(test_env, userGroupEnv.write(), pygit2.GIT_FILEMODE_TREE)
 
     groupsFolder = artifacts.repo.TreeBuilder()
-    groupsFolder.insert(test_group, testGroup.write(),
-                        pygit2.GIT_FILEMODE_TREE)
+    groupsFolder.insert(
+        test_group, testGroup.write(), pygit2.GIT_FILEMODE_TREE
+    )
 
     environments = artifacts.repo.TreeBuilder()
-    environments.insert(artifacts.users_folder_name,
-                        usersFolder.write(), pygit2.GIT_FILEMODE_TREE)
-    environments.insert(artifacts.groups_folder_name,
-                        groupsFolder.write(), pygit2.GIT_FILEMODE_TREE)
+    environments.insert(
+        artifacts.users_folder_name,
+        usersFolder.write(),
+        pygit2.GIT_FILEMODE_TREE,
+    )
+    environments.insert(
+        artifacts.groups_folder_name,
+        groupsFolder.write(),
+        pygit2.GIT_FILEMODE_TREE,
+    )
 
     tree = artifacts.repo.head.peel(pygit2.Tree)
     treeBuilder = artifacts.repo.TreeBuilder(tree)
-    treeBuilder.insert(artifacts.environments_root,
-                       environments.write(), pygit2.GIT_FILEMODE_TREE)
+    treeBuilder.insert(
+        artifacts.environments_root,
+        environments.write(),
+        pygit2.GIT_FILEMODE_TREE,
+    )
 
-    oid = commit_and_push_test_repo_changes(artifacts, treeBuilder.write(),
-                                            "Add test environments")
+    oid = commit_and_push_test_repo_changes(
+        artifacts, treeBuilder.write(), "Add test environments"
+    )
 
     dict: artifacts_dict = {
         "initial_commit_oid": oid,

From 258dfb880acf20f01637bb64df9eb6f0f4224dbc Mon Sep 17 00:00:00 2001
From: Sendu Bala <sb10@sanger.ac.uk>
Date: Fri, 1 Sep 2023 10:50:58 +0100
Subject: [PATCH 077/129] Make --repo flag work for all tests.

---
 tests/conftest.py             | 37 +++++++++++++++++++++++++++++++++++
 tests/integration/conftest.py | 30 ----------------------------
 2 files changed, 37 insertions(+), 30 deletions(-)
 create mode 100644 tests/conftest.py

diff --git a/tests/conftest.py b/tests/conftest.py
new file mode 100644
index 0000000..9b968da
--- /dev/null
+++ b/tests/conftest.py
@@ -0,0 +1,37 @@
+"""Copyright (c) 2023 Genome Research Ltd.
+
+This source code is licensed under the MIT license found in the
+LICENSE file in the root directory of this source tree.
+"""
+
+import pytest
+
+
+def pytest_addoption(parser):
+    parser.addoption(
+        "--repo",
+        action="store_true",
+        default=False,
+        help=("run integration tests that alter your real git repo"),
+    )
+
+
+def pytest_configure(config):
+    config.addinivalue_line(
+        "markers", "repo: mark test as altering a real git repo"
+    )
+
+
+def pytest_collection_modifyitems(config, items):
+    if config.getoption("--repo"):
+        return
+    skip_repo = pytest.mark.skip(
+        reason=(
+            "specify --repo to run integration "
+            "tests that will alter your "
+            "configured git repo"
+        )
+    )
+    for item in items:
+        if "repo" in item.keywords:
+            item.add_marker(skip_repo)
diff --git a/tests/integration/conftest.py b/tests/integration/conftest.py
index 090e70b..136122f 100644
--- a/tests/integration/conftest.py
+++ b/tests/integration/conftest.py
@@ -11,36 +11,6 @@
 from softpack_core.artifacts import app
 
 
-def pytest_addoption(parser):
-    parser.addoption(
-        "--repo",
-        action="store_true",
-        default=False,
-        help=("run integration tests that alter your real git repo"),
-    )
-
-
-def pytest_configure(config):
-    config.addinivalue_line(
-        "markers", "repo: mark test as altering a real git repo"
-    )
-
-
-def pytest_collection_modifyitems(config, items):
-    if config.getoption("--repo"):
-        return
-    skip_repo = pytest.mark.skip(
-        reason=(
-            "specify --repo to run integration "
-            "tests that will alter your "
-            "configured git repo"
-        )
-    )
-    for item in items:
-        if "repo" in item.keywords:
-            item.add_marker(skip_repo)
-
-
 @pytest.fixture(scope="package", autouse=True)
 def testable_artifacts_setup():
     user = app.settings.artifacts.repo.username.split('@', 1)[0]

From 5b762242b73c9d26ce9e4a79534222538c9eee94 Mon Sep 17 00:00:00 2001
From: Sendu Bala <sb10@sanger.ac.uk>
Date: Fri, 1 Sep 2023 10:51:29 +0100
Subject: [PATCH 078/129] Disable (unused) ldap init if no ldap config.

---
 softpack_core/ldapapi.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/softpack_core/ldapapi.py b/softpack_core/ldapapi.py
index dff5d48..9c6c2f2 100644
--- a/softpack_core/ldapapi.py
+++ b/softpack_core/ldapapi.py
@@ -19,8 +19,9 @@ class LDAP:
 
     def __init__(self) -> None:
         """Constructor."""
-        self.settings = cast(LDAPConfig, app.settings.ldap)
-        self.initialize()
+        if app.settings.ldap is not None:
+            self.settings = cast(LDAPConfig, app.settings.ldap)
+            self.initialize()
 
     def initialize(self) -> None:
         """Initialize an LDAP client.
@@ -28,7 +29,6 @@ def initialize(self) -> None:
         Returns:
             None.
         """
-        return
         try:
             self.ldap = ldap.initialize(self.settings.server)
             self.group_regex = re.compile(self.settings.group.pattern)

From 94e08fb18108ac6b796c74639d7e03fceb672698 Mon Sep 17 00:00:00 2001
From: Sendu Bala <sb10@sanger.ac.uk>
Date: Fri, 1 Sep 2023 17:04:56 +0100
Subject: [PATCH 079/129] Add new module file parser.

---
 tests/files/ldsc.module   | 210 ++++++++++++++++++++++++++++++++++++++
 tests/files/ldsc.yml      |  54 ++++++++++
 tests/files/simple.module |  24 +++++
 tests/files/simple.yml    |   6 ++
 tests/test_moduleparse.py |  23 +++++
 5 files changed, 317 insertions(+)
 create mode 100644 tests/files/ldsc.module
 create mode 100644 tests/files/ldsc.yml
 create mode 100644 tests/files/simple.module
 create mode 100644 tests/files/simple.yml
 create mode 100644 tests/test_moduleparse.py

diff --git a/tests/files/ldsc.module b/tests/files/ldsc.module
new file mode 100644
index 0000000..0feb40c
--- /dev/null
+++ b/tests/files/ldsc.module
@@ -0,0 +1,210 @@
+#%Module
+
+#=====
+# Created by singularity-hpc (https://github.com/singularityhub/singularity-hpc)
+# ##
+# quay.io/biocontainers/ldsc:1.0.1--pyhdfd78af_2 on 2023-08-15 12:08:41.851818
+#=====
+
+proc ModulesHelp { } {
+
+    puts stderr "This module is a singularity container wrapper for quay.io/biocontainers/ldsc:1.0.1--pyhdfd78af_2 v1.0.1--pyhdfd78af_2"
+    
+    puts stderr ""
+    puts stderr "Container (available through variable SINGULARITY_CONTAINER):"
+    puts stderr ""
+    puts stderr " - /software/hgi/containers/shpc/quay.io/biocontainers/ldsc/1.0.1--pyhdfd78af_2/quay.io-biocontainers-ldsc-1.0.1--pyhdfd78af_2-sha256:308ddebaa643d50306779ce42752eb4c4a3e1635be74531594013959e312af2c.sif"
+    puts stderr ""
+    puts stderr "Commands include:"
+    puts stderr ""
+    puts stderr " - ldsc-run:"
+    puts stderr "       singularity run -B <moduleDir>/99-shpc.sh:/.singularity.d/env/99-shpc.sh <container> \"\$@\""
+    puts stderr " - ldsc-shell:"
+    puts stderr "       singularity shell -s /bin/sh -B <moduleDir>/99-shpc.sh:/.singularity.d/env/99-shpc.sh <container>"
+    puts stderr " - ldsc-exec:"
+    puts stderr "       singularity exec -B <moduleDir>/99-shpc.sh:/.singularity.d/env/99-shpc.sh <container> \"\$@\""
+    puts stderr " - ldsc-inspect-runscript:"
+    puts stderr "       singularity inspect -r <container>"
+    puts stderr " - ldsc-inspect-deffile:"
+    puts stderr "       singularity inspect -d <container>"
+    puts stderr " - ldsc-container:"
+    puts stderr "       echo \"\$SINGULARITY_CONTAINER\""
+    puts stderr ""
+    puts stderr " - ldsc.py:"
+    puts stderr "       singularity exec -B <moduleDir>/99-shpc.sh:/.singularity.d/env/99-shpc.sh <container> /usr/local/bin/ldsc.py \"\$@\""
+    puts stderr " - munge_sumstats.py:"
+    puts stderr "       singularity exec -B <moduleDir>/99-shpc.sh:/.singularity.d/env/99-shpc.sh <container> /usr/local/bin/munge_sumstats.py \"\$@\""
+    puts stderr " - f2py2:"
+    puts stderr "       singularity exec -B <moduleDir>/99-shpc.sh:/.singularity.d/env/99-shpc.sh <container> /usr/local/bin/f2py2 \"\$@\""
+    puts stderr " - f2py2.7:"
+    puts stderr "       singularity exec -B <moduleDir>/99-shpc.sh:/.singularity.d/env/99-shpc.sh <container> /usr/local/bin/f2py2.7 \"\$@\""
+    puts stderr " - shiftBed:"
+    puts stderr "       singularity exec -B <moduleDir>/99-shpc.sh:/.singularity.d/env/99-shpc.sh <container> /usr/local/bin/shiftBed \"\$@\""
+    puts stderr " - annotateBed:"
+    puts stderr "       singularity exec -B <moduleDir>/99-shpc.sh:/.singularity.d/env/99-shpc.sh <container> /usr/local/bin/annotateBed \"\$@\""
+    puts stderr " - bamToBed:"
+    puts stderr "       singularity exec -B <moduleDir>/99-shpc.sh:/.singularity.d/env/99-shpc.sh <container> /usr/local/bin/bamToBed \"\$@\""
+    puts stderr " - bamToFastq:"
+    puts stderr "       singularity exec -B <moduleDir>/99-shpc.sh:/.singularity.d/env/99-shpc.sh <container> /usr/local/bin/bamToFastq \"\$@\""
+    puts stderr " - bed12ToBed6:"
+    puts stderr "       singularity exec -B <moduleDir>/99-shpc.sh:/.singularity.d/env/99-shpc.sh <container> /usr/local/bin/bed12ToBed6 \"\$@\""
+    puts stderr " - bedToBam:"
+    puts stderr "       singularity exec -B <moduleDir>/99-shpc.sh:/.singularity.d/env/99-shpc.sh <container> /usr/local/bin/bedToBam \"\$@\""
+    puts stderr " - bedToIgv:"
+    puts stderr "       singularity exec -B <moduleDir>/99-shpc.sh:/.singularity.d/env/99-shpc.sh <container> /usr/local/bin/bedToIgv \"\$@\""
+    puts stderr " - bedpeToBam:"
+    puts stderr "       singularity exec -B <moduleDir>/99-shpc.sh:/.singularity.d/env/99-shpc.sh <container> /usr/local/bin/bedpeToBam \"\$@\""
+
+    puts stderr ""
+    puts stderr "For each of the above, you can export:"
+    puts stderr ""
+    puts stderr " - SINGULARITY_OPTS: to define custom options for singularity (e.g., --debug)"
+    puts stderr " - SINGULARITY_COMMAND_OPTS: to define custom options for the command (e.g., -b)"
+    puts stderr " - SINGULARITY_CONTAINER: The Singularity (sif) path"
+
+}
+
+set view_dir "[file dirname [file dirname ${ModulesCurrentModulefile}] ]"
+set view_name "[file tail ${view_dir}]"
+set view_module ".view_module"
+set view_modulefile "${view_dir}/${view_module}"
+
+if {[file exists ${view_modulefile}]} {
+    source ${view_modulefile}
+}
+
+# Environment - only set if not already defined
+if { ![info exists ::env(SINGULARITY_OPTS)] } {
+    setenv SINGULARITY_OPTS ""
+}
+if { ![info exists ::env(SINGULARITY_COMMAND_OPTS)] } {
+    setenv SINGULARITY_COMMAND_OPTS ""
+}
+
+# Variables
+
+set name        quay.io/biocontainers/ldsc:1.0.1--pyhdfd78af_2
+set version     1.0.1--pyhdfd78af_2
+set description "$name - $version"
+set containerPath /software/hgi/containers/shpc/quay.io/biocontainers/ldsc/1.0.1--pyhdfd78af_2/quay.io-biocontainers-ldsc-1.0.1--pyhdfd78af_2-sha256:308ddebaa643d50306779ce42752eb4c4a3e1635be74531594013959e312af2c.sif
+
+
+set helpcommand "This module is a singularity container wrapper for quay.io/biocontainers/ldsc:1.0.1--pyhdfd78af_2 v1.0.1--pyhdfd78af_2. "
+set busybox "BusyBox v1.32.1 (2021-04-13 11:15:36 UTC) multi-call binary."
+set deb-list "gcc-8-base_8.3.0-6_amd64.deb, libc6_2.28-10_amd64.deb, libgcc1_1%3a8.3.0-6_amd64.deb, bash_5.0-4_amd64.deb, libc-bin_2.28-10_amd64.deb, libtinfo6_6.1+20181013-2+deb10u2_amd64.deb, ncurses-base_6.1+20181013-2+deb10u2_all.deb, base-files_10.3+deb10u9_amd64.deb"
+set glibc "GNU C Library (Debian GLIBC 2.28-10) stable release version 2.28."
+set io.buildah.version "1.19.6"
+set org.label-schema.build-arch "amd64"
+set org.label-schema.build-date "Tuesday_15_August_2023_12:8:5_BST"
+set org.label-schema.schema-version "1.0"
+set org.label-schema.usage.singularity.deffile.bootstrap "docker"
+set org.label-schema.usage.singularity.deffile.from "quay.io/biocontainers/ldsc@sha256:308ddebaa643d50306779ce42752eb4c4a3e1635be74531594013959e312af2c"
+set org.label-schema.usage.singularity.version "3.10.0"
+set pkg-list "gcc-8-base, libc6, libgcc1, bash, libc-bin, libtinfo6, ncurses-base, base-files"
+
+
+# directory containing this modulefile, once symlinks resolved (dynamically defined)
+set moduleDir   [file dirname [expr { [string equal [file type ${ModulesCurrentModulefile}] "link"] ? [file readlink ${ModulesCurrentModulefile}] : ${ModulesCurrentModulefile} }]]
+
+# conflict with modules with the same alias name
+conflict ldsc
+conflict quay.io/biocontainers/ldsc:1.0.1--pyhdfd78af_2
+conflict ldsc.py
+conflict munge_sumstats.py
+conflict f2py2
+conflict f2py2.7
+conflict shiftBed
+conflict annotateBed
+conflict bamToBed
+conflict bamToFastq
+conflict bed12ToBed6
+conflict bedToBam
+conflict bedToIgv
+conflict bedpeToBam
+
+
+# singularity environment variable to set shell
+setenv SINGULARITY_SHELL /bin/sh
+
+# service environment variable to access full SIF image path
+setenv SINGULARITY_CONTAINER "${containerPath}"
+
+# interactive shell to any container, plus exec for aliases
+set shellCmd "singularity \${SINGULARITY_OPTS} shell \${SINGULARITY_COMMAND_OPTS} -s /bin/sh -B ${moduleDir}/99-shpc.sh:/.singularity.d/env/99-shpc.sh  ${containerPath}"
+set execCmd "singularity \${SINGULARITY_OPTS} exec \${SINGULARITY_COMMAND_OPTS} -B ${moduleDir}/99-shpc.sh:/.singularity.d/env/99-shpc.sh  "
+set runCmd "singularity \${SINGULARITY_OPTS} run \${SINGULARITY_COMMAND_OPTS} -B ${moduleDir}/99-shpc.sh:/.singularity.d/env/99-shpc.sh  ${containerPath}"
+set inspectCmd "singularity \${SINGULARITY_OPTS} inspect \${SINGULARITY_COMMAND_OPTS} "
+
+# if we have any wrapper scripts, add bin to path
+prepend-path PATH "${moduleDir}/bin"
+
+# "aliases" to module commands
+if { [ module-info shell bash ] } {
+  if { [ module-info mode load ] } {
+ 
+ 
+ 
+ 
+ 
+ 
+ 
+ 
+ 
+ 
+ 
+ 
+
+  }
+  if { [ module-info mode remove ] } {
+ 
+ 
+ 
+ 
+ 
+ 
+ 
+ 
+ 
+ 
+ 
+ 
+
+  }
+} else {
+
+
+
+
+
+
+
+
+
+
+
+
+
+}
+
+
+
+#=====
+# Module options
+#=====
+module-whatis "    Name: quay.io/biocontainers/ldsc:1.0.1--pyhdfd78af_2"
+module-whatis "    Version: 1.0.1--pyhdfd78af_2"
+
+
+module-whatis "    busybox: BusyBox v1.32.1 (2021-04-13 11:15:36 UTC) multi-call binary."
+module-whatis "    deb-list: gcc-8-base_8.3.0-6_amd64.deb, libc6_2.28-10_amd64.deb, libgcc1_1%3a8.3.0-6_amd64.deb, bash_5.0-4_amd64.deb, libc-bin_2.28-10_amd64.deb, libtinfo6_6.1+20181013-2+deb10u2_amd64.deb, ncurses-base_6.1+20181013-2+deb10u2_all.deb, base-files_10.3+deb10u9_amd64.deb"
+module-whatis "    glibc: GNU C Library (Debian GLIBC 2.28-10) stable release version 2.28."
+module-whatis "    io.buildah.version: 1.19.6"
+module-whatis "    org.label-schema.build-arch: amd64"
+module-whatis "    org.label-schema.build-date: Tuesday_15_August_2023_12:8:5_BST"
+module-whatis "    org.label-schema.schema-version: 1.0"
+module-whatis "    org.label-schema.usage.singularity.deffile.bootstrap: docker"
+module-whatis "    org.label-schema.usage.singularity.deffile.from: quay.io/biocontainers/ldsc@sha256:308ddebaa643d50306779ce42752eb4c4a3e1635be74531594013959e312af2c"
+module-whatis "    org.label-schema.usage.singularity.version: 3.10.0"
+module-whatis "    pkg-list: gcc-8-base, libc6, libgcc1, bash, libc-bin, libtinfo6, ncurses-base, base-files"
+
+module load /software/modules/ISG/singularity/3.10.0
\ No newline at end of file
diff --git a/tests/files/ldsc.yml b/tests/files/ldsc.yml
new file mode 100644
index 0000000..81a872f
--- /dev/null
+++ b/tests/files/ldsc.yml
@@ -0,0 +1,54 @@
+description: |
+  This module is a singularity container wrapper for quay.io/biocontainers/ldsc:1.0.1--pyhdfd78af_2 v1.0.1--pyhdfd78af_2
+  
+  Container (available through variable SINGULARITY_CONTAINER):
+  
+   - /software/hgi/containers/shpc/quay.io/biocontainers/ldsc/1.0.1--pyhdfd78af_2/quay.io-biocontainers-ldsc-1.0.1--pyhdfd78af_2-sha256:308ddebaa643d50306779ce42752eb4c4a3e1635be74531594013959e312af2c.sif
+  
+  Commands include:
+  
+   - ldsc-run:
+         singularity run -B <moduleDir>/99-shpc.sh:/.singularity.d/env/99-shpc.sh <container> "$@"
+   - ldsc-shell:
+         singularity shell -s /bin/sh -B <moduleDir>/99-shpc.sh:/.singularity.d/env/99-shpc.sh <container>
+   - ldsc-exec:
+         singularity exec -B <moduleDir>/99-shpc.sh:/.singularity.d/env/99-shpc.sh <container> "$@"
+   - ldsc-inspect-runscript:
+         singularity inspect -r <container>
+   - ldsc-inspect-deffile:
+         singularity inspect -d <container>
+   - ldsc-container:
+         echo "$SINGULARITY_CONTAINER"
+  
+   - ldsc.py:
+         singularity exec -B <moduleDir>/99-shpc.sh:/.singularity.d/env/99-shpc.sh <container> /usr/local/bin/ldsc.py "$@"
+   - munge_sumstats.py:
+         singularity exec -B <moduleDir>/99-shpc.sh:/.singularity.d/env/99-shpc.sh <container> /usr/local/bin/munge_sumstats.py "$@"
+   - f2py2:
+         singularity exec -B <moduleDir>/99-shpc.sh:/.singularity.d/env/99-shpc.sh <container> /usr/local/bin/f2py2 "$@"
+   - f2py2.7:
+         singularity exec -B <moduleDir>/99-shpc.sh:/.singularity.d/env/99-shpc.sh <container> /usr/local/bin/f2py2.7 "$@"
+   - shiftBed:
+         singularity exec -B <moduleDir>/99-shpc.sh:/.singularity.d/env/99-shpc.sh <container> /usr/local/bin/shiftBed "$@"
+   - annotateBed:
+         singularity exec -B <moduleDir>/99-shpc.sh:/.singularity.d/env/99-shpc.sh <container> /usr/local/bin/annotateBed "$@"
+   - bamToBed:
+         singularity exec -B <moduleDir>/99-shpc.sh:/.singularity.d/env/99-shpc.sh <container> /usr/local/bin/bamToBed "$@"
+   - bamToFastq:
+         singularity exec -B <moduleDir>/99-shpc.sh:/.singularity.d/env/99-shpc.sh <container> /usr/local/bin/bamToFastq "$@"
+   - bed12ToBed6:
+         singularity exec -B <moduleDir>/99-shpc.sh:/.singularity.d/env/99-shpc.sh <container> /usr/local/bin/bed12ToBed6 "$@"
+   - bedToBam:
+         singularity exec -B <moduleDir>/99-shpc.sh:/.singularity.d/env/99-shpc.sh <container> /usr/local/bin/bedToBam "$@"
+   - bedToIgv:
+         singularity exec -B <moduleDir>/99-shpc.sh:/.singularity.d/env/99-shpc.sh <container> /usr/local/bin/bedToIgv "$@"
+   - bedpeToBam:
+         singularity exec -B <moduleDir>/99-shpc.sh:/.singularity.d/env/99-shpc.sh <container> /usr/local/bin/bedpeToBam "$@"
+  
+  For each of the above, you can export:
+  
+   - SINGULARITY_OPTS: to define custom options for singularity (e.g., --debug)
+   - SINGULARITY_COMMAND_OPTS: to define custom options for the command (e.g., -b)
+   - SINGULARITY_CONTAINER: The Singularity (sif) path
+packages:
+  - quay.io/biocontainers/ldsc@1.0.1--pyhdfd78af_2
diff --git a/tests/files/simple.module b/tests/files/simple.module
new file mode 100644
index 0000000..0b0fd17
--- /dev/null
+++ b/tests/files/simple.module
@@ -0,0 +1,24 @@
+#%Module
+
+proc ModulesHelp { } {
+
+    puts stderr "Line 1"
+    
+    puts stderr ""
+    puts stderr " - line 2 \"\$@\""
+
+}
+
+set view_dir "[file dirname [file dirname ${ModulesCurrentModulefile}] ]"
+
+#=====
+# Module options
+#=====
+module-whatis "    Name: quay.io/biocontainers/ldsc:1.0.1--pyhdfd78af_2"
+module-whatis "    Version: 1.0.1--pyhdfd78af_2"
+
+
+module-whatis "    busybox: BusyBox v1.32.1 (2021-04-13 11:15:36 UTC) multi-call binary."
+module-whatis "    Packages: gcc-8-base, libc6, libgcc1, bash, libc-bin, libtinfo6, ncurses-base, base-files"
+
+module load /software/modules/ISG/singularity/3.10.0
diff --git a/tests/files/simple.yml b/tests/files/simple.yml
new file mode 100644
index 0000000..4f1a29c
--- /dev/null
+++ b/tests/files/simple.yml
@@ -0,0 +1,6 @@
+description: |
+  Line 1
+  
+   - line 2 "$@"
+packages:
+  - quay.io/biocontainers/ldsc@1.0.1--pyhdfd78af_2
diff --git a/tests/test_moduleparse.py b/tests/test_moduleparse.py
new file mode 100644
index 0000000..0cef5f3
--- /dev/null
+++ b/tests/test_moduleparse.py
@@ -0,0 +1,23 @@
+"""Copyright (c) 2023 Genome Research Ltd.
+
+This source code is licensed under the MIT license found in the
+LICENSE file in the root directory of this source tree.
+"""
+
+from pathlib import Path
+import pytest
+from softpack_core.moduleparse import ToSoftpackYML
+
+
+def test_tosoftpack() -> None:
+    test_files_dir = Path(Path(__file__).parent, "files")
+
+    with open(Path(test_files_dir, "ldsc.module"), "rb") as fh:
+        module_data = fh.read()
+
+    with open(Path(test_files_dir, "ldsc.yml"), "rb") as fh:
+        expected_yml = fh.read()
+
+    yml = ToSoftpackYML(module_data)
+
+    assert yml == expected_yml

From 9cd290ea8edeceb462e1f56826dd4846d06822f4 Mon Sep 17 00:00:00 2001
From: Sendu Bala <sb10@sanger.ac.uk>
Date: Fri, 1 Sep 2023 17:05:25 +0100
Subject: [PATCH 080/129] Remove unused test files.

---
 tests/files/simple.module | 24 ------------------------
 tests/files/simple.yml    |  6 ------
 2 files changed, 30 deletions(-)
 delete mode 100644 tests/files/simple.module
 delete mode 100644 tests/files/simple.yml

diff --git a/tests/files/simple.module b/tests/files/simple.module
deleted file mode 100644
index 0b0fd17..0000000
--- a/tests/files/simple.module
+++ /dev/null
@@ -1,24 +0,0 @@
-#%Module
-
-proc ModulesHelp { } {
-
-    puts stderr "Line 1"
-    
-    puts stderr ""
-    puts stderr " - line 2 \"\$@\""
-
-}
-
-set view_dir "[file dirname [file dirname ${ModulesCurrentModulefile}] ]"
-
-#=====
-# Module options
-#=====
-module-whatis "    Name: quay.io/biocontainers/ldsc:1.0.1--pyhdfd78af_2"
-module-whatis "    Version: 1.0.1--pyhdfd78af_2"
-
-
-module-whatis "    busybox: BusyBox v1.32.1 (2021-04-13 11:15:36 UTC) multi-call binary."
-module-whatis "    Packages: gcc-8-base, libc6, libgcc1, bash, libc-bin, libtinfo6, ncurses-base, base-files"
-
-module load /software/modules/ISG/singularity/3.10.0
diff --git a/tests/files/simple.yml b/tests/files/simple.yml
deleted file mode 100644
index 4f1a29c..0000000
--- a/tests/files/simple.yml
+++ /dev/null
@@ -1,6 +0,0 @@
-description: |
-  Line 1
-  
-   - line 2 "$@"
-packages:
-  - quay.io/biocontainers/ldsc@1.0.1--pyhdfd78af_2

From eb824486e549559ae38621f23b30873319d88afa Mon Sep 17 00:00:00 2001
From: Sendu Bala <sb10@sanger.ac.uk>
Date: Fri, 1 Sep 2023 17:06:45 +0100
Subject: [PATCH 081/129] wip for create_from_module.

---
 softpack_core/artifacts.py            |   1 +
 softpack_core/schemas/environment.py  | 123 +++++++++++++++++++++++---
 tests/integration/test_environment.py |  35 ++++++++
 3 files changed, 146 insertions(+), 13 deletions(-)

diff --git a/softpack_core/artifacts.py b/softpack_core/artifacts.py
index 4c4f31b..eeecfb8 100644
--- a/softpack_core/artifacts.py
+++ b/softpack_core/artifacts.py
@@ -22,6 +22,7 @@ class Artifacts:
 
     environments_root = "environments"
     environments_file = "softpack.yml"
+    module_file = "module"
     users_folder_name = "users"
     groups_folder_name = "groups"
     credentials_callback = None
diff --git a/softpack_core/schemas/environment.py b/softpack_core/schemas/environment.py
index eaada15..2a2a343 100644
--- a/softpack_core/schemas/environment.py
+++ b/softpack_core/schemas/environment.py
@@ -6,15 +6,19 @@
 
 from dataclasses import dataclass
 from pathlib import Path
+import io
 from typing import Iterable, Optional
 
 import httpx
 import strawberry
 from strawberry.file_uploads import Upload
 
+from starlette.datastructures import UploadFile
+
 from softpack_core.artifacts import Artifacts
 from softpack_core.schemas.base import BaseSchema
 from softpack_core.spack import Spack
+from softpack_core.moduleparse import ToSoftpackYML
 
 
 # Interfaces
@@ -219,6 +223,28 @@ def create(cls, env: EnvironmentInput) -> CreateResponse:
         if any(len(value) == 0 for value in vars(env).values()):
             return InvalidInputError(message="all fields must be filled in")
 
+        response = cls.create_new_env()
+        if not isinstance(response, CreateEnvironmentSuccess):
+            return response
+
+        # Send build request
+        httpx.post(
+            "http://0.0.0.0:7080/environments/build",
+            json={
+                "name": f"{env.path}/{env.name}",
+                "model": {
+                    "description": env.description,
+                    "packages": [f"{pkg.name}" for pkg in env.packages],
+                },
+            },
+        )
+
+        return CreateEnvironmentSuccess(
+            message="Successfully scheduled environment creation"
+        )
+
+    @classmethod
+    def create_new_env(cls, env: EnvironmentInput) -> CreateResponse:
         # Check if a valid path has been provided. TODO: improve this to check
         # that they can only create stuff in their own users folder, or in
         # group folders of unix groups they belong to.
@@ -250,20 +276,8 @@ def create(cls, env: EnvironmentInput) -> CreateResponse:
         except RuntimeError as e:
             return InvalidInputError(message=str(e))
 
-        # Send build request
-        httpx.post(
-            "http://0.0.0.0:7080/environments/build",
-            json={
-                "name": f"{env.path}/{env.name}",
-                "model": {
-                    "description": env.description,
-                    "packages": [f"{pkg.name}" for pkg in env.packages],
-                },
-            },
-        )
-
         return CreateEnvironmentSuccess(
-            message="Successfully scheduled environment creation"
+            message="Successfully created environment in artifacts repo"
         )
 
     @classmethod
@@ -344,6 +358,86 @@ def delete(cls, name: str, path: str) -> DeleteResponse:
             name=name,
         )
 
+    @classmethod
+    async def create_from_module(cls, file: Upload, module_path: str,
+                                 environment_path: str) -> CreateResponse:
+        """Create an Environment based on an existing module.
+
+        The environment will not be built; a "fake" softpack.yml and the
+        supplied module file will be written as artifacts in a newly created
+        environment instead, so that they can be discovered.
+
+        Args:
+            file: the module file to add to the repo, and to parse to fake a
+                  corresponding softpack.yml. It should have a format similar
+                  to that produced by shpc, with `module whatis` outputting
+                  a "Name: " line, a "Version: " line, and optionally a
+                  "Packages: " line to say what packages are available.
+                  `module help` output will be translated into the description
+                  in the softpack.yml.
+            module_path: the local path that users can `module load` - this is
+                         used to auto-generate usage help text for this
+                         environment.
+            environment_path: the subdirectories of environments folder that
+                              artifacts will be stored in, eg.
+                              users/username/software_name
+
+        Returns:
+            A message confirming the success or failure of the operation.
+        """
+        environment_dirs = environment_path.split("/")
+        environment_name = environment_dirs.pop()
+
+        contents = (await file.read()).decode()
+        yml = ToSoftpackYML(contents)
+
+        env = EnvironmentInput(
+            name=environment_name,
+            path="/".join(environment_dirs),
+        )
+
+        response = cls.create_new_env(env)
+        if not isinstance(response, CreateEnvironmentSuccess):
+            return response
+
+        module_file = UploadFile(file=io.StringIO(contents))
+        softpack_file = UploadFile(file=io.StringIO(yml))
+
+        result = cls.write_module_artifacts(
+            module_file=module_file,
+            softpack_file=softpack_file,
+            environment_path=environment_path
+        )
+
+        if not isinstance(result, WriteArtifactSuccess):
+            cls.delete(name=environment_name, path=environment_path)
+            return InvalidInputError(
+                msg="Write of module file failed: " + result.msg
+            )
+
+        return CreateEnvironmentSuccess(
+            message="Successfully created environment in artifacts repo"
+        )
+
+    @classmethod
+    async def write_module_artifacts(
+        cls, module_file: Upload, softpack_file: Upload, environment_path: str
+    ) -> WriteArtifactResponse:
+        result = await cls.write_artifact(
+            file=module_file,
+            folder_path=environment_path,
+            file_name=cls.artifacts.module_file
+        )
+
+        if not isinstance(result, WriteArtifactSuccess):
+            return result
+
+        return await cls.write_artifact(
+            file=softpack_file,
+            folder_path=environment_path,
+            file_name=cls.artifacts.environments_file
+        )
+
     @classmethod
     async def write_artifact(
         cls, file: Upload, folder_path: str, file_name: str
@@ -391,3 +485,6 @@ class Mutation:
         writeArtifact: WriteArtifactResponse = (
             Environment.write_artifact
         )  # type: ignore
+        createFromModule: CreateResponse = (
+            Environment.create_from_module
+        )  # type: ignore
diff --git a/tests/integration/test_environment.py b/tests/integration/test_environment.py
index ad964fa..97e45e3 100644
--- a/tests/integration/test_environment.py
+++ b/tests/integration/test_environment.py
@@ -230,3 +230,38 @@ async def test_iter(post, testable_environment, upload):
         count += 1
 
     assert count == 1
+
+
+@pytest.mark.asyncio
+async def test_create_from_module(post, testable_environment: tetype, upload):
+    environment, _ = testable_environment
+
+    test_file_path = Path(Path(__file__).parent.parent, "files", "ldsc.module")
+
+    with open(test_file_path, "rb") as fh:
+        upload.filename = "ldsc.module"
+        upload.content_type = "text/plain"
+        upload.read.return_value = fh.read()
+
+    name = "groups/hgi/some-environment"
+    module_path = "HGI/common/some_environment"
+
+    result = await environment.create_from_module(
+        file=upload,
+        module_path=module_path,
+        environment_path=name,
+    )
+
+    assert isinstance(result, CreateEnvironmentSuccess)
+
+    parent_path = Path(
+        environment.artifacts.environments_root,
+        environment.artifacts.group_folder,
+        "hgi",
+        name
+    )
+
+    assert file_was_pushed(
+        Path(parent_path, environment.artifacts.environments_file),
+        Path(parent_path, environment.artifacts.module_file)
+    )

From f00b97ba337b5051fb6950581b77916fd16dd7b2 Mon Sep 17 00:00:00 2001
From: Sendu Bala <sb10@sanger.ac.uk>
Date: Mon, 4 Sep 2023 09:07:54 +0100
Subject: [PATCH 082/129] Add new module file parser.

---
 softpack_core/moduleparse.py | 50 ++++++++++++++++++++++++++++++++++++
 1 file changed, 50 insertions(+)
 create mode 100644 softpack_core/moduleparse.py

diff --git a/softpack_core/moduleparse.py b/softpack_core/moduleparse.py
new file mode 100644
index 0000000..06ec936
--- /dev/null
+++ b/softpack_core/moduleparse.py
@@ -0,0 +1,50 @@
+"""Copyright (c) 2023 Genome Research Ltd.
+
+This source code is licensed under the MIT license found in the
+LICENSE file in the root directory of this source tree.
+"""
+
+
+def ToSoftpackYML(contents: bytes) -> bytes:
+    mode = 0
+
+    name = ""
+    version = ""
+    packages: list[str] = []
+    description = ""
+
+    for line in contents.splitlines():
+        line = line.lstrip()
+        match mode:
+            case 0:
+                if line.startswith(b"proc ModulesHelp"):
+                    mode = 1
+                elif line.startswith(b"module-whatis "):
+                    line = line.removeprefix(
+                        b"module-whatis ").decode('unicode_escape').removeprefix("\"").removesuffix("\"").lstrip()
+                    print(line)
+                    if line.startswith("Name: "):
+                        nv = line.removeprefix("Name: ").split(":")
+                        name = nv[0]
+                        if len(nv) > 1:
+                            version = nv[1]
+                    elif line.startswith("Version: "):
+                        version = line.removeprefix("Version: ")
+                    elif line.startswith("Packages: "):
+                        packages = line.removeprefix("Packages: ").split(", ")
+            case 1:
+                if line == b"}":
+                    mode = 0
+                elif line.startswith(b"puts stderr "):
+                    line = line.removeprefix(b"puts stderr ").decode(
+                        'unicode_escape').replace("\\$", "$").removeprefix("\"").removesuffix("\"")
+                    description += "  " + line + "\n"
+
+    if version != "":
+        name += f"@{version}"
+
+    packages.insert(0, name)
+
+    package_str = "\n  - ".join(packages)
+
+    return f"description: |\n{description}packages:\n  - {package_str}\n".encode()

From a9b755d1cdb5eee90c3f0af624fb397adc93893a Mon Sep 17 00:00:00 2001
From: Michael Woolnough <mw31@sanger.ac.uk>
Date: Mon, 4 Sep 2023 14:27:41 +0100
Subject: [PATCH 083/129] Move marker definition to ini.

---
 pyproject.toml    | 1 +
 tests/conftest.py | 6 ------
 2 files changed, 1 insertion(+), 6 deletions(-)

diff --git a/pyproject.toml b/pyproject.toml
index f1299b0..b9ae929 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -139,6 +139,7 @@ ignore_missing_imports = true
 filterwarnings = [
     "ignore::DeprecationWarning:starlette"
 ]
+markers = "repo: mark test as altering a real git repo"
 
 [build-system]
 requires = ["poetry-core>=1.0.0"]
diff --git a/tests/conftest.py b/tests/conftest.py
index 9b968da..af3f163 100644
--- a/tests/conftest.py
+++ b/tests/conftest.py
@@ -16,12 +16,6 @@ def pytest_addoption(parser):
     )
 
 
-def pytest_configure(config):
-    config.addinivalue_line(
-        "markers", "repo: mark test as altering a real git repo"
-    )
-
-
 def pytest_collection_modifyitems(config, items):
     if config.getoption("--repo"):
         return

From f6d8ec470e29b9b112b0718e48ca3466140ab3a9 Mon Sep 17 00:00:00 2001
From: Michael Woolnough <mw31@sanger.ac.uk>
Date: Mon, 4 Sep 2023 14:28:21 +0100
Subject: [PATCH 084/129] Make compatible with python 3.9.

---
 softpack_core/moduleparse.py | 49 ++++++++++++++++++------------------
 tests/test_moduleparse.py    |  1 +
 2 files changed, 25 insertions(+), 25 deletions(-)

diff --git a/softpack_core/moduleparse.py b/softpack_core/moduleparse.py
index 06ec936..f34985f 100644
--- a/softpack_core/moduleparse.py
+++ b/softpack_core/moduleparse.py
@@ -6,7 +6,7 @@
 
 
 def ToSoftpackYML(contents: bytes) -> bytes:
-    mode = 0
+    mode = "0"
 
     name = ""
     version = ""
@@ -15,30 +15,29 @@ def ToSoftpackYML(contents: bytes) -> bytes:
 
     for line in contents.splitlines():
         line = line.lstrip()
-        match mode:
-            case 0:
-                if line.startswith(b"proc ModulesHelp"):
-                    mode = 1
-                elif line.startswith(b"module-whatis "):
-                    line = line.removeprefix(
-                        b"module-whatis ").decode('unicode_escape').removeprefix("\"").removesuffix("\"").lstrip()
-                    print(line)
-                    if line.startswith("Name: "):
-                        nv = line.removeprefix("Name: ").split(":")
-                        name = nv[0]
-                        if len(nv) > 1:
-                            version = nv[1]
-                    elif line.startswith("Version: "):
-                        version = line.removeprefix("Version: ")
-                    elif line.startswith("Packages: "):
-                        packages = line.removeprefix("Packages: ").split(", ")
-            case 1:
-                if line == b"}":
-                    mode = 0
-                elif line.startswith(b"puts stderr "):
-                    line = line.removeprefix(b"puts stderr ").decode(
-                        'unicode_escape').replace("\\$", "$").removeprefix("\"").removesuffix("\"")
-                    description += "  " + line + "\n"
+        if mode is "0":
+            if line.startswith(b"proc ModulesHelp"):
+                mode = "1"
+            elif line.startswith(b"module-whatis "):
+                line = line.removeprefix(
+                    b"module-whatis ").decode('unicode_escape').removeprefix("\"").removesuffix("\"").lstrip()
+                print(line)
+                if line.startswith("Name: "):
+                    nv = line.removeprefix("Name: ").split(":")
+                    name = nv[0]
+                    if len(nv) > 1:
+                        version = nv[1]
+                elif line.startswith("Version: "):
+                    version = line.removeprefix("Version: ")
+                elif line.startswith("Packages: "):
+                    packages = line.removeprefix("Packages: ").split(", ")
+        else:
+            if line == b"}":
+                mode = "0"
+            elif line.startswith(b"puts stderr "):
+                line = line.removeprefix(b"puts stderr ").decode(
+                    'unicode_escape').replace("\\$", "$").removeprefix("\"").removesuffix("\"")
+                description += "  " + line + "\n"
 
     if version != "":
         name += f"@{version}"
diff --git a/tests/test_moduleparse.py b/tests/test_moduleparse.py
index 0cef5f3..589e395 100644
--- a/tests/test_moduleparse.py
+++ b/tests/test_moduleparse.py
@@ -9,6 +9,7 @@
 from softpack_core.moduleparse import ToSoftpackYML
 
 
+@pytest.mark.skip
 def test_tosoftpack() -> None:
     test_files_dir = Path(Path(__file__).parent, "files")
 

From f369897227132d0c1e026bb73b38031d0348636d Mon Sep 17 00:00:00 2001
From: Michael Woolnough <mw31@sanger.ac.uk>
Date: Mon, 4 Sep 2023 14:28:58 +0100
Subject: [PATCH 085/129] Make compatible with python 3.9, reformat and fix
 tests.

---
 softpack_core/schemas/environment.py  | 13 +++++++------
 tests/integration/test_environment.py |  5 +++--
 tests/integration/utils.py            |  5 +++--
 3 files changed, 13 insertions(+), 10 deletions(-)

diff --git a/softpack_core/schemas/environment.py b/softpack_core/schemas/environment.py
index 2a2a343..ef2ba83 100644
--- a/softpack_core/schemas/environment.py
+++ b/softpack_core/schemas/environment.py
@@ -223,7 +223,7 @@ def create(cls, env: EnvironmentInput) -> CreateResponse:
         if any(len(value) == 0 for value in vars(env).values()):
             return InvalidInputError(message="all fields must be filled in")
 
-        response = cls.create_new_env()
+        response = cls.create_new_env(env)
         if not isinstance(response, CreateEnvironmentSuccess):
             return response
 
@@ -359,8 +359,9 @@ def delete(cls, name: str, path: str) -> DeleteResponse:
         )
 
     @classmethod
-    async def create_from_module(cls, file: Upload, module_path: str,
-                                 environment_path: str) -> CreateResponse:
+    async def create_from_module(
+        cls, file: Upload, module_path: str, environment_path: str
+    ) -> CreateResponse:
         """Create an Environment based on an existing module.
 
         The environment will not be built; a "fake" softpack.yml and the
@@ -406,7 +407,7 @@ async def create_from_module(cls, file: Upload, module_path: str,
         result = cls.write_module_artifacts(
             module_file=module_file,
             softpack_file=softpack_file,
-            environment_path=environment_path
+            environment_path=environment_path,
         )
 
         if not isinstance(result, WriteArtifactSuccess):
@@ -426,7 +427,7 @@ async def write_module_artifacts(
         result = await cls.write_artifact(
             file=module_file,
             folder_path=environment_path,
-            file_name=cls.artifacts.module_file
+            file_name=cls.artifacts.module_file,
         )
 
         if not isinstance(result, WriteArtifactSuccess):
@@ -435,7 +436,7 @@ async def write_module_artifacts(
         return await cls.write_artifact(
             file=softpack_file,
             folder_path=environment_path,
-            file_name=cls.artifacts.environments_file
+            file_name=cls.artifacts.environments_file,
         )
 
     @classmethod
diff --git a/tests/integration/test_environment.py b/tests/integration/test_environment.py
index 97e45e3..39e58d8 100644
--- a/tests/integration/test_environment.py
+++ b/tests/integration/test_environment.py
@@ -233,6 +233,7 @@ async def test_iter(post, testable_environment, upload):
 
 
 @pytest.mark.asyncio
+@pytest.mark.skip
 async def test_create_from_module(post, testable_environment: tetype, upload):
     environment, _ = testable_environment
 
@@ -258,10 +259,10 @@ async def test_create_from_module(post, testable_environment: tetype, upload):
         environment.artifacts.environments_root,
         environment.artifacts.group_folder,
         "hgi",
-        name
+        name,
     )
 
     assert file_was_pushed(
         Path(parent_path, environment.artifacts.environments_file),
-        Path(parent_path, environment.artifacts.module_file)
+        Path(parent_path, environment.artifacts.module_file),
     )
diff --git a/tests/integration/utils.py b/tests/integration/utils.py
index 9ee065a..7601137 100644
--- a/tests/integration/utils.py
+++ b/tests/integration/utils.py
@@ -6,6 +6,7 @@
 
 import tempfile
 from pathlib import Path
+from typing import Union
 
 import pygit2
 import pytest
@@ -13,7 +14,7 @@
 from softpack_core.artifacts import Artifacts, app
 
 artifacts_dict = dict[
-    str, str | pygit2.Oid | Path | Artifacts | tempfile.TemporaryDirectory[str]
+    str, Union[str, pygit2.Oid, Path, Artifacts, tempfile.TemporaryDirectory[str]]
 ]
 
 
@@ -164,7 +165,7 @@ def get_user_path_without_environments(
     return Path(*(artifacts.user_folder(user).parts[1:]))
 
 
-def file_was_pushed(*paths_with_environment: str | Path) -> bool:
+def file_was_pushed(*paths_with_environment: Union[str, Path]) -> bool:
     temp_dir = tempfile.TemporaryDirectory()
     app.settings.artifacts.path = Path(temp_dir.name)
     artifacts = Artifacts()

From 45ede5a86e19beafa598302f5f2e85f7cb83d62a Mon Sep 17 00:00:00 2001
From: Michael Woolnough <mw31@sanger.ac.uk>
Date: Mon, 4 Sep 2023 14:52:17 +0100
Subject: [PATCH 086/129] Add docstrings and reformat.

---
 softpack_core/moduleparse.py         | 37 +++++++++++++++++++++++-----
 softpack_core/schemas/environment.py | 32 +++++++++++++++++++++---
 tests/integration/utils.py           |  3 ++-
 tests/test_moduleparse.py            |  2 ++
 4 files changed, 63 insertions(+), 11 deletions(-)

diff --git a/softpack_core/moduleparse.py b/softpack_core/moduleparse.py
index f34985f..ec4d043 100644
--- a/softpack_core/moduleparse.py
+++ b/softpack_core/moduleparse.py
@@ -6,6 +6,19 @@
 
 
 def ToSoftpackYML(contents: bytes) -> bytes:
+    """Converts an shpc-style module file to a softpack.yml file.
+
+    It should have a format similar to that produced by shpc, with `module
+    whatis` outputting a "Name: " line, a "Version: " line, and optionally a
+    "Packages: " line to say what packages are available. `module help` output
+    will be translated into the description in the softpack.yml.
+
+    Args:
+        contents (bytes): The byte content of the module file.
+
+    Returns:
+        bytes: The byte content of the softpack.yml file.
+    """
     mode = "0"
 
     name = ""
@@ -15,12 +28,17 @@ def ToSoftpackYML(contents: bytes) -> bytes:
 
     for line in contents.splitlines():
         line = line.lstrip()
-        if mode is "0":
+        if mode == "0":
             if line.startswith(b"proc ModulesHelp"):
                 mode = "1"
             elif line.startswith(b"module-whatis "):
-                line = line.removeprefix(
-                    b"module-whatis ").decode('unicode_escape').removeprefix("\"").removesuffix("\"").lstrip()
+                line = (
+                    line.removeprefix(b"module-whatis ")
+                    .decode('unicode_escape')
+                    .removeprefix("\"")
+                    .removesuffix("\"")
+                    .lstrip()
+                )
                 print(line)
                 if line.startswith("Name: "):
                     nv = line.removeprefix("Name: ").split(":")
@@ -35,8 +53,13 @@ def ToSoftpackYML(contents: bytes) -> bytes:
             if line == b"}":
                 mode = "0"
             elif line.startswith(b"puts stderr "):
-                line = line.removeprefix(b"puts stderr ").decode(
-                    'unicode_escape').replace("\\$", "$").removeprefix("\"").removesuffix("\"")
+                line = (
+                    line.removeprefix(b"puts stderr ")
+                    .decode('unicode_escape')
+                    .replace("\\$", "$")
+                    .removeprefix("\"")
+                    .removesuffix("\"")
+                )
                 description += "  " + line + "\n"
 
     if version != "":
@@ -46,4 +69,6 @@ def ToSoftpackYML(contents: bytes) -> bytes:
 
     package_str = "\n  - ".join(packages)
 
-    return f"description: |\n{description}packages:\n  - {package_str}\n".encode()
+    return (
+        f"description: |\n{description}packages:\n  - {package_str}\n".encode()
+    )
diff --git a/softpack_core/schemas/environment.py b/softpack_core/schemas/environment.py
index ef2ba83..f561b72 100644
--- a/softpack_core/schemas/environment.py
+++ b/softpack_core/schemas/environment.py
@@ -4,21 +4,20 @@
 LICENSE file in the root directory of this source tree.
 """
 
+import io
 from dataclasses import dataclass
 from pathlib import Path
-import io
 from typing import Iterable, Optional
 
 import httpx
 import strawberry
-from strawberry.file_uploads import Upload
-
 from starlette.datastructures import UploadFile
+from strawberry.file_uploads import Upload
 
 from softpack_core.artifacts import Artifacts
+from softpack_core.moduleparse import ToSoftpackYML
 from softpack_core.schemas.base import BaseSchema
 from softpack_core.spack import Spack
-from softpack_core.moduleparse import ToSoftpackYML
 
 
 # Interfaces
@@ -245,6 +244,18 @@ def create(cls, env: EnvironmentInput) -> CreateResponse:
 
     @classmethod
     def create_new_env(cls, env: EnvironmentInput) -> CreateResponse:
+        """Create a new environment in the repository.
+
+        Adds an empty .created file in the desired location. Fails if this
+        already exists.
+
+        Args:
+            env (EnvironmentInput): Details of the new environment.
+
+        Returns:
+            CreateResponse: a CreateEnvironmentSuccess on success, or one of
+            (InvalidInputError, EnvironmentAlreadyExistsError) on error.
+        """
         # Check if a valid path has been provided. TODO: improve this to check
         # that they can only create stuff in their own users folder, or in
         # group folders of unix groups they belong to.
@@ -424,6 +435,19 @@ async def create_from_module(
     async def write_module_artifacts(
         cls, module_file: Upload, softpack_file: Upload, environment_path: str
     ) -> WriteArtifactResponse:
+        """Writes the given module and softpack files to the artifacts repo.
+
+        Args:
+            module_file (Upload): An shpc-style module file.
+            softpack_file (Upload): A "fake" softpack.yml file describing what
+            the module file offers.
+            environment_path (str): Path to the environment, eg.
+            users/user/env.
+
+        Returns:
+            WriteArtifactResponse: contains message and commit hash of
+            softpack.yml upload.
+        """
         result = await cls.write_artifact(
             file=module_file,
             folder_path=environment_path,
diff --git a/tests/integration/utils.py b/tests/integration/utils.py
index 7601137..2fca0a2 100644
--- a/tests/integration/utils.py
+++ b/tests/integration/utils.py
@@ -14,7 +14,8 @@
 from softpack_core.artifacts import Artifacts, app
 
 artifacts_dict = dict[
-    str, Union[str, pygit2.Oid, Path, Artifacts, tempfile.TemporaryDirectory[str]]
+    str,
+    Union[str, pygit2.Oid, Path, Artifacts, tempfile.TemporaryDirectory[str]],
 ]
 
 
diff --git a/tests/test_moduleparse.py b/tests/test_moduleparse.py
index 589e395..e548139 100644
--- a/tests/test_moduleparse.py
+++ b/tests/test_moduleparse.py
@@ -5,7 +5,9 @@
 """
 
 from pathlib import Path
+
 import pytest
+
 from softpack_core.moduleparse import ToSoftpackYML
 
 

From 26fed98bf880f9df76e5362e36e89dd66029ab56 Mon Sep 17 00:00:00 2001
From: Michael Woolnough <mw31@sanger.ac.uk>
Date: Mon, 4 Sep 2023 15:06:53 +0100
Subject: [PATCH 087/129] Remove redundant id field from Package.

---
 softpack_core/schemas/environment.py  | 4 +---
 softpack_core/spack.py                | 2 --
 tests/integration/test_environment.py | 4 ++--
 3 files changed, 3 insertions(+), 7 deletions(-)

diff --git a/softpack_core/schemas/environment.py b/softpack_core/schemas/environment.py
index f561b72..5f462a0 100644
--- a/softpack_core/schemas/environment.py
+++ b/softpack_core/schemas/environment.py
@@ -139,8 +139,6 @@ class Package(Spack.PackageBase):
 class PackageInput(Package):
     """A Strawberry input model representing a package."""
 
-    id: Optional[str] = None
-
     def to_package(self) -> Package:
         """Create a Package object from a PackageInput object.
 
@@ -200,7 +198,7 @@ def from_artifact(cls, obj: Artifacts.Object) -> Optional["Environment"]:
                 path=obj.path.parent,
                 description=spec.description,
                 packages=map(
-                    lambda package: Package(id=package, name=package),
+                    lambda package: Package(name=package),
                     spec.packages,
                 ),  # type: ignore [call-arg]
                 state=None,
diff --git a/softpack_core/spack.py b/softpack_core/spack.py
index 641eab9..6bc13fc 100644
--- a/softpack_core/spack.py
+++ b/softpack_core/spack.py
@@ -63,7 +63,6 @@ def load_repo_list(self) -> list:
     class PackageBase:
         """Wrapper for a spack package."""
 
-        id: str
         name: str
 
     @dataclass
@@ -77,7 +76,6 @@ def load_package_list(self) -> list[Package]:
         return list(
             map(
                 lambda package: self.Package(
-                    id=uuid.uuid4().hex,
                     name=package.name,
                     versions=[
                         str(ver) for ver in list(package.versions.keys())
diff --git a/tests/integration/test_environment.py b/tests/integration/test_environment.py
index 39e58d8..90bee5b 100644
--- a/tests/integration/test_environment.py
+++ b/tests/integration/test_environment.py
@@ -41,11 +41,11 @@ def testable_environment(mocker) -> tetype:
     mocker.patch.object(Environment, 'artifacts', new=artifacts)
 
     environment = Environment(
-        id="",
+        id="test env id",
         name="test_env_create",
         path=str(get_user_path_without_environments(artifacts, user)),
         description="description",
-        packages=[Package(id="", name="pkg_test")],
+        packages=[Package(name="pkg_test")],
         state=None,
     )
 

From 77346940ca5430d8a4aa00937abb0813350f21fa Mon Sep 17 00:00:00 2001
From: Michael Woolnough <mw31@sanger.ac.uk>
Date: Mon, 4 Sep 2023 16:03:53 +0100
Subject: [PATCH 088/129] Remove module feature for now; fix some type linting
 issues.

---
 pyproject.toml                              |   1 +
 softpack_core/moduleparse.py                |  74 ------------
 softpack_core/schemas/environment.py        | 124 +++-----------------
 softpack_core/schemas/package_collection.py |   4 +-
 tests/integration/test_environment.py       |  36 ------
 tests/test_moduleparse.py                   |  26 ----
 6 files changed, 17 insertions(+), 248 deletions(-)
 delete mode 100644 softpack_core/moduleparse.py
 delete mode 100644 tests/test_moduleparse.py

diff --git a/pyproject.toml b/pyproject.toml
index b9ae929..3c43236 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -134,6 +134,7 @@ skip_gitignore = true
 disallow_untyped_calls = true
 disallow_untyped_defs = true
 ignore_missing_imports = true
+plugins = "strawberry.ext.mypy_plugin"
 
 [tool.pytest.ini_options]
 filterwarnings = [
diff --git a/softpack_core/moduleparse.py b/softpack_core/moduleparse.py
deleted file mode 100644
index ec4d043..0000000
--- a/softpack_core/moduleparse.py
+++ /dev/null
@@ -1,74 +0,0 @@
-"""Copyright (c) 2023 Genome Research Ltd.
-
-This source code is licensed under the MIT license found in the
-LICENSE file in the root directory of this source tree.
-"""
-
-
-def ToSoftpackYML(contents: bytes) -> bytes:
-    """Converts an shpc-style module file to a softpack.yml file.
-
-    It should have a format similar to that produced by shpc, with `module
-    whatis` outputting a "Name: " line, a "Version: " line, and optionally a
-    "Packages: " line to say what packages are available. `module help` output
-    will be translated into the description in the softpack.yml.
-
-    Args:
-        contents (bytes): The byte content of the module file.
-
-    Returns:
-        bytes: The byte content of the softpack.yml file.
-    """
-    mode = "0"
-
-    name = ""
-    version = ""
-    packages: list[str] = []
-    description = ""
-
-    for line in contents.splitlines():
-        line = line.lstrip()
-        if mode == "0":
-            if line.startswith(b"proc ModulesHelp"):
-                mode = "1"
-            elif line.startswith(b"module-whatis "):
-                line = (
-                    line.removeprefix(b"module-whatis ")
-                    .decode('unicode_escape')
-                    .removeprefix("\"")
-                    .removesuffix("\"")
-                    .lstrip()
-                )
-                print(line)
-                if line.startswith("Name: "):
-                    nv = line.removeprefix("Name: ").split(":")
-                    name = nv[0]
-                    if len(nv) > 1:
-                        version = nv[1]
-                elif line.startswith("Version: "):
-                    version = line.removeprefix("Version: ")
-                elif line.startswith("Packages: "):
-                    packages = line.removeprefix("Packages: ").split(", ")
-        else:
-            if line == b"}":
-                mode = "0"
-            elif line.startswith(b"puts stderr "):
-                line = (
-                    line.removeprefix(b"puts stderr ")
-                    .decode('unicode_escape')
-                    .replace("\\$", "$")
-                    .removeprefix("\"")
-                    .removesuffix("\"")
-                )
-                description += "  " + line + "\n"
-
-    if version != "":
-        name += f"@{version}"
-
-    packages.insert(0, name)
-
-    package_str = "\n  - ".join(packages)
-
-    return (
-        f"description: |\n{description}packages:\n  - {package_str}\n".encode()
-    )
diff --git a/softpack_core/schemas/environment.py b/softpack_core/schemas/environment.py
index 5f462a0..85067bf 100644
--- a/softpack_core/schemas/environment.py
+++ b/softpack_core/schemas/environment.py
@@ -15,7 +15,6 @@
 from strawberry.file_uploads import Upload
 
 from softpack_core.artifacts import Artifacts
-from softpack_core.moduleparse import ToSoftpackYML
 from softpack_core.schemas.base import BaseSchema
 from softpack_core.spack import Spack
 
@@ -70,8 +69,6 @@ class WriteArtifactSuccess(Success):
 class InvalidInputError(Error):
     """Invalid input data."""
 
-    message: str
-
 
 @strawberry.type
 class EnvironmentNotFoundError(Error):
@@ -178,7 +175,7 @@ def iter(cls) -> Iterable["Environment"]:
         """
         environment_folders = cls.artifacts.iter()
         environment_objects = map(cls.from_artifact, environment_folders)
-        return filter(lambda x: x is not None, environment_objects)
+        return filter(None, environment_objects)
 
     @classmethod
     def from_artifact(cls, obj: Artifacts.Object) -> Optional["Environment"]:
@@ -195,11 +192,13 @@ def from_artifact(cls, obj: Artifacts.Object) -> Optional["Environment"]:
             return Environment(
                 id=obj.oid,
                 name=obj.name,
-                path=obj.path.parent,
+                path=str(obj.path.parent),
                 description=spec.description,
-                packages=map(
-                    lambda package: Package(name=package),
-                    spec.packages,
+                packages=list(
+                    map(
+                        lambda package: Package(name=package),
+                        spec.packages,
+                    )
                 ),  # type: ignore [call-arg]
                 state=None,
             )
@@ -207,7 +206,7 @@ def from_artifact(cls, obj: Artifacts.Object) -> Optional["Environment"]:
             return None
 
     @classmethod
-    def create(cls, env: EnvironmentInput) -> CreateResponse:
+    def create(cls, env: EnvironmentInput) -> CreateResponse:  # type: ignore
         """Create an Environment.
 
         Args:
@@ -241,7 +240,9 @@ def create(cls, env: EnvironmentInput) -> CreateResponse:
         )
 
     @classmethod
-    def create_new_env(cls, env: EnvironmentInput) -> CreateResponse:
+    def create_new_env(
+        cls, env: EnvironmentInput
+    ) -> CreateResponse:  # type: ignore
         """Create a new environment in the repository.
 
         Adds an empty .created file in the desired location. Fails if this
@@ -295,7 +296,7 @@ def update(
         env: EnvironmentInput,
         current_path: str,
         current_name: str,
-    ) -> UpdateResponse:
+    ) -> UpdateResponse:  # type: ignore
         """Update an Environment.
 
         Args:
@@ -344,7 +345,7 @@ def update(
         )
 
     @classmethod
-    def delete(cls, name: str, path: str) -> DeleteResponse:
+    def delete(cls, name: str, path: str) -> DeleteResponse:  # type: ignore
         """Delete an Environment.
 
         Args:
@@ -367,104 +368,10 @@ def delete(cls, name: str, path: str) -> DeleteResponse:
             name=name,
         )
 
-    @classmethod
-    async def create_from_module(
-        cls, file: Upload, module_path: str, environment_path: str
-    ) -> CreateResponse:
-        """Create an Environment based on an existing module.
-
-        The environment will not be built; a "fake" softpack.yml and the
-        supplied module file will be written as artifacts in a newly created
-        environment instead, so that they can be discovered.
-
-        Args:
-            file: the module file to add to the repo, and to parse to fake a
-                  corresponding softpack.yml. It should have a format similar
-                  to that produced by shpc, with `module whatis` outputting
-                  a "Name: " line, a "Version: " line, and optionally a
-                  "Packages: " line to say what packages are available.
-                  `module help` output will be translated into the description
-                  in the softpack.yml.
-            module_path: the local path that users can `module load` - this is
-                         used to auto-generate usage help text for this
-                         environment.
-            environment_path: the subdirectories of environments folder that
-                              artifacts will be stored in, eg.
-                              users/username/software_name
-
-        Returns:
-            A message confirming the success or failure of the operation.
-        """
-        environment_dirs = environment_path.split("/")
-        environment_name = environment_dirs.pop()
-
-        contents = (await file.read()).decode()
-        yml = ToSoftpackYML(contents)
-
-        env = EnvironmentInput(
-            name=environment_name,
-            path="/".join(environment_dirs),
-        )
-
-        response = cls.create_new_env(env)
-        if not isinstance(response, CreateEnvironmentSuccess):
-            return response
-
-        module_file = UploadFile(file=io.StringIO(contents))
-        softpack_file = UploadFile(file=io.StringIO(yml))
-
-        result = cls.write_module_artifacts(
-            module_file=module_file,
-            softpack_file=softpack_file,
-            environment_path=environment_path,
-        )
-
-        if not isinstance(result, WriteArtifactSuccess):
-            cls.delete(name=environment_name, path=environment_path)
-            return InvalidInputError(
-                msg="Write of module file failed: " + result.msg
-            )
-
-        return CreateEnvironmentSuccess(
-            message="Successfully created environment in artifacts repo"
-        )
-
-    @classmethod
-    async def write_module_artifacts(
-        cls, module_file: Upload, softpack_file: Upload, environment_path: str
-    ) -> WriteArtifactResponse:
-        """Writes the given module and softpack files to the artifacts repo.
-
-        Args:
-            module_file (Upload): An shpc-style module file.
-            softpack_file (Upload): A "fake" softpack.yml file describing what
-            the module file offers.
-            environment_path (str): Path to the environment, eg.
-            users/user/env.
-
-        Returns:
-            WriteArtifactResponse: contains message and commit hash of
-            softpack.yml upload.
-        """
-        result = await cls.write_artifact(
-            file=module_file,
-            folder_path=environment_path,
-            file_name=cls.artifacts.module_file,
-        )
-
-        if not isinstance(result, WriteArtifactSuccess):
-            return result
-
-        return await cls.write_artifact(
-            file=softpack_file,
-            folder_path=environment_path,
-            file_name=cls.artifacts.environments_file,
-        )
-
     @classmethod
     async def write_artifact(
         cls, file: Upload, folder_path: str, file_name: str
-    ) -> WriteArtifactResponse:
+    ) -> WriteArtifactResponse:  # type: ignore
         """Add a file to the Artifacts repo.
 
         Args:
@@ -508,6 +415,3 @@ class Mutation:
         writeArtifact: WriteArtifactResponse = (
             Environment.write_artifact
         )  # type: ignore
-        createFromModule: CreateResponse = (
-            Environment.create_from_module
-        )  # type: ignore
diff --git a/softpack_core/schemas/package_collection.py b/softpack_core/schemas/package_collection.py
index 289a859..24093de 100644
--- a/softpack_core/schemas/package_collection.py
+++ b/softpack_core/schemas/package_collection.py
@@ -52,7 +52,7 @@ def from_collection(
         return PackageCollection(
             id=collection.id,
             name=collection.name,
-            packages=map(cls.from_package, collection.packages),
+            packages=list(map(cls.from_package, collection.packages)),
         )  # type: ignore [call-arg]
 
     @classmethod
@@ -67,7 +67,7 @@ def from_package(cls, package: Spack.Package) -> PackageMultiVersion:
 
         """
         return PackageMultiVersion(
-            id=package.id, name=package.name, versions=package.versions
+            name=package.name, versions=package.versions
         )  # type: ignore [call-arg]
 
 
diff --git a/tests/integration/test_environment.py b/tests/integration/test_environment.py
index 90bee5b..97a3df6 100644
--- a/tests/integration/test_environment.py
+++ b/tests/integration/test_environment.py
@@ -230,39 +230,3 @@ async def test_iter(post, testable_environment, upload):
         count += 1
 
     assert count == 1
-
-
-@pytest.mark.asyncio
-@pytest.mark.skip
-async def test_create_from_module(post, testable_environment: tetype, upload):
-    environment, _ = testable_environment
-
-    test_file_path = Path(Path(__file__).parent.parent, "files", "ldsc.module")
-
-    with open(test_file_path, "rb") as fh:
-        upload.filename = "ldsc.module"
-        upload.content_type = "text/plain"
-        upload.read.return_value = fh.read()
-
-    name = "groups/hgi/some-environment"
-    module_path = "HGI/common/some_environment"
-
-    result = await environment.create_from_module(
-        file=upload,
-        module_path=module_path,
-        environment_path=name,
-    )
-
-    assert isinstance(result, CreateEnvironmentSuccess)
-
-    parent_path = Path(
-        environment.artifacts.environments_root,
-        environment.artifacts.group_folder,
-        "hgi",
-        name,
-    )
-
-    assert file_was_pushed(
-        Path(parent_path, environment.artifacts.environments_file),
-        Path(parent_path, environment.artifacts.module_file),
-    )
diff --git a/tests/test_moduleparse.py b/tests/test_moduleparse.py
deleted file mode 100644
index e548139..0000000
--- a/tests/test_moduleparse.py
+++ /dev/null
@@ -1,26 +0,0 @@
-"""Copyright (c) 2023 Genome Research Ltd.
-
-This source code is licensed under the MIT license found in the
-LICENSE file in the root directory of this source tree.
-"""
-
-from pathlib import Path
-
-import pytest
-
-from softpack_core.moduleparse import ToSoftpackYML
-
-
-@pytest.mark.skip
-def test_tosoftpack() -> None:
-    test_files_dir = Path(Path(__file__).parent, "files")
-
-    with open(Path(test_files_dir, "ldsc.module"), "rb") as fh:
-        module_data = fh.read()
-
-    with open(Path(test_files_dir, "ldsc.yml"), "rb") as fh:
-        expected_yml = fh.read()
-
-    yml = ToSoftpackYML(module_data)
-
-    assert yml == expected_yml

From 9c0cd2c83b9435db5a1f887686928a4d43004803 Mon Sep 17 00:00:00 2001
From: Michael Woolnough <mw31@sanger.ac.uk>
Date: Mon, 4 Sep 2023 16:07:00 +0100
Subject: [PATCH 089/129] Delint type issue.

---
 softpack_core/schemas/environment.py | 6 ++----
 1 file changed, 2 insertions(+), 4 deletions(-)

diff --git a/softpack_core/schemas/environment.py b/softpack_core/schemas/environment.py
index 85067bf..18a4d83 100644
--- a/softpack_core/schemas/environment.py
+++ b/softpack_core/schemas/environment.py
@@ -4,14 +4,12 @@
 LICENSE file in the root directory of this source tree.
 """
 
-import io
 from dataclasses import dataclass
 from pathlib import Path
 from typing import Iterable, Optional
 
 import httpx
 import strawberry
-from starlette.datastructures import UploadFile
 from strawberry.file_uploads import Upload
 
 from softpack_core.artifacts import Artifacts
@@ -412,6 +410,6 @@ class Mutation:
         createEnvironment: CreateResponse = Environment.create  # type: ignore
         updateEnvironment: UpdateResponse = Environment.update  # type: ignore
         deleteEnvironment: DeleteResponse = Environment.delete  # type: ignore
-        writeArtifact: WriteArtifactResponse = (
+        writeArtifact: WriteArtifactResponse = (  # type: ignore
             Environment.write_artifact
-        )  # type: ignore
+        )

From a61e28216b854c9ac890e3eb8248015d1b1eb3ec Mon Sep 17 00:00:00 2001
From: Michael Woolnough <mw31@sanger.ac.uk>
Date: Mon, 4 Sep 2023 16:21:54 +0100
Subject: [PATCH 090/129] Note TODO that we should not hardcode URL.

---
 softpack_core/schemas/environment.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/softpack_core/schemas/environment.py b/softpack_core/schemas/environment.py
index 18a4d83..3f8af21 100644
--- a/softpack_core/schemas/environment.py
+++ b/softpack_core/schemas/environment.py
@@ -221,6 +221,7 @@ def create(cls, env: EnvironmentInput) -> CreateResponse:  # type: ignore
         if not isinstance(response, CreateEnvironmentSuccess):
             return response
 
+        # TODO: remove hard-coding of URL.
         # Send build request
         httpx.post(
             "http://0.0.0.0:7080/environments/build",

From 3bc78fc5716b1be9481a0ffc6800978104f0cacc Mon Sep 17 00:00:00 2001
From: Michael Woolnough <mw31@sanger.ac.uk>
Date: Mon, 4 Sep 2023 16:22:37 +0100
Subject: [PATCH 091/129] Provide default user for git repo config and rename
 post fixture to httpx_post.

---
 tests/integration/conftest.py         | 10 +++++++---
 tests/integration/test_environment.py | 24 ++++++++++++------------
 2 files changed, 19 insertions(+), 15 deletions(-)

diff --git a/tests/integration/conftest.py b/tests/integration/conftest.py
index 136122f..e264357 100644
--- a/tests/integration/conftest.py
+++ b/tests/integration/conftest.py
@@ -4,7 +4,7 @@
 LICENSE file in the root directory of this source tree.
 """
 
-
+import os
 import pytest
 from starlette.datastructures import UploadFile
 
@@ -14,6 +14,10 @@
 @pytest.fixture(scope="package", autouse=True)
 def testable_artifacts_setup():
     user = app.settings.artifacts.repo.username.split('@', 1)[0]
+
+    if user is None:
+        user = os.getlogin()
+
     if user is None or user == "main":
         pytest.skip(
             ("Your artifacts repo username must be defined in your config.")
@@ -27,8 +31,8 @@ def testable_artifacts_setup():
     app.settings.artifacts.repo.branch = user
 
 
-@pytest.fixture(autouse=True)
-def post(mocker):
+@pytest.fixture()
+def httpx_post(mocker):
     post_mock = mocker.patch('httpx.post')
     return post_mock
 
diff --git a/tests/integration/test_environment.py b/tests/integration/test_environment.py
index 97a3df6..fbf0c5b 100644
--- a/tests/integration/test_environment.py
+++ b/tests/integration/test_environment.py
@@ -59,7 +59,7 @@ def testable_environment(mocker) -> tetype:
     yield environment, env_input
 
 
-def test_create(post, testable_environment: tetype) -> None:
+def test_create(httpx_post, testable_environment: tetype) -> None:
     environment, env_input = testable_environment
 
     result = environment.create(env_input)
@@ -73,8 +73,8 @@ def test_create(post, testable_environment: tetype) -> None:
     )
     assert file_was_pushed(path)
 
-    post.assert_called_once()
-    builder_called_correctly(post, env_input)
+    httpx_post.assert_called_once()
+    builder_called_correctly(httpx_post, env_input)
 
     result = environment.create(env_input)
     assert isinstance(result, EnvironmentAlreadyExistsError)
@@ -104,18 +104,18 @@ def builder_called_correctly(post_mock, env_input: EnvironmentInput) -> None:
     )
 
 
-def test_update(post, testable_environment) -> None:
+def test_update(httpx_post, testable_environment) -> None:
     environment, env_input = testable_environment
 
     result = environment.create(env_input)
     assert isinstance(result, CreateEnvironmentSuccess)
-    post.assert_called_once()
+    httpx_post.assert_called_once()
 
     env_input.description = "updated description"
     result = environment.update(env_input, env_input.path, env_input.name)
     assert isinstance(result, UpdateEnvironmentSuccess)
 
-    builder_called_correctly(post, env_input)
+    builder_called_correctly(httpx_post, env_input)
 
     result = environment.update(env_input, "invalid/path", "invalid_name")
     assert isinstance(result, InvalidInputError)
@@ -130,7 +130,7 @@ def test_update(post, testable_environment) -> None:
     assert isinstance(result, EnvironmentNotFoundError)
 
 
-def test_delete(post, testable_environment) -> None:
+def test_delete(httpx_post, testable_environment) -> None:
     environment, env_input = testable_environment
 
     result = environment.delete(env_input.name, env_input.path)
@@ -138,7 +138,7 @@ def test_delete(post, testable_environment) -> None:
 
     result = environment.create(env_input)
     assert isinstance(result, CreateEnvironmentSuccess)
-    post.assert_called_once()
+    httpx_post.assert_called_once()
 
     path = Path(
         environment.artifacts.environments_root,
@@ -155,7 +155,7 @@ def test_delete(post, testable_environment) -> None:
 
 
 @pytest.mark.asyncio
-async def test_write_artifact(post, testable_environment, upload):
+async def test_write_artifact(httpx_post, testable_environment, upload):
     environment, env_input = testable_environment
 
     upload.filename = "example.txt"
@@ -171,7 +171,7 @@ async def test_write_artifact(post, testable_environment, upload):
 
     result = environment.create(env_input)
     assert isinstance(result, CreateEnvironmentSuccess)
-    post.assert_called_once()
+    httpx_post.assert_called_once()
 
     result = await environment.write_artifact(
         file=upload,
@@ -197,7 +197,7 @@ async def test_write_artifact(post, testable_environment, upload):
 
 
 @pytest.mark.asyncio
-async def test_iter(post, testable_environment, upload):
+async def test_iter(httpx_post, testable_environment, upload):
     environment, env_input = testable_environment
 
     envs_filter = environment.iter()
@@ -209,7 +209,7 @@ async def test_iter(post, testable_environment, upload):
 
     result = environment.create(env_input)
     assert isinstance(result, CreateEnvironmentSuccess)
-    post.assert_called_once()
+    httpx_post.assert_called_once()
 
     upload.filename = Artifacts.environments_file
     upload.content_type = "text/plain"

From 68b942a7b853d100c223b8e37439611f9e8f89bb Mon Sep 17 00:00:00 2001
From: Michael Woolnough <mw31@sanger.ac.uk>
Date: Mon, 4 Sep 2023 16:56:15 +0100
Subject: [PATCH 092/129] Change testable_environment to return a single value.

---
 tests/integration/test_environment.py | 153 +++++++++++++-------------
 1 file changed, 78 insertions(+), 75 deletions(-)

diff --git a/tests/integration/test_environment.py b/tests/integration/test_environment.py
index fbf0c5b..25f8546 100644
--- a/tests/integration/test_environment.py
+++ b/tests/integration/test_environment.py
@@ -29,166 +29,171 @@
 
 pytestmark = pytest.mark.repo
 
-tetype = tuple[Environment, EnvironmentInput]
-
 
 @pytest.fixture
-def testable_environment(mocker) -> tetype:
+def testable_environment_input(mocker) -> EnvironmentInput:
     ad = new_test_artifacts()
     artifacts: Artifacts = ad["artifacts"]
     user = ad["test_user"]
 
     mocker.patch.object(Environment, 'artifacts', new=artifacts)
 
-    environment = Environment(
-        id="test env id",
+    testable_environment_input = EnvironmentInput(
         name="test_env_create",
         path=str(get_user_path_without_environments(artifacts, user)),
         description="description",
         packages=[Package(name="pkg_test")],
-        state=None,
-    )
-
-    env_input = EnvironmentInput(
-        name=environment.name,
-        path=environment.path,
-        description=environment.description,
-        packages=environment.packages,
     )
 
-    yield environment, env_input
-
+    yield testable_environment_input
 
-def test_create(httpx_post, testable_environment: tetype) -> None:
-    environment, env_input = testable_environment
 
-    result = environment.create(env_input)
+def test_create(
+    httpx_post, testable_environment_input: EnvironmentInput
+) -> None:
+    result = Environment.create(testable_environment_input)
     assert isinstance(result, CreateEnvironmentSuccess)
 
     path = Path(
-        environment.artifacts.environments_root,
-        env_input.path,
-        env_input.name,
+        Environment.artifacts.environments_root,
+        testable_environment_input.path,
+        testable_environment_input.name,
         ".created",
     )
     assert file_was_pushed(path)
 
     httpx_post.assert_called_once()
-    builder_called_correctly(httpx_post, env_input)
+    builder_called_correctly(httpx_post, testable_environment_input)
 
-    result = environment.create(env_input)
+    result = Environment.create(testable_environment_input)
     assert isinstance(result, EnvironmentAlreadyExistsError)
 
-    env_input.name = ""
-    result = environment.create(env_input)
+    orig_name = testable_environment_input.name
+    testable_environment_input.name = ""
+    result = Environment.create(testable_environment_input)
     assert isinstance(result, InvalidInputError)
 
-    env_input.name = environment.name
-    env_input.path = "invalid/path"
-    result = environment.create(env_input)
+    testable_environment_input.name = orig_name
+    testable_environment_input.path = "invalid/path"
+    result = Environment.create(testable_environment_input)
     assert isinstance(result, InvalidInputError)
 
 
-def builder_called_correctly(post_mock, env_input: EnvironmentInput) -> None:
+def builder_called_correctly(
+    post_mock, testable_environment_input: EnvironmentInput
+) -> None:
     # TODO: don't mock this; actually have a real builder service to test with?
     # Also need to not hard-code the url here.
     post_mock.assert_called_with(
         "http://0.0.0.0:7080/environments/build",
         json={
-            "name": f"{env_input.path}/{env_input.name}",
+            "name": f"{testable_environment_input.path}/{testable_environment_input.name}",
             "model": {
-                "description": env_input.description,
-                "packages": [f"{pkg.name}" for pkg in env_input.packages],
+                "description": testable_environment_input.description,
+                "packages": [
+                    f"{pkg.name}"
+                    for pkg in testable_environment_input.packages
+                ],
             },
         },
     )
 
 
-def test_update(httpx_post, testable_environment) -> None:
-    environment, env_input = testable_environment
-
-    result = environment.create(env_input)
+def test_update(httpx_post, testable_environment_input) -> None:
+    result = Environment.create(testable_environment_input)
     assert isinstance(result, CreateEnvironmentSuccess)
     httpx_post.assert_called_once()
 
-    env_input.description = "updated description"
-    result = environment.update(env_input, env_input.path, env_input.name)
+    testable_environment_input.description = "updated description"
+    result = Environment.update(
+        testable_environment_input,
+        testable_environment_input.path,
+        testable_environment_input.name,
+    )
     assert isinstance(result, UpdateEnvironmentSuccess)
 
-    builder_called_correctly(httpx_post, env_input)
+    builder_called_correctly(httpx_post, testable_environment_input)
 
-    result = environment.update(env_input, "invalid/path", "invalid_name")
+    result = Environment.update(
+        testable_environment_input, "invalid/path", "invalid_name"
+    )
     assert isinstance(result, InvalidInputError)
 
-    env_input.name = ""
-    result = environment.update(env_input, env_input.path, env_input.name)
+    testable_environment_input.name = ""
+    result = Environment.update(
+        testable_environment_input,
+        testable_environment_input.path,
+        testable_environment_input.name,
+    )
     assert isinstance(result, InvalidInputError)
 
-    env_input.name = "invalid_name"
-    env_input.path = "invalid/path"
-    result = environment.update(env_input, "invalid/path", "invalid_name")
+    testable_environment_input.name = "invalid_name"
+    testable_environment_input.path = "invalid/path"
+    result = Environment.update(
+        testable_environment_input, "invalid/path", "invalid_name"
+    )
     assert isinstance(result, EnvironmentNotFoundError)
 
 
-def test_delete(httpx_post, testable_environment) -> None:
-    environment, env_input = testable_environment
-
-    result = environment.delete(env_input.name, env_input.path)
+def test_delete(httpx_post, testable_environment_input) -> None:
+    result = Environment.delete(
+        testable_environment_input.name, testable_environment_input.path
+    )
     assert isinstance(result, EnvironmentNotFoundError)
 
-    result = environment.create(env_input)
+    result = Environment.create(testable_environment_input)
     assert isinstance(result, CreateEnvironmentSuccess)
     httpx_post.assert_called_once()
 
     path = Path(
-        environment.artifacts.environments_root,
-        env_input.path,
-        env_input.name,
+        Environment.artifacts.environments_root,
+        testable_environment_input.path,
+        testable_environment_input.name,
         ".created",
     )
     assert file_was_pushed(path)
 
-    result = environment.delete(env_input.name, env_input.path)
+    result = Environment.delete(
+        testable_environment_input.name, testable_environment_input.path
+    )
     assert isinstance(result, DeleteEnvironmentSuccess)
 
     assert not file_was_pushed(path)
 
 
 @pytest.mark.asyncio
-async def test_write_artifact(httpx_post, testable_environment, upload):
-    environment, env_input = testable_environment
-
+async def test_write_artifact(httpx_post, testable_environment_input, upload):
     upload.filename = "example.txt"
     upload.content_type = "text/plain"
     upload.read.return_value = b"mock data"
 
-    result = await environment.write_artifact(
+    result = await Environment.write_artifact(
         file=upload,
-        folder_path=f"{env_input.path}/{env_input.name}",
+        folder_path=f"{testable_environment_input.path}/{testable_environment_input.name}",
         file_name=upload.filename,
     )
     assert isinstance(result, InvalidInputError)
 
-    result = environment.create(env_input)
+    result = Environment.create(testable_environment_input)
     assert isinstance(result, CreateEnvironmentSuccess)
     httpx_post.assert_called_once()
 
-    result = await environment.write_artifact(
+    result = await Environment.write_artifact(
         file=upload,
-        folder_path=f"{env_input.path}/{env_input.name}",
+        folder_path=f"{testable_environment_input.path}/{testable_environment_input.name}",
         file_name=upload.filename,
     )
     assert isinstance(result, WriteArtifactSuccess)
 
     path = Path(
-        environment.artifacts.environments_root,
-        env_input.path,
-        env_input.name,
+        Environment.artifacts.environments_root,
+        testable_environment_input.path,
+        testable_environment_input.name,
         upload.filename,
     )
     assert file_was_pushed(path)
 
-    result = await environment.write_artifact(
+    result = await Environment.write_artifact(
         file=upload,
         folder_path="invalid/env/path",
         file_name=upload.filename,
@@ -197,17 +202,15 @@ async def test_write_artifact(httpx_post, testable_environment, upload):
 
 
 @pytest.mark.asyncio
-async def test_iter(httpx_post, testable_environment, upload):
-    environment, env_input = testable_environment
-
-    envs_filter = environment.iter()
+async def test_iter(httpx_post, testable_environment_input, upload):
+    envs_filter = Environment.iter()
     count = 0
     for env in envs_filter:
         count += 1
 
     assert count == 0
 
-    result = environment.create(env_input)
+    result = Environment.create(testable_environment_input)
     assert isinstance(result, CreateEnvironmentSuccess)
     httpx_post.assert_called_once()
 
@@ -215,17 +218,17 @@ async def test_iter(httpx_post, testable_environment, upload):
     upload.content_type = "text/plain"
     upload.read.return_value = b"description: test env\npackages:\n- zlib\n"
 
-    result = await environment.write_artifact(
+    result = await Environment.write_artifact(
         file=upload,
-        folder_path=f"{env_input.path}/{env_input.name}",
+        folder_path=f"{testable_environment_input.path}/{testable_environment_input.name}",
         file_name=upload.filename,
     )
     assert isinstance(result, WriteArtifactSuccess)
 
-    envs_filter = environment.iter()
+    envs_filter = Environment.iter()
     count = 0
     for env in envs_filter:
-        assert env.name == env_input.name
+        assert env.name == testable_environment_input.name
         assert any(p.name == "zlib" for p in env.packages)
         count += 1
 

From e2bfe67205874002e1384a5d79ed48016dedb251 Mon Sep 17 00:00:00 2001
From: Michael Woolnough <mw31@sanger.ac.uk>
Date: Tue, 5 Sep 2023 09:56:19 +0100
Subject: [PATCH 093/129] Reformat.

---
 tests/integration/conftest.py         |   1 +
 tests/integration/test_environment.py | 103 +++++++++++++-------------
 2 files changed, 51 insertions(+), 53 deletions(-)

diff --git a/tests/integration/conftest.py b/tests/integration/conftest.py
index e264357..1ea552a 100644
--- a/tests/integration/conftest.py
+++ b/tests/integration/conftest.py
@@ -5,6 +5,7 @@
 """
 
 import os
+
 import pytest
 from starlette.datastructures import UploadFile
 
diff --git a/tests/integration/test_environment.py b/tests/integration/test_environment.py
index 25f8546..f698852 100644
--- a/tests/integration/test_environment.py
+++ b/tests/integration/test_environment.py
@@ -31,130 +31,127 @@
 
 
 @pytest.fixture
-def testable_environment_input(mocker) -> EnvironmentInput:
+def testable_env_input(mocker) -> EnvironmentInput:
     ad = new_test_artifacts()
     artifacts: Artifacts = ad["artifacts"]
     user = ad["test_user"]
 
     mocker.patch.object(Environment, 'artifacts', new=artifacts)
 
-    testable_environment_input = EnvironmentInput(
+    testable_env_input = EnvironmentInput(
         name="test_env_create",
         path=str(get_user_path_without_environments(artifacts, user)),
         description="description",
         packages=[Package(name="pkg_test")],
     )
 
-    yield testable_environment_input
+    yield testable_env_input
 
 
-def test_create(
-    httpx_post, testable_environment_input: EnvironmentInput
-) -> None:
-    result = Environment.create(testable_environment_input)
+def test_create(httpx_post, testable_env_input: EnvironmentInput) -> None:
+    result = Environment.create(testable_env_input)
     assert isinstance(result, CreateEnvironmentSuccess)
 
     path = Path(
         Environment.artifacts.environments_root,
-        testable_environment_input.path,
-        testable_environment_input.name,
+        testable_env_input.path,
+        testable_env_input.name,
         ".created",
     )
     assert file_was_pushed(path)
 
     httpx_post.assert_called_once()
-    builder_called_correctly(httpx_post, testable_environment_input)
+    builder_called_correctly(httpx_post, testable_env_input)
 
-    result = Environment.create(testable_environment_input)
+    result = Environment.create(testable_env_input)
     assert isinstance(result, EnvironmentAlreadyExistsError)
 
-    orig_name = testable_environment_input.name
-    testable_environment_input.name = ""
-    result = Environment.create(testable_environment_input)
+    orig_name = testable_env_input.name
+    testable_env_input.name = ""
+    result = Environment.create(testable_env_input)
     assert isinstance(result, InvalidInputError)
 
-    testable_environment_input.name = orig_name
-    testable_environment_input.path = "invalid/path"
-    result = Environment.create(testable_environment_input)
+    testable_env_input.name = orig_name
+    testable_env_input.path = "invalid/path"
+    result = Environment.create(testable_env_input)
     assert isinstance(result, InvalidInputError)
 
 
 def builder_called_correctly(
-    post_mock, testable_environment_input: EnvironmentInput
+    post_mock, testable_env_input: EnvironmentInput
 ) -> None:
     # TODO: don't mock this; actually have a real builder service to test with?
     # Also need to not hard-code the url here.
     post_mock.assert_called_with(
         "http://0.0.0.0:7080/environments/build",
         json={
-            "name": f"{testable_environment_input.path}/{testable_environment_input.name}",
+            "name": f"{testable_env_input.path}/{testable_env_input.name}",
             "model": {
-                "description": testable_environment_input.description,
+                "description": testable_env_input.description,
                 "packages": [
-                    f"{pkg.name}"
-                    for pkg in testable_environment_input.packages
+                    f"{pkg.name}" for pkg in testable_env_input.packages
                 ],
             },
         },
     )
 
 
-def test_update(httpx_post, testable_environment_input) -> None:
-    result = Environment.create(testable_environment_input)
+def test_update(httpx_post, testable_env_input) -> None:
+    result = Environment.create(testable_env_input)
     assert isinstance(result, CreateEnvironmentSuccess)
     httpx_post.assert_called_once()
 
-    testable_environment_input.description = "updated description"
+    testable_env_input.description = "updated description"
     result = Environment.update(
-        testable_environment_input,
-        testable_environment_input.path,
-        testable_environment_input.name,
+        testable_env_input,
+        testable_env_input.path,
+        testable_env_input.name,
     )
     assert isinstance(result, UpdateEnvironmentSuccess)
 
-    builder_called_correctly(httpx_post, testable_environment_input)
+    builder_called_correctly(httpx_post, testable_env_input)
 
     result = Environment.update(
-        testable_environment_input, "invalid/path", "invalid_name"
+        testable_env_input, "invalid/path", "invalid_name"
     )
     assert isinstance(result, InvalidInputError)
 
-    testable_environment_input.name = ""
+    testable_env_input.name = ""
     result = Environment.update(
-        testable_environment_input,
-        testable_environment_input.path,
-        testable_environment_input.name,
+        testable_env_input,
+        testable_env_input.path,
+        testable_env_input.name,
     )
     assert isinstance(result, InvalidInputError)
 
-    testable_environment_input.name = "invalid_name"
-    testable_environment_input.path = "invalid/path"
+    testable_env_input.name = "invalid_name"
+    testable_env_input.path = "invalid/path"
     result = Environment.update(
-        testable_environment_input, "invalid/path", "invalid_name"
+        testable_env_input, "invalid/path", "invalid_name"
     )
     assert isinstance(result, EnvironmentNotFoundError)
 
 
-def test_delete(httpx_post, testable_environment_input) -> None:
+def test_delete(httpx_post, testable_env_input) -> None:
     result = Environment.delete(
-        testable_environment_input.name, testable_environment_input.path
+        testable_env_input.name, testable_env_input.path
     )
     assert isinstance(result, EnvironmentNotFoundError)
 
-    result = Environment.create(testable_environment_input)
+    result = Environment.create(testable_env_input)
     assert isinstance(result, CreateEnvironmentSuccess)
     httpx_post.assert_called_once()
 
     path = Path(
         Environment.artifacts.environments_root,
-        testable_environment_input.path,
-        testable_environment_input.name,
+        testable_env_input.path,
+        testable_env_input.name,
         ".created",
     )
     assert file_was_pushed(path)
 
     result = Environment.delete(
-        testable_environment_input.name, testable_environment_input.path
+        testable_env_input.name, testable_env_input.path
     )
     assert isinstance(result, DeleteEnvironmentSuccess)
 
@@ -162,33 +159,33 @@ def test_delete(httpx_post, testable_environment_input) -> None:
 
 
 @pytest.mark.asyncio
-async def test_write_artifact(httpx_post, testable_environment_input, upload):
+async def test_write_artifact(httpx_post, testable_env_input, upload):
     upload.filename = "example.txt"
     upload.content_type = "text/plain"
     upload.read.return_value = b"mock data"
 
     result = await Environment.write_artifact(
         file=upload,
-        folder_path=f"{testable_environment_input.path}/{testable_environment_input.name}",
+        folder_path=f"{testable_env_input.path}/{testable_env_input.name}",
         file_name=upload.filename,
     )
     assert isinstance(result, InvalidInputError)
 
-    result = Environment.create(testable_environment_input)
+    result = Environment.create(testable_env_input)
     assert isinstance(result, CreateEnvironmentSuccess)
     httpx_post.assert_called_once()
 
     result = await Environment.write_artifact(
         file=upload,
-        folder_path=f"{testable_environment_input.path}/{testable_environment_input.name}",
+        folder_path=f"{testable_env_input.path}/{testable_env_input.name}",
         file_name=upload.filename,
     )
     assert isinstance(result, WriteArtifactSuccess)
 
     path = Path(
         Environment.artifacts.environments_root,
-        testable_environment_input.path,
-        testable_environment_input.name,
+        testable_env_input.path,
+        testable_env_input.name,
         upload.filename,
     )
     assert file_was_pushed(path)
@@ -202,7 +199,7 @@ async def test_write_artifact(httpx_post, testable_environment_input, upload):
 
 
 @pytest.mark.asyncio
-async def test_iter(httpx_post, testable_environment_input, upload):
+async def test_iter(httpx_post, testable_env_input, upload):
     envs_filter = Environment.iter()
     count = 0
     for env in envs_filter:
@@ -210,7 +207,7 @@ async def test_iter(httpx_post, testable_environment_input, upload):
 
     assert count == 0
 
-    result = Environment.create(testable_environment_input)
+    result = Environment.create(testable_env_input)
     assert isinstance(result, CreateEnvironmentSuccess)
     httpx_post.assert_called_once()
 
@@ -220,7 +217,7 @@ async def test_iter(httpx_post, testable_environment_input, upload):
 
     result = await Environment.write_artifact(
         file=upload,
-        folder_path=f"{testable_environment_input.path}/{testable_environment_input.name}",
+        folder_path=f"{testable_env_input.path}/{testable_env_input.name}",
         file_name=upload.filename,
     )
     assert isinstance(result, WriteArtifactSuccess)
@@ -228,7 +225,7 @@ async def test_iter(httpx_post, testable_environment_input, upload):
     envs_filter = Environment.iter()
     count = 0
     for env in envs_filter:
-        assert env.name == testable_environment_input.name
+        assert env.name == testable_env_input.name
         assert any(p.name == "zlib" for p in env.packages)
         count += 1
 

From 70950f17c10cf4662708a789f09cd42a95dfdf22 Mon Sep 17 00:00:00 2001
From: Michael Woolnough <mw31@sanger.ac.uk>
Date: Tue, 5 Sep 2023 10:33:59 +0100
Subject: [PATCH 094/129] Remove redundant 'message' fields.

---
 softpack_core/schemas/environment.py | 9 ---------
 1 file changed, 9 deletions(-)

diff --git a/softpack_core/schemas/environment.py b/softpack_core/schemas/environment.py
index 3f8af21..9e9e937 100644
--- a/softpack_core/schemas/environment.py
+++ b/softpack_core/schemas/environment.py
@@ -37,28 +37,21 @@ class Error:
 class CreateEnvironmentSuccess(Success):
     """Environment successfully scheduled."""
 
-    message: str
-
 
 @strawberry.type
 class UpdateEnvironmentSuccess(Success):
     """Environment successfully updated."""
 
-    message: str
-
 
 @strawberry.type
 class DeleteEnvironmentSuccess(Success):
     """Environment successfully deleted."""
 
-    message: str
-
 
 @strawberry.type
 class WriteArtifactSuccess(Success):
     """Artifact successfully created."""
 
-    message: str
     commit_oid: str
 
 
@@ -72,7 +65,6 @@ class InvalidInputError(Error):
 class EnvironmentNotFoundError(Error):
     """Environment not found."""
 
-    message: str
     path: str
     name: str
 
@@ -81,7 +73,6 @@ class EnvironmentNotFoundError(Error):
 class EnvironmentAlreadyExistsError(Error):
     """Environment name already exists."""
 
-    message: str
     path: str
     name: str
 

From 18887224f425f3d5198a723a38cce3434b0416dc Mon Sep 17 00:00:00 2001
From: Michael Woolnough <mw31@sanger.ac.uk>
Date: Tue, 5 Sep 2023 10:34:22 +0100
Subject: [PATCH 095/129] Remove test data for removed test.

---
 tests/files/ldsc.module | 210 ----------------------------------------
 tests/files/ldsc.yml    |  54 -----------
 2 files changed, 264 deletions(-)
 delete mode 100644 tests/files/ldsc.module
 delete mode 100644 tests/files/ldsc.yml

diff --git a/tests/files/ldsc.module b/tests/files/ldsc.module
deleted file mode 100644
index 0feb40c..0000000
--- a/tests/files/ldsc.module
+++ /dev/null
@@ -1,210 +0,0 @@
-#%Module
-
-#=====
-# Created by singularity-hpc (https://github.com/singularityhub/singularity-hpc)
-# ##
-# quay.io/biocontainers/ldsc:1.0.1--pyhdfd78af_2 on 2023-08-15 12:08:41.851818
-#=====
-
-proc ModulesHelp { } {
-
-    puts stderr "This module is a singularity container wrapper for quay.io/biocontainers/ldsc:1.0.1--pyhdfd78af_2 v1.0.1--pyhdfd78af_2"
-    
-    puts stderr ""
-    puts stderr "Container (available through variable SINGULARITY_CONTAINER):"
-    puts stderr ""
-    puts stderr " - /software/hgi/containers/shpc/quay.io/biocontainers/ldsc/1.0.1--pyhdfd78af_2/quay.io-biocontainers-ldsc-1.0.1--pyhdfd78af_2-sha256:308ddebaa643d50306779ce42752eb4c4a3e1635be74531594013959e312af2c.sif"
-    puts stderr ""
-    puts stderr "Commands include:"
-    puts stderr ""
-    puts stderr " - ldsc-run:"
-    puts stderr "       singularity run -B <moduleDir>/99-shpc.sh:/.singularity.d/env/99-shpc.sh <container> \"\$@\""
-    puts stderr " - ldsc-shell:"
-    puts stderr "       singularity shell -s /bin/sh -B <moduleDir>/99-shpc.sh:/.singularity.d/env/99-shpc.sh <container>"
-    puts stderr " - ldsc-exec:"
-    puts stderr "       singularity exec -B <moduleDir>/99-shpc.sh:/.singularity.d/env/99-shpc.sh <container> \"\$@\""
-    puts stderr " - ldsc-inspect-runscript:"
-    puts stderr "       singularity inspect -r <container>"
-    puts stderr " - ldsc-inspect-deffile:"
-    puts stderr "       singularity inspect -d <container>"
-    puts stderr " - ldsc-container:"
-    puts stderr "       echo \"\$SINGULARITY_CONTAINER\""
-    puts stderr ""
-    puts stderr " - ldsc.py:"
-    puts stderr "       singularity exec -B <moduleDir>/99-shpc.sh:/.singularity.d/env/99-shpc.sh <container> /usr/local/bin/ldsc.py \"\$@\""
-    puts stderr " - munge_sumstats.py:"
-    puts stderr "       singularity exec -B <moduleDir>/99-shpc.sh:/.singularity.d/env/99-shpc.sh <container> /usr/local/bin/munge_sumstats.py \"\$@\""
-    puts stderr " - f2py2:"
-    puts stderr "       singularity exec -B <moduleDir>/99-shpc.sh:/.singularity.d/env/99-shpc.sh <container> /usr/local/bin/f2py2 \"\$@\""
-    puts stderr " - f2py2.7:"
-    puts stderr "       singularity exec -B <moduleDir>/99-shpc.sh:/.singularity.d/env/99-shpc.sh <container> /usr/local/bin/f2py2.7 \"\$@\""
-    puts stderr " - shiftBed:"
-    puts stderr "       singularity exec -B <moduleDir>/99-shpc.sh:/.singularity.d/env/99-shpc.sh <container> /usr/local/bin/shiftBed \"\$@\""
-    puts stderr " - annotateBed:"
-    puts stderr "       singularity exec -B <moduleDir>/99-shpc.sh:/.singularity.d/env/99-shpc.sh <container> /usr/local/bin/annotateBed \"\$@\""
-    puts stderr " - bamToBed:"
-    puts stderr "       singularity exec -B <moduleDir>/99-shpc.sh:/.singularity.d/env/99-shpc.sh <container> /usr/local/bin/bamToBed \"\$@\""
-    puts stderr " - bamToFastq:"
-    puts stderr "       singularity exec -B <moduleDir>/99-shpc.sh:/.singularity.d/env/99-shpc.sh <container> /usr/local/bin/bamToFastq \"\$@\""
-    puts stderr " - bed12ToBed6:"
-    puts stderr "       singularity exec -B <moduleDir>/99-shpc.sh:/.singularity.d/env/99-shpc.sh <container> /usr/local/bin/bed12ToBed6 \"\$@\""
-    puts stderr " - bedToBam:"
-    puts stderr "       singularity exec -B <moduleDir>/99-shpc.sh:/.singularity.d/env/99-shpc.sh <container> /usr/local/bin/bedToBam \"\$@\""
-    puts stderr " - bedToIgv:"
-    puts stderr "       singularity exec -B <moduleDir>/99-shpc.sh:/.singularity.d/env/99-shpc.sh <container> /usr/local/bin/bedToIgv \"\$@\""
-    puts stderr " - bedpeToBam:"
-    puts stderr "       singularity exec -B <moduleDir>/99-shpc.sh:/.singularity.d/env/99-shpc.sh <container> /usr/local/bin/bedpeToBam \"\$@\""
-
-    puts stderr ""
-    puts stderr "For each of the above, you can export:"
-    puts stderr ""
-    puts stderr " - SINGULARITY_OPTS: to define custom options for singularity (e.g., --debug)"
-    puts stderr " - SINGULARITY_COMMAND_OPTS: to define custom options for the command (e.g., -b)"
-    puts stderr " - SINGULARITY_CONTAINER: The Singularity (sif) path"
-
-}
-
-set view_dir "[file dirname [file dirname ${ModulesCurrentModulefile}] ]"
-set view_name "[file tail ${view_dir}]"
-set view_module ".view_module"
-set view_modulefile "${view_dir}/${view_module}"
-
-if {[file exists ${view_modulefile}]} {
-    source ${view_modulefile}
-}
-
-# Environment - only set if not already defined
-if { ![info exists ::env(SINGULARITY_OPTS)] } {
-    setenv SINGULARITY_OPTS ""
-}
-if { ![info exists ::env(SINGULARITY_COMMAND_OPTS)] } {
-    setenv SINGULARITY_COMMAND_OPTS ""
-}
-
-# Variables
-
-set name        quay.io/biocontainers/ldsc:1.0.1--pyhdfd78af_2
-set version     1.0.1--pyhdfd78af_2
-set description "$name - $version"
-set containerPath /software/hgi/containers/shpc/quay.io/biocontainers/ldsc/1.0.1--pyhdfd78af_2/quay.io-biocontainers-ldsc-1.0.1--pyhdfd78af_2-sha256:308ddebaa643d50306779ce42752eb4c4a3e1635be74531594013959e312af2c.sif
-
-
-set helpcommand "This module is a singularity container wrapper for quay.io/biocontainers/ldsc:1.0.1--pyhdfd78af_2 v1.0.1--pyhdfd78af_2. "
-set busybox "BusyBox v1.32.1 (2021-04-13 11:15:36 UTC) multi-call binary."
-set deb-list "gcc-8-base_8.3.0-6_amd64.deb, libc6_2.28-10_amd64.deb, libgcc1_1%3a8.3.0-6_amd64.deb, bash_5.0-4_amd64.deb, libc-bin_2.28-10_amd64.deb, libtinfo6_6.1+20181013-2+deb10u2_amd64.deb, ncurses-base_6.1+20181013-2+deb10u2_all.deb, base-files_10.3+deb10u9_amd64.deb"
-set glibc "GNU C Library (Debian GLIBC 2.28-10) stable release version 2.28."
-set io.buildah.version "1.19.6"
-set org.label-schema.build-arch "amd64"
-set org.label-schema.build-date "Tuesday_15_August_2023_12:8:5_BST"
-set org.label-schema.schema-version "1.0"
-set org.label-schema.usage.singularity.deffile.bootstrap "docker"
-set org.label-schema.usage.singularity.deffile.from "quay.io/biocontainers/ldsc@sha256:308ddebaa643d50306779ce42752eb4c4a3e1635be74531594013959e312af2c"
-set org.label-schema.usage.singularity.version "3.10.0"
-set pkg-list "gcc-8-base, libc6, libgcc1, bash, libc-bin, libtinfo6, ncurses-base, base-files"
-
-
-# directory containing this modulefile, once symlinks resolved (dynamically defined)
-set moduleDir   [file dirname [expr { [string equal [file type ${ModulesCurrentModulefile}] "link"] ? [file readlink ${ModulesCurrentModulefile}] : ${ModulesCurrentModulefile} }]]
-
-# conflict with modules with the same alias name
-conflict ldsc
-conflict quay.io/biocontainers/ldsc:1.0.1--pyhdfd78af_2
-conflict ldsc.py
-conflict munge_sumstats.py
-conflict f2py2
-conflict f2py2.7
-conflict shiftBed
-conflict annotateBed
-conflict bamToBed
-conflict bamToFastq
-conflict bed12ToBed6
-conflict bedToBam
-conflict bedToIgv
-conflict bedpeToBam
-
-
-# singularity environment variable to set shell
-setenv SINGULARITY_SHELL /bin/sh
-
-# service environment variable to access full SIF image path
-setenv SINGULARITY_CONTAINER "${containerPath}"
-
-# interactive shell to any container, plus exec for aliases
-set shellCmd "singularity \${SINGULARITY_OPTS} shell \${SINGULARITY_COMMAND_OPTS} -s /bin/sh -B ${moduleDir}/99-shpc.sh:/.singularity.d/env/99-shpc.sh  ${containerPath}"
-set execCmd "singularity \${SINGULARITY_OPTS} exec \${SINGULARITY_COMMAND_OPTS} -B ${moduleDir}/99-shpc.sh:/.singularity.d/env/99-shpc.sh  "
-set runCmd "singularity \${SINGULARITY_OPTS} run \${SINGULARITY_COMMAND_OPTS} -B ${moduleDir}/99-shpc.sh:/.singularity.d/env/99-shpc.sh  ${containerPath}"
-set inspectCmd "singularity \${SINGULARITY_OPTS} inspect \${SINGULARITY_COMMAND_OPTS} "
-
-# if we have any wrapper scripts, add bin to path
-prepend-path PATH "${moduleDir}/bin"
-
-# "aliases" to module commands
-if { [ module-info shell bash ] } {
-  if { [ module-info mode load ] } {
- 
- 
- 
- 
- 
- 
- 
- 
- 
- 
- 
- 
-
-  }
-  if { [ module-info mode remove ] } {
- 
- 
- 
- 
- 
- 
- 
- 
- 
- 
- 
- 
-
-  }
-} else {
-
-
-
-
-
-
-
-
-
-
-
-
-
-}
-
-
-
-#=====
-# Module options
-#=====
-module-whatis "    Name: quay.io/biocontainers/ldsc:1.0.1--pyhdfd78af_2"
-module-whatis "    Version: 1.0.1--pyhdfd78af_2"
-
-
-module-whatis "    busybox: BusyBox v1.32.1 (2021-04-13 11:15:36 UTC) multi-call binary."
-module-whatis "    deb-list: gcc-8-base_8.3.0-6_amd64.deb, libc6_2.28-10_amd64.deb, libgcc1_1%3a8.3.0-6_amd64.deb, bash_5.0-4_amd64.deb, libc-bin_2.28-10_amd64.deb, libtinfo6_6.1+20181013-2+deb10u2_amd64.deb, ncurses-base_6.1+20181013-2+deb10u2_all.deb, base-files_10.3+deb10u9_amd64.deb"
-module-whatis "    glibc: GNU C Library (Debian GLIBC 2.28-10) stable release version 2.28."
-module-whatis "    io.buildah.version: 1.19.6"
-module-whatis "    org.label-schema.build-arch: amd64"
-module-whatis "    org.label-schema.build-date: Tuesday_15_August_2023_12:8:5_BST"
-module-whatis "    org.label-schema.schema-version: 1.0"
-module-whatis "    org.label-schema.usage.singularity.deffile.bootstrap: docker"
-module-whatis "    org.label-schema.usage.singularity.deffile.from: quay.io/biocontainers/ldsc@sha256:308ddebaa643d50306779ce42752eb4c4a3e1635be74531594013959e312af2c"
-module-whatis "    org.label-schema.usage.singularity.version: 3.10.0"
-module-whatis "    pkg-list: gcc-8-base, libc6, libgcc1, bash, libc-bin, libtinfo6, ncurses-base, base-files"
-
-module load /software/modules/ISG/singularity/3.10.0
\ No newline at end of file
diff --git a/tests/files/ldsc.yml b/tests/files/ldsc.yml
deleted file mode 100644
index 81a872f..0000000
--- a/tests/files/ldsc.yml
+++ /dev/null
@@ -1,54 +0,0 @@
-description: |
-  This module is a singularity container wrapper for quay.io/biocontainers/ldsc:1.0.1--pyhdfd78af_2 v1.0.1--pyhdfd78af_2
-  
-  Container (available through variable SINGULARITY_CONTAINER):
-  
-   - /software/hgi/containers/shpc/quay.io/biocontainers/ldsc/1.0.1--pyhdfd78af_2/quay.io-biocontainers-ldsc-1.0.1--pyhdfd78af_2-sha256:308ddebaa643d50306779ce42752eb4c4a3e1635be74531594013959e312af2c.sif
-  
-  Commands include:
-  
-   - ldsc-run:
-         singularity run -B <moduleDir>/99-shpc.sh:/.singularity.d/env/99-shpc.sh <container> "$@"
-   - ldsc-shell:
-         singularity shell -s /bin/sh -B <moduleDir>/99-shpc.sh:/.singularity.d/env/99-shpc.sh <container>
-   - ldsc-exec:
-         singularity exec -B <moduleDir>/99-shpc.sh:/.singularity.d/env/99-shpc.sh <container> "$@"
-   - ldsc-inspect-runscript:
-         singularity inspect -r <container>
-   - ldsc-inspect-deffile:
-         singularity inspect -d <container>
-   - ldsc-container:
-         echo "$SINGULARITY_CONTAINER"
-  
-   - ldsc.py:
-         singularity exec -B <moduleDir>/99-shpc.sh:/.singularity.d/env/99-shpc.sh <container> /usr/local/bin/ldsc.py "$@"
-   - munge_sumstats.py:
-         singularity exec -B <moduleDir>/99-shpc.sh:/.singularity.d/env/99-shpc.sh <container> /usr/local/bin/munge_sumstats.py "$@"
-   - f2py2:
-         singularity exec -B <moduleDir>/99-shpc.sh:/.singularity.d/env/99-shpc.sh <container> /usr/local/bin/f2py2 "$@"
-   - f2py2.7:
-         singularity exec -B <moduleDir>/99-shpc.sh:/.singularity.d/env/99-shpc.sh <container> /usr/local/bin/f2py2.7 "$@"
-   - shiftBed:
-         singularity exec -B <moduleDir>/99-shpc.sh:/.singularity.d/env/99-shpc.sh <container> /usr/local/bin/shiftBed "$@"
-   - annotateBed:
-         singularity exec -B <moduleDir>/99-shpc.sh:/.singularity.d/env/99-shpc.sh <container> /usr/local/bin/annotateBed "$@"
-   - bamToBed:
-         singularity exec -B <moduleDir>/99-shpc.sh:/.singularity.d/env/99-shpc.sh <container> /usr/local/bin/bamToBed "$@"
-   - bamToFastq:
-         singularity exec -B <moduleDir>/99-shpc.sh:/.singularity.d/env/99-shpc.sh <container> /usr/local/bin/bamToFastq "$@"
-   - bed12ToBed6:
-         singularity exec -B <moduleDir>/99-shpc.sh:/.singularity.d/env/99-shpc.sh <container> /usr/local/bin/bed12ToBed6 "$@"
-   - bedToBam:
-         singularity exec -B <moduleDir>/99-shpc.sh:/.singularity.d/env/99-shpc.sh <container> /usr/local/bin/bedToBam "$@"
-   - bedToIgv:
-         singularity exec -B <moduleDir>/99-shpc.sh:/.singularity.d/env/99-shpc.sh <container> /usr/local/bin/bedToIgv "$@"
-   - bedpeToBam:
-         singularity exec -B <moduleDir>/99-shpc.sh:/.singularity.d/env/99-shpc.sh <container> /usr/local/bin/bedpeToBam "$@"
-  
-  For each of the above, you can export:
-  
-   - SINGULARITY_OPTS: to define custom options for singularity (e.g., --debug)
-   - SINGULARITY_COMMAND_OPTS: to define custom options for the command (e.g., -b)
-   - SINGULARITY_CONTAINER: The Singularity (sif) path
-packages:
-  - quay.io/biocontainers/ldsc@1.0.1--pyhdfd78af_2

From 04afd847a6059ce22058e3f7558e9bbe9807cc86 Mon Sep 17 00:00:00 2001
From: Michael Woolnough <mw31@sanger.ac.uk>
Date: Tue, 5 Sep 2023 10:41:37 +0100
Subject: [PATCH 096/129] Rename file_was_pushed to better reflect intent.

---
 tests/integration/test_artifacts.py   |  6 +++---
 tests/integration/test_environment.py | 10 +++++-----
 tests/integration/utils.py            |  2 +-
 3 files changed, 9 insertions(+), 9 deletions(-)

diff --git a/tests/integration/test_artifacts.py b/tests/integration/test_artifacts.py
index bc346b7..587fb6c 100644
--- a/tests/integration/test_artifacts.py
+++ b/tests/integration/test_artifacts.py
@@ -15,7 +15,7 @@
 from tests.integration.utils import (
     commit_and_push_test_repo_changes,
     file_in_repo,
-    file_was_pushed,
+    file_in_remote,
     get_user_path_without_environments,
     new_test_artifacts,
 )
@@ -61,7 +61,7 @@ def test_commit_and_push() -> None:
     assert old_commit_oid != new_commit_oid
     assert new_commit_oid == repo_head
 
-    assert file_was_pushed(file_path)
+    assert file_in_remote(file_path)
 
 
 def add_test_file_to_repo(artifacts: Artifacts) -> tuple[pygit2.Oid, Path]:
@@ -140,7 +140,7 @@ def test_create_file() -> None:
     assert basename in [obj.name for obj in user_envs_tree[new_test_env]]
     assert user_envs_tree[new_test_env][basename].data.decode() == "override"
 
-    assert file_was_pushed(
+    assert file_in_remote(
         Path(artifacts.environments_root, folder_path, basename),
         Path(artifacts.environments_root, folder_path, basename2),
     )
diff --git a/tests/integration/test_environment.py b/tests/integration/test_environment.py
index f698852..87164f4 100644
--- a/tests/integration/test_environment.py
+++ b/tests/integration/test_environment.py
@@ -22,7 +22,7 @@
     WriteArtifactSuccess,
 )
 from tests.integration.utils import (
-    file_was_pushed,
+    file_in_remote,
     get_user_path_without_environments,
     new_test_artifacts,
 )
@@ -58,7 +58,7 @@ def test_create(httpx_post, testable_env_input: EnvironmentInput) -> None:
         testable_env_input.name,
         ".created",
     )
-    assert file_was_pushed(path)
+    assert file_in_remote(path)
 
     httpx_post.assert_called_once()
     builder_called_correctly(httpx_post, testable_env_input)
@@ -148,14 +148,14 @@ def test_delete(httpx_post, testable_env_input) -> None:
         testable_env_input.name,
         ".created",
     )
-    assert file_was_pushed(path)
+    assert file_in_remote(path)
 
     result = Environment.delete(
         testable_env_input.name, testable_env_input.path
     )
     assert isinstance(result, DeleteEnvironmentSuccess)
 
-    assert not file_was_pushed(path)
+    assert not file_in_remote(path)
 
 
 @pytest.mark.asyncio
@@ -188,7 +188,7 @@ async def test_write_artifact(httpx_post, testable_env_input, upload):
         testable_env_input.name,
         upload.filename,
     )
-    assert file_was_pushed(path)
+    assert file_in_remote(path)
 
     result = await Environment.write_artifact(
         file=upload,
diff --git a/tests/integration/utils.py b/tests/integration/utils.py
index 2fca0a2..200730c 100644
--- a/tests/integration/utils.py
+++ b/tests/integration/utils.py
@@ -166,7 +166,7 @@ def get_user_path_without_environments(
     return Path(*(artifacts.user_folder(user).parts[1:]))
 
 
-def file_was_pushed(*paths_with_environment: Union[str, Path]) -> bool:
+def file_in_remote(*paths_with_environment: Union[str, Path]) -> bool:
     temp_dir = tempfile.TemporaryDirectory()
     app.settings.artifacts.path = Path(temp_dir.name)
     artifacts = Artifacts()

From 5ad16b97fa9542e170e03ea5b2a8d6d64957fb9a Mon Sep 17 00:00:00 2001
From: Michael Woolnough <mw31@sanger.ac.uk>
Date: Tue, 5 Sep 2023 10:42:01 +0100
Subject: [PATCH 097/129] Correct softpack.yaml format.

---
 tests/integration/test_environment.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/integration/test_environment.py b/tests/integration/test_environment.py
index 87164f4..98aff12 100644
--- a/tests/integration/test_environment.py
+++ b/tests/integration/test_environment.py
@@ -213,7 +213,7 @@ async def test_iter(httpx_post, testable_env_input, upload):
 
     upload.filename = Artifacts.environments_file
     upload.content_type = "text/plain"
-    upload.read.return_value = b"description: test env\npackages:\n- zlib\n"
+    upload.read.return_value = b"description: test env\npackages:\n  - zlib\n"
 
     result = await Environment.write_artifact(
         file=upload,

From 7f8f7174ed964fb333959cbcd904205e0dc30d56 Mon Sep 17 00:00:00 2001
From: Michael Woolnough <mw31@sanger.ac.uk>
Date: Tue, 5 Sep 2023 10:59:31 +0100
Subject: [PATCH 098/129] Reformat.

---
 tests/integration/test_artifacts.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/integration/test_artifacts.py b/tests/integration/test_artifacts.py
index 587fb6c..320611c 100644
--- a/tests/integration/test_artifacts.py
+++ b/tests/integration/test_artifacts.py
@@ -14,8 +14,8 @@
 from softpack_core.artifacts import Artifacts, app
 from tests.integration.utils import (
     commit_and_push_test_repo_changes,
-    file_in_repo,
     file_in_remote,
+    file_in_repo,
     get_user_path_without_environments,
     new_test_artifacts,
 )

From 27b418ed68f441470ee3d25acb42b29f901061ab Mon Sep 17 00:00:00 2001
From: Michael Woolnough <mw31@sanger.ac.uk>
Date: Tue, 5 Sep 2023 14:36:49 +0100
Subject: [PATCH 099/129] Fix creating new envs.

---
 softpack_core/artifacts.py            |  8 +++-----
 tests/integration/test_environment.py | 16 +++++++++++++---
 2 files changed, 16 insertions(+), 8 deletions(-)

diff --git a/softpack_core/artifacts.py b/softpack_core/artifacts.py
index eeecfb8..28c8a53 100644
--- a/softpack_core/artifacts.py
+++ b/softpack_core/artifacts.py
@@ -293,16 +293,14 @@ def build_tree(
         """
         while str(path) != ".":
             try:
-                sub_tree = (
+                sub_treebuilder = repo.TreeBuilder(
                     root_tree[str(path.parent)]
                     if str(path.parent) != "."
                     else root_tree
                 )
             except KeyError:
-                raise KeyError(
-                    f"{path.parent} does not exist in the repository"
-                )
-            sub_treebuilder = repo.TreeBuilder(sub_tree)
+                sub_treebuilder = repo.TreeBuilder()
+
             sub_treebuilder.insert(
                 path.name, new_tree, pygit2.GIT_FILEMODE_TREE
             )
diff --git a/tests/integration/test_environment.py b/tests/integration/test_environment.py
index 98aff12..6240914 100644
--- a/tests/integration/test_environment.py
+++ b/tests/integration/test_environment.py
@@ -51,6 +51,19 @@ def testable_env_input(mocker) -> EnvironmentInput:
 def test_create(httpx_post, testable_env_input: EnvironmentInput) -> None:
     result = Environment.create(testable_env_input)
     assert isinstance(result, CreateEnvironmentSuccess)
+    httpx_post.assert_called_once()
+    builder_called_correctly(httpx_post, testable_env_input)
+
+    result = Environment.create(
+        EnvironmentInput(
+            name="test_env_create2",
+            path="groups/not_already_in_repo",
+            description="description2",
+            packages=[Package(name="pkg_test2")],
+        )
+    )
+    assert isinstance(result, CreateEnvironmentSuccess)
+    httpx_post.assert_called()
 
     path = Path(
         Environment.artifacts.environments_root,
@@ -60,9 +73,6 @@ def test_create(httpx_post, testable_env_input: EnvironmentInput) -> None:
     )
     assert file_in_remote(path)
 
-    httpx_post.assert_called_once()
-    builder_called_correctly(httpx_post, testable_env_input)
-
     result = Environment.create(testable_env_input)
     assert isinstance(result, EnvironmentAlreadyExistsError)
 

From 4ddb78f21ec1ac82ce0051e98300dd78af8a68e1 Mon Sep 17 00:00:00 2001
From: Michael Woolnough <mw31@sanger.ac.uk>
Date: Tue, 5 Sep 2023 15:14:49 +0100
Subject: [PATCH 100/129] Add GraphQL endpoint to recieve module file to be
 parsed into pseudo-environment.

---
 softpack_core/moduleparse.py          |  73 +++++++++
 softpack_core/schemas/environment.py  |  97 ++++++++++++
 tests/files/ldsc.module               | 210 ++++++++++++++++++++++++++
 tests/files/ldsc.yml                  |  54 +++++++
 tests/integration/test_environment.py |  33 ++++
 tests/test_moduleparse.py             |  24 +++
 6 files changed, 491 insertions(+)
 create mode 100644 softpack_core/moduleparse.py
 create mode 100644 tests/files/ldsc.module
 create mode 100644 tests/files/ldsc.yml
 create mode 100644 tests/test_moduleparse.py

diff --git a/softpack_core/moduleparse.py b/softpack_core/moduleparse.py
new file mode 100644
index 0000000..e3707cd
--- /dev/null
+++ b/softpack_core/moduleparse.py
@@ -0,0 +1,73 @@
+"""Copyright (c) 2023 Genome Research Ltd.
+This source code is licensed under the MIT license found in the
+LICENSE file in the root directory of this source tree.
+"""
+
+
+def ToSoftpackYML(contents: bytes | str) -> bytes:
+    """Converts an shpc-style module file to a softpack.yml file.
+    It should have a format similar to that produced by shpc, with `module
+    whatis` outputting a "Name: " line, a "Version: " line, and optionally a
+    "Packages: " line to say what packages are available. `module help` output
+    will be translated into the description in the softpack.yml.
+    Args:
+        contents (bytes): The byte content of the module file.
+    Returns:
+        bytes: The byte content of the softpack.yml file.
+    """
+    in_help = False
+
+    name = ""
+    version = ""
+    packages: list[str] = []
+    description = ""
+
+    if type(contents) == str:
+        contents = contents.encode()
+
+    for line in contents.splitlines():
+        line = line.lstrip()
+        if in_help:
+            if line == b"}":
+                in_help = False
+            elif line.startswith(b"puts stderr "):
+                line = (
+                    line.removeprefix(b"puts stderr ")
+                    .decode('unicode_escape')
+                    .replace("\\$", "$")
+                    .removeprefix("\"")
+                    .removesuffix("\"")
+                )
+                description += "  " + line + "\n"
+        else:
+            if line.startswith(b"proc ModulesHelp"):
+                in_help = True
+            elif line.startswith(b"module-whatis "):
+                line = (
+                    line.removeprefix(b"module-whatis ")
+                    .decode('unicode_escape')
+                    .removeprefix("\"")
+                    .removesuffix("\"")
+                    .lstrip()
+                )
+
+                if line.startswith("Name: "):
+                    nv = line.removeprefix("Name: ").split(":")
+                    name = nv[0]
+                    if len(nv) > 1:
+                        version = nv[1]
+                elif line.startswith("Version: "):
+                    version = line.removeprefix("Version: ")
+                elif line.startswith("Packages: "):
+                    packages = line.removeprefix("Packages: ").split(", ")
+
+    if version != "":
+        name += f"@{version}"
+
+    packages.insert(0, name)
+
+    package_str = "\n  - ".join(packages)
+
+    return (
+        f"description: |\n{description}packages:\n  - {package_str}\n".encode()
+    )
diff --git a/softpack_core/schemas/environment.py b/softpack_core/schemas/environment.py
index 9e9e937..2e32581 100644
--- a/softpack_core/schemas/environment.py
+++ b/softpack_core/schemas/environment.py
@@ -5,14 +5,17 @@
 """
 
 from dataclasses import dataclass
+import io
 from pathlib import Path
 from typing import Iterable, Optional
 
 import httpx
 import strawberry
 from strawberry.file_uploads import Upload
+from starlette.datastructures import UploadFile
 
 from softpack_core.artifacts import Artifacts
+from softpack_core.moduleparse import ToSoftpackYML
 from softpack_core.schemas.base import BaseSchema
 from softpack_core.spack import Spack
 
@@ -358,6 +361,97 @@ def delete(cls, name: str, path: str) -> DeleteResponse:  # type: ignore
             name=name,
         )
 
+    @classmethod
+    async def create_from_module(
+        cls, file: Upload, module_path: str, environment_path: str
+    ) -> CreateResponse:
+        """Create an Environment based on an existing module.
+        The environment will not be built; a "fake" softpack.yml and the
+        supplied module file will be written as artifacts in a newly created
+        environment instead, so that they can be discovered.
+        Args:
+            file: the module file to add to the repo, and to parse to fake a
+                  corresponding softpack.yml. It should have a format similar
+                  to that produced by shpc, with `module whatis` outputting
+                  a "Name: " line, a "Version: " line, and optionally a
+                  "Packages: " line to say what packages are available.
+                  `module help` output will be translated into the description
+                  in the softpack.yml.
+            module_path: the local path that users can `module load` - this is
+                         used to auto-generate usage help text for this
+                         environment.
+            environment_path: the subdirectories of environments folder that
+                              artifacts will be stored in, eg.
+                              users/username/software_name
+        Returns:
+            A message confirming the success or failure of the operation.
+        """
+        environment_dirs = environment_path.split("/")
+        environment_name = environment_dirs.pop()
+
+        contents = await file.read()
+        yml = ToSoftpackYML(contents)
+
+        env = EnvironmentInput(
+            name=environment_name,
+            path="/".join(environment_dirs),
+            description="",
+            packages=(),
+        )
+
+        response = cls.create_new_env(env)
+        if not isinstance(response, CreateEnvironmentSuccess):
+            return response
+
+        module_file = UploadFile(file=io.BytesIO(contents))
+        softpack_file = UploadFile(file=io.BytesIO(yml))
+
+        result = await cls.write_module_artifacts(
+            module_file=module_file,
+            softpack_file=softpack_file,
+            environment_path=environment_path,
+        )
+
+        if not isinstance(result, WriteArtifactSuccess):
+            cls.delete(name=environment_name, path=environment_path)
+            return InvalidInputError(
+                message="Write of module file failed: " + result.message
+            )
+
+        return CreateEnvironmentSuccess(
+            message="Successfully created environment in artifacts repo"
+        )
+
+    @classmethod
+    async def write_module_artifacts(
+        cls, module_file: Upload, softpack_file: Upload, environment_path: str
+    ) -> WriteArtifactResponse:
+        """Writes the given module and softpack files to the artifacts repo.
+        Args:
+            module_file (Upload): An shpc-style module file.
+            softpack_file (Upload): A "fake" softpack.yml file describing what
+            the module file offers.
+            environment_path (str): Path to the environment, eg.
+            users/user/env.
+        Returns:
+            WriteArtifactResponse: contains message and commit hash of
+            softpack.yml upload.
+        """
+        result = await cls.write_artifact(
+            file=module_file,
+            folder_path=environment_path,
+            file_name=cls.artifacts.module_file,
+        )
+
+        if not isinstance(result, WriteArtifactSuccess):
+            return result
+
+        return await cls.write_artifact(
+            file=softpack_file,
+            folder_path=environment_path,
+            file_name=cls.artifacts.environments_file,
+        )
+
     @classmethod
     async def write_artifact(
         cls, file: Upload, folder_path: str, file_name: str
@@ -405,3 +499,6 @@ class Mutation:
         writeArtifact: WriteArtifactResponse = (  # type: ignore
             Environment.write_artifact
         )
+        createFromModule: CreateResponse = (  # type: ignore
+            Environment.create_from_module
+        )
diff --git a/tests/files/ldsc.module b/tests/files/ldsc.module
new file mode 100644
index 0000000..0feb40c
--- /dev/null
+++ b/tests/files/ldsc.module
@@ -0,0 +1,210 @@
+#%Module
+
+#=====
+# Created by singularity-hpc (https://github.com/singularityhub/singularity-hpc)
+# ##
+# quay.io/biocontainers/ldsc:1.0.1--pyhdfd78af_2 on 2023-08-15 12:08:41.851818
+#=====
+
+proc ModulesHelp { } {
+
+    puts stderr "This module is a singularity container wrapper for quay.io/biocontainers/ldsc:1.0.1--pyhdfd78af_2 v1.0.1--pyhdfd78af_2"
+    
+    puts stderr ""
+    puts stderr "Container (available through variable SINGULARITY_CONTAINER):"
+    puts stderr ""
+    puts stderr " - /software/hgi/containers/shpc/quay.io/biocontainers/ldsc/1.0.1--pyhdfd78af_2/quay.io-biocontainers-ldsc-1.0.1--pyhdfd78af_2-sha256:308ddebaa643d50306779ce42752eb4c4a3e1635be74531594013959e312af2c.sif"
+    puts stderr ""
+    puts stderr "Commands include:"
+    puts stderr ""
+    puts stderr " - ldsc-run:"
+    puts stderr "       singularity run -B <moduleDir>/99-shpc.sh:/.singularity.d/env/99-shpc.sh <container> \"\$@\""
+    puts stderr " - ldsc-shell:"
+    puts stderr "       singularity shell -s /bin/sh -B <moduleDir>/99-shpc.sh:/.singularity.d/env/99-shpc.sh <container>"
+    puts stderr " - ldsc-exec:"
+    puts stderr "       singularity exec -B <moduleDir>/99-shpc.sh:/.singularity.d/env/99-shpc.sh <container> \"\$@\""
+    puts stderr " - ldsc-inspect-runscript:"
+    puts stderr "       singularity inspect -r <container>"
+    puts stderr " - ldsc-inspect-deffile:"
+    puts stderr "       singularity inspect -d <container>"
+    puts stderr " - ldsc-container:"
+    puts stderr "       echo \"\$SINGULARITY_CONTAINER\""
+    puts stderr ""
+    puts stderr " - ldsc.py:"
+    puts stderr "       singularity exec -B <moduleDir>/99-shpc.sh:/.singularity.d/env/99-shpc.sh <container> /usr/local/bin/ldsc.py \"\$@\""
+    puts stderr " - munge_sumstats.py:"
+    puts stderr "       singularity exec -B <moduleDir>/99-shpc.sh:/.singularity.d/env/99-shpc.sh <container> /usr/local/bin/munge_sumstats.py \"\$@\""
+    puts stderr " - f2py2:"
+    puts stderr "       singularity exec -B <moduleDir>/99-shpc.sh:/.singularity.d/env/99-shpc.sh <container> /usr/local/bin/f2py2 \"\$@\""
+    puts stderr " - f2py2.7:"
+    puts stderr "       singularity exec -B <moduleDir>/99-shpc.sh:/.singularity.d/env/99-shpc.sh <container> /usr/local/bin/f2py2.7 \"\$@\""
+    puts stderr " - shiftBed:"
+    puts stderr "       singularity exec -B <moduleDir>/99-shpc.sh:/.singularity.d/env/99-shpc.sh <container> /usr/local/bin/shiftBed \"\$@\""
+    puts stderr " - annotateBed:"
+    puts stderr "       singularity exec -B <moduleDir>/99-shpc.sh:/.singularity.d/env/99-shpc.sh <container> /usr/local/bin/annotateBed \"\$@\""
+    puts stderr " - bamToBed:"
+    puts stderr "       singularity exec -B <moduleDir>/99-shpc.sh:/.singularity.d/env/99-shpc.sh <container> /usr/local/bin/bamToBed \"\$@\""
+    puts stderr " - bamToFastq:"
+    puts stderr "       singularity exec -B <moduleDir>/99-shpc.sh:/.singularity.d/env/99-shpc.sh <container> /usr/local/bin/bamToFastq \"\$@\""
+    puts stderr " - bed12ToBed6:"
+    puts stderr "       singularity exec -B <moduleDir>/99-shpc.sh:/.singularity.d/env/99-shpc.sh <container> /usr/local/bin/bed12ToBed6 \"\$@\""
+    puts stderr " - bedToBam:"
+    puts stderr "       singularity exec -B <moduleDir>/99-shpc.sh:/.singularity.d/env/99-shpc.sh <container> /usr/local/bin/bedToBam \"\$@\""
+    puts stderr " - bedToIgv:"
+    puts stderr "       singularity exec -B <moduleDir>/99-shpc.sh:/.singularity.d/env/99-shpc.sh <container> /usr/local/bin/bedToIgv \"\$@\""
+    puts stderr " - bedpeToBam:"
+    puts stderr "       singularity exec -B <moduleDir>/99-shpc.sh:/.singularity.d/env/99-shpc.sh <container> /usr/local/bin/bedpeToBam \"\$@\""
+
+    puts stderr ""
+    puts stderr "For each of the above, you can export:"
+    puts stderr ""
+    puts stderr " - SINGULARITY_OPTS: to define custom options for singularity (e.g., --debug)"
+    puts stderr " - SINGULARITY_COMMAND_OPTS: to define custom options for the command (e.g., -b)"
+    puts stderr " - SINGULARITY_CONTAINER: The Singularity (sif) path"
+
+}
+
+set view_dir "[file dirname [file dirname ${ModulesCurrentModulefile}] ]"
+set view_name "[file tail ${view_dir}]"
+set view_module ".view_module"
+set view_modulefile "${view_dir}/${view_module}"
+
+if {[file exists ${view_modulefile}]} {
+    source ${view_modulefile}
+}
+
+# Environment - only set if not already defined
+if { ![info exists ::env(SINGULARITY_OPTS)] } {
+    setenv SINGULARITY_OPTS ""
+}
+if { ![info exists ::env(SINGULARITY_COMMAND_OPTS)] } {
+    setenv SINGULARITY_COMMAND_OPTS ""
+}
+
+# Variables
+
+set name        quay.io/biocontainers/ldsc:1.0.1--pyhdfd78af_2
+set version     1.0.1--pyhdfd78af_2
+set description "$name - $version"
+set containerPath /software/hgi/containers/shpc/quay.io/biocontainers/ldsc/1.0.1--pyhdfd78af_2/quay.io-biocontainers-ldsc-1.0.1--pyhdfd78af_2-sha256:308ddebaa643d50306779ce42752eb4c4a3e1635be74531594013959e312af2c.sif
+
+
+set helpcommand "This module is a singularity container wrapper for quay.io/biocontainers/ldsc:1.0.1--pyhdfd78af_2 v1.0.1--pyhdfd78af_2. "
+set busybox "BusyBox v1.32.1 (2021-04-13 11:15:36 UTC) multi-call binary."
+set deb-list "gcc-8-base_8.3.0-6_amd64.deb, libc6_2.28-10_amd64.deb, libgcc1_1%3a8.3.0-6_amd64.deb, bash_5.0-4_amd64.deb, libc-bin_2.28-10_amd64.deb, libtinfo6_6.1+20181013-2+deb10u2_amd64.deb, ncurses-base_6.1+20181013-2+deb10u2_all.deb, base-files_10.3+deb10u9_amd64.deb"
+set glibc "GNU C Library (Debian GLIBC 2.28-10) stable release version 2.28."
+set io.buildah.version "1.19.6"
+set org.label-schema.build-arch "amd64"
+set org.label-schema.build-date "Tuesday_15_August_2023_12:8:5_BST"
+set org.label-schema.schema-version "1.0"
+set org.label-schema.usage.singularity.deffile.bootstrap "docker"
+set org.label-schema.usage.singularity.deffile.from "quay.io/biocontainers/ldsc@sha256:308ddebaa643d50306779ce42752eb4c4a3e1635be74531594013959e312af2c"
+set org.label-schema.usage.singularity.version "3.10.0"
+set pkg-list "gcc-8-base, libc6, libgcc1, bash, libc-bin, libtinfo6, ncurses-base, base-files"
+
+
+# directory containing this modulefile, once symlinks resolved (dynamically defined)
+set moduleDir   [file dirname [expr { [string equal [file type ${ModulesCurrentModulefile}] "link"] ? [file readlink ${ModulesCurrentModulefile}] : ${ModulesCurrentModulefile} }]]
+
+# conflict with modules with the same alias name
+conflict ldsc
+conflict quay.io/biocontainers/ldsc:1.0.1--pyhdfd78af_2
+conflict ldsc.py
+conflict munge_sumstats.py
+conflict f2py2
+conflict f2py2.7
+conflict shiftBed
+conflict annotateBed
+conflict bamToBed
+conflict bamToFastq
+conflict bed12ToBed6
+conflict bedToBam
+conflict bedToIgv
+conflict bedpeToBam
+
+
+# singularity environment variable to set shell
+setenv SINGULARITY_SHELL /bin/sh
+
+# service environment variable to access full SIF image path
+setenv SINGULARITY_CONTAINER "${containerPath}"
+
+# interactive shell to any container, plus exec for aliases
+set shellCmd "singularity \${SINGULARITY_OPTS} shell \${SINGULARITY_COMMAND_OPTS} -s /bin/sh -B ${moduleDir}/99-shpc.sh:/.singularity.d/env/99-shpc.sh  ${containerPath}"
+set execCmd "singularity \${SINGULARITY_OPTS} exec \${SINGULARITY_COMMAND_OPTS} -B ${moduleDir}/99-shpc.sh:/.singularity.d/env/99-shpc.sh  "
+set runCmd "singularity \${SINGULARITY_OPTS} run \${SINGULARITY_COMMAND_OPTS} -B ${moduleDir}/99-shpc.sh:/.singularity.d/env/99-shpc.sh  ${containerPath}"
+set inspectCmd "singularity \${SINGULARITY_OPTS} inspect \${SINGULARITY_COMMAND_OPTS} "
+
+# if we have any wrapper scripts, add bin to path
+prepend-path PATH "${moduleDir}/bin"
+
+# "aliases" to module commands
+if { [ module-info shell bash ] } {
+  if { [ module-info mode load ] } {
+ 
+ 
+ 
+ 
+ 
+ 
+ 
+ 
+ 
+ 
+ 
+ 
+
+  }
+  if { [ module-info mode remove ] } {
+ 
+ 
+ 
+ 
+ 
+ 
+ 
+ 
+ 
+ 
+ 
+ 
+
+  }
+} else {
+
+
+
+
+
+
+
+
+
+
+
+
+
+}
+
+
+
+#=====
+# Module options
+#=====
+module-whatis "    Name: quay.io/biocontainers/ldsc:1.0.1--pyhdfd78af_2"
+module-whatis "    Version: 1.0.1--pyhdfd78af_2"
+
+
+module-whatis "    busybox: BusyBox v1.32.1 (2021-04-13 11:15:36 UTC) multi-call binary."
+module-whatis "    deb-list: gcc-8-base_8.3.0-6_amd64.deb, libc6_2.28-10_amd64.deb, libgcc1_1%3a8.3.0-6_amd64.deb, bash_5.0-4_amd64.deb, libc-bin_2.28-10_amd64.deb, libtinfo6_6.1+20181013-2+deb10u2_amd64.deb, ncurses-base_6.1+20181013-2+deb10u2_all.deb, base-files_10.3+deb10u9_amd64.deb"
+module-whatis "    glibc: GNU C Library (Debian GLIBC 2.28-10) stable release version 2.28."
+module-whatis "    io.buildah.version: 1.19.6"
+module-whatis "    org.label-schema.build-arch: amd64"
+module-whatis "    org.label-schema.build-date: Tuesday_15_August_2023_12:8:5_BST"
+module-whatis "    org.label-schema.schema-version: 1.0"
+module-whatis "    org.label-schema.usage.singularity.deffile.bootstrap: docker"
+module-whatis "    org.label-schema.usage.singularity.deffile.from: quay.io/biocontainers/ldsc@sha256:308ddebaa643d50306779ce42752eb4c4a3e1635be74531594013959e312af2c"
+module-whatis "    org.label-schema.usage.singularity.version: 3.10.0"
+module-whatis "    pkg-list: gcc-8-base, libc6, libgcc1, bash, libc-bin, libtinfo6, ncurses-base, base-files"
+
+module load /software/modules/ISG/singularity/3.10.0
\ No newline at end of file
diff --git a/tests/files/ldsc.yml b/tests/files/ldsc.yml
new file mode 100644
index 0000000..81a872f
--- /dev/null
+++ b/tests/files/ldsc.yml
@@ -0,0 +1,54 @@
+description: |
+  This module is a singularity container wrapper for quay.io/biocontainers/ldsc:1.0.1--pyhdfd78af_2 v1.0.1--pyhdfd78af_2
+  
+  Container (available through variable SINGULARITY_CONTAINER):
+  
+   - /software/hgi/containers/shpc/quay.io/biocontainers/ldsc/1.0.1--pyhdfd78af_2/quay.io-biocontainers-ldsc-1.0.1--pyhdfd78af_2-sha256:308ddebaa643d50306779ce42752eb4c4a3e1635be74531594013959e312af2c.sif
+  
+  Commands include:
+  
+   - ldsc-run:
+         singularity run -B <moduleDir>/99-shpc.sh:/.singularity.d/env/99-shpc.sh <container> "$@"
+   - ldsc-shell:
+         singularity shell -s /bin/sh -B <moduleDir>/99-shpc.sh:/.singularity.d/env/99-shpc.sh <container>
+   - ldsc-exec:
+         singularity exec -B <moduleDir>/99-shpc.sh:/.singularity.d/env/99-shpc.sh <container> "$@"
+   - ldsc-inspect-runscript:
+         singularity inspect -r <container>
+   - ldsc-inspect-deffile:
+         singularity inspect -d <container>
+   - ldsc-container:
+         echo "$SINGULARITY_CONTAINER"
+  
+   - ldsc.py:
+         singularity exec -B <moduleDir>/99-shpc.sh:/.singularity.d/env/99-shpc.sh <container> /usr/local/bin/ldsc.py "$@"
+   - munge_sumstats.py:
+         singularity exec -B <moduleDir>/99-shpc.sh:/.singularity.d/env/99-shpc.sh <container> /usr/local/bin/munge_sumstats.py "$@"
+   - f2py2:
+         singularity exec -B <moduleDir>/99-shpc.sh:/.singularity.d/env/99-shpc.sh <container> /usr/local/bin/f2py2 "$@"
+   - f2py2.7:
+         singularity exec -B <moduleDir>/99-shpc.sh:/.singularity.d/env/99-shpc.sh <container> /usr/local/bin/f2py2.7 "$@"
+   - shiftBed:
+         singularity exec -B <moduleDir>/99-shpc.sh:/.singularity.d/env/99-shpc.sh <container> /usr/local/bin/shiftBed "$@"
+   - annotateBed:
+         singularity exec -B <moduleDir>/99-shpc.sh:/.singularity.d/env/99-shpc.sh <container> /usr/local/bin/annotateBed "$@"
+   - bamToBed:
+         singularity exec -B <moduleDir>/99-shpc.sh:/.singularity.d/env/99-shpc.sh <container> /usr/local/bin/bamToBed "$@"
+   - bamToFastq:
+         singularity exec -B <moduleDir>/99-shpc.sh:/.singularity.d/env/99-shpc.sh <container> /usr/local/bin/bamToFastq "$@"
+   - bed12ToBed6:
+         singularity exec -B <moduleDir>/99-shpc.sh:/.singularity.d/env/99-shpc.sh <container> /usr/local/bin/bed12ToBed6 "$@"
+   - bedToBam:
+         singularity exec -B <moduleDir>/99-shpc.sh:/.singularity.d/env/99-shpc.sh <container> /usr/local/bin/bedToBam "$@"
+   - bedToIgv:
+         singularity exec -B <moduleDir>/99-shpc.sh:/.singularity.d/env/99-shpc.sh <container> /usr/local/bin/bedToIgv "$@"
+   - bedpeToBam:
+         singularity exec -B <moduleDir>/99-shpc.sh:/.singularity.d/env/99-shpc.sh <container> /usr/local/bin/bedpeToBam "$@"
+  
+  For each of the above, you can export:
+  
+   - SINGULARITY_OPTS: to define custom options for singularity (e.g., --debug)
+   - SINGULARITY_COMMAND_OPTS: to define custom options for the command (e.g., -b)
+   - SINGULARITY_CONTAINER: The Singularity (sif) path
+packages:
+  - quay.io/biocontainers/ldsc@1.0.1--pyhdfd78af_2
diff --git a/tests/integration/test_environment.py b/tests/integration/test_environment.py
index 6240914..a77c1b2 100644
--- a/tests/integration/test_environment.py
+++ b/tests/integration/test_environment.py
@@ -240,3 +240,36 @@ async def test_iter(httpx_post, testable_env_input, upload):
         count += 1
 
     assert count == 1
+
+
+@pytest.mark.asyncio
+async def test_create_from_module(httpx_post, testable_env_input, upload):
+    test_file_path = Path(Path(__file__).parent.parent, "files", "ldsc.module")
+
+    with open(test_file_path, "rb") as fh:
+        upload.filename = "ldsc.module"
+        upload.content_type = "text/plain"
+        upload.read.return_value = fh.read()
+
+    env_name = "some-environment"
+    name = "groups/hgi/" + env_name
+    module_path = "HGI/common/some_environment"
+
+    result = await Environment.create_from_module(
+        file=upload,
+        module_path=module_path,
+        environment_path=name,
+    )
+
+    assert isinstance(result, CreateEnvironmentSuccess)
+
+    parent_path = Path(
+        Environment.artifacts.group_folder(),
+        "hgi",
+        env_name,
+    )
+
+    assert file_in_remote(
+        Path(parent_path, Environment.artifacts.environments_file),
+        Path(parent_path, Environment.artifacts.module_file),
+    )
diff --git a/tests/test_moduleparse.py b/tests/test_moduleparse.py
new file mode 100644
index 0000000..d6ef18e
--- /dev/null
+++ b/tests/test_moduleparse.py
@@ -0,0 +1,24 @@
+"""Copyright (c) 2023 Genome Research Ltd.
+This source code is licensed under the MIT license found in the
+LICENSE file in the root directory of this source tree.
+"""
+
+from pathlib import Path
+
+import pytest
+
+from softpack_core.moduleparse import ToSoftpackYML
+
+
+def test_tosoftpack() -> None:
+    test_files_dir = Path(Path(__file__).parent, "files")
+
+    with open(Path(test_files_dir, "ldsc.module"), "rb") as fh:
+        module_data = fh.read()
+
+    with open(Path(test_files_dir, "ldsc.yml"), "rb") as fh:
+        expected_yml = fh.read()
+
+    yml = ToSoftpackYML(module_data)
+
+    assert yml == expected_yml

From baaec719cbd464d039c2a5ab8b1d081cf42a47a6 Mon Sep 17 00:00:00 2001
From: Michael Woolnough <mw31@sanger.ac.uk>
Date: Tue, 5 Sep 2023 16:24:50 +0100
Subject: [PATCH 101/129] Allow tests to be run with additional flags (such as
 --repo).

---
 README.md | 7 ++++++-
 tox.ini   | 2 +-
 2 files changed, 7 insertions(+), 2 deletions(-)

diff --git a/README.md b/README.md
index 204c93b..6523460 100644
--- a/README.md
+++ b/README.md
@@ -120,7 +120,12 @@ artifacts:
     writer: [your-token]
 ```
 
-Then enable the integration tests by suppling --repo to `poetry run pytest`.
+Then enable the integration tests by suppling --repo to `poetry run pytest`, or
+to tox like this:
+
+```
+poetry run tox -- -- --repo
+```
 
 To discover all tests and run them (skipping integration tests with no --repo):
 
diff --git a/tox.ini b/tox.ini
index 3ccaa24..bb1214b 100644
--- a/tox.ini
+++ b/tox.ini
@@ -63,7 +63,7 @@ commands =
         --cov-branch \
         --cov-report=xml \
         --cov-report=term-missing \
-        tests
+        tests {posargs}
 
 [testenv:format]
 skip_install = true

From 0f9f7aef83e7596d345a46d01c0e6d075bca8724 Mon Sep 17 00:00:00 2001
From: Michael Woolnough <mw31@sanger.ac.uk>
Date: Tue, 5 Sep 2023 16:25:22 +0100
Subject: [PATCH 102/129] De-linted.

---
 softpack_core/moduleparse.py         | 34 ++++++++++++++++++----------
 softpack_core/schemas/environment.py | 15 ++++++++----
 tests/test_moduleparse.py            |  3 +--
 3 files changed, 33 insertions(+), 19 deletions(-)

diff --git a/softpack_core/moduleparse.py b/softpack_core/moduleparse.py
index e3707cd..12cd1de 100644
--- a/softpack_core/moduleparse.py
+++ b/softpack_core/moduleparse.py
@@ -1,17 +1,23 @@
 """Copyright (c) 2023 Genome Research Ltd.
+
 This source code is licensed under the MIT license found in the
 LICENSE file in the root directory of this source tree.
 """
 
+from typing import Union, cast
+
 
-def ToSoftpackYML(contents: bytes | str) -> bytes:
+def ToSoftpackYML(contents: Union[bytes, str]) -> bytes:
     """Converts an shpc-style module file to a softpack.yml file.
+
     It should have a format similar to that produced by shpc, with `module
     whatis` outputting a "Name: " line, a "Version: " line, and optionally a
     "Packages: " line to say what packages are available. `module help` output
     will be translated into the description in the softpack.yml.
+
     Args:
         contents (bytes): The byte content of the module file.
+
     Returns:
         bytes: The byte content of the softpack.yml file.
     """
@@ -22,28 +28,32 @@ def ToSoftpackYML(contents: bytes | str) -> bytes:
     packages: list[str] = []
     description = ""
 
+    contents_bytes: bytes
+
     if type(contents) == str:
-        contents = contents.encode()
+        contents_bytes = contents.encode()
+    else:
+        contents_bytes = cast(bytes, contents)
 
-    for line in contents.splitlines():
+    for line in contents_bytes.splitlines():
         line = line.lstrip()
         if in_help:
             if line == b"}":
                 in_help = False
             elif line.startswith(b"puts stderr "):
-                line = (
+                line_str = (
                     line.removeprefix(b"puts stderr ")
                     .decode('unicode_escape')
                     .replace("\\$", "$")
                     .removeprefix("\"")
                     .removesuffix("\"")
                 )
-                description += "  " + line + "\n"
+                description += "  " + line_str + "\n"
         else:
             if line.startswith(b"proc ModulesHelp"):
                 in_help = True
             elif line.startswith(b"module-whatis "):
-                line = (
+                line_str = (
                     line.removeprefix(b"module-whatis ")
                     .decode('unicode_escape')
                     .removeprefix("\"")
@@ -51,15 +61,15 @@ def ToSoftpackYML(contents: bytes | str) -> bytes:
                     .lstrip()
                 )
 
-                if line.startswith("Name: "):
-                    nv = line.removeprefix("Name: ").split(":")
+                if line_str.startswith("Name: "):
+                    nv = line_str.removeprefix("Name: ").split(":")
                     name = nv[0]
                     if len(nv) > 1:
                         version = nv[1]
-                elif line.startswith("Version: "):
-                    version = line.removeprefix("Version: ")
-                elif line.startswith("Packages: "):
-                    packages = line.removeprefix("Packages: ").split(", ")
+                elif line_str.startswith("Version: "):
+                    version = line_str.removeprefix("Version: ")
+                elif line_str.startswith("Packages: "):
+                    packages = line_str.removeprefix("Packages: ").split(", ")
 
     if version != "":
         name += f"@{version}"
diff --git a/softpack_core/schemas/environment.py b/softpack_core/schemas/environment.py
index 2e32581..c17fa8e 100644
--- a/softpack_core/schemas/environment.py
+++ b/softpack_core/schemas/environment.py
@@ -4,15 +4,15 @@
 LICENSE file in the root directory of this source tree.
 """
 
-from dataclasses import dataclass
 import io
+from dataclasses import dataclass
 from pathlib import Path
 from typing import Iterable, Optional
 
 import httpx
 import strawberry
-from strawberry.file_uploads import Upload
 from starlette.datastructures import UploadFile
+from strawberry.file_uploads import Upload
 
 from softpack_core.artifacts import Artifacts
 from softpack_core.moduleparse import ToSoftpackYML
@@ -364,11 +364,13 @@ def delete(cls, name: str, path: str) -> DeleteResponse:  # type: ignore
     @classmethod
     async def create_from_module(
         cls, file: Upload, module_path: str, environment_path: str
-    ) -> CreateResponse:
+    ) -> CreateResponse:  # type: ignore
         """Create an Environment based on an existing module.
+
         The environment will not be built; a "fake" softpack.yml and the
         supplied module file will be written as artifacts in a newly created
         environment instead, so that they can be discovered.
+
         Args:
             file: the module file to add to the repo, and to parse to fake a
                   corresponding softpack.yml. It should have a format similar
@@ -383,6 +385,7 @@ async def create_from_module(
             environment_path: the subdirectories of environments folder that
                               artifacts will be stored in, eg.
                               users/username/software_name
+
         Returns:
             A message confirming the success or failure of the operation.
         """
@@ -396,7 +399,7 @@ async def create_from_module(
             name=environment_name,
             path="/".join(environment_dirs),
             description="",
-            packages=(),
+            packages=list(),
         )
 
         response = cls.create_new_env(env)
@@ -425,14 +428,16 @@ async def create_from_module(
     @classmethod
     async def write_module_artifacts(
         cls, module_file: Upload, softpack_file: Upload, environment_path: str
-    ) -> WriteArtifactResponse:
+    ) -> WriteArtifactResponse:  # type: ignore
         """Writes the given module and softpack files to the artifacts repo.
+
         Args:
             module_file (Upload): An shpc-style module file.
             softpack_file (Upload): A "fake" softpack.yml file describing what
             the module file offers.
             environment_path (str): Path to the environment, eg.
             users/user/env.
+
         Returns:
             WriteArtifactResponse: contains message and commit hash of
             softpack.yml upload.
diff --git a/tests/test_moduleparse.py b/tests/test_moduleparse.py
index d6ef18e..218f0ef 100644
--- a/tests/test_moduleparse.py
+++ b/tests/test_moduleparse.py
@@ -1,12 +1,11 @@
 """Copyright (c) 2023 Genome Research Ltd.
+
 This source code is licensed under the MIT license found in the
 LICENSE file in the root directory of this source tree.
 """
 
 from pathlib import Path
 
-import pytest
-
 from softpack_core.moduleparse import ToSoftpackYML
 
 

From 831923719f8b17e103ccdf4112f74f6b109456cb Mon Sep 17 00:00:00 2001
From: Michael Woolnough <mw31@sanger.ac.uk>
Date: Tue, 5 Sep 2023 17:00:25 +0100
Subject: [PATCH 103/129] WIP: Add parameterised tests for ToSoftPack.

---
 tests/data/specs/modules/tests.yml |  6 +++++
 tests/test_moduleparse.py          | 36 +++++++++++++++++++++++-------
 2 files changed, 34 insertions(+), 8 deletions(-)
 create mode 100644 tests/data/specs/modules/tests.yml

diff --git a/tests/data/specs/modules/tests.yml b/tests/data/specs/modules/tests.yml
new file mode 100644
index 0000000..dcb3c0e
--- /dev/null
+++ b/tests/data/specs/modules/tests.yml
@@ -0,0 +1,6 @@
+tests:
+  - path: data/specs/env-a.yml
+    expect: CreateEnvironmentSuccess
+  
+  - path: data/specs/env-b.yml
+    expect: dfdffddfdEnvironmentSuccess
\ No newline at end of file
diff --git a/tests/test_moduleparse.py b/tests/test_moduleparse.py
index 218f0ef..ca45120 100644
--- a/tests/test_moduleparse.py
+++ b/tests/test_moduleparse.py
@@ -6,18 +6,38 @@
 
 from pathlib import Path
 
+import yaml
+
 from softpack_core.moduleparse import ToSoftpackYML
 
 
-def test_tosoftpack() -> None:
-    test_files_dir = Path(Path(__file__).parent, "files")
+def pytest_generate_tests(metafunc):
+    if "module_spec" not in metafunc.fixturenames:
+        return
 
-    with open(Path(test_files_dir, "ldsc.module"), "rb") as fh:
-        module_data = fh.read()
+    with open(Path(__file__).parent / "data/specs/modules/tests.yml") as f:
+        yml = yaml.safe_load(f)
+
+    metafunc.parametrize(
+        "module_spec",
+        yml["tests"],
+    )
 
-    with open(Path(test_files_dir, "ldsc.yml"), "rb") as fh:
-        expected_yml = fh.read()
 
-    yml = ToSoftpackYML(module_data)
+def test_tosoftpack(module_spec) -> None:
+    path = module_spec["path"]
+    output = module_spec.get("output")
+    fail_message = module_spec.get("fail")
 
-    assert yml == expected_yml
+    with open(Path(Path(__file__).parent, path), "rb") as fh:
+        module_data = fh.read()
+
+    try:
+        yml = ToSoftpackYML(module_data)
+    except any as e:
+        assert e.message == fail_message
+        return
+
+    with open(Path(Path(__file__).parent, output), "rb") as fh:
+        expected_yml = fh.read()
+        assert yml == expected_yml

From cad405e822cbcfa6784db38ecd1271a921d2fc36 Mon Sep 17 00:00:00 2001
From: Michael Woolnough <mw31@sanger.ac.uk>
Date: Wed, 6 Sep 2023 10:20:24 +0100
Subject: [PATCH 104/129] Test edge cases for module parser.

---
 softpack_core/moduleparse.py                  | 52 ++++++++++++++-----
 softpack_core/schemas/environment.py          |  2 +-
 tests/data/specs/modules/tests.yml            |  6 ---
 tests/files/modules/all_fields.mod            | 17 ++++++
 tests/files/modules/all_fields.yml            | 10 ++++
 tests/files/modules/bad_name.mod              | 17 ++++++
 tests/files/modules/bad_name.yml              | 10 ++++
 tests/files/modules/bad_packages.mod          | 17 ++++++
 tests/files/modules/bad_packages.yml          | 11 ++++
 tests/files/modules/bad_version.mod           | 17 ++++++
 tests/files/modules/bad_version.yml           | 10 ++++
 tests/files/modules/empty_name.mod            | 17 ++++++
 tests/files/modules/empty_name.yml            | 10 ++++
 tests/files/modules/minimal.mod               |  0
 tests/files/modules/minimal.yml               |  3 ++
 tests/files/modules/no_description.mod        |  7 +++
 tests/files/modules/no_description.yml        |  3 ++
 tests/files/modules/no_name.mod               | 12 +++++
 tests/files/modules/no_name.yml               |  6 +++
 tests/files/modules/no_pkgs.mod               | 15 ++++++
 tests/files/modules/no_pkgs.yml               |  6 +++
 tests/files/modules/no_version.mod            | 14 +++++
 tests/files/modules/no_version.yml            |  6 +++
 tests/files/{ldsc.module => modules/shpc.mod} |  0
 tests/files/{ldsc.yml => modules/shpc.yml}    |  0
 tests/files/modules/version_in_name.mod       | 15 ++++++
 tests/files/modules/version_in_name.yml       |  6 +++
 tests/test_moduleparse.py                     | 25 +++------
 28 files changed, 277 insertions(+), 37 deletions(-)
 delete mode 100644 tests/data/specs/modules/tests.yml
 create mode 100644 tests/files/modules/all_fields.mod
 create mode 100644 tests/files/modules/all_fields.yml
 create mode 100644 tests/files/modules/bad_name.mod
 create mode 100644 tests/files/modules/bad_name.yml
 create mode 100644 tests/files/modules/bad_packages.mod
 create mode 100644 tests/files/modules/bad_packages.yml
 create mode 100644 tests/files/modules/bad_version.mod
 create mode 100644 tests/files/modules/bad_version.yml
 create mode 100644 tests/files/modules/empty_name.mod
 create mode 100644 tests/files/modules/empty_name.yml
 create mode 100644 tests/files/modules/minimal.mod
 create mode 100644 tests/files/modules/minimal.yml
 create mode 100644 tests/files/modules/no_description.mod
 create mode 100644 tests/files/modules/no_description.yml
 create mode 100644 tests/files/modules/no_name.mod
 create mode 100644 tests/files/modules/no_name.yml
 create mode 100644 tests/files/modules/no_pkgs.mod
 create mode 100644 tests/files/modules/no_pkgs.yml
 create mode 100644 tests/files/modules/no_version.mod
 create mode 100644 tests/files/modules/no_version.yml
 rename tests/files/{ldsc.module => modules/shpc.mod} (100%)
 rename tests/files/{ldsc.yml => modules/shpc.yml} (100%)
 create mode 100644 tests/files/modules/version_in_name.mod
 create mode 100644 tests/files/modules/version_in_name.yml

diff --git a/softpack_core/moduleparse.py b/softpack_core/moduleparse.py
index 12cd1de..ed593e1 100644
--- a/softpack_core/moduleparse.py
+++ b/softpack_core/moduleparse.py
@@ -6,14 +6,19 @@
 
 from typing import Union, cast
 
+import re
 
-def ToSoftpackYML(contents: Union[bytes, str]) -> bytes:
+
+def ToSoftpackYML(name: str, contents: Union[bytes, str]) -> bytes:
     """Converts an shpc-style module file to a softpack.yml file.
 
     It should have a format similar to that produced by shpc, with `module
     whatis` outputting a "Name: " line, a "Version: " line, and optionally a
-    "Packages: " line to say what packages are available. `module help` output
-    will be translated into the description in the softpack.yml.
+    "Packages: " line to say what packages are available. Each package should be
+    separated by a comma.
+
+    `module help` output will be translated into the description in the
+    softpack.yml.
 
     Args:
         contents (bytes): The byte content of the module file.
@@ -23,7 +28,6 @@ def ToSoftpackYML(contents: Union[bytes, str]) -> bytes:
     """
     in_help = False
 
-    name = ""
     version = ""
     packages: list[str] = []
     description = ""
@@ -61,15 +65,37 @@ def ToSoftpackYML(contents: Union[bytes, str]) -> bytes:
                     .lstrip()
                 )
 
-                if line_str.startswith("Name: "):
-                    nv = line_str.removeprefix("Name: ").split(":")
-                    name = nv[0]
-                    if len(nv) > 1:
-                        version = nv[1]
-                elif line_str.startswith("Version: "):
-                    version = line_str.removeprefix("Version: ")
-                elif line_str.startswith("Packages: "):
-                    packages = line_str.removeprefix("Packages: ").split(", ")
+                if line_str.startswith("Name:"):
+                    nv = line_str.removeprefix("Name:")
+                    if nv != "":
+                        name_value = list(
+                            map(lambda x: x.strip().split()[0], nv.split(":"))
+                        )
+
+                        if name_value[0] is not None:
+                            name = name_value[0]
+
+                        if len(name_value) > 1 and name_value[1] != "":
+                            version = name_value[1].strip()
+                elif line_str.startswith("Version:"):
+                    ver = line_str.removeprefix("Version:")
+                    if ver != "":
+                        vers = ver.split()[0]
+                        if vers is not None and vers != "":
+                            version = vers
+                elif line_str.startswith("Packages:"):
+                    packages = list(
+                        filter(
+                            None,
+                            map(
+                                lambda x: x.strip(),
+                                re.split(
+                                    r'[,\s]+',
+                                    line_str.removeprefix("Packages:"),
+                                ),
+                            ),
+                        )
+                    )
 
     if version != "":
         name += f"@{version}"
diff --git a/softpack_core/schemas/environment.py b/softpack_core/schemas/environment.py
index c17fa8e..7e6d6a6 100644
--- a/softpack_core/schemas/environment.py
+++ b/softpack_core/schemas/environment.py
@@ -393,7 +393,7 @@ async def create_from_module(
         environment_name = environment_dirs.pop()
 
         contents = await file.read()
-        yml = ToSoftpackYML(contents)
+        yml = ToSoftpackYML(environment_name, contents)
 
         env = EnvironmentInput(
             name=environment_name,
diff --git a/tests/data/specs/modules/tests.yml b/tests/data/specs/modules/tests.yml
deleted file mode 100644
index dcb3c0e..0000000
--- a/tests/data/specs/modules/tests.yml
+++ /dev/null
@@ -1,6 +0,0 @@
-tests:
-  - path: data/specs/env-a.yml
-    expect: CreateEnvironmentSuccess
-  
-  - path: data/specs/env-b.yml
-    expect: dfdffddfdEnvironmentSuccess
\ No newline at end of file
diff --git a/tests/files/modules/all_fields.mod b/tests/files/modules/all_fields.mod
new file mode 100644
index 0000000..f9ec71e
--- /dev/null
+++ b/tests/files/modules/all_fields.mod
@@ -0,0 +1,17 @@
+#%Module
+
+proc ModulesHelp { } {
+
+    puts stderr "Help text line 1"
+    
+    puts stderr ""
+    puts stderr "Help text line 2"
+
+}
+
+module-whatis "Name:   name_of_container  "
+module-whatis "Version:1.0.1"
+
+module-whatis "Foo: bar"
+
+module-whatis "Packages: pkg1, pkg2,pkg3 pkg4 "
\ No newline at end of file
diff --git a/tests/files/modules/all_fields.yml b/tests/files/modules/all_fields.yml
new file mode 100644
index 0000000..2ddaa4c
--- /dev/null
+++ b/tests/files/modules/all_fields.yml
@@ -0,0 +1,10 @@
+description: |
+  Help text line 1
+  
+  Help text line 2
+packages:
+  - name_of_container@1.0.1
+  - pkg1
+  - pkg2
+  - pkg3
+  - pkg4
diff --git a/tests/files/modules/bad_name.mod b/tests/files/modules/bad_name.mod
new file mode 100644
index 0000000..ef6da5e
--- /dev/null
+++ b/tests/files/modules/bad_name.mod
@@ -0,0 +1,17 @@
+#%Module
+
+proc ModulesHelp { } {
+
+    puts stderr "Help text line 1"
+    
+    puts stderr ""
+    puts stderr "Help text line 2"
+
+}
+
+module-whatis "Name:   name_of_container\nmore_name  "
+module-whatis "Version:1.0.1"
+
+module-whatis "Foo: bar"
+
+module-whatis "Packages: pkg1, pkg2,pkg3 pkg4 "
\ No newline at end of file
diff --git a/tests/files/modules/bad_name.yml b/tests/files/modules/bad_name.yml
new file mode 100644
index 0000000..2ddaa4c
--- /dev/null
+++ b/tests/files/modules/bad_name.yml
@@ -0,0 +1,10 @@
+description: |
+  Help text line 1
+  
+  Help text line 2
+packages:
+  - name_of_container@1.0.1
+  - pkg1
+  - pkg2
+  - pkg3
+  - pkg4
diff --git a/tests/files/modules/bad_packages.mod b/tests/files/modules/bad_packages.mod
new file mode 100644
index 0000000..f7018f5
--- /dev/null
+++ b/tests/files/modules/bad_packages.mod
@@ -0,0 +1,17 @@
+#%Module
+
+proc ModulesHelp { } {
+
+    puts stderr "Help text line 1"
+    
+    puts stderr ""
+    puts stderr "Help text line 2"
+
+}
+
+module-whatis "Name:   name_of_container  "
+module-whatis "Version:1.0.1"
+
+module-whatis "Foo: bar"
+
+module-whatis "Packages: pkg1, pkg\n2,pkg3 pkg4 "
\ No newline at end of file
diff --git a/tests/files/modules/bad_packages.yml b/tests/files/modules/bad_packages.yml
new file mode 100644
index 0000000..d57aafd
--- /dev/null
+++ b/tests/files/modules/bad_packages.yml
@@ -0,0 +1,11 @@
+description: |
+  Help text line 1
+  
+  Help text line 2
+packages:
+  - name_of_container@1.0.1
+  - pkg1
+  - pkg
+  - 2
+  - pkg3
+  - pkg4
diff --git a/tests/files/modules/bad_version.mod b/tests/files/modules/bad_version.mod
new file mode 100644
index 0000000..a9fd5c6
--- /dev/null
+++ b/tests/files/modules/bad_version.mod
@@ -0,0 +1,17 @@
+#%Module
+
+proc ModulesHelp { } {
+
+    puts stderr "Help text line 1"
+    
+    puts stderr ""
+    puts stderr "Help text line 2"
+
+}
+
+module-whatis "Name:   name_of_container  "
+module-whatis "Version:   1.0.1\na"
+
+module-whatis "Foo: bar"
+
+module-whatis "Packages: pkg1, pkg2,pkg3 pkg4 "
\ No newline at end of file
diff --git a/tests/files/modules/bad_version.yml b/tests/files/modules/bad_version.yml
new file mode 100644
index 0000000..2ddaa4c
--- /dev/null
+++ b/tests/files/modules/bad_version.yml
@@ -0,0 +1,10 @@
+description: |
+  Help text line 1
+  
+  Help text line 2
+packages:
+  - name_of_container@1.0.1
+  - pkg1
+  - pkg2
+  - pkg3
+  - pkg4
diff --git a/tests/files/modules/empty_name.mod b/tests/files/modules/empty_name.mod
new file mode 100644
index 0000000..e7b6335
--- /dev/null
+++ b/tests/files/modules/empty_name.mod
@@ -0,0 +1,17 @@
+#%Module
+
+proc ModulesHelp { } {
+
+    puts stderr "Help text line 1"
+    
+    puts stderr ""
+    puts stderr "Help text line 2"
+
+}
+
+module-whatis "Name:"
+module-whatis "Version:1.0.1"
+
+module-whatis "Foo: bar"
+
+module-whatis "Packages: pkg1, pkg2,pkg3 pkg4 "
\ No newline at end of file
diff --git a/tests/files/modules/empty_name.yml b/tests/files/modules/empty_name.yml
new file mode 100644
index 0000000..2350c8c
--- /dev/null
+++ b/tests/files/modules/empty_name.yml
@@ -0,0 +1,10 @@
+description: |
+  Help text line 1
+  
+  Help text line 2
+packages:
+  - empty_name@1.0.1
+  - pkg1
+  - pkg2
+  - pkg3
+  - pkg4
diff --git a/tests/files/modules/minimal.mod b/tests/files/modules/minimal.mod
new file mode 100644
index 0000000..e69de29
diff --git a/tests/files/modules/minimal.yml b/tests/files/modules/minimal.yml
new file mode 100644
index 0000000..c625ae5
--- /dev/null
+++ b/tests/files/modules/minimal.yml
@@ -0,0 +1,3 @@
+description: |
+packages:
+  - minimal
diff --git a/tests/files/modules/no_description.mod b/tests/files/modules/no_description.mod
new file mode 100644
index 0000000..ef5e493
--- /dev/null
+++ b/tests/files/modules/no_description.mod
@@ -0,0 +1,7 @@
+#%Module
+
+
+
+module-whatis "Name:   name_of_container  "
+
+module-whatis "Foo: bar"
diff --git a/tests/files/modules/no_description.yml b/tests/files/modules/no_description.yml
new file mode 100644
index 0000000..0ca0685
--- /dev/null
+++ b/tests/files/modules/no_description.yml
@@ -0,0 +1,3 @@
+description: |
+packages:
+  - name_of_container
diff --git a/tests/files/modules/no_name.mod b/tests/files/modules/no_name.mod
new file mode 100644
index 0000000..62c8892
--- /dev/null
+++ b/tests/files/modules/no_name.mod
@@ -0,0 +1,12 @@
+#%Module
+
+proc ModulesHelp { } {
+
+    puts stderr "Help text line 1"
+    
+    puts stderr ""
+    puts stderr "Help text line 2"
+
+}
+
+module-whatis "Foo: bar"
diff --git a/tests/files/modules/no_name.yml b/tests/files/modules/no_name.yml
new file mode 100644
index 0000000..9a29b05
--- /dev/null
+++ b/tests/files/modules/no_name.yml
@@ -0,0 +1,6 @@
+description: |
+  Help text line 1
+  
+  Help text line 2
+packages:
+  - no_name
diff --git a/tests/files/modules/no_pkgs.mod b/tests/files/modules/no_pkgs.mod
new file mode 100644
index 0000000..4dd4b62
--- /dev/null
+++ b/tests/files/modules/no_pkgs.mod
@@ -0,0 +1,15 @@
+#%Module
+
+proc ModulesHelp { } {
+
+    puts stderr "Help text line 1"
+    
+    puts stderr ""
+    puts stderr "Help text line 2"
+
+}
+
+module-whatis "Name:   name_of_container  "
+module-whatis "Version:1.0.1"
+
+module-whatis "Foo: bar"
diff --git a/tests/files/modules/no_pkgs.yml b/tests/files/modules/no_pkgs.yml
new file mode 100644
index 0000000..3e07a3d
--- /dev/null
+++ b/tests/files/modules/no_pkgs.yml
@@ -0,0 +1,6 @@
+description: |
+  Help text line 1
+  
+  Help text line 2
+packages:
+  - name_of_container@1.0.1
diff --git a/tests/files/modules/no_version.mod b/tests/files/modules/no_version.mod
new file mode 100644
index 0000000..cbbd338
--- /dev/null
+++ b/tests/files/modules/no_version.mod
@@ -0,0 +1,14 @@
+#%Module
+
+proc ModulesHelp { } {
+
+    puts stderr "Help text line 1"
+    
+    puts stderr ""
+    puts stderr "Help text line 2"
+
+}
+
+module-whatis "Name:   name_of_container  "
+
+module-whatis "Foo: bar"
diff --git a/tests/files/modules/no_version.yml b/tests/files/modules/no_version.yml
new file mode 100644
index 0000000..d6afca5
--- /dev/null
+++ b/tests/files/modules/no_version.yml
@@ -0,0 +1,6 @@
+description: |
+  Help text line 1
+  
+  Help text line 2
+packages:
+  - name_of_container
diff --git a/tests/files/ldsc.module b/tests/files/modules/shpc.mod
similarity index 100%
rename from tests/files/ldsc.module
rename to tests/files/modules/shpc.mod
diff --git a/tests/files/ldsc.yml b/tests/files/modules/shpc.yml
similarity index 100%
rename from tests/files/ldsc.yml
rename to tests/files/modules/shpc.yml
diff --git a/tests/files/modules/version_in_name.mod b/tests/files/modules/version_in_name.mod
new file mode 100644
index 0000000..854b071
--- /dev/null
+++ b/tests/files/modules/version_in_name.mod
@@ -0,0 +1,15 @@
+#%Module
+
+proc ModulesHelp { } {
+
+    puts stderr "Help text line 1"
+    
+    puts stderr ""
+    puts stderr "Help text line 2"
+
+}
+
+module-whatis "Name:   name_of_container:1.0.2  "
+module-whatis "Version:"
+
+module-whatis "Foo: bar"
diff --git a/tests/files/modules/version_in_name.yml b/tests/files/modules/version_in_name.yml
new file mode 100644
index 0000000..948a7d6
--- /dev/null
+++ b/tests/files/modules/version_in_name.yml
@@ -0,0 +1,6 @@
+description: |
+  Help text line 1
+  
+  Help text line 2
+packages:
+  - name_of_container@1.0.2
diff --git a/tests/test_moduleparse.py b/tests/test_moduleparse.py
index ca45120..43e3470 100644
--- a/tests/test_moduleparse.py
+++ b/tests/test_moduleparse.py
@@ -12,32 +12,23 @@
 
 
 def pytest_generate_tests(metafunc):
-    if "module_spec" not in metafunc.fixturenames:
+    if "module_input" not in metafunc.fixturenames:
         return
 
-    with open(Path(__file__).parent / "data/specs/modules/tests.yml") as f:
-        yml = yaml.safe_load(f)
-
     metafunc.parametrize(
-        "module_spec",
-        yml["tests"],
+        "module_input",
+        list((Path(__file__).parent / "files" / "modules").glob("*.mod")),
     )
 
 
-def test_tosoftpack(module_spec) -> None:
-    path = module_spec["path"]
-    output = module_spec.get("output")
-    fail_message = module_spec.get("fail")
+def test_tosoftpack(module_input: Path) -> None:
+    output = str(module_input).removesuffix(".mod") + ".yml"
 
-    with open(Path(Path(__file__).parent, path), "rb") as fh:
+    with open(module_input, "rb") as fh:
         module_data = fh.read()
 
-    try:
-        yml = ToSoftpackYML(module_data)
-    except any as e:
-        assert e.message == fail_message
-        return
+    yml = ToSoftpackYML(module_input.name.removesuffix(".mod"), module_data)
 
-    with open(Path(Path(__file__).parent, output), "rb") as fh:
+    with open(output, "rb") as fh:
         expected_yml = fh.read()
         assert yml == expected_yml

From ddcaa6d91e8dc6f6a1e5a1a9fa5fb5b562b869be Mon Sep 17 00:00:00 2001
From: Michael Woolnough <mw31@sanger.ac.uk>
Date: Wed, 6 Sep 2023 10:28:07 +0100
Subject: [PATCH 105/129] Fix location of test data.

---
 tests/integration/test_environment.py | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/tests/integration/test_environment.py b/tests/integration/test_environment.py
index a77c1b2..0382597 100644
--- a/tests/integration/test_environment.py
+++ b/tests/integration/test_environment.py
@@ -244,10 +244,12 @@ async def test_iter(httpx_post, testable_env_input, upload):
 
 @pytest.mark.asyncio
 async def test_create_from_module(httpx_post, testable_env_input, upload):
-    test_file_path = Path(Path(__file__).parent.parent, "files", "ldsc.module")
+    test_file_path = Path(
+        Path(__file__).parent.parent, "files", "modules", "shpc.mod"
+    )
 
     with open(test_file_path, "rb") as fh:
-        upload.filename = "ldsc.module"
+        upload.filename = "shpc.mod"
         upload.content_type = "text/plain"
         upload.read.return_value = fh.read()
 

From e63bfed33b871874870a4ab0322b51f0f2502742 Mon Sep 17 00:00:00 2001
From: Michael Woolnough <mw31@sanger.ac.uk>
Date: Wed, 6 Sep 2023 10:28:23 +0100
Subject: [PATCH 106/129] Reformat.

---
 softpack_core/moduleparse.py | 7 +++----
 tests/test_moduleparse.py    | 2 --
 2 files changed, 3 insertions(+), 6 deletions(-)

diff --git a/softpack_core/moduleparse.py b/softpack_core/moduleparse.py
index ed593e1..3f8e22f 100644
--- a/softpack_core/moduleparse.py
+++ b/softpack_core/moduleparse.py
@@ -4,9 +4,8 @@
 LICENSE file in the root directory of this source tree.
 """
 
-from typing import Union, cast
-
 import re
+from typing import Union, cast
 
 
 def ToSoftpackYML(name: str, contents: Union[bytes, str]) -> bytes:
@@ -14,8 +13,8 @@ def ToSoftpackYML(name: str, contents: Union[bytes, str]) -> bytes:
 
     It should have a format similar to that produced by shpc, with `module
     whatis` outputting a "Name: " line, a "Version: " line, and optionally a
-    "Packages: " line to say what packages are available. Each package should be
-    separated by a comma.
+    "Packages: " line to say what packages are available. Each package should
+    be separated by a comma.
 
     `module help` output will be translated into the description in the
     softpack.yml.
diff --git a/tests/test_moduleparse.py b/tests/test_moduleparse.py
index 43e3470..d18eb78 100644
--- a/tests/test_moduleparse.py
+++ b/tests/test_moduleparse.py
@@ -6,8 +6,6 @@
 
 from pathlib import Path
 
-import yaml
-
 from softpack_core.moduleparse import ToSoftpackYML
 
 

From 6e1a9c77e1555976731f56e81bac9e1e31ace193 Mon Sep 17 00:00:00 2001
From: Michael Woolnough <mw31@sanger.ac.uk>
Date: Wed, 6 Sep 2023 14:39:44 +0100
Subject: [PATCH 107/129] Generate README.md file from module path.

---
 softpack_core/artifacts.py                    |  1 +
 softpack_core/{moduleparse.py => module.py}   | 25 +++++++++++++++++++
 softpack_core/schemas/environment.py          | 22 ++++++++++++++--
 tests/files/modules/shpc.readme               | 15 +++++++++++
 tests/integration/test_environment.py         | 16 +++++++++---
 tests/{test_moduleparse.py => test_module.py} | 14 ++++++++++-
 6 files changed, 87 insertions(+), 6 deletions(-)
 rename softpack_core/{moduleparse.py => module.py} (89%)
 create mode 100644 tests/files/modules/shpc.readme
 rename tests/{test_moduleparse.py => test_module.py} (66%)

diff --git a/softpack_core/artifacts.py b/softpack_core/artifacts.py
index 28c8a53..324ec6c 100644
--- a/softpack_core/artifacts.py
+++ b/softpack_core/artifacts.py
@@ -23,6 +23,7 @@ class Artifacts:
     environments_root = "environments"
     environments_file = "softpack.yml"
     module_file = "module"
+    readme_file = "README.md"
     users_folder_name = "users"
     groups_folder_name = "groups"
     credentials_callback = None
diff --git a/softpack_core/moduleparse.py b/softpack_core/module.py
similarity index 89%
rename from softpack_core/moduleparse.py
rename to softpack_core/module.py
index 3f8e22f..2213ea3 100644
--- a/softpack_core/moduleparse.py
+++ b/softpack_core/module.py
@@ -106,3 +106,28 @@ def ToSoftpackYML(name: str, contents: Union[bytes, str]) -> bytes:
     return (
         f"description: |\n{description}packages:\n  - {package_str}\n".encode()
     )
+
+
+def GenerateEnvReadme(module_path: str) -> bytes:
+    return (
+        """# Usage
+
+To use this environment, run:
+
+``` 
+module load """
+        + module_path
+        + """
+```
+
+This will usually add your desired software to your PATH. Check the description
+of the environement for more information, which might also be available by
+running:
+
+```
+module help """
+        + module_path
+        + """
+```
+"""
+    ).encode()
diff --git a/softpack_core/schemas/environment.py b/softpack_core/schemas/environment.py
index 7e6d6a6..f625116 100644
--- a/softpack_core/schemas/environment.py
+++ b/softpack_core/schemas/environment.py
@@ -15,7 +15,7 @@
 from strawberry.file_uploads import Upload
 
 from softpack_core.artifacts import Artifacts
-from softpack_core.moduleparse import ToSoftpackYML
+from softpack_core.module import ToSoftpackYML, GenerateEnvReadme
 from softpack_core.schemas.base import BaseSchema
 from softpack_core.spack import Spack
 
@@ -394,6 +394,7 @@ async def create_from_module(
 
         contents = await file.read()
         yml = ToSoftpackYML(environment_name, contents)
+        readme = GenerateEnvReadme(module_path)
 
         env = EnvironmentInput(
             name=environment_name,
@@ -408,10 +409,12 @@ async def create_from_module(
 
         module_file = UploadFile(file=io.BytesIO(contents))
         softpack_file = UploadFile(file=io.BytesIO(yml))
+        readme_file = UploadFile(file=io.BytesIO(readme))
 
         result = await cls.write_module_artifacts(
             module_file=module_file,
             softpack_file=softpack_file,
+            readme_file=readme_file,
             environment_path=environment_path,
         )
 
@@ -427,7 +430,11 @@ async def create_from_module(
 
     @classmethod
     async def write_module_artifacts(
-        cls, module_file: Upload, softpack_file: Upload, environment_path: str
+        cls,
+        module_file: Upload,
+        softpack_file: Upload,
+        readme_file: Upload,
+        environment_path: str,
     ) -> WriteArtifactResponse:  # type: ignore
         """Writes the given module and softpack files to the artifacts repo.
 
@@ -435,6 +442,8 @@ async def write_module_artifacts(
             module_file (Upload): An shpc-style module file.
             softpack_file (Upload): A "fake" softpack.yml file describing what
             the module file offers.
+            readme_file (Upload): An README.md file containing usage
+            instructions.
             environment_path (str): Path to the environment, eg.
             users/user/env.
 
@@ -448,6 +457,15 @@ async def write_module_artifacts(
             file_name=cls.artifacts.module_file,
         )
 
+        if not isinstance(result, WriteArtifactSuccess):
+            return result
+
+        result = await cls.write_artifact(
+            file=readme_file,
+            folder_path=environment_path,
+            file_name=cls.artifacts.readme_file,
+        )
+
         if not isinstance(result, WriteArtifactSuccess):
             return result
 
diff --git a/tests/files/modules/shpc.readme b/tests/files/modules/shpc.readme
new file mode 100644
index 0000000..5e64793
--- /dev/null
+++ b/tests/files/modules/shpc.readme
@@ -0,0 +1,15 @@
+# Usage
+
+To use this environment, run:
+
+``` 
+module load HGI/common/some_environment
+```
+
+This will usually add your desired software to your PATH. Check the description
+of the environement for more information, which might also be available by
+running:
+
+```
+module help HGI/common/some_environment
+```
diff --git a/tests/integration/test_environment.py b/tests/integration/test_environment.py
index 0382597..13cfd08 100644
--- a/tests/integration/test_environment.py
+++ b/tests/integration/test_environment.py
@@ -7,6 +7,7 @@
 from pathlib import Path
 
 import pytest
+import pygit2
 
 from softpack_core.artifacts import Artifacts
 from softpack_core.schemas.environment import (
@@ -244,9 +245,8 @@ async def test_iter(httpx_post, testable_env_input, upload):
 
 @pytest.mark.asyncio
 async def test_create_from_module(httpx_post, testable_env_input, upload):
-    test_file_path = Path(
-        Path(__file__).parent.parent, "files", "modules", "shpc.mod"
-    )
+    test_files_dir = Path(__file__).parent.parent / "files" / "modules"
+    test_file_path = test_files_dir / "shpc.mod"
 
     with open(test_file_path, "rb") as fh:
         upload.filename = "shpc.mod"
@@ -271,7 +271,17 @@ async def test_create_from_module(httpx_post, testable_env_input, upload):
         env_name,
     )
 
+    readme_path = Path(parent_path, Environment.artifacts.readme_file)
     assert file_in_remote(
         Path(parent_path, Environment.artifacts.environments_file),
         Path(parent_path, Environment.artifacts.module_file),
+        readme_path,
     )
+
+    with open(test_files_dir / "shpc.readme", "rb") as fh:
+        expected_readme_data = fh.read()
+
+    tree = Environment.artifacts.repo.head.peel(pygit2.Tree)
+    obj = tree[str(readme_path)]
+    assert obj is not None
+    assert obj.data == expected_readme_data
diff --git a/tests/test_moduleparse.py b/tests/test_module.py
similarity index 66%
rename from tests/test_moduleparse.py
rename to tests/test_module.py
index d18eb78..026504d 100644
--- a/tests/test_moduleparse.py
+++ b/tests/test_module.py
@@ -5,8 +5,9 @@
 """
 
 from pathlib import Path
+import pygit2
 
-from softpack_core.moduleparse import ToSoftpackYML
+from softpack_core.module import GenerateEnvReadme, ToSoftpackYML
 
 
 def pytest_generate_tests(metafunc):
@@ -30,3 +31,14 @@ def test_tosoftpack(module_input: Path) -> None:
     with open(output, "rb") as fh:
         expected_yml = fh.read()
         assert yml == expected_yml
+
+
+def test_generate_env_readme() -> None:
+    test_files_dir = Path(__file__).parent / "files" / "modules"
+
+    readme_data = GenerateEnvReadme("HGI/common/some_environment")
+
+    with open(test_files_dir / "shpc.readme", "rb") as fh:
+        expected_readme_data = fh.read()
+
+    assert readme_data == expected_readme_data

From a524f4d9cd1a21c8c73013c8b1329e5c88e34537 Mon Sep 17 00:00:00 2001
From: Michael Woolnough <mw31@sanger.ac.uk>
Date: Wed, 6 Sep 2023 15:23:06 +0100
Subject: [PATCH 108/129] Remove extra whitespace after backticks.

---
 tests/files/modules/shpc.readme | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/files/modules/shpc.readme b/tests/files/modules/shpc.readme
index 5e64793..ae095e1 100644
--- a/tests/files/modules/shpc.readme
+++ b/tests/files/modules/shpc.readme
@@ -2,7 +2,7 @@
 
 To use this environment, run:
 
-``` 
+```
 module load HGI/common/some_environment
 ```
 

From 8dcb34287c68347a70b6daa5857e2bbbae820513 Mon Sep 17 00:00:00 2001
From: Michael Woolnough <mw31@sanger.ac.uk>
Date: Wed, 6 Sep 2023 15:23:16 +0100
Subject: [PATCH 109/129] Add docstring.

---
 softpack_core/module.py | 10 +++++++++-
 1 file changed, 9 insertions(+), 1 deletion(-)

diff --git a/softpack_core/module.py b/softpack_core/module.py
index 2213ea3..d525f3f 100644
--- a/softpack_core/module.py
+++ b/softpack_core/module.py
@@ -109,12 +109,20 @@ def ToSoftpackYML(name: str, contents: Union[bytes, str]) -> bytes:
 
 
 def GenerateEnvReadme(module_path: str) -> bytes:
+    """Generates a simple README file for the environment.
+
+    Args:
+        module_path (str): The module path as used by the module command.
+
+    Returns:
+        bytes: The byte content of the README.md file.
+    """
     return (
         """# Usage
 
 To use this environment, run:
 
-``` 
+```
 module load """
         + module_path
         + """

From e043039c418a1d907cf7b753c5d108f93bc35f8f Mon Sep 17 00:00:00 2001
From: Michael Woolnough <mw31@sanger.ac.uk>
Date: Wed, 6 Sep 2023 15:23:47 +0100
Subject: [PATCH 110/129] Reformat.

---
 softpack_core/schemas/environment.py  | 2 +-
 tests/integration/test_environment.py | 2 +-
 tests/test_module.py                  | 1 -
 3 files changed, 2 insertions(+), 3 deletions(-)

diff --git a/softpack_core/schemas/environment.py b/softpack_core/schemas/environment.py
index f625116..a97a1bc 100644
--- a/softpack_core/schemas/environment.py
+++ b/softpack_core/schemas/environment.py
@@ -15,7 +15,7 @@
 from strawberry.file_uploads import Upload
 
 from softpack_core.artifacts import Artifacts
-from softpack_core.module import ToSoftpackYML, GenerateEnvReadme
+from softpack_core.module import GenerateEnvReadme, ToSoftpackYML
 from softpack_core.schemas.base import BaseSchema
 from softpack_core.spack import Spack
 
diff --git a/tests/integration/test_environment.py b/tests/integration/test_environment.py
index 13cfd08..d177d17 100644
--- a/tests/integration/test_environment.py
+++ b/tests/integration/test_environment.py
@@ -6,8 +6,8 @@
 
 from pathlib import Path
 
-import pytest
 import pygit2
+import pytest
 
 from softpack_core.artifacts import Artifacts
 from softpack_core.schemas.environment import (
diff --git a/tests/test_module.py b/tests/test_module.py
index 026504d..4f2006a 100644
--- a/tests/test_module.py
+++ b/tests/test_module.py
@@ -5,7 +5,6 @@
 """
 
 from pathlib import Path
-import pygit2
 
 from softpack_core.module import GenerateEnvReadme, ToSoftpackYML
 

From ab51f663475b8b83ccf6236e3592d09340ac0104 Mon Sep 17 00:00:00 2001
From: Michael Woolnough <mw31@sanger.ac.uk>
Date: Wed, 6 Sep 2023 15:29:50 +0100
Subject: [PATCH 111/129] Correct typo.

---
 tests/files/modules/shpc.readme | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/files/modules/shpc.readme b/tests/files/modules/shpc.readme
index ae095e1..8ecb4f3 100644
--- a/tests/files/modules/shpc.readme
+++ b/tests/files/modules/shpc.readme
@@ -7,7 +7,7 @@ module load HGI/common/some_environment
 ```
 
 This will usually add your desired software to your PATH. Check the description
-of the environement for more information, which might also be available by
+of the environment for more information, which might also be available by
 running:
 
 ```

From deda76e57017989bf9b3a67e0b4e396bbd84aaf9 Mon Sep 17 00:00:00 2001
From: Michael Woolnough <mw31@sanger.ac.uk>
Date: Wed, 6 Sep 2023 16:03:01 +0100
Subject: [PATCH 112/129] Use template to generate README.md file.

---
 softpack_core/module.py             | 26 +++++---------------------
 softpack_core/templates/readme.tmpl | 15 +++++++++++++++
 2 files changed, 20 insertions(+), 21 deletions(-)
 create mode 100644 softpack_core/templates/readme.tmpl

diff --git a/softpack_core/module.py b/softpack_core/module.py
index d525f3f..6b83313 100644
--- a/softpack_core/module.py
+++ b/softpack_core/module.py
@@ -5,6 +5,8 @@
 """
 
 import re
+from pathlib import Path
+from string import Template
 from typing import Union, cast
 
 
@@ -117,25 +119,7 @@ def GenerateEnvReadme(module_path: str) -> bytes:
     Returns:
         bytes: The byte content of the README.md file.
     """
-    return (
-        """# Usage
-
-To use this environment, run:
-
-```
-module load """
-        + module_path
-        + """
-```
+    with open(Path(__file__).parent / "templates" / "readme.tmpl", "r") as fh:
+        tmpl = Template(fh.read())
 
-This will usually add your desired software to your PATH. Check the description
-of the environement for more information, which might also be available by
-running:
-
-```
-module help """
-        + module_path
-        + """
-```
-"""
-    ).encode()
+    return tmpl.substitute({"module_path": module_path}).encode()
diff --git a/softpack_core/templates/readme.tmpl b/softpack_core/templates/readme.tmpl
new file mode 100644
index 0000000..9e41bf0
--- /dev/null
+++ b/softpack_core/templates/readme.tmpl
@@ -0,0 +1,15 @@
+# Usage
+
+To use this environment, run:
+
+```
+module load $module_path
+```
+
+This will usually add your desired software to your PATH. Check the description
+of the environment for more information, which might also be available by
+running:
+
+```
+module help $module_path
+```

From f6f9a537da29b84eb400cf3e71b18dee25108f1a Mon Sep 17 00:00:00 2001
From: Michael Woolnough <mw31@sanger.ac.uk>
Date: Wed, 6 Sep 2023 16:49:38 +0100
Subject: [PATCH 113/129] Add README string to environments endpoint.

---
 softpack_core/artifacts.py            | 12 +++++++++---
 softpack_core/schemas/environment.py  |  2 ++
 tests/integration/test_environment.py | 12 ++++++++++++
 3 files changed, 23 insertions(+), 3 deletions(-)

diff --git a/softpack_core/artifacts.py b/softpack_core/artifacts.py
index 324ec6c..a473074 100644
--- a/softpack_core/artifacts.py
+++ b/softpack_core/artifacts.py
@@ -75,13 +75,19 @@ def get(self, key: str) -> "Artifacts.Object":
             return Artifacts.Object(path=self.path, obj=self.obj[key])
 
         def spec(self) -> Box:
-            """Get spec dictionary.
+            """Get dictionary of the softpack.yml file contents.
+
+            Also includes the contents of any README.md file.
 
             Returns:
                 Box: A boxed dictionary.
             """
-            spec = self.obj[Artifacts.environments_file]
-            return Box.from_yaml(spec.data)
+            info = Box.from_yaml(self.obj[Artifacts.environments_file].data)
+
+            if Artifacts.readme_file in self.obj:
+                info["readme"] = self.obj[Artifacts.readme_file].data.decode()
+
+            return info
 
         def __iter__(self) -> Iterator["Artifacts.Object"]:
             """A generator for returning items under an artifacts.
diff --git a/softpack_core/schemas/environment.py b/softpack_core/schemas/environment.py
index a97a1bc..ec2a46b 100644
--- a/softpack_core/schemas/environment.py
+++ b/softpack_core/schemas/environment.py
@@ -154,6 +154,7 @@ class Environment:
     name: str
     path: str
     description: str
+    readme: str
     packages: list[Package]
     state: Optional[str]
     artifacts = Artifacts()
@@ -193,6 +194,7 @@ def from_artifact(cls, obj: Artifacts.Object) -> Optional["Environment"]:
                     )
                 ),  # type: ignore [call-arg]
                 state=None,
+                readme=spec.get("readme", ""),
             )
         except KeyError:
             return None
diff --git a/tests/integration/test_environment.py b/tests/integration/test_environment.py
index d177d17..638bfa3 100644
--- a/tests/integration/test_environment.py
+++ b/tests/integration/test_environment.py
@@ -285,3 +285,15 @@ async def test_create_from_module(httpx_post, testable_env_input, upload):
     obj = tree[str(readme_path)]
     assert obj is not None
     assert obj.data == expected_readme_data
+
+    envs = list(Environment.iter())
+
+    assert len(envs) == 1
+
+    env = envs[0]
+
+    package_name = "quay.io/biocontainers/ldsc@1.0.1--pyhdfd78af_2"
+
+    assert env.name == env_name
+    assert len(env.packages) == 1 and env.packages[0].name == package_name
+    assert "module load " + module_path in env.readme

From c942b884fb14a06da7daec3c72eb60137dcc377b Mon Sep 17 00:00:00 2001
From: Michael Woolnough <mw31@sanger.ac.uk>
Date: Thu, 7 Sep 2023 13:53:19 +0100
Subject: [PATCH 114/129] Add Environment type (softpack or module).

---
 softpack_core/artifacts.py            |  9 +++++++++
 softpack_core/schemas/environment.py  | 18 ++++++++++++------
 tests/integration/test_environment.py |  7 +++++--
 3 files changed, 26 insertions(+), 8 deletions(-)

diff --git a/softpack_core/artifacts.py b/softpack_core/artifacts.py
index a473074..3fa601a 100644
--- a/softpack_core/artifacts.py
+++ b/softpack_core/artifacts.py
@@ -24,6 +24,10 @@ class Artifacts:
     environments_file = "softpack.yml"
     module_file = "module"
     readme_file = "README.md"
+    built_by_softpack_file = ".built_by_softpack"
+    built_by_softpack = "softpack"
+    generated_from_module_file = ".generated_from_module"
+    generated_from_module = "module"
     users_folder_name = "users"
     groups_folder_name = "groups"
     credentials_callback = None
@@ -87,6 +91,11 @@ def spec(self) -> Box:
             if Artifacts.readme_file in self.obj:
                 info["readme"] = self.obj[Artifacts.readme_file].data.decode()
 
+            if Artifacts.generated_from_module_file in self.obj:
+                info["type"] = Artifacts.generated_from_module
+            else:
+                info["type"] = Artifacts.built_by_softpack
+
             return info
 
         def __iter__(self) -> Iterator["Artifacts.Object"]:
diff --git a/softpack_core/schemas/environment.py b/softpack_core/schemas/environment.py
index ec2a46b..58c1400 100644
--- a/softpack_core/schemas/environment.py
+++ b/softpack_core/schemas/environment.py
@@ -155,6 +155,7 @@ class Environment:
     path: str
     description: str
     readme: str
+    type: str
     packages: list[Package]
     state: Optional[str]
     artifacts = Artifacts()
@@ -195,6 +196,7 @@ def from_artifact(cls, obj: Artifacts.Object) -> Optional["Environment"]:
                 ),  # type: ignore [call-arg]
                 state=None,
                 readme=spec.get("readme", ""),
+                type=spec.get("type", ""),
             )
         except KeyError:
             return None
@@ -213,7 +215,7 @@ def create(cls, env: EnvironmentInput) -> CreateResponse:  # type: ignore
         if any(len(value) == 0 for value in vars(env).values()):
             return InvalidInputError(message="all fields must be filled in")
 
-        response = cls.create_new_env(env)
+        response = cls.create_new_env(env, Artifacts.built_by_softpack_file)
         if not isinstance(response, CreateEnvironmentSuccess):
             return response
 
@@ -236,7 +238,7 @@ def create(cls, env: EnvironmentInput) -> CreateResponse:  # type: ignore
 
     @classmethod
     def create_new_env(
-        cls, env: EnvironmentInput
+        cls, env: EnvironmentInput, env_type: str
     ) -> CreateResponse:  # type: ignore
         """Create a new environment in the repository.
 
@@ -244,7 +246,10 @@ def create_new_env(
         already exists.
 
         Args:
-            env (EnvironmentInput): Details of the new environment.
+            env (EnvironmentInput): Details of the new environment. env_type
+            (str): One of Artifacts.built_by_softpack_file or
+            Artifacts.generated_from_module_file that denotes how the
+            environment was made.
 
         Returns:
             CreateResponse: a CreateEnvironmentSuccess on success, or one of
@@ -270,10 +275,9 @@ def create_new_env(
 
         # Create folder with place-holder file
         new_folder_path = Path(env.path, env.name)
-        file_name = ".created"
         try:
             tree_oid = cls.artifacts.create_file(
-                new_folder_path, file_name, "", True
+                new_folder_path, env_type, "", True
             )
             cls.artifacts.commit_and_push(
                 tree_oid, "create environment folder"
@@ -405,7 +409,9 @@ async def create_from_module(
             packages=list(),
         )
 
-        response = cls.create_new_env(env)
+        response = cls.create_new_env(
+            env, Artifacts.generated_from_module_file
+        )
         if not isinstance(response, CreateEnvironmentSuccess):
             return response
 
diff --git a/tests/integration/test_environment.py b/tests/integration/test_environment.py
index 638bfa3..680fe69 100644
--- a/tests/integration/test_environment.py
+++ b/tests/integration/test_environment.py
@@ -70,7 +70,7 @@ def test_create(httpx_post, testable_env_input: EnvironmentInput) -> None:
         Environment.artifacts.environments_root,
         testable_env_input.path,
         testable_env_input.name,
-        ".created",
+        Environment.artifacts.built_by_softpack_file,
     )
     assert file_in_remote(path)
 
@@ -157,7 +157,7 @@ def test_delete(httpx_post, testable_env_input) -> None:
         Environment.artifacts.environments_root,
         testable_env_input.path,
         testable_env_input.name,
-        ".created",
+        Artifacts.built_by_softpack_file,
     )
     assert file_in_remote(path)
 
@@ -238,6 +238,7 @@ async def test_iter(httpx_post, testable_env_input, upload):
     for env in envs_filter:
         assert env.name == testable_env_input.name
         assert any(p.name == "zlib" for p in env.packages)
+        assert env.type == Artifacts.built_by_softpack
         count += 1
 
     assert count == 1
@@ -276,6 +277,7 @@ async def test_create_from_module(httpx_post, testable_env_input, upload):
         Path(parent_path, Environment.artifacts.environments_file),
         Path(parent_path, Environment.artifacts.module_file),
         readme_path,
+        Path(parent_path, Environment.artifacts.generated_from_module_file),
     )
 
     with open(test_files_dir / "shpc.readme", "rb") as fh:
@@ -297,3 +299,4 @@ async def test_create_from_module(httpx_post, testable_env_input, upload):
     assert env.name == env_name
     assert len(env.packages) == 1 and env.packages[0].name == package_name
     assert "module load " + module_path in env.readme
+    assert env.type == Artifacts.generated_from_module

From 7b82d85f623c84638803f8957816f23c0585e67f Mon Sep 17 00:00:00 2001
From: Michael Woolnough <mw31@sanger.ac.uk>
Date: Thu, 7 Sep 2023 14:53:12 +0100
Subject: [PATCH 115/129] Fix bootstrapping problem requiring at least one
 environment in both the users and groups directories.

---
 README.md                           | 3 +--
 softpack_core/artifacts.py          | 5 ++++-
 tests/integration/test_artifacts.py | 9 +++++++++
 3 files changed, 14 insertions(+), 3 deletions(-)

diff --git a/README.md b/README.md
index 6523460..3992145 100644
--- a/README.md
+++ b/README.md
@@ -36,8 +36,7 @@ $ source spack/share/spack/setup-env.sh
 ```
 
 To start the service, you will also need to configure a git repository to store
-artifacts. That respository must have at least 1 file in
-environments/users/<subdir> and another file in environments/groups/<subdir>.
+artifacts.
 
 ### Stable release
 
diff --git a/softpack_core/artifacts.py b/softpack_core/artifacts.py
index 3fa601a..b555715 100644
--- a/softpack_core/artifacts.py
+++ b/softpack_core/artifacts.py
@@ -220,7 +220,10 @@ def iter_environments(self, path: Path) -> list[pygit2.Tree]:
         Returns:
             list[pygit2.Tree]: List of environments
         """
-        return [path / folder.name for folder in self.tree(str(path))]
+        try:
+            return [path / folder.name for folder in self.tree(str(path))]
+        except KeyError:
+            return list(())
 
     def tree(self, path: str) -> pygit2.Tree:
         """Return a Tree object.
diff --git a/tests/integration/test_artifacts.py b/tests/integration/test_artifacts.py
index 320611c..bc82aee 100644
--- a/tests/integration/test_artifacts.py
+++ b/tests/integration/test_artifacts.py
@@ -14,6 +14,7 @@
 from softpack_core.artifacts import Artifacts, app
 from tests.integration.utils import (
     commit_and_push_test_repo_changes,
+    delete_environments_folder_from_test_repo,
     file_in_remote,
     file_in_repo,
     get_user_path_without_environments,
@@ -47,6 +48,14 @@ def test_clone() -> None:
 
     assert file_in_repo(artifacts, file_path)
 
+    delete_environments_folder_from_test_repo(artifacts)
+
+    try:
+        artifacts.iter()
+    except BaseException as e:
+        print(e)
+        assert False
+
 
 def test_commit_and_push() -> None:
     ad = new_test_artifacts()

From 91acc3cccd4e85e1abaec76a54620ce666329b81 Mon Sep 17 00:00:00 2001
From: Michael Woolnough <mw31@sanger.ac.uk>
Date: Thu, 7 Sep 2023 16:30:35 +0100
Subject: [PATCH 116/129] Add missing dependency to poetry lock file.

---
 poetry.lock    | 222 +++++++------------------------------------------
 pyproject.toml |   1 +
 2 files changed, 31 insertions(+), 192 deletions(-)

diff --git a/poetry.lock b/poetry.lock
index 2c72d5d..3a2eacc 100644
--- a/poetry.lock
+++ b/poetry.lock
@@ -1,10 +1,9 @@
-# This file is automatically @generated by Poetry 1.4.2 and should not be changed by hand.
+# This file is automatically @generated by Poetry 1.6.1 and should not be changed by hand.
 
 [[package]]
 name = "aiosqlite"
 version = "0.19.0"
 description = "asyncio bridge to the standard sqlite3 module"
-category = "main"
 optional = false
 python-versions = ">=3.7"
 files = [
@@ -20,7 +19,6 @@ docs = ["sphinx (==6.1.3)", "sphinx-mdinclude (==0.5.3)"]
 name = "alembic"
 version = "1.11.0"
 description = "A database migration tool for SQLAlchemy."
-category = "main"
 optional = false
 python-versions = ">=3.7"
 files = [
@@ -40,7 +38,6 @@ tz = ["python-dateutil"]
 name = "altgraph"
 version = "0.17.3"
 description = "Python graph (network) package"
-category = "main"
 optional = false
 python-versions = "*"
 files = [
@@ -52,7 +49,6 @@ files = [
 name = "anyio"
 version = "3.6.2"
 description = "High level compatibility layer for multiple asynchronous event loop implementations"
-category = "main"
 optional = false
 python-versions = ">=3.6.2"
 files = [
@@ -73,7 +69,6 @@ trio = ["trio (>=0.16,<0.22)"]
 name = "apprise"
 version = "1.4.0"
 description = "Push Notifications that work with just about every platform!"
-category = "main"
 optional = false
 python-versions = ">=3.6"
 files = [
@@ -93,7 +88,6 @@ requests-oauthlib = "*"
 name = "archspec"
 version = "0.2.1"
 description = "A library to query system architecture"
-category = "main"
 optional = false
 python-versions = ">=3.6, !=2.7.*, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*, !=3.5.*"
 files = [
@@ -105,7 +99,6 @@ files = [
 name = "asgi-lifespan"
 version = "2.1.0"
 description = "Programmatic startup/shutdown of ASGI apps."
-category = "main"
 optional = false
 python-versions = ">=3.7"
 files = [
@@ -120,7 +113,6 @@ sniffio = "*"
 name = "asyncpg"
 version = "0.27.0"
 description = "An asyncio PostgreSQL driver"
-category = "main"
 optional = false
 python-versions = ">=3.7.0"
 files = [
@@ -171,7 +163,6 @@ test = ["flake8 (>=5.0.4,<5.1.0)", "uvloop (>=0.15.3)"]
 name = "attrs"
 version = "23.1.0"
 description = "Classes Without Boilerplate"
-category = "main"
 optional = false
 python-versions = ">=3.7"
 files = [
@@ -190,7 +181,6 @@ tests-no-zope = ["cloudpickle", "hypothesis", "mypy (>=1.1.1)", "pympler", "pyte
 name = "autoflake"
 version = "1.7.8"
 description = "Removes unused imports and unused variables"
-category = "dev"
 optional = false
 python-versions = ">=3.7"
 files = [
@@ -206,7 +196,6 @@ tomli = {version = ">=2.0.1", markers = "python_version < \"3.11\""}
 name = "black"
 version = "23.3.0"
 description = "The uncompromising code formatter."
-category = "dev"
 optional = false
 python-versions = ">=3.7"
 files = [
@@ -256,7 +245,6 @@ uvloop = ["uvloop (>=0.15.2)"]
 name = "bleach"
 version = "6.0.0"
 description = "An easy safelist-based HTML-sanitizing tool."
-category = "dev"
 optional = false
 python-versions = ">=3.7"
 files = [
@@ -275,7 +263,6 @@ css = ["tinycss2 (>=1.1.0,<1.2)"]
 name = "bokeh"
 version = "2.4.3"
 description = "Interactive plots and applications in the browser from Python"
-category = "main"
 optional = false
 python-versions = ">=3.7"
 files = [
@@ -296,7 +283,6 @@ typing-extensions = ">=3.10.0"
 name = "bump2version"
 version = "1.0.1"
 description = "Version-bump your software with a single command!"
-category = "dev"
 optional = false
 python-versions = ">=3.5"
 files = [
@@ -308,7 +294,6 @@ files = [
 name = "cachetools"
 version = "5.3.0"
 description = "Extensible memoizing collections and decorators"
-category = "main"
 optional = false
 python-versions = "~=3.7"
 files = [
@@ -320,7 +305,6 @@ files = [
 name = "certifi"
 version = "2023.5.7"
 description = "Python package for providing Mozilla's CA Bundle."
-category = "main"
 optional = false
 python-versions = ">=3.6"
 files = [
@@ -332,7 +316,6 @@ files = [
 name = "cffi"
 version = "1.15.1"
 description = "Foreign Function Interface for Python calling C code."
-category = "main"
 optional = false
 python-versions = "*"
 files = [
@@ -409,7 +392,6 @@ pycparser = "*"
 name = "cfgv"
 version = "3.3.1"
 description = "Validate configuration and produce human readable error messages."
-category = "dev"
 optional = false
 python-versions = ">=3.6.1"
 files = [
@@ -421,7 +403,6 @@ files = [
 name = "chardet"
 version = "5.1.0"
 description = "Universal encoding detector for Python 3"
-category = "dev"
 optional = false
 python-versions = ">=3.7"
 files = [
@@ -433,7 +414,6 @@ files = [
 name = "charset-normalizer"
 version = "3.1.0"
 description = "The Real First Universal Charset Detector. Open, modern and actively maintained alternative to Chardet."
-category = "main"
 optional = false
 python-versions = ">=3.7.0"
 files = [
@@ -518,7 +498,6 @@ files = [
 name = "click"
 version = "8.1.3"
 description = "Composable command line interface toolkit"
-category = "main"
 optional = false
 python-versions = ">=3.7"
 files = [
@@ -533,7 +512,6 @@ colorama = {version = "*", markers = "platform_system == \"Windows\""}
 name = "cloudpickle"
 version = "2.2.1"
 description = "Extended pickling support for Python objects"
-category = "main"
 optional = false
 python-versions = ">=3.6"
 files = [
@@ -545,7 +523,6 @@ files = [
 name = "colorama"
 version = "0.4.6"
 description = "Cross-platform colored terminal text."
-category = "main"
 optional = false
 python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,!=3.6.*,>=2.7"
 files = [
@@ -557,7 +534,6 @@ files = [
 name = "coolname"
 version = "2.2.0"
 description = "Random name and slug generator"
-category = "main"
 optional = false
 python-versions = "*"
 files = [
@@ -569,7 +545,6 @@ files = [
 name = "coverage"
 version = "7.2.5"
 description = "Code coverage measurement for Python"
-category = "dev"
 optional = false
 python-versions = ">=3.7"
 files = [
@@ -636,7 +611,6 @@ toml = ["tomli"]
 name = "croniter"
 version = "1.3.14"
 description = "croniter provides iteration for datetime object with cron like format"
-category = "main"
 optional = false
 python-versions = ">=2.6, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*"
 files = [
@@ -651,7 +625,6 @@ python-dateutil = "*"
 name = "cryptography"
 version = "40.0.2"
 description = "cryptography is a package which provides cryptographic recipes and primitives to Python developers."
-category = "main"
 optional = false
 python-versions = ">=3.6"
 files = [
@@ -693,7 +666,6 @@ tox = ["tox"]
 name = "dask"
 version = "2023.3.1"
 description = "Parallel PyData with Task Scheduling"
-category = "main"
 optional = false
 python-versions = ">=3.8"
 files = [
@@ -722,7 +694,6 @@ test = ["pandas[test]", "pre-commit", "pytest", "pytest-rerunfailures", "pytest-
 name = "dateparser"
 version = "1.1.8"
 description = "Date parsing library designed to parse dates from HTML pages"
-category = "main"
 optional = false
 python-versions = ">=3.7"
 files = [
@@ -745,7 +716,6 @@ langdetect = ["langdetect"]
 name = "decopatch"
 version = "1.4.10"
 description = "Create decorators easily in python."
-category = "dev"
 optional = false
 python-versions = "*"
 files = [
@@ -760,7 +730,6 @@ makefun = ">=1.5.0"
 name = "distlib"
 version = "0.3.6"
 description = "Distribution utilities"
-category = "dev"
 optional = false
 python-versions = "*"
 files = [
@@ -772,7 +741,6 @@ files = [
 name = "distributed"
 version = "2023.3.1"
 description = "Distributed scheduler for Dask"
-category = "main"
 optional = false
 python-versions = ">=3.8"
 files = [
@@ -801,7 +769,6 @@ zict = ">=2.1.0"
 name = "docker"
 version = "6.1.2"
 description = "A Python library for the Docker Engine API."
-category = "main"
 optional = false
 python-versions = ">=3.7"
 files = [
@@ -823,7 +790,6 @@ ssh = ["paramiko (>=2.4.3)"]
 name = "docutils"
 version = "0.20"
 description = "Docutils -- Python Documentation Utilities"
-category = "dev"
 optional = false
 python-versions = ">=3.7"
 files = [
@@ -835,7 +801,6 @@ files = [
 name = "exceptiongroup"
 version = "1.1.1"
 description = "Backport of PEP 654 (exception groups)"
-category = "main"
 optional = false
 python-versions = ">=3.7"
 files = [
@@ -850,7 +815,6 @@ test = ["pytest (>=6)"]
 name = "fastapi"
 version = "0.94.1"
 description = "FastAPI framework, high performance, easy to learn, fast to code, ready for production"
-category = "main"
 optional = false
 python-versions = ">=3.7"
 files = [
@@ -872,7 +836,6 @@ test = ["anyio[trio] (>=3.2.1,<4.0.0)", "black (==23.1.0)", "coverage[toml] (>=6
 name = "filelock"
 version = "3.12.0"
 description = "A platform independent file lock."
-category = "dev"
 optional = false
 python-versions = ">=3.7"
 files = [
@@ -888,7 +851,6 @@ testing = ["covdefaults (>=2.3)", "coverage (>=7.2.3)", "diff-cover (>=7.5)", "p
 name = "flake8"
 version = "5.0.4"
 description = "the modular source code checker: pep8 pyflakes and co"
-category = "dev"
 optional = false
 python-versions = ">=3.6.1"
 files = [
@@ -905,7 +867,6 @@ pyflakes = ">=2.5.0,<2.6.0"
 name = "flake8-docstrings"
 version = "1.7.0"
 description = "Extension for flake8 which uses pydocstyle to check docstrings"
-category = "dev"
 optional = false
 python-versions = ">=3.7"
 files = [
@@ -921,7 +882,6 @@ pydocstyle = ">=2.1"
 name = "fsspec"
 version = "2023.5.0"
 description = "File-system specification"
-category = "main"
 optional = false
 python-versions = ">=3.8"
 files = [
@@ -957,7 +917,6 @@ tqdm = ["tqdm"]
 name = "ghp-import"
 version = "2.1.0"
 description = "Copy your docs directly to the gh-pages branch."
-category = "dev"
 optional = false
 python-versions = "*"
 files = [
@@ -975,7 +934,6 @@ dev = ["flake8", "markdown", "twine", "wheel"]
 name = "google-auth"
 version = "2.18.0"
 description = "Google Authentication Library"
-category = "main"
 optional = false
 python-versions = ">=2.7,!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*"
 files = [
@@ -1001,7 +959,6 @@ requests = ["requests (>=2.20.0,<3.0.0dev)"]
 name = "graphql-core"
 version = "3.2.3"
 description = "GraphQL implementation for Python, a port of GraphQL.js, the JavaScript reference implementation for GraphQL."
-category = "main"
 optional = false
 python-versions = ">=3.6,<4"
 files = [
@@ -1013,7 +970,6 @@ files = [
 name = "greenlet"
 version = "2.0.2"
 description = "Lightweight in-process concurrent programming"
-category = "main"
 optional = false
 python-versions = ">=2.7,!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*"
 files = [
@@ -1022,6 +978,7 @@ files = [
     {file = "greenlet-2.0.2-cp27-cp27m-win32.whl", hash = "sha256:6c3acb79b0bfd4fe733dff8bc62695283b57949ebcca05ae5c129eb606ff2d74"},
     {file = "greenlet-2.0.2-cp27-cp27m-win_amd64.whl", hash = "sha256:283737e0da3f08bd637b5ad058507e578dd462db259f7f6e4c5c365ba4ee9343"},
     {file = "greenlet-2.0.2-cp27-cp27mu-manylinux2010_x86_64.whl", hash = "sha256:d27ec7509b9c18b6d73f2f5ede2622441de812e7b1a80bbd446cb0633bd3d5ae"},
+    {file = "greenlet-2.0.2-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:d967650d3f56af314b72df7089d96cda1083a7fc2da05b375d2bc48c82ab3f3c"},
     {file = "greenlet-2.0.2-cp310-cp310-macosx_11_0_x86_64.whl", hash = "sha256:30bcf80dda7f15ac77ba5af2b961bdd9dbc77fd4ac6105cee85b0d0a5fcf74df"},
     {file = "greenlet-2.0.2-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:26fbfce90728d82bc9e6c38ea4d038cba20b7faf8a0ca53a9c07b67318d46088"},
     {file = "greenlet-2.0.2-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:9190f09060ea4debddd24665d6804b995a9c122ef5917ab26e1566dcc712ceeb"},
@@ -1030,6 +987,7 @@ files = [
     {file = "greenlet-2.0.2-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:76ae285c8104046b3a7f06b42f29c7b73f77683df18c49ab5af7983994c2dd91"},
     {file = "greenlet-2.0.2-cp310-cp310-win_amd64.whl", hash = "sha256:2d4686f195e32d36b4d7cf2d166857dbd0ee9f3d20ae349b6bf8afc8485b3645"},
     {file = "greenlet-2.0.2-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:c4302695ad8027363e96311df24ee28978162cdcdd2006476c43970b384a244c"},
+    {file = "greenlet-2.0.2-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:d4606a527e30548153be1a9f155f4e283d109ffba663a15856089fb55f933e47"},
     {file = "greenlet-2.0.2-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c48f54ef8e05f04d6eff74b8233f6063cb1ed960243eacc474ee73a2ea8573ca"},
     {file = "greenlet-2.0.2-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:a1846f1b999e78e13837c93c778dcfc3365902cfb8d1bdb7dd73ead37059f0d0"},
     {file = "greenlet-2.0.2-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3a06ad5312349fec0ab944664b01d26f8d1f05009566339ac6f63f56589bc1a2"},
@@ -1059,6 +1017,7 @@ files = [
     {file = "greenlet-2.0.2-cp37-cp37m-win32.whl", hash = "sha256:3f6ea9bd35eb450837a3d80e77b517ea5bc56b4647f5502cd28de13675ee12f7"},
     {file = "greenlet-2.0.2-cp37-cp37m-win_amd64.whl", hash = "sha256:7492e2b7bd7c9b9916388d9df23fa49d9b88ac0640db0a5b4ecc2b653bf451e3"},
     {file = "greenlet-2.0.2-cp38-cp38-macosx_10_15_x86_64.whl", hash = "sha256:b864ba53912b6c3ab6bcb2beb19f19edd01a6bfcbdfe1f37ddd1778abfe75a30"},
+    {file = "greenlet-2.0.2-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:1087300cf9700bbf455b1b97e24db18f2f77b55302a68272c56209d5587c12d1"},
     {file = "greenlet-2.0.2-cp38-cp38-manylinux2010_x86_64.whl", hash = "sha256:ba2956617f1c42598a308a84c6cf021a90ff3862eddafd20c3333d50f0edb45b"},
     {file = "greenlet-2.0.2-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:fc3a569657468b6f3fb60587e48356fe512c1754ca05a564f11366ac9e306526"},
     {file = "greenlet-2.0.2-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:8eab883b3b2a38cc1e050819ef06a7e6344d4a990d24d45bc6f2cf959045a45b"},
@@ -1067,6 +1026,7 @@ files = [
     {file = "greenlet-2.0.2-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:b0ef99cdbe2b682b9ccbb964743a6aca37905fda5e0452e5ee239b1654d37f2a"},
     {file = "greenlet-2.0.2-cp38-cp38-win32.whl", hash = "sha256:b80f600eddddce72320dbbc8e3784d16bd3fb7b517e82476d8da921f27d4b249"},
     {file = "greenlet-2.0.2-cp38-cp38-win_amd64.whl", hash = "sha256:4d2e11331fc0c02b6e84b0d28ece3a36e0548ee1a1ce9ddde03752d9b79bba40"},
+    {file = "greenlet-2.0.2-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:8512a0c38cfd4e66a858ddd1b17705587900dd760c6003998e9472b77b56d417"},
     {file = "greenlet-2.0.2-cp39-cp39-macosx_11_0_x86_64.whl", hash = "sha256:88d9ab96491d38a5ab7c56dd7a3cc37d83336ecc564e4e8816dbed12e5aaefc8"},
     {file = "greenlet-2.0.2-cp39-cp39-manylinux2010_x86_64.whl", hash = "sha256:561091a7be172ab497a3527602d467e2b3fbe75f9e783d8b8ce403fa414f71a6"},
     {file = "greenlet-2.0.2-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:971ce5e14dc5e73715755d0ca2975ac88cfdaefcaab078a284fea6cfabf866df"},
@@ -1087,7 +1047,6 @@ test = ["objgraph", "psutil"]
 name = "griffe"
 version = "0.27.5"
 description = "Signatures for entire Python programs. Extract the structure, the frame, the skeleton of your project, to generate API documentation or find breaking changes in your API."
-category = "main"
 optional = false
 python-versions = ">=3.7"
 files = [
@@ -1102,7 +1061,6 @@ colorama = ">=0.4"
 name = "h11"
 version = "0.14.0"
 description = "A pure-Python, bring-your-own-I/O implementation of HTTP/1.1"
-category = "main"
 optional = false
 python-versions = ">=3.7"
 files = [
@@ -1114,7 +1072,6 @@ files = [
 name = "h2"
 version = "4.1.0"
 description = "HTTP/2 State-Machine based protocol implementation"
-category = "main"
 optional = false
 python-versions = ">=3.6.1"
 files = [
@@ -1130,7 +1087,6 @@ hyperframe = ">=6.0,<7"
 name = "hpack"
 version = "4.0.0"
 description = "Pure-Python HPACK header compression"
-category = "main"
 optional = false
 python-versions = ">=3.6.1"
 files = [
@@ -1142,7 +1098,6 @@ files = [
 name = "httpcore"
 version = "0.16.3"
 description = "A minimal low-level HTTP client."
-category = "main"
 optional = false
 python-versions = ">=3.7"
 files = [
@@ -1154,17 +1109,16 @@ files = [
 anyio = ">=3.0,<5.0"
 certifi = "*"
 h11 = ">=0.13,<0.15"
-sniffio = ">=1.0.0,<2.0.0"
+sniffio = "==1.*"
 
 [package.extras]
 http2 = ["h2 (>=3,<5)"]
-socks = ["socksio (>=1.0.0,<2.0.0)"]
+socks = ["socksio (==1.*)"]
 
 [[package]]
 name = "httpx"
 version = "0.23.3"
 description = "The next generation HTTP client."
-category = "main"
 optional = false
 python-versions = ">=3.7"
 files = [
@@ -1181,15 +1135,14 @@ sniffio = "*"
 
 [package.extras]
 brotli = ["brotli", "brotlicffi"]
-cli = ["click (>=8.0.0,<9.0.0)", "pygments (>=2.0.0,<3.0.0)", "rich (>=10,<13)"]
+cli = ["click (==8.*)", "pygments (==2.*)", "rich (>=10,<13)"]
 http2 = ["h2 (>=3,<5)"]
-socks = ["socksio (>=1.0.0,<2.0.0)"]
+socks = ["socksio (==1.*)"]
 
 [[package]]
 name = "hvac"
 version = "1.1.0"
 description = "HashiCorp Vault API client"
-category = "main"
 optional = false
 python-versions = ">=3.6.2,<4.0.0"
 files = [
@@ -1205,7 +1158,6 @@ requests = ">=2.27.1,<3.0.0"
 name = "hyperframe"
 version = "6.0.1"
 description = "HTTP/2 framing layer for Python"
-category = "main"
 optional = false
 python-versions = ">=3.6.1"
 files = [
@@ -1217,7 +1169,6 @@ files = [
 name = "identify"
 version = "2.5.24"
 description = "File identification library for Python"
-category = "dev"
 optional = false
 python-versions = ">=3.7"
 files = [
@@ -1232,7 +1183,6 @@ license = ["ukkonen"]
 name = "idna"
 version = "3.4"
 description = "Internationalized Domain Names in Applications (IDNA)"
-category = "main"
 optional = false
 python-versions = ">=3.5"
 files = [
@@ -1244,7 +1194,6 @@ files = [
 name = "importlib-metadata"
 version = "6.6.0"
 description = "Read metadata from Python packages"
-category = "main"
 optional = false
 python-versions = ">=3.7"
 files = [
@@ -1264,7 +1213,6 @@ testing = ["flake8 (<5)", "flufl.flake8", "importlib-resources (>=1.3)", "packag
 name = "iniconfig"
 version = "2.0.0"
 description = "brain-dead simple config-ini parsing"
-category = "main"
 optional = false
 python-versions = ">=3.7"
 files = [
@@ -1276,7 +1224,6 @@ files = [
 name = "isort"
 version = "5.12.0"
 description = "A Python utility / library to sort Python imports."
-category = "dev"
 optional = false
 python-versions = ">=3.8.0"
 files = [
@@ -1294,7 +1241,6 @@ requirements-deprecated-finder = ["pip-api", "pipreqs"]
 name = "jaraco-classes"
 version = "3.2.3"
 description = "Utility functions for Python class constructs"
-category = "dev"
 optional = false
 python-versions = ">=3.7"
 files = [
@@ -1313,7 +1259,6 @@ testing = ["flake8 (<5)", "pytest (>=6)", "pytest-black (>=0.3.7)", "pytest-chec
 name = "jeepney"
 version = "0.8.0"
 description = "Low-level, pure Python DBus protocol wrapper."
-category = "dev"
 optional = false
 python-versions = ">=3.7"
 files = [
@@ -1329,7 +1274,6 @@ trio = ["async_generator", "trio"]
 name = "jinja2"
 version = "3.1.2"
 description = "A very fast and expressive template engine."
-category = "main"
 optional = false
 python-versions = ">=3.7"
 files = [
@@ -1347,7 +1291,6 @@ i18n = ["Babel (>=2.7)"]
 name = "jsonpatch"
 version = "1.32"
 description = "Apply JSON-Patches (RFC 6902)"
-category = "main"
 optional = false
 python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*"
 files = [
@@ -1362,7 +1305,6 @@ jsonpointer = ">=1.9"
 name = "jsonpointer"
 version = "2.3"
 description = "Identify specific nodes in a JSON document (RFC 6901)"
-category = "main"
 optional = false
 python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*"
 files = [
@@ -1374,7 +1316,6 @@ files = [
 name = "jsonschema"
 version = "4.17.3"
 description = "An implementation of JSON Schema validation for Python"
-category = "main"
 optional = false
 python-versions = ">=3.7"
 files = [
@@ -1394,7 +1335,6 @@ format-nongpl = ["fqdn", "idna", "isoduration", "jsonpointer (>1.13)", "rfc3339-
 name = "keyring"
 version = "23.13.1"
 description = "Store and access your passwords safely."
-category = "dev"
 optional = false
 python-versions = ">=3.7"
 files = [
@@ -1418,7 +1358,6 @@ testing = ["flake8 (<5)", "pytest (>=6)", "pytest-black (>=0.3.7)", "pytest-chec
 name = "kubernetes"
 version = "26.1.0"
 description = "Kubernetes python client"
-category = "main"
 optional = false
 python-versions = ">=3.6"
 files = [
@@ -1436,7 +1375,7 @@ requests-oauthlib = "*"
 setuptools = ">=21.0.0"
 six = ">=1.9.0"
 urllib3 = ">=1.24.2"
-websocket-client = ">=0.32.0,<0.40.0 || >0.40.0,<0.41.0 || >=0.43.0"
+websocket-client = ">=0.32.0,<0.40.0 || >0.40.0,<0.41.dev0 || >=0.43.dev0"
 
 [package.extras]
 adal = ["adal (>=1.0.2)"]
@@ -1445,7 +1384,6 @@ adal = ["adal (>=1.0.2)"]
 name = "locket"
 version = "1.0.0"
 description = "File-based locks for Python on Linux and Windows"
-category = "main"
 optional = false
 python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*"
 files = [
@@ -1457,7 +1395,6 @@ files = [
 name = "macholib"
 version = "1.16.2"
 description = "Mach-O header analysis and editing"
-category = "main"
 optional = false
 python-versions = "*"
 files = [
@@ -1472,7 +1409,6 @@ altgraph = ">=0.17"
 name = "makefun"
 version = "1.15.1"
 description = "Small library to dynamically create python functions."
-category = "dev"
 optional = false
 python-versions = "*"
 files = [
@@ -1484,7 +1420,6 @@ files = [
 name = "mako"
 version = "1.2.4"
 description = "A super-fast templating language that borrows the best ideas from the existing templating languages."
-category = "main"
 optional = false
 python-versions = ">=3.7"
 files = [
@@ -1504,7 +1439,6 @@ testing = ["pytest"]
 name = "markdown"
 version = "3.3.7"
 description = "Python implementation of Markdown."
-category = "main"
 optional = false
 python-versions = ">=3.6"
 files = [
@@ -1522,7 +1456,6 @@ testing = ["coverage", "pyyaml"]
 name = "markdown-it-py"
 version = "2.2.0"
 description = "Python port of markdown-it. Markdown parsing, done right!"
-category = "main"
 optional = false
 python-versions = ">=3.7"
 files = [
@@ -1547,7 +1480,6 @@ testing = ["coverage", "pytest", "pytest-cov", "pytest-regressions"]
 name = "markupsafe"
 version = "2.1.2"
 description = "Safely add untrusted strings to HTML/XML markup."
-category = "main"
 optional = false
 python-versions = ">=3.7"
 files = [
@@ -1607,7 +1539,6 @@ files = [
 name = "mccabe"
 version = "0.7.0"
 description = "McCabe checker, plugin for flake8"
-category = "dev"
 optional = false
 python-versions = ">=3.6"
 files = [
@@ -1619,7 +1550,6 @@ files = [
 name = "mdurl"
 version = "0.1.2"
 description = "Markdown URL utilities"
-category = "main"
 optional = false
 python-versions = ">=3.7"
 files = [
@@ -1631,7 +1561,6 @@ files = [
 name = "mergedeep"
 version = "1.3.4"
 description = "A deep merge function for 🐍."
-category = "main"
 optional = false
 python-versions = ">=3.6"
 files = [
@@ -1643,7 +1572,6 @@ files = [
 name = "mkdocs"
 version = "1.4.3"
 description = "Project documentation with Markdown."
-category = "dev"
 optional = false
 python-versions = ">=3.7"
 files = [
@@ -1672,7 +1600,6 @@ min-versions = ["babel (==2.9.0)", "click (==7.0)", "colorama (==0.4)", "ghp-imp
 name = "mkdocs-autorefs"
 version = "0.4.1"
 description = "Automatically link across pages in MkDocs."
-category = "dev"
 optional = false
 python-versions = ">=3.7"
 files = [
@@ -1688,7 +1615,6 @@ mkdocs = ">=1.1"
 name = "mkdocs-include-markdown-plugin"
 version = "3.9.1"
 description = "Mkdocs Markdown includer plugin."
-category = "dev"
 optional = false
 python-versions = ">=3.6"
 files = [
@@ -1704,7 +1630,6 @@ test = ["mkdocs (==1.4.0)", "pytest (==7.1.3)", "pytest-cov (==3.0.0)"]
 name = "mkdocs-material"
 version = "9.1.12"
 description = "Documentation that simply works"
-category = "dev"
 optional = false
 python-versions = ">=3.7"
 files = [
@@ -1727,7 +1652,6 @@ requests = ">=2.26"
 name = "mkdocs-material-extensions"
 version = "1.1.1"
 description = "Extension pack for Python Markdown and MkDocs Material."
-category = "dev"
 optional = false
 python-versions = ">=3.7"
 files = [
@@ -1739,7 +1663,6 @@ files = [
 name = "mkdocstrings"
 version = "0.20.0"
 description = "Automatic documentation from sources, for MkDocs."
-category = "dev"
 optional = false
 python-versions = ">=3.7"
 files = [
@@ -1764,7 +1687,6 @@ python-legacy = ["mkdocstrings-python-legacy (>=0.2.1)"]
 name = "mkdocstrings-python"
 version = "0.8.3"
 description = "A Python handler for mkdocstrings."
-category = "dev"
 optional = false
 python-versions = ">=3.7"
 files = [
@@ -1780,7 +1702,6 @@ mkdocstrings = ">=0.19"
 name = "more-itertools"
 version = "9.1.0"
 description = "More routines for operating on iterables, beyond itertools"
-category = "dev"
 optional = false
 python-versions = ">=3.7"
 files = [
@@ -1792,7 +1713,6 @@ files = [
 name = "msgpack"
 version = "1.0.5"
 description = "MessagePack serializer"
-category = "main"
 optional = false
 python-versions = "*"
 files = [
@@ -1865,7 +1785,6 @@ files = [
 name = "mypy"
 version = "1.3.0"
 description = "Optional static typing for Python"
-category = "dev"
 optional = false
 python-versions = ">=3.7"
 files = [
@@ -1912,7 +1831,6 @@ reports = ["lxml"]
 name = "mypy-extensions"
 version = "1.0.0"
 description = "Type system extensions for programs checked with the mypy type checker."
-category = "dev"
 optional = false
 python-versions = ">=3.5"
 files = [
@@ -1924,7 +1842,6 @@ files = [
 name = "nodeenv"
 version = "1.8.0"
 description = "Node.js virtual environment builder"
-category = "dev"
 optional = false
 python-versions = ">=2.7,!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,!=3.6.*"
 files = [
@@ -1939,7 +1856,6 @@ setuptools = "*"
 name = "numpy"
 version = "1.24.3"
 description = "Fundamental package for array computing in Python"
-category = "main"
 optional = false
 python-versions = ">=3.8"
 files = [
@@ -1977,7 +1893,6 @@ files = [
 name = "oauthlib"
 version = "3.2.2"
 description = "A generic, spec-compliant, thorough implementation of the OAuth request-signing logic"
-category = "main"
 optional = false
 python-versions = ">=3.6"
 files = [
@@ -1994,7 +1909,6 @@ signedtoken = ["cryptography (>=3.0.0)", "pyjwt (>=2.0.0,<3)"]
 name = "oras"
 version = "0.1.17"
 description = "OCI Registry as Storage Python SDK"
-category = "main"
 optional = false
 python-versions = "*"
 files = [
@@ -2015,7 +1929,6 @@ tests = ["black", "isort", "mypy", "pyflakes", "pytest (>=4.6.2)", "types-reques
 name = "orjson"
 version = "3.8.12"
 description = "Fast, correct Python JSON library supporting dataclasses, datetimes, and numpy"
-category = "main"
 optional = false
 python-versions = ">=3.7"
 files = [
@@ -2071,7 +1984,6 @@ files = [
 name = "packaging"
 version = "23.1"
 description = "Core utilities for Python packages"
-category = "main"
 optional = false
 python-versions = ">=3.7"
 files = [
@@ -2083,7 +1995,6 @@ files = [
 name = "partd"
 version = "1.4.0"
 description = "Appendable key-value storage"
-category = "main"
 optional = false
 python-versions = ">=3.7"
 files = [
@@ -2102,7 +2013,6 @@ complete = ["blosc", "numpy (>=1.9.0)", "pandas (>=0.19.0)", "pyzmq"]
 name = "pathspec"
 version = "0.11.1"
 description = "Utility library for gitignore style pattern matching of file paths."
-category = "main"
 optional = false
 python-versions = ">=3.7"
 files = [
@@ -2114,7 +2024,6 @@ files = [
 name = "pendulum"
 version = "2.1.2"
 description = "Python datetimes made easy"
-category = "main"
 optional = false
 python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*"
 files = [
@@ -2149,7 +2058,6 @@ pytzdata = ">=2020.1"
 name = "pillow"
 version = "9.5.0"
 description = "Python Imaging Library (Fork)"
-category = "main"
 optional = false
 python-versions = ">=3.7"
 files = [
@@ -2229,7 +2137,6 @@ tests = ["check-manifest", "coverage", "defusedxml", "markdown2", "olefile", "pa
 name = "pip"
 version = "23.1.2"
 description = "The PyPA recommended tool for installing Python packages."
-category = "main"
 optional = false
 python-versions = ">=3.7"
 files = [
@@ -2241,7 +2148,6 @@ files = [
 name = "pkginfo"
 version = "1.9.6"
 description = "Query metadata from sdists / bdists / installed packages."
-category = "dev"
 optional = false
 python-versions = ">=3.6"
 files = [
@@ -2256,7 +2162,6 @@ testing = ["pytest", "pytest-cov"]
 name = "platformdirs"
 version = "3.5.1"
 description = "A small Python package for determining appropriate platform-specific dirs, e.g. a \"user data dir\"."
-category = "dev"
 optional = false
 python-versions = ">=3.7"
 files = [
@@ -2272,7 +2177,6 @@ test = ["appdirs (==1.4.4)", "covdefaults (>=2.3)", "pytest (>=7.3.1)", "pytest-
 name = "pluggy"
 version = "1.0.0"
 description = "plugin and hook calling mechanisms for python"
-category = "main"
 optional = false
 python-versions = ">=3.6"
 files = [
@@ -2288,7 +2192,6 @@ testing = ["pytest", "pytest-benchmark"]
 name = "pre-commit"
 version = "3.3.1"
 description = "A framework for managing and maintaining multi-language pre-commit hooks."
-category = "dev"
 optional = false
 python-versions = ">=3.8"
 files = [
@@ -2307,7 +2210,6 @@ virtualenv = ">=20.10.0"
 name = "prefect"
 version = "2.8.6"
 description = "Workflow orchestration and management."
-category = "main"
 optional = false
 python-versions = ">=3.7"
 files = [
@@ -2362,7 +2264,6 @@ dev = ["autoflake8", "cairosvg", "flake8", "flaky", "ipython", "jinja2", "mike",
 name = "prefect-dask"
 version = "0.2.4"
 description = "Prefect integrations with the Dask execution framework."
-category = "main"
 optional = false
 python-versions = ">=3.7"
 files = [
@@ -2381,7 +2282,6 @@ dev = ["black", "coverage", "flake8", "flaky", "interrogate", "isort", "mkdocs",
 name = "prefect-shell"
 version = "0.1.5"
 description = "Prefect tasks and subflows for interacting with shell commands."
-category = "main"
 optional = false
 python-versions = ">=3.7"
 files = [
@@ -2399,7 +2299,6 @@ dev = ["black", "coverage", "flake8", "interrogate", "isort", "mkdocs", "mkdocs-
 name = "psutil"
 version = "5.9.5"
 description = "Cross-platform lib for process and system monitoring in Python."
-category = "main"
 optional = false
 python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*"
 files = [
@@ -2426,7 +2325,6 @@ test = ["enum34", "ipaddress", "mock", "pywin32", "wmi"]
 name = "pyasn1"
 version = "0.5.0"
 description = "Pure-Python implementation of ASN.1 types and DER/BER/CER codecs (X.208)"
-category = "main"
 optional = false
 python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,>=2.7"
 files = [
@@ -2438,7 +2336,6 @@ files = [
 name = "pyasn1-modules"
 version = "0.3.0"
 description = "A collection of ASN.1-based protocols modules"
-category = "main"
 optional = false
 python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,>=2.7"
 files = [
@@ -2453,7 +2350,6 @@ pyasn1 = ">=0.4.6,<0.6.0"
 name = "pycodestyle"
 version = "2.9.1"
 description = "Python style guide checker"
-category = "dev"
 optional = false
 python-versions = ">=3.6"
 files = [
@@ -2465,7 +2361,6 @@ files = [
 name = "pycparser"
 version = "2.21"
 description = "C parser in Python"
-category = "main"
 optional = false
 python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*"
 files = [
@@ -2477,7 +2372,6 @@ files = [
 name = "pydantic"
 version = "1.10.7"
 description = "Data validation and settings management using python type hints"
-category = "main"
 optional = false
 python-versions = ">=3.7"
 files = [
@@ -2530,7 +2424,6 @@ email = ["email-validator (>=1.0.3)"]
 name = "pydocstyle"
 version = "6.3.0"
 description = "Python docstring style checker"
-category = "dev"
 optional = false
 python-versions = ">=3.6"
 files = [
@@ -2548,7 +2441,6 @@ toml = ["tomli (>=1.2.3)"]
 name = "pyflakes"
 version = "2.5.0"
 description = "passive checker of Python programs"
-category = "dev"
 optional = false
 python-versions = ">=3.6"
 files = [
@@ -2560,7 +2452,6 @@ files = [
 name = "pygit2"
 version = "1.12.1"
 description = "Python bindings for libgit2."
-category = "main"
 optional = false
 python-versions = ">=3.8"
 files = [
@@ -2604,7 +2495,6 @@ cffi = ">=1.9.1"
 name = "pygments"
 version = "2.15.1"
 description = "Pygments is a syntax highlighting package written in Python."
-category = "main"
 optional = false
 python-versions = ">=3.7"
 files = [
@@ -2619,7 +2509,6 @@ plugins = ["importlib-metadata"]
 name = "pyhcl"
 version = "0.4.4"
 description = "HCL configuration parser for python"
-category = "main"
 optional = false
 python-versions = "*"
 files = [
@@ -2630,7 +2519,6 @@ files = [
 name = "pymdown-extensions"
 version = "10.0.1"
 description = "Extension pack for Python Markdown."
-category = "dev"
 optional = false
 python-versions = ">=3.7"
 files = [
@@ -2646,7 +2534,6 @@ pyyaml = "*"
 name = "pyproject-api"
 version = "1.5.1"
 description = "API to interact with the python pyproject.toml based projects"
-category = "dev"
 optional = false
 python-versions = ">=3.7"
 files = [
@@ -2666,7 +2553,6 @@ testing = ["covdefaults (>=2.2.2)", "importlib-metadata (>=6)", "pytest (>=7.2.1
 name = "pyrsistent"
 version = "0.19.3"
 description = "Persistent/Functional/Immutable data structures"
-category = "main"
 optional = false
 python-versions = ">=3.7"
 files = [
@@ -2703,7 +2589,6 @@ files = [
 name = "pytest"
 version = "7.3.1"
 description = "pytest: simple powerful testing with Python"
-category = "main"
 optional = false
 python-versions = ">=3.7"
 files = [
@@ -2726,7 +2611,6 @@ testing = ["argcomplete", "attrs (>=19.2.0)", "hypothesis (>=3.56)", "mock", "no
 name = "pytest-asyncio"
 version = "0.21.1"
 description = "Pytest support for asyncio"
-category = "main"
 optional = false
 python-versions = ">=3.7"
 files = [
@@ -2745,7 +2629,6 @@ testing = ["coverage (>=6.2)", "flaky (>=3.5.0)", "hypothesis (>=5.7.1)", "mypy
 name = "pytest-cases"
 version = "3.6.14"
 description = "Separate test code from test cases in pytest."
-category = "dev"
 optional = false
 python-versions = "*"
 files = [
@@ -2761,7 +2644,6 @@ makefun = ">=1.9.5"
 name = "pytest-cov"
 version = "4.0.0"
 description = "Pytest plugin for measuring coverage."
-category = "dev"
 optional = false
 python-versions = ">=3.6"
 files = [
@@ -2780,7 +2662,6 @@ testing = ["fields", "hunter", "process-tests", "pytest-xdist", "six", "virtuale
 name = "pytest-httpx"
 version = "0.21.3"
 description = "Send responses to httpx."
-category = "dev"
 optional = false
 python-versions = ">=3.7"
 files = [
@@ -2789,17 +2670,16 @@ files = [
 ]
 
 [package.dependencies]
-httpx = ">=0.23.0,<0.24.0"
+httpx = "==0.23.*"
 pytest = ">=6.0,<8.0"
 
 [package.extras]
-testing = ["pytest-asyncio (>=0.20.0,<0.21.0)", "pytest-cov (>=4.0.0,<5.0.0)"]
+testing = ["pytest-asyncio (==0.20.*)", "pytest-cov (==4.*)"]
 
 [[package]]
 name = "pytest-mock"
 version = "3.11.1"
 description = "Thin-wrapper around the mock package for easier use with pytest"
-category = "main"
 optional = false
 python-versions = ">=3.7"
 files = [
@@ -2817,7 +2697,6 @@ dev = ["pre-commit", "pytest-asyncio", "tox"]
 name = "python-box"
 version = "7.0.1"
 description = "Advanced Python dictionaries with dot notation access"
-category = "main"
 optional = false
 python-versions = ">=3.7"
 files = [
@@ -2852,7 +2731,6 @@ yaml = ["ruamel.yaml (>=0.17)"]
 name = "python-dateutil"
 version = "2.8.2"
 description = "Extensions to the standard Python datetime module"
-category = "main"
 optional = false
 python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,>=2.7"
 files = [
@@ -2867,7 +2745,6 @@ six = ">=1.5"
 name = "python-ldap"
 version = "3.4.3"
 description = "Python modules for implementing LDAP clients"
-category = "main"
 optional = false
 python-versions = ">=3.6"
 files = [
@@ -2878,11 +2755,24 @@ files = [
 pyasn1 = ">=0.3.7"
 pyasn1_modules = ">=0.1.5"
 
+[[package]]
+name = "python-multipart"
+version = "0.0.6"
+description = "A streaming multipart parser for Python"
+optional = false
+python-versions = ">=3.7"
+files = [
+    {file = "python_multipart-0.0.6-py3-none-any.whl", hash = "sha256:ee698bab5ef148b0a760751c261902cd096e57e10558e11aca17646b74ee1c18"},
+    {file = "python_multipart-0.0.6.tar.gz", hash = "sha256:e9925a80bb668529f1b67c7fdb0a5dacdd7cbfc6fb0bff3ea443fe22bdd62132"},
+]
+
+[package.extras]
+dev = ["atomicwrites (==1.2.1)", "attrs (==19.2.0)", "coverage (==6.5.0)", "hatch", "invoke (==1.7.3)", "more-itertools (==4.3.0)", "pbr (==4.3.0)", "pluggy (==1.0.0)", "py (==1.11.0)", "pytest (==7.2.0)", "pytest-cov (==4.0.0)", "pytest-timeout (==2.1.0)", "pyyaml (==5.1)"]
+
 [[package]]
 name = "python-slugify"
 version = "8.0.1"
 description = "A Python slugify application that also handles Unicode"
-category = "main"
 optional = false
 python-versions = ">=3.7"
 files = [
@@ -2900,7 +2790,6 @@ unidecode = ["Unidecode (>=1.1.1)"]
 name = "pytz"
 version = "2023.3"
 description = "World timezone definitions, modern and historical"
-category = "main"
 optional = false
 python-versions = "*"
 files = [
@@ -2912,7 +2801,6 @@ files = [
 name = "pytzdata"
 version = "2020.1"
 description = "The Olson timezone database for Python."
-category = "main"
 optional = false
 python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*"
 files = [
@@ -2924,7 +2812,6 @@ files = [
 name = "pywin32"
 version = "306"
 description = "Python for Window Extensions"
-category = "main"
 optional = false
 python-versions = "*"
 files = [
@@ -2948,7 +2835,6 @@ files = [
 name = "pywin32-ctypes"
 version = "0.2.0"
 description = ""
-category = "dev"
 optional = false
 python-versions = "*"
 files = [
@@ -2960,7 +2846,6 @@ files = [
 name = "pyyaml"
 version = "6.0"
 description = "YAML parser and emitter for Python"
-category = "main"
 optional = false
 python-versions = ">=3.6"
 files = [
@@ -3010,7 +2895,6 @@ files = [
 name = "pyyaml-env-tag"
 version = "0.1"
 description = "A custom YAML tag for referencing environment variables in YAML files. "
-category = "dev"
 optional = false
 python-versions = ">=3.6"
 files = [
@@ -3025,7 +2909,6 @@ pyyaml = "*"
 name = "readchar"
 version = "4.0.5"
 description = "Library to easily read single chars and key strokes"
-category = "main"
 optional = false
 python-versions = ">=3.7"
 files = [
@@ -3040,7 +2923,6 @@ setuptools = ">=41.0"
 name = "readme-renderer"
 version = "37.3"
 description = "readme_renderer is a library for rendering \"readme\" descriptions for Warehouse"
-category = "dev"
 optional = false
 python-versions = ">=3.7"
 files = [
@@ -3060,7 +2942,6 @@ md = ["cmarkgfm (>=0.8.0)"]
 name = "regex"
 version = "2023.5.5"
 description = "Alternative regular expression module, to replace re."
-category = "main"
 optional = false
 python-versions = ">=3.6"
 files = [
@@ -3158,7 +3039,6 @@ files = [
 name = "requests"
 version = "2.29.0"
 description = "Python HTTP for Humans."
-category = "main"
 optional = false
 python-versions = ">=3.7"
 files = [
@@ -3180,7 +3060,6 @@ use-chardet-on-py3 = ["chardet (>=3.0.2,<6)"]
 name = "requests-oauthlib"
 version = "1.3.1"
 description = "OAuthlib authentication support for Requests."
-category = "main"
 optional = false
 python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*"
 files = [
@@ -3199,7 +3078,6 @@ rsa = ["oauthlib[signedtoken] (>=3.0.0)"]
 name = "requests-toolbelt"
 version = "1.0.0"
 description = "A utility belt for advanced users of python-requests"
-category = "dev"
 optional = false
 python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*"
 files = [
@@ -3214,7 +3092,6 @@ requests = ">=2.0.1,<3.0.0"
 name = "rfc3986"
 version = "1.5.0"
 description = "Validating URI References per RFC 3986"
-category = "main"
 optional = false
 python-versions = "*"
 files = [
@@ -3232,7 +3109,6 @@ idna2008 = ["idna"]
 name = "rich"
 version = "13.3.5"
 description = "Render rich text, tables, progress bars, syntax highlighting, markdown and more to the terminal"
-category = "main"
 optional = false
 python-versions = ">=3.7.0"
 files = [
@@ -3251,7 +3127,6 @@ jupyter = ["ipywidgets (>=7.5.1,<9)"]
 name = "rsa"
 version = "4.9"
 description = "Pure-Python RSA implementation"
-category = "main"
 optional = false
 python-versions = ">=3.6,<4"
 files = [
@@ -3266,7 +3141,6 @@ pyasn1 = ">=0.1.3"
 name = "ruamel-yaml"
 version = "0.17.26"
 description = "ruamel.yaml is a YAML parser/emitter that supports roundtrip preservation of comments, seq/map flow style, and map key order"
-category = "main"
 optional = false
 python-versions = ">=3"
 files = [
@@ -3285,7 +3159,6 @@ jinja2 = ["ruamel.yaml.jinja2 (>=0.2)"]
 name = "ruamel-yaml-clib"
 version = "0.2.7"
 description = "C version of reader, parser and emitter for ruamel.yaml derived from libyaml"
-category = "main"
 optional = false
 python-versions = ">=3.5"
 files = [
@@ -3296,7 +3169,8 @@ files = [
     {file = "ruamel.yaml.clib-0.2.7-cp310-cp310-win32.whl", hash = "sha256:763d65baa3b952479c4e972669f679fe490eee058d5aa85da483ebae2009d231"},
     {file = "ruamel.yaml.clib-0.2.7-cp310-cp310-win_amd64.whl", hash = "sha256:d000f258cf42fec2b1bbf2863c61d7b8918d31ffee905da62dede869254d3b8a"},
     {file = "ruamel.yaml.clib-0.2.7-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:045e0626baf1c52e5527bd5db361bc83180faaba2ff586e763d3d5982a876a9e"},
-    {file = "ruamel.yaml.clib-0.2.7-cp311-cp311-macosx_12_6_arm64.whl", hash = "sha256:721bc4ba4525f53f6a611ec0967bdcee61b31df5a56801281027a3a6d1c2daf5"},
+    {file = "ruamel.yaml.clib-0.2.7-cp311-cp311-macosx_13_0_arm64.whl", hash = "sha256:1a6391a7cabb7641c32517539ca42cf84b87b667bad38b78d4d42dd23e957c81"},
+    {file = "ruamel.yaml.clib-0.2.7-cp311-cp311-manylinux2014_aarch64.whl", hash = "sha256:9c7617df90c1365638916b98cdd9be833d31d337dbcd722485597b43c4a215bf"},
     {file = "ruamel.yaml.clib-0.2.7-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_24_x86_64.whl", hash = "sha256:41d0f1fa4c6830176eef5b276af04c89320ea616655d01327d5ce65e50575c94"},
     {file = "ruamel.yaml.clib-0.2.7-cp311-cp311-win32.whl", hash = "sha256:f6d3d39611ac2e4f62c3128a9eed45f19a6608670c5a2f4f07f24e8de3441d38"},
     {file = "ruamel.yaml.clib-0.2.7-cp311-cp311-win_amd64.whl", hash = "sha256:da538167284de58a52109a9b89b8f6a53ff8437dd6dc26d33b57bf6699153122"},
@@ -3331,7 +3205,6 @@ files = [
 name = "secretstorage"
 version = "3.3.3"
 description = "Python bindings to FreeDesktop.org Secret Service API"
-category = "dev"
 optional = false
 python-versions = ">=3.6"
 files = [
@@ -3347,7 +3220,6 @@ jeepney = ">=0.6"
 name = "semver"
 version = "3.0.0"
 description = "Python helper for Semantic Versioning (https://semver.org)"
-category = "main"
 optional = false
 python-versions = ">=3.7"
 files = [
@@ -3359,7 +3231,6 @@ files = [
 name = "setuptools"
 version = "67.7.2"
 description = "Easily download, build, install, upgrade, and uninstall Python packages"
-category = "main"
 optional = false
 python-versions = ">=3.7"
 files = [
@@ -3376,7 +3247,6 @@ testing-integration = ["build[virtualenv]", "filelock (>=3.4.0)", "jaraco.envs (
 name = "singleton-decorator"
 version = "1.0.0"
 description = "A testable singleton decorator"
-category = "main"
 optional = false
 python-versions = "*"
 files = [
@@ -3387,7 +3257,6 @@ files = [
 name = "six"
 version = "1.16.0"
 description = "Python 2 and 3 compatibility utilities"
-category = "main"
 optional = false
 python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*"
 files = [
@@ -3399,7 +3268,6 @@ files = [
 name = "sniffio"
 version = "1.3.0"
 description = "Sniff out which async library your code is running under"
-category = "main"
 optional = false
 python-versions = ">=3.7"
 files = [
@@ -3411,7 +3279,6 @@ files = [
 name = "snowballstemmer"
 version = "2.2.0"
 description = "This package provides 29 stemmers for 28 languages generated from Snowball algorithms."
-category = "dev"
 optional = false
 python-versions = "*"
 files = [
@@ -3423,7 +3290,6 @@ files = [
 name = "sortedcontainers"
 version = "2.4.0"
 description = "Sorted Containers -- Sorted List, Sorted Dict, Sorted Set"
-category = "main"
 optional = false
 python-versions = "*"
 files = [
@@ -3435,7 +3301,6 @@ files = [
 name = "sqlalchemy"
 version = "1.4.45"
 description = "Database Abstraction Library"
-category = "main"
 optional = false
 python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,>=2.7"
 files = [
@@ -3483,7 +3348,7 @@ files = [
 ]
 
 [package.dependencies]
-greenlet = {version = "!=0.4.17", optional = true, markers = "python_version >= \"3\" and platform_machine == \"aarch64\" or python_version >= \"3\" and platform_machine == \"ppc64le\" or python_version >= \"3\" and platform_machine == \"x86_64\" or python_version >= \"3\" and platform_machine == \"amd64\" or python_version >= \"3\" and platform_machine == \"AMD64\" or python_version >= \"3\" and platform_machine == \"win32\" or python_version >= \"3\" and platform_machine == \"WIN32\" or python_version >= \"3\" and extra == \"asyncio\""}
+greenlet = {version = "!=0.4.17", optional = true, markers = "python_version >= \"3\" and (platform_machine == \"aarch64\" or platform_machine == \"ppc64le\" or platform_machine == \"x86_64\" or platform_machine == \"amd64\" or platform_machine == \"AMD64\" or platform_machine == \"win32\" or platform_machine == \"WIN32\" or extra == \"asyncio\")"}
 
 [package.extras]
 aiomysql = ["aiomysql", "greenlet (!=0.4.17)"]
@@ -3510,7 +3375,6 @@ sqlcipher = ["sqlcipher3-binary"]
 name = "starlette"
 version = "0.26.1"
 description = "The little ASGI library that shines."
-category = "main"
 optional = false
 python-versions = ">=3.7"
 files = [
@@ -3529,7 +3393,6 @@ full = ["httpx (>=0.22.0)", "itsdangerous", "jinja2", "python-multipart", "pyyam
 name = "strawberry-graphql"
 version = "0.177.1"
 description = "A library for creating GraphQL APIs"
-category = "main"
 optional = false
 python-versions = ">=3.7,<4.0"
 files = [
@@ -3563,7 +3426,6 @@ starlite = ["starlite (>=1.48.0)"]
 name = "tblib"
 version = "1.7.0"
 description = "Traceback serialization library."
-category = "main"
 optional = false
 python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*"
 files = [
@@ -3575,7 +3437,6 @@ files = [
 name = "text-unidecode"
 version = "1.3"
 description = "The most basic Text::Unidecode port"
-category = "main"
 optional = false
 python-versions = "*"
 files = [
@@ -3587,7 +3448,6 @@ files = [
 name = "toml"
 version = "0.10.2"
 description = "Python Library for Tom's Obvious, Minimal Language"
-category = "main"
 optional = false
 python-versions = ">=2.6, !=3.0.*, !=3.1.*, !=3.2.*"
 files = [
@@ -3599,7 +3459,6 @@ files = [
 name = "tomli"
 version = "2.0.1"
 description = "A lil' TOML parser"
-category = "main"
 optional = false
 python-versions = ">=3.7"
 files = [
@@ -3611,7 +3470,6 @@ files = [
 name = "toolz"
 version = "0.12.0"
 description = "List processing tools and functional utilities"
-category = "main"
 optional = false
 python-versions = ">=3.5"
 files = [
@@ -3623,7 +3481,6 @@ files = [
 name = "tornado"
 version = "6.3.2"
 description = "Tornado is a Python web framework and asynchronous networking library, originally developed at FriendFeed."
-category = "main"
 optional = false
 python-versions = ">= 3.8"
 files = [
@@ -3644,7 +3501,6 @@ files = [
 name = "tox"
 version = "4.5.1"
 description = "tox is a generic virtualenv management and test command line tool"
-category = "dev"
 optional = false
 python-versions = ">=3.7"
 files = [
@@ -3672,7 +3528,6 @@ testing = ["build[virtualenv] (>=0.10)", "covdefaults (>=2.3)", "devpi-process (
 name = "twine"
 version = "4.0.2"
 description = "Collection of utilities for publishing packages on PyPI"
-category = "dev"
 optional = false
 python-versions = ">=3.7"
 files = [
@@ -3695,7 +3550,6 @@ urllib3 = ">=1.26.0"
 name = "typer"
 version = "0.9.0"
 description = "Typer, build great CLIs. Easy to code. Based on Python type hints."
-category = "main"
 optional = false
 python-versions = ">=3.6"
 files = [
@@ -3717,7 +3571,6 @@ test = ["black (>=22.3.0,<23.0.0)", "coverage (>=6.2,<7.0)", "isort (>=5.0.6,<6.
 name = "types-pyyaml"
 version = "6.0.12.9"
 description = "Typing stubs for PyYAML"
-category = "dev"
 optional = false
 python-versions = "*"
 files = [
@@ -3729,7 +3582,6 @@ files = [
 name = "types-requests"
 version = "2.30.0.0"
 description = "Typing stubs for requests"
-category = "dev"
 optional = false
 python-versions = "*"
 files = [
@@ -3744,7 +3596,6 @@ types-urllib3 = "*"
 name = "types-setuptools"
 version = "67.7.0.2"
 description = "Typing stubs for setuptools"
-category = "dev"
 optional = false
 python-versions = "*"
 files = [
@@ -3756,7 +3607,6 @@ files = [
 name = "types-urllib3"
 version = "1.26.25.13"
 description = "Typing stubs for urllib3"
-category = "dev"
 optional = false
 python-versions = "*"
 files = [
@@ -3768,7 +3618,6 @@ files = [
 name = "typing-extensions"
 version = "4.5.0"
 description = "Backported and Experimental Type Hints for Python 3.7+"
-category = "main"
 optional = false
 python-versions = ">=3.7"
 files = [
@@ -3780,7 +3629,6 @@ files = [
 name = "tzdata"
 version = "2023.3"
 description = "Provider of IANA time zone data"
-category = "main"
 optional = false
 python-versions = ">=2"
 files = [
@@ -3792,7 +3640,6 @@ files = [
 name = "tzlocal"
 version = "5.0.1"
 description = "tzinfo object for the local timezone"
-category = "main"
 optional = false
 python-versions = ">=3.7"
 files = [
@@ -3810,7 +3657,6 @@ devenv = ["black", "check-manifest", "flake8", "pyroma", "pytest (>=4.3)", "pyte
 name = "urllib3"
 version = "1.26.15"
 description = "HTTP library with thread-safe connection pooling, file post, and more."
-category = "main"
 optional = false
 python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*, !=3.5.*"
 files = [
@@ -3827,7 +3673,6 @@ socks = ["PySocks (>=1.5.6,!=1.5.7,<2.0)"]
 name = "uvicorn"
 version = "0.22.0"
 description = "The lightning-fast ASGI server."
-category = "main"
 optional = false
 python-versions = ">=3.7"
 files = [
@@ -3846,7 +3691,6 @@ standard = ["colorama (>=0.4)", "httptools (>=0.5.0)", "python-dotenv (>=0.13)",
 name = "virtualenv"
 version = "20.23.0"
 description = "Virtual Python Environment builder"
-category = "dev"
 optional = false
 python-versions = ">=3.7"
 files = [
@@ -3867,7 +3711,6 @@ test = ["covdefaults (>=2.3)", "coverage (>=7.2.3)", "coverage-enable-subprocess
 name = "watchdog"
 version = "3.0.0"
 description = "Filesystem events monitoring"
-category = "dev"
 optional = false
 python-versions = ">=3.7"
 files = [
@@ -3907,7 +3750,6 @@ watchmedo = ["PyYAML (>=3.10)"]
 name = "webencodings"
 version = "0.5.1"
 description = "Character encoding aliases for legacy web content"
-category = "dev"
 optional = false
 python-versions = "*"
 files = [
@@ -3919,7 +3761,6 @@ files = [
 name = "websocket-client"
 version = "1.5.1"
 description = "WebSocket client for Python with low level API options"
-category = "main"
 optional = false
 python-versions = ">=3.7"
 files = [
@@ -3936,7 +3777,6 @@ test = ["websockets"]
 name = "websockets"
 version = "11.0.3"
 description = "An implementation of the WebSocket Protocol (RFC 6455 & 7692)"
-category = "main"
 optional = false
 python-versions = ">=3.7"
 files = [
@@ -4016,7 +3856,6 @@ files = [
 name = "zict"
 version = "3.0.0"
 description = "Mutable mapping tools"
-category = "main"
 optional = false
 python-versions = ">=3.8"
 files = [
@@ -4028,7 +3867,6 @@ files = [
 name = "zipp"
 version = "3.15.0"
 description = "Backport of pathlib-compatible object wrapper for zip files"
-category = "main"
 optional = false
 python-versions = ">=3.7"
 files = [
@@ -4043,4 +3881,4 @@ testing = ["big-O", "flake8 (<5)", "jaraco.functools", "jaraco.itertools", "more
 [metadata]
 lock-version = "2.0"
 python-versions = ">=3.9,<4.0"
-content-hash = "d9352aa50d84b36bfeae85c3339f95cd4a6a6d0d2bb6cfb60b85f03a2398392a"
+content-hash = "275c6e07b55140f0db60185c15cd07c6cc3e6210bb46a879d04aebb528628743"
diff --git a/pyproject.toml b/pyproject.toml
index 3c43236..e44e433 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -50,6 +50,7 @@ strawberry-graphql = "^0.177.1"
 typer = "^0.9.0"
 pytest-mock = "^3.11.1"
 pytest-asyncio = "^0.21.1"
+python-multipart = "^0.0.6"
 
 [tool.poetry.group.dev]
 optional = true

From 68e377d6c1ac3834e4aa48ee314e02216946eb35 Mon Sep 17 00:00:00 2001
From: Michael Woolnough <mw31@sanger.ac.uk>
Date: Fri, 8 Sep 2023 14:32:31 +0100
Subject: [PATCH 117/129] Split version strings from name.

---
 softpack_core/artifacts.py            | 32 ++++++++++++++++++++++++
 softpack_core/schemas/environment.py  | 17 ++-----------
 tests/integration/test_artifacts.py   | 10 ++++++++
 tests/integration/test_environment.py | 36 ++++++++++++++++-----------
 tests/integration/utils.py            | 11 +++++---
 5 files changed, 73 insertions(+), 33 deletions(-)

diff --git a/softpack_core/artifacts.py b/softpack_core/artifacts.py
index b555715..ed2bbd8 100644
--- a/softpack_core/artifacts.py
+++ b/softpack_core/artifacts.py
@@ -11,12 +11,40 @@
 from typing import Iterable, Iterator, Optional
 
 import pygit2
+import strawberry
 from box import Box
 
+from softpack_core.spack import Spack
+
 from .app import app
 from .ldapapi import LDAP
 
 
+@strawberry.type
+class Package(Spack.PackageBase):
+    """A Strawberry model representing a package."""
+
+    version: Optional[str] = None
+
+    @classmethod
+    def from_name(cls, name: str) -> 'Package':
+        """Makes a new Package based on the name.
+
+        Args:
+            name (str): Combined name and version string, deliniated by an '@'.
+
+        Returns:
+            Package: A Package with name set, and version set if given name had
+                     a version.
+        """
+        parts = name.split("@", 2)
+
+        if len(parts) == 2:
+            return Package(name=parts[0], version=parts[1])
+
+        return Package(name=name)
+
+
 class Artifacts:
     """Artifacts repo access class."""
 
@@ -96,6 +124,10 @@ def spec(self) -> Box:
             else:
                 info["type"] = Artifacts.built_by_softpack
 
+            info.packages = list(
+                map(lambda p: Package.from_name(p), info.packages)
+            )
+
             return info
 
         def __iter__(self) -> Iterator["Artifacts.Object"]:
diff --git a/softpack_core/schemas/environment.py b/softpack_core/schemas/environment.py
index 58c1400..7ac2f72 100644
--- a/softpack_core/schemas/environment.py
+++ b/softpack_core/schemas/environment.py
@@ -14,10 +14,9 @@
 from starlette.datastructures import UploadFile
 from strawberry.file_uploads import Upload
 
-from softpack_core.artifacts import Artifacts
+from softpack_core.artifacts import Artifacts, Package
 from softpack_core.module import GenerateEnvReadme, ToSoftpackYML
 from softpack_core.schemas.base import BaseSchema
-from softpack_core.spack import Spack
 
 
 # Interfaces
@@ -117,13 +116,6 @@ class EnvironmentAlreadyExistsError(Error):
 )
 
 
-@strawberry.type
-class Package(Spack.PackageBase):
-    """A Strawberry model representing a package."""
-
-    version: Optional[str] = None
-
-
 @strawberry.input
 class PackageInput(Package):
     """A Strawberry input model representing a package."""
@@ -188,12 +180,7 @@ def from_artifact(cls, obj: Artifacts.Object) -> Optional["Environment"]:
                 name=obj.name,
                 path=str(obj.path.parent),
                 description=spec.description,
-                packages=list(
-                    map(
-                        lambda package: Package(name=package),
-                        spec.packages,
-                    )
-                ),  # type: ignore [call-arg]
+                packages=spec.packages,
                 state=None,
                 readme=spec.get("readme", ""),
                 type=spec.get("type", ""),
diff --git a/tests/integration/test_artifacts.py b/tests/integration/test_artifacts.py
index bc82aee..92a8923 100644
--- a/tests/integration/test_artifacts.py
+++ b/tests/integration/test_artifacts.py
@@ -215,3 +215,13 @@ def test_iter() -> None:
     assert user_found is True
     assert num_user_envs == 1
     assert num_group_envs == 1
+
+    envs = artifacts.iter()
+    pkgs = list(envs)[0].spec().packages
+    assert len(pkgs) == 3
+    assert pkgs[0].name == "pck1"
+    assert pkgs[0].version == "1"
+    assert pkgs[1].name == "pck2"
+    assert pkgs[1].version == "v2.0.1"
+    assert pkgs[2].name == "pck3"
+    assert pkgs[2].version is None
diff --git a/tests/integration/test_environment.py b/tests/integration/test_environment.py
index 680fe69..fc64a18 100644
--- a/tests/integration/test_environment.py
+++ b/tests/integration/test_environment.py
@@ -211,12 +211,12 @@ async def test_write_artifact(httpx_post, testable_env_input, upload):
 
 @pytest.mark.asyncio
 async def test_iter(httpx_post, testable_env_input, upload):
-    envs_filter = Environment.iter()
+    envs = Environment.iter()
     count = 0
-    for env in envs_filter:
+    for env in envs:
         count += 1
 
-    assert count == 0
+    assert count == 2
 
     result = Environment.create(testable_env_input)
     assert isinstance(result, CreateEnvironmentSuccess)
@@ -224,7 +224,9 @@ async def test_iter(httpx_post, testable_env_input, upload):
 
     upload.filename = Artifacts.environments_file
     upload.content_type = "text/plain"
-    upload.read.return_value = b"description: test env\npackages:\n  - zlib\n"
+    upload.read.return_value = (
+        b"description: test env\n" b"packages:\n  - zlib@v1.1\n"
+    )
 
     result = await Environment.write_artifact(
         file=upload,
@@ -233,13 +235,14 @@ async def test_iter(httpx_post, testable_env_input, upload):
     )
     assert isinstance(result, WriteArtifactSuccess)
 
-    envs_filter = Environment.iter()
+    envs = Environment.iter()
     count = 0
-    for env in envs_filter:
-        assert env.name == testable_env_input.name
-        assert any(p.name == "zlib" for p in env.packages)
-        assert env.type == Artifacts.built_by_softpack
-        count += 1
+    for env in envs:
+        if env.name == testable_env_input.name:
+            assert any(p.name == "zlib" for p in env.packages)
+            assert any(p.version == "v1.1" for p in env.packages)
+            assert env.type == Artifacts.built_by_softpack
+            count += 1
 
     assert count == 1
 
@@ -290,13 +293,16 @@ async def test_create_from_module(httpx_post, testable_env_input, upload):
 
     envs = list(Environment.iter())
 
-    assert len(envs) == 1
+    assert len(envs) == 3
 
-    env = envs[0]
+    env = next((env for env in envs if env.name == env_name), None)
+    assert env is not None
 
-    package_name = "quay.io/biocontainers/ldsc@1.0.1--pyhdfd78af_2"
+    package_name = "quay.io/biocontainers/ldsc"
+    package_version = "1.0.1--pyhdfd78af_2"
 
-    assert env.name == env_name
-    assert len(env.packages) == 1 and env.packages[0].name == package_name
+    assert len(env.packages) == 1
+    assert env.packages[0].name == package_name
+    assert env.packages[0].version == package_version
     assert "module load " + module_path in env.readme
     assert env.type == Artifacts.generated_from_module
diff --git a/tests/integration/utils.py b/tests/integration/utils.py
index 200730c..6948588 100644
--- a/tests/integration/utils.py
+++ b/tests/integration/utils.py
@@ -98,9 +98,14 @@ def create_initial_test_repo_state(artifacts: Artifacts) -> artifacts_dict:
         test_group,
         test_env,
     )
-    file_basename = "file.txt"
+    file_basename = Artifacts.environments_file
 
-    oid = artifacts.repo.create_blob(b"")
+    softpack_yml_data = (
+        b"description: \"desc\"\npackages:\n"
+        b"  - pck1@1\n  - pck2@v2.0.1\n  - pck3"
+    )
+
+    oid = artifacts.repo.create_blob(softpack_yml_data)
 
     userTestEnv = artifacts.repo.TreeBuilder()
     userTestEnv.insert(file_basename, oid, pygit2.GIT_FILEMODE_BLOB)
@@ -111,7 +116,7 @@ def create_initial_test_repo_state(artifacts: Artifacts) -> artifacts_dict:
     usersFolder = artifacts.repo.TreeBuilder()
     usersFolder.insert(test_user, testUser.write(), pygit2.GIT_FILEMODE_TREE)
 
-    oid = artifacts.repo.create_blob(b"")
+    oid = artifacts.repo.create_blob(softpack_yml_data)
 
     userGroupEnv = artifacts.repo.TreeBuilder()
     userGroupEnv.insert(file_basename, oid, pygit2.GIT_FILEMODE_BLOB)

From 5c01150aad9d4181dadcb9c55b7cbfb96e762400 Mon Sep 17 00:00:00 2001
From: Michael Woolnough <mw31@sanger.ac.uk>
Date: Fri, 8 Sep 2023 15:15:44 +0100
Subject: [PATCH 118/129] Allow multiple spaces after module-whatis and puts
 stderr statements.

---
 softpack_core/module.py            | 6 ++++--
 tests/files/modules/all_fields.mod | 2 +-
 2 files changed, 5 insertions(+), 3 deletions(-)

diff --git a/softpack_core/module.py b/softpack_core/module.py
index 6b83313..4af4043 100644
--- a/softpack_core/module.py
+++ b/softpack_core/module.py
@@ -47,7 +47,8 @@ def ToSoftpackYML(name: str, contents: Union[bytes, str]) -> bytes:
                 in_help = False
             elif line.startswith(b"puts stderr "):
                 line_str = (
-                    line.removeprefix(b"puts stderr ")
+                    line.removeprefix(b"puts stderr")
+                    .lstrip()
                     .decode('unicode_escape')
                     .replace("\\$", "$")
                     .removeprefix("\"")
@@ -59,7 +60,8 @@ def ToSoftpackYML(name: str, contents: Union[bytes, str]) -> bytes:
                 in_help = True
             elif line.startswith(b"module-whatis "):
                 line_str = (
-                    line.removeprefix(b"module-whatis ")
+                    line.removeprefix(b"module-whatis")
+                    .lstrip()
                     .decode('unicode_escape')
                     .removeprefix("\"")
                     .removesuffix("\"")
diff --git a/tests/files/modules/all_fields.mod b/tests/files/modules/all_fields.mod
index f9ec71e..9d51de2 100644
--- a/tests/files/modules/all_fields.mod
+++ b/tests/files/modules/all_fields.mod
@@ -10,7 +10,7 @@ proc ModulesHelp { } {
 }
 
 module-whatis "Name:   name_of_container  "
-module-whatis "Version:1.0.1"
+module-whatis   "Version:1.0.1"
 
 module-whatis "Foo: bar"
 

From c3aa147b0d73537f17f4b4a06e5815e81e1c71cd Mon Sep 17 00:00:00 2001
From: Michael Woolnough <mw31@sanger.ac.uk>
Date: Mon, 11 Sep 2023 13:50:14 +0100
Subject: [PATCH 119/129] Determine environment state from artifacts.

---
 softpack_core/artifacts.py            | 14 ++++++++
 softpack_core/schemas/environment.py  |  6 ++--
 tests/integration/test_environment.py | 51 ++++++++++++++++++---------
 3 files changed, 51 insertions(+), 20 deletions(-)

diff --git a/softpack_core/artifacts.py b/softpack_core/artifacts.py
index ed2bbd8..89d16dd 100644
--- a/softpack_core/artifacts.py
+++ b/softpack_core/artifacts.py
@@ -7,6 +7,7 @@
 import itertools
 import shutil
 from dataclasses import dataclass
+from enum import Enum
 from pathlib import Path
 from typing import Iterable, Iterator, Optional
 
@@ -45,6 +46,14 @@ def from_name(cls, name: str) -> 'Package':
         return Package(name=name)
 
 
+@strawberry.enum
+class State(Enum):
+    """Environment states."""
+
+    ready = 'ready'
+    queued = 'queued'
+
+
 class Artifacts:
     """Artifacts repo access class."""
 
@@ -119,6 +128,11 @@ def spec(self) -> Box:
             if Artifacts.readme_file in self.obj:
                 info["readme"] = self.obj[Artifacts.readme_file].data.decode()
 
+            if Artifacts.module_file in self.obj:
+                info["state"] = State.ready
+            else:
+                info["state"] = State.queued
+
             if Artifacts.generated_from_module_file in self.obj:
                 info["type"] = Artifacts.generated_from_module
             else:
diff --git a/softpack_core/schemas/environment.py b/softpack_core/schemas/environment.py
index 7ac2f72..e83ea81 100644
--- a/softpack_core/schemas/environment.py
+++ b/softpack_core/schemas/environment.py
@@ -14,7 +14,7 @@
 from starlette.datastructures import UploadFile
 from strawberry.file_uploads import Upload
 
-from softpack_core.artifacts import Artifacts, Package
+from softpack_core.artifacts import Artifacts, Package, State
 from softpack_core.module import GenerateEnvReadme, ToSoftpackYML
 from softpack_core.schemas.base import BaseSchema
 
@@ -149,7 +149,7 @@ class Environment:
     readme: str
     type: str
     packages: list[Package]
-    state: Optional[str]
+    state: Optional[State]
     artifacts = Artifacts()
 
     @classmethod
@@ -181,7 +181,7 @@ def from_artifact(cls, obj: Artifacts.Object) -> Optional["Environment"]:
                 path=str(obj.path.parent),
                 description=spec.description,
                 packages=spec.packages,
-                state=None,
+                state=spec.state,
                 readme=spec.get("readme", ""),
                 type=spec.get("type", ""),
             )
diff --git a/tests/integration/test_environment.py b/tests/integration/test_environment.py
index fc64a18..fede6cc 100644
--- a/tests/integration/test_environment.py
+++ b/tests/integration/test_environment.py
@@ -5,6 +5,7 @@
 """
 
 from pathlib import Path
+from typing import Optional
 
 import pygit2
 import pytest
@@ -19,6 +20,7 @@
     EnvironmentNotFoundError,
     InvalidInputError,
     Package,
+    State,
     UpdateEnvironmentSuccess,
     WriteArtifactSuccess,
 )
@@ -209,15 +211,13 @@ async def test_write_artifact(httpx_post, testable_env_input, upload):
     assert isinstance(result, InvalidInputError)
 
 
-@pytest.mark.asyncio
-async def test_iter(httpx_post, testable_env_input, upload):
+def test_iter(testable_env_input):
     envs = Environment.iter()
-    count = 0
-    for env in envs:
-        count += 1
+    assert len(list(envs)) == 2
 
-    assert count == 2
 
+@pytest.mark.asyncio
+async def test_states(httpx_post, testable_env_input, upload):
     result = Environment.create(testable_env_input)
     assert isinstance(result, CreateEnvironmentSuccess)
     httpx_post.assert_called_once()
@@ -235,16 +235,33 @@ async def test_iter(httpx_post, testable_env_input, upload):
     )
     assert isinstance(result, WriteArtifactSuccess)
 
-    envs = Environment.iter()
-    count = 0
-    for env in envs:
-        if env.name == testable_env_input.name:
-            assert any(p.name == "zlib" for p in env.packages)
-            assert any(p.version == "v1.1" for p in env.packages)
-            assert env.type == Artifacts.built_by_softpack
-            count += 1
+    env = get_env_from_iter(testable_env_input.name)
+    assert env is not None
+    assert any(p.name == "zlib" for p in env.packages)
+    assert any(p.version == "v1.1" for p in env.packages)
+    assert env.type == Artifacts.built_by_softpack
+    assert env.state == State.queued
+
+    upload.filename = Artifacts.module_file
+    upload.content_type = "text/plain"
+    upload.read.return_value = b"#%Module"
+
+    result = await Environment.write_artifact(
+        file=upload,
+        folder_path=f"{testable_env_input.path}/{testable_env_input.name}",
+        file_name=upload.filename,
+    )
+    assert isinstance(result, WriteArtifactSuccess)
 
-    assert count == 1
+    env = get_env_from_iter(testable_env_input.name)
+    assert env is not None
+    assert env.type == Artifacts.built_by_softpack
+    assert env.state == State.ready
+
+
+def get_env_from_iter(name: str) -> Optional[Environment]:
+    envs = Environment.iter()
+    return next((env for env in envs if env.name == name), None)
 
 
 @pytest.mark.asyncio
@@ -292,10 +309,9 @@ async def test_create_from_module(httpx_post, testable_env_input, upload):
     assert obj.data == expected_readme_data
 
     envs = list(Environment.iter())
-
     assert len(envs) == 3
 
-    env = next((env for env in envs if env.name == env_name), None)
+    env = get_env_from_iter(env_name)
     assert env is not None
 
     package_name = "quay.io/biocontainers/ldsc"
@@ -306,3 +322,4 @@ async def test_create_from_module(httpx_post, testable_env_input, upload):
     assert env.packages[0].version == package_version
     assert "module load " + module_path in env.readme
     assert env.type == Artifacts.generated_from_module
+    assert env.state == State.ready

From c9ca423c496fbe5518e62f00624fa9993c7b5e80 Mon Sep 17 00:00:00 2001
From: Michael Woolnough <mw31@sanger.ac.uk>
Date: Tue, 12 Sep 2023 10:27:28 +0100
Subject: [PATCH 120/129] New updateFromModule endpoint.

---
 softpack_core/schemas/environment.py  | 215 +++++++++++++++++++-------
 tests/integration/test_environment.py |  44 ++++++
 2 files changed, 205 insertions(+), 54 deletions(-)

diff --git a/softpack_core/schemas/environment.py b/softpack_core/schemas/environment.py
index e83ea81..ad919b3 100644
--- a/softpack_core/schemas/environment.py
+++ b/softpack_core/schemas/environment.py
@@ -7,7 +7,7 @@
 import io
 from dataclasses import dataclass
 from pathlib import Path
-from typing import Iterable, Optional
+from typing import Iterable, Optional, Union
 
 import httpx
 import strawberry
@@ -95,7 +95,6 @@ class EnvironmentAlreadyExistsError(Error):
         UpdateEnvironmentSuccess,
         InvalidInputError,
         EnvironmentNotFoundError,
-        EnvironmentAlreadyExistsError,
     ],
 )
 
@@ -137,6 +136,38 @@ class EnvironmentInput:
     description: str
     packages: list[PackageInput]
 
+    def validate(cls) -> Union[None, InvalidInputError]:
+        """Validate that all values have been supplied.
+
+        Returns:
+            None if good, or InvalidInputError if not all values supplied.
+        """
+        if any(len(value) == 0 for value in vars(cls).values()):
+            return InvalidInputError(message="all fields must be filled in")
+
+        return None
+
+    @classmethod
+    def from_path(cls, environment_path: str) -> 'EnvironmentInput':
+        """from_path creates a new EnvironmentInput based on an env path.
+
+        Args:
+            environment_path (str): path of the environment.
+
+        Returns:
+            EnvironmentInput: a package-less, description-less
+                              EnvironmentInput.
+        """
+        environment_dirs = environment_path.split("/")
+        environment_name = environment_dirs.pop()
+
+        return EnvironmentInput(
+            name=environment_name,
+            path="/".join(environment_dirs),
+            description="",
+            packages=list(),
+        )
+
 
 @strawberry.type
 class Environment:
@@ -198,9 +229,9 @@ def create(cls, env: EnvironmentInput) -> CreateResponse:  # type: ignore
         Returns:
             A message confirming the success or failure of the operation.
         """
-        # Check if any field has been left empty
-        if any(len(value) == 0 for value in vars(env).values()):
-            return InvalidInputError(message="all fields must be filled in")
+        result = env.validate()
+        if result is not None:
+            return result
 
         response = cls.create_new_env(env, Artifacts.built_by_softpack_file)
         if not isinstance(response, CreateEnvironmentSuccess):
@@ -293,41 +324,58 @@ def update(
         Returns:
             A message confirming the success or failure of the operation.
         """
-        # Check if any field has been left empty
-        if (
-            any(len(value) == 0 for value in vars(env).values())
-            or current_name == ""
-            or current_path == ""
-        ):
-            return InvalidInputError(message="all fields must be filled in")
+        result = env.validate()
+        if result is not None:
+            return result
+
+        if current_name == "" or current_path == "":
+            return InvalidInputError(message="current values must be supplied")
 
-        # Check name and path have not been changed.
         if env.path != current_path or env.name != current_name:
             return InvalidInputError(
-                message=("change of name or path not " "currently supported")
+                message=("change of name or path not currently supported")
             )
 
-        # Check if an environment exists at the specified path and name
-        if cls.artifacts.get(Path(current_path), current_name):
-            httpx.post(
-                "http://0.0.0.0:7080/environments/build",
-                json={
-                    "name": f"{env.path}/{env.name}",
-                    "model": {
-                        "description": env.description,
-                        "packages": [pkg.name for pkg in env.packages or []],
-                    },
+        result2 = cls.check_env_exists(Path(current_path, current_name))
+        if result2 is not None:
+            return result2
+
+        httpx.post(
+            "http://0.0.0.0:7080/environments/build",
+            json={
+                "name": f"{env.path}/{env.name}",
+                "model": {
+                    "description": env.description,
+                    "packages": [pkg.name for pkg in env.packages or []],
                 },
-            )
+            },
+        )
 
-            return UpdateEnvironmentSuccess(
-                message="Successfully updated environment"
-            )
+        # TODO: validate the post worked
+
+        return UpdateEnvironmentSuccess(
+            message="Successfully updated environment"
+        )
+
+    @classmethod
+    def check_env_exists(
+        cls, path: Path
+    ) -> Union[None, EnvironmentNotFoundError]:
+        """check_env_exists checks if an env with the given path exists.
+
+        Args:
+            path (Path): path of the environment
+
+        Returns:
+            Union[None, EnvironmentNotFoundError]: an error if env not found.
+        """
+        if cls.artifacts.get(path.parent, path.name):
+            return None
 
         return EnvironmentNotFoundError(
-            message="No environment with this name found in this location.",
-            path=current_path,
-            name=current_name,
+            message="No environment with this path and name found.",
+            path=str(path.parent),
+            name=path.name,
         )
 
     @classmethod
@@ -382,19 +430,7 @@ async def create_from_module(
         Returns:
             A message confirming the success or failure of the operation.
         """
-        environment_dirs = environment_path.split("/")
-        environment_name = environment_dirs.pop()
-
-        contents = await file.read()
-        yml = ToSoftpackYML(environment_name, contents)
-        readme = GenerateEnvReadme(module_path)
-
-        env = EnvironmentInput(
-            name=environment_name,
-            path="/".join(environment_dirs),
-            description="",
-            packages=list(),
-        )
+        env = EnvironmentInput.from_path(environment_path)
 
         response = cls.create_new_env(
             env, Artifacts.generated_from_module_file
@@ -402,19 +438,12 @@ async def create_from_module(
         if not isinstance(response, CreateEnvironmentSuccess):
             return response
 
-        module_file = UploadFile(file=io.BytesIO(contents))
-        softpack_file = UploadFile(file=io.BytesIO(yml))
-        readme_file = UploadFile(file=io.BytesIO(readme))
-
-        result = await cls.write_module_artifacts(
-            module_file=module_file,
-            softpack_file=softpack_file,
-            readme_file=readme_file,
-            environment_path=environment_path,
+        result = await cls.convert_module_file_to_artifacts(
+            file, env.name, environment_path, module_path
         )
 
         if not isinstance(result, WriteArtifactSuccess):
-            cls.delete(name=environment_name, path=environment_path)
+            cls.delete(name=env.name, path=environment_path)
             return InvalidInputError(
                 message="Write of module file failed: " + result.message
             )
@@ -423,6 +452,36 @@ async def create_from_module(
             message="Successfully created environment in artifacts repo"
         )
 
+    @classmethod
+    async def convert_module_file_to_artifacts(
+        cls, file: Upload, env_name: str, env_path: str, module_path: str
+    ) -> WriteArtifactResponse:  # type: ignore
+        """convert_module_file_to_artifacts parses a module and writes to repo.
+
+        Args:
+            file (Upload): shpc-style module file contents.
+            env_name (str): name of the environment.
+            env_path (str): path of the environment.
+            module_path (str): the `module load` path users will use.
+
+        Returns:
+            WriteArtifactResponse: success or failure indicator.
+        """
+        contents = await file.read()
+        yml = ToSoftpackYML(env_name, contents)
+        readme = GenerateEnvReadme(module_path)
+
+        module_file = UploadFile(file=io.BytesIO(contents))
+        softpack_file = UploadFile(file=io.BytesIO(yml))
+        readme_file = UploadFile(file=io.BytesIO(readme))
+
+        return await cls.write_module_artifacts(
+            module_file=module_file,
+            softpack_file=softpack_file,
+            readme_file=readme_file,
+            environment_path=env_path,
+        )
+
     @classmethod
     async def write_module_artifacts(
         cls,
@@ -497,6 +556,51 @@ async def write_artifact(
         except Exception as e:
             return InvalidInputError(message=str(e))
 
+    @classmethod
+    async def update_from_module(
+        cls, file: Upload, module_path: str, environment_path: str
+    ) -> UpdateResponse:  # type: ignore
+        """Update an Environment based on an existing module.
+
+        Same as create_from_module, but only works for an existing environment.
+
+        Args:
+            file: the module file to add to the repo, and to parse to fake a
+                  corresponding softpack.yml. It should have a format similar
+                  to that produced by shpc, with `module whatis` outputting
+                  a "Name: " line, a "Version: " line, and optionally a
+                  "Packages: " line to say what packages are available.
+                  `module help` output will be translated into the description
+                  in the softpack.yml.
+            module_path: the local path that users can `module load` - this is
+                         used to auto-generate usage help text for this
+                         environment.
+            environment_path: the subdirectories of environments folder that
+                              artifacts will be stored in, eg.
+                              users/username/software_name
+
+        Returns:
+            A message confirming the success or failure of the operation.
+        """
+        env = EnvironmentInput.from_path(environment_path)
+
+        result = cls.check_env_exists(Path(environment_path))
+        if result is not None:
+            return result
+
+        result = await cls.convert_module_file_to_artifacts(
+            file, env.name, environment_path, module_path
+        )
+
+        if not isinstance(result, WriteArtifactSuccess):
+            return InvalidInputError(
+                message="Write of module file failed: " + result.message
+            )
+
+        return UpdateEnvironmentSuccess(
+            message="Successfully updated environment in artifacts repo"
+        )
+
 
 class EnvironmentSchema(BaseSchema):
     """Environment schema."""
@@ -520,3 +624,6 @@ class Mutation:
         createFromModule: CreateResponse = (  # type: ignore
             Environment.create_from_module
         )
+        updateFromModule: UpdateResponse = (  # type: ignore
+            Environment.update_from_module
+        )
diff --git a/tests/integration/test_environment.py b/tests/integration/test_environment.py
index fede6cc..7cf5d85 100644
--- a/tests/integration/test_environment.py
+++ b/tests/integration/test_environment.py
@@ -286,6 +286,14 @@ async def test_create_from_module(httpx_post, testable_env_input, upload):
 
     assert isinstance(result, CreateEnvironmentSuccess)
 
+    result = await Environment.create_from_module(
+        file=upload,
+        module_path=module_path,
+        environment_path=name,
+    )
+
+    assert isinstance(result, EnvironmentAlreadyExistsError)
+
     parent_path = Path(
         Environment.artifacts.group_folder(),
         "hgi",
@@ -323,3 +331,39 @@ async def test_create_from_module(httpx_post, testable_env_input, upload):
     assert "module load " + module_path in env.readme
     assert env.type == Artifacts.generated_from_module
     assert env.state == State.ready
+
+    test_modifiy_file_path = test_files_dir / "all_fields.mod"
+
+    with open(test_modifiy_file_path, "rb") as fh:
+        upload.filename = "all_fields.mod"
+        upload.content_type = "text/plain"
+        upload.read.return_value = fh.read()
+
+    module_path = "HGI/common/all_fields"
+
+    result = await Environment.update_from_module(
+        file=upload,
+        module_path=module_path,
+        environment_path=name,
+    )
+
+    assert isinstance(result, UpdateEnvironmentSuccess)
+    env = get_env_from_iter(env_name)
+    assert env is not None
+
+    package_name = "name_of_container"
+    package_version = "1.0.1"
+
+    assert len(env.packages) == 5
+    assert env.packages[0].name == package_name
+    assert env.packages[0].version == package_version
+    assert "module load " + module_path in env.readme
+    assert env.type == Artifacts.generated_from_module
+    assert env.state == State.ready
+
+    result = await Environment.update_from_module(
+        file=upload,
+        module_path=module_path,
+        environment_path="users/non/existant",
+    )
+    assert isinstance(result, EnvironmentNotFoundError)

From 9b85e01cf83dd00fdbbbf9c2ecf37fd4551e3cfe Mon Sep 17 00:00:00 2001
From: Michael Woolnough <mw31@sanger.ac.uk>
Date: Tue, 12 Sep 2023 11:31:10 +0100
Subject: [PATCH 121/129] Write multiple artifacts in a single commit.

---
 softpack_core/artifacts.py           | 56 +++++++++++++++++---------
 softpack_core/schemas/environment.py | 59 +++++++++++++++-------------
 2 files changed, 69 insertions(+), 46 deletions(-)

diff --git a/softpack_core/artifacts.py b/softpack_core/artifacts.py
index 89d16dd..358a114 100644
--- a/softpack_core/artifacts.py
+++ b/softpack_core/artifacts.py
@@ -9,7 +9,7 @@
 from dataclasses import dataclass
 from enum import Enum
 from pathlib import Path
-from typing import Iterable, Iterator, Optional
+from typing import Iterable, Iterator, Optional, Tuple
 
 import pygit2
 import strawberry
@@ -381,7 +381,7 @@ def create_file(
         new_folder: bool = False,
         overwrite: bool = False,
     ) -> pygit2.Oid:
-        """Create a file in the artifacts repo.
+        """Create one or more file in the artifacts repo.
 
         Args:
             folder_path: the path to the folder the file will be placed in
@@ -393,22 +393,49 @@ def create_file(
         Returns:
             the OID of the new tree structure of the repository
         """
-        if not overwrite and self.get(Path(folder_path), file_name):
-            raise FileExistsError("File already exists")
+        return self.create_files(
+            folder_path, [(file_name, contents)], new_folder, overwrite
+        )
+
+    def create_files(
+        self,
+        folder_path: Path,
+        files: list[Tuple[str, str]],
+        new_folder: bool = False,
+        overwrite: bool = False,
+    ) -> pygit2.Oid:
+        """Create one or more files in the artifacts repo.
+
+        Args:
+            folder_path: the path to the folder the files will be placed
+            files: Array of tuples, containing file name and contents.
+            file_name: the name of the file
+            contents: the contents of the file
+            new_folder: if True, create the file's parent folder as well
+            overwrite: if True, overwrite the file at the specified path
+
+        Returns:
+            the OID of the new tree structure of the repository
+        """
+        for file_name, _ in files:
+            if not overwrite and self.get(Path(folder_path), file_name):
+                raise FileExistsError("File already exists")
 
         root_tree = self.repo.head.peel(pygit2.Tree)
         full_path = Path(self.environments_root, folder_path)
 
-        # Create file
-        file_oid = self.repo.create_blob(contents.encode())
-
-        # Put file in folder
         if new_folder:
             new_treebuilder = self.repo.TreeBuilder()
         else:
             folder = root_tree[full_path]
             new_treebuilder = self.repo.TreeBuilder(folder)
-        new_treebuilder.insert(file_name, file_oid, pygit2.GIT_FILEMODE_BLOB)
+
+        for file_name, contents in files:
+            file_oid = self.repo.create_blob(contents.encode())
+            new_treebuilder.insert(
+                file_name, file_oid, pygit2.GIT_FILEMODE_BLOB
+            )
+
         new_tree = new_treebuilder.write()
 
         # Expand to include the whole repo
@@ -416,19 +443,12 @@ def create_file(
 
         # Check for errors in the new tree
         new_tree = self.repo.get(full_tree)
-        path = Path(self.environments_root, folder_path, file_name)
+        Path(self.environments_root, folder_path, file_name)
         diff = self.repo.diff(new_tree, root_tree)
-        if len(diff) > 1:
+        if len(diff) > len(files):
             raise RuntimeError("Too many changes to the repo")
         elif len(diff) < 1:
             raise RuntimeError("No changes made to the environment")
-        elif len(diff) == 1:
-            new_file = diff[0].delta.new_file
-            if new_file.path != str(path):
-                raise RuntimeError(
-                    f"New file added to incorrect path: \
-                        {new_file.path} instead of {str(path)}"
-                )
 
         return full_tree
 
diff --git a/softpack_core/schemas/environment.py b/softpack_core/schemas/environment.py
index ad919b3..16b7235 100644
--- a/softpack_core/schemas/environment.py
+++ b/softpack_core/schemas/environment.py
@@ -7,7 +7,7 @@
 import io
 from dataclasses import dataclass
 from pathlib import Path
-from typing import Iterable, Optional, Union
+from typing import Iterable, List, Optional, Tuple, Union, cast
 
 import httpx
 import strawberry
@@ -505,28 +505,13 @@ async def write_module_artifacts(
             WriteArtifactResponse: contains message and commit hash of
             softpack.yml upload.
         """
-        result = await cls.write_artifact(
-            file=module_file,
-            folder_path=environment_path,
-            file_name=cls.artifacts.module_file,
-        )
-
-        if not isinstance(result, WriteArtifactSuccess):
-            return result
-
-        result = await cls.write_artifact(
-            file=readme_file,
-            folder_path=environment_path,
-            file_name=cls.artifacts.readme_file,
-        )
-
-        if not isinstance(result, WriteArtifactSuccess):
-            return result
+        module_file.name = cls.artifacts.module_file
+        readme_file.name = cls.artifacts.readme_file
+        softpack_file.name = cls.artifacts.environments_file
 
-        return await cls.write_artifact(
-            file=softpack_file,
+        return await cls.write_artifacts(
             folder_path=environment_path,
-            file_name=cls.artifacts.environments_file,
+            files=[module_file, readme_file, softpack_file],
         )
 
     @classmethod
@@ -536,20 +521,38 @@ async def write_artifact(
         """Add a file to the Artifacts repo.
 
         Args:
-            file: the file to add to the repo
-            folder_path: the path to the folder that the file will be added to
-            file_name: the name of the file
+            file: the file to be added to the repo.
+            folder_path: the path to the folder that the file will be added to.
+            file_name: the name of the file to be added.
+        """
+        file.name = file_name
+
+        return await cls.write_artifacts(folder_path, [file])
+
+    @classmethod
+    async def write_artifacts(
+        cls, folder_path: str, files: list[Upload]
+    ) -> WriteArtifactResponse:  # type: ignore
+        """Add one or more files to the Artifacts repo.
+
+        Args:
+            folder_path: the path to the folder that the file will be added to.
+            files: the files to add to the repo.
         """
         try:
-            contents = (await file.read()).decode()
-            tree_oid = cls.artifacts.create_file(
-                Path(folder_path), file_name, contents, overwrite=True
+            new_files: List[Tuple[str, str]] = []
+            for file in files:
+                contents = cast(str, (await file.read()).decode())
+                new_files.append((file.name, contents))
+
+            tree_oid = cls.artifacts.create_files(
+                Path(folder_path), new_files, overwrite=True
             )
             commit_oid = cls.artifacts.commit_and_push(
                 tree_oid, "write artifact"
             )
             return WriteArtifactSuccess(
-                message="Successfully written artifact",
+                message="Successfully written artifact(s)",
                 commit_oid=str(commit_oid),
             )
 

From 0f55d2cf9899992a7430df2cf2c6bc8068826674 Mon Sep 17 00:00:00 2001
From: Michael Woolnough <mw31@sanger.ac.uk>
Date: Tue, 12 Sep 2023 11:36:32 +0100
Subject: [PATCH 122/129] Add endpoint to create multiple artifacts in a single
 commit.

---
 softpack_core/schemas/environment.py | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/softpack_core/schemas/environment.py b/softpack_core/schemas/environment.py
index 16b7235..d9744d8 100644
--- a/softpack_core/schemas/environment.py
+++ b/softpack_core/schemas/environment.py
@@ -624,6 +624,9 @@ class Mutation:
         writeArtifact: WriteArtifactResponse = (  # type: ignore
             Environment.write_artifact
         )
+        writeArtifacts: WriteArtifactResponse = (  # type: ignore
+            Environment.write_artifacts
+        )
         createFromModule: CreateResponse = (  # type: ignore
             Environment.create_from_module
         )

From e5cb64273806886a42e4352cd919703fe3d76d91 Mon Sep 17 00:00:00 2001
From: Michael Woolnough <mw31@sanger.ac.uk>
Date: Tue, 12 Sep 2023 11:39:51 +0100
Subject: [PATCH 123/129] Remove unused Path call.

---
 softpack_core/artifacts.py | 1 -
 1 file changed, 1 deletion(-)

diff --git a/softpack_core/artifacts.py b/softpack_core/artifacts.py
index 358a114..1b776ab 100644
--- a/softpack_core/artifacts.py
+++ b/softpack_core/artifacts.py
@@ -443,7 +443,6 @@ def create_files(
 
         # Check for errors in the new tree
         new_tree = self.repo.get(full_tree)
-        Path(self.environments_root, folder_path, file_name)
         diff = self.repo.diff(new_tree, root_tree)
         if len(diff) > len(files):
             raise RuntimeError("Too many changes to the repo")

From c1fd7e6c125acd46d3bee65f039a6c889d84097f Mon Sep 17 00:00:00 2001
From: Michael Woolnough <mw31@sanger.ac.uk>
Date: Mon, 25 Sep 2023 10:58:28 +0100
Subject: [PATCH 124/129] Add upload endpoint for builder to pass artifacts to
 be uploaded to repo.

---
 softpack_core/schemas/environment.py    | 10 +++--
 softpack_core/service.py                | 42 ++++++++++++++++++
 tests/integration/conftest.py           | 25 ++++++++++-
 tests/integration/test_builderupload.py | 58 +++++++++++++++++++++++++
 tests/integration/test_environment.py   | 24 +---------
 5 files changed, 132 insertions(+), 27 deletions(-)
 create mode 100644 tests/integration/test_builderupload.py

diff --git a/softpack_core/schemas/environment.py b/softpack_core/schemas/environment.py
index d9744d8..4113379 100644
--- a/softpack_core/schemas/environment.py
+++ b/softpack_core/schemas/environment.py
@@ -264,8 +264,8 @@ def create_new_env(
         already exists.
 
         Args:
-            env (EnvironmentInput): Details of the new environment. env_type
-            (str): One of Artifacts.built_by_softpack_file or
+            env (EnvironmentInput): Details of the new environment.
+            env_type (str): One of Artifacts.built_by_softpack_file or
             Artifacts.generated_from_module_file that denotes how the
             environment was made.
 
@@ -543,7 +543,11 @@ async def write_artifacts(
             new_files: List[Tuple[str, str]] = []
             for file in files:
                 contents = cast(str, (await file.read()).decode())
-                new_files.append((file.name, contents))
+                try:
+                    name = file.name
+                except BaseException:
+                    name = file.filename
+                new_files.append((name, contents))
 
             tree_oid = cls.artifacts.create_files(
                 Path(folder_path), new_files, overwrite=True
diff --git a/softpack_core/service.py b/softpack_core/service.py
index eaafe32..a477099 100644
--- a/softpack_core/service.py
+++ b/softpack_core/service.py
@@ -5,11 +5,22 @@
 """
 
 
+import urllib.parse
+from pathlib import Path
+from typing import List
+
 import typer
 import uvicorn
+from fastapi import APIRouter, Request, UploadFile
 from typer import Typer
 from typing_extensions import Annotated
 
+from softpack_core.schemas.environment import (
+    CreateEnvironmentSuccess,
+    Environment,
+    EnvironmentInput,
+)
+
 from .api import API
 from .app import app
 
@@ -19,6 +30,7 @@ class ServiceAPI(API):
 
     prefix = "/service"
     commands = Typer(help="Commands for managing core service.")
+    router = APIRouter()
 
     @staticmethod
     @commands.command(help="Start the SoftPack Core API service.")
@@ -46,3 +58,33 @@ def run(
             reload=reload,
             log_level="debug",
         )
+
+    @staticmethod
+    @router.post("/upload")
+    async def upload_artifacts(  # type: ignore[no-untyped-def]
+        file: List[UploadFile], request: Request
+    ):
+        """upload_artifacts is a POST fn that adds files to an environment.
+
+          The environment does not need to exist already.
+
+        Args:
+            file (List[MUploadFile]): The files to be uploaded.
+            request (Request): The POST request which contains the environment
+            path in the query.
+
+        Returns:
+            WriteArtifactResponse
+        """
+        env_path = urllib.parse.unquote(request.url.query)
+
+        if Environment.check_env_exists(Path(env_path)) is not None:
+            create_response = Environment.create_new_env(
+                EnvironmentInput.from_path(env_path),
+                Environment.artifacts.built_by_softpack,
+            )
+
+            if not isinstance(create_response, CreateEnvironmentSuccess):
+                return create_response
+
+        return await Environment.write_artifacts(env_path, file)
diff --git a/tests/integration/conftest.py b/tests/integration/conftest.py
index 1ea552a..40c7857 100644
--- a/tests/integration/conftest.py
+++ b/tests/integration/conftest.py
@@ -9,7 +9,12 @@
 import pytest
 from starlette.datastructures import UploadFile
 
-from softpack_core.artifacts import app
+from softpack_core.artifacts import Artifacts, Package, app
+from softpack_core.schemas.environment import Environment, EnvironmentInput
+from tests.integration.utils import (
+    get_user_path_without_environments,
+    new_test_artifacts,
+)
 
 
 @pytest.fixture(scope="package", autouse=True)
@@ -41,3 +46,21 @@ def httpx_post(mocker):
 @pytest.fixture()
 def upload(mocker):
     return mocker.Mock(spec=UploadFile)
+
+
+@pytest.fixture
+def testable_env_input(mocker) -> EnvironmentInput:
+    ad = new_test_artifacts()
+    artifacts: Artifacts = ad["artifacts"]
+    user = ad["test_user"]
+
+    mocker.patch.object(Environment, 'artifacts', new=artifacts)
+
+    testable_env_input = EnvironmentInput(
+        name="test_env_create",
+        path=str(get_user_path_without_environments(artifacts, user)),
+        description="description",
+        packages=[Package(name="pkg_test")],
+    )
+
+    yield testable_env_input
diff --git a/tests/integration/test_builderupload.py b/tests/integration/test_builderupload.py
new file mode 100644
index 0000000..b250177
--- /dev/null
+++ b/tests/integration/test_builderupload.py
@@ -0,0 +1,58 @@
+"""Copyright (c) 2023 Genome Research Ltd.
+
+This source code is licensed under the MIT license found in the
+LICENSE file in the root directory of this source tree.
+"""
+
+from pathlib import Path
+
+import pytest
+from fastapi.testclient import TestClient
+
+from softpack_core.app import app
+from softpack_core.schemas.environment import Environment
+from softpack_core.service import ServiceAPI
+from tests.integration.utils import file_in_repo
+
+pytestmark = pytest.mark.repo
+
+
+def test_builder_upload(testable_env_input):
+    ServiceAPI.register()
+    client = TestClient(app.router)
+
+    env_parent = "groups/hgi"
+    env_name = "unknown-env"
+    env_path = env_parent + "/" + env_name
+
+    softpackYaml = "softpack.yaml"
+    softpackYamlContents = b"softpack yaml file"
+
+    spackLock = "spack.lock"
+    spackLockContents = b"spack lock file"
+
+    assert Environment.check_env_exists(Path(env_path)) is not None
+    resp = client.post(
+        url="/upload?" + env_path,
+        files=[
+            ("file", (softpackYaml, softpackYamlContents)),
+            ("file", (spackLock, spackLockContents)),
+        ],
+    )
+    assert resp.status_code == 200
+    assert resp.json().get("message") == "Successfully written artifact(s)"
+    assert Environment.check_env_exists(Path(env_path)) is None
+    assert file_in_repo(
+        Environment.artifacts,
+        Path(Environment.artifacts.environments_root, env_path, softpackYaml),
+    )
+    assert file_in_repo(
+        Environment.artifacts,
+        Path(Environment.artifacts.environments_root, env_path, spackLock),
+    )
+
+    tree = Environment.artifacts.get(env_parent, env_name)
+    assert tree is not None
+
+    assert tree[softpackYaml].data == softpackYamlContents
+    assert tree[spackLock].data == spackLockContents
diff --git a/tests/integration/test_environment.py b/tests/integration/test_environment.py
index 7cf5d85..060bc59 100644
--- a/tests/integration/test_environment.py
+++ b/tests/integration/test_environment.py
@@ -24,33 +24,11 @@
     UpdateEnvironmentSuccess,
     WriteArtifactSuccess,
 )
-from tests.integration.utils import (
-    file_in_remote,
-    get_user_path_without_environments,
-    new_test_artifacts,
-)
+from tests.integration.utils import file_in_remote
 
 pytestmark = pytest.mark.repo
 
 
-@pytest.fixture
-def testable_env_input(mocker) -> EnvironmentInput:
-    ad = new_test_artifacts()
-    artifacts: Artifacts = ad["artifacts"]
-    user = ad["test_user"]
-
-    mocker.patch.object(Environment, 'artifacts', new=artifacts)
-
-    testable_env_input = EnvironmentInput(
-        name="test_env_create",
-        path=str(get_user_path_without_environments(artifacts, user)),
-        description="description",
-        packages=[Package(name="pkg_test")],
-    )
-
-    yield testable_env_input
-
-
 def test_create(httpx_post, testable_env_input: EnvironmentInput) -> None:
     result = Environment.create(testable_env_input)
     assert isinstance(result, CreateEnvironmentSuccess)

From d649b6b47425ee06c2b07c1fa8c9f522b4fed635 Mon Sep 17 00:00:00 2001
From: Michael Woolnough <mw31@sanger.ac.uk>
Date: Mon, 25 Sep 2023 11:58:49 +0100
Subject: [PATCH 125/129] Use correct placeholder filename.

---
 softpack_core/service.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/softpack_core/service.py b/softpack_core/service.py
index a477099..35f21e2 100644
--- a/softpack_core/service.py
+++ b/softpack_core/service.py
@@ -81,7 +81,7 @@ async def upload_artifacts(  # type: ignore[no-untyped-def]
         if Environment.check_env_exists(Path(env_path)) is not None:
             create_response = Environment.create_new_env(
                 EnvironmentInput.from_path(env_path),
-                Environment.artifacts.built_by_softpack,
+                Environment.artifacts.built_by_softpack_file,
             )
 
             if not isinstance(create_response, CreateEnvironmentSuccess):

From 5d289dee8aab856899112c9606590cf536bca2a7 Mon Sep 17 00:00:00 2001
From: Michael Woolnough <mw31@sanger.ac.uk>
Date: Tue, 26 Sep 2023 10:19:14 +0100
Subject: [PATCH 126/129] Support large file uploads.

---
 softpack_core/artifacts.py            | 10 ++++--
 softpack_core/schemas/environment.py  | 45 ++++++++++++++++-----------
 softpack_core/service.py              | 14 ++++++---
 tests/integration/conftest.py         |  6 ----
 tests/integration/test_environment.py | 43 ++++++++++++++-----------
 tox.ini                               |  4 +--
 6 files changed, 68 insertions(+), 54 deletions(-)

diff --git a/softpack_core/artifacts.py b/softpack_core/artifacts.py
index 1b776ab..3d9d9b9 100644
--- a/softpack_core/artifacts.py
+++ b/softpack_core/artifacts.py
@@ -9,11 +9,12 @@
 from dataclasses import dataclass
 from enum import Enum
 from pathlib import Path
-from typing import Iterable, Iterator, Optional, Tuple
+from typing import Iterable, Iterator, List, Optional, Tuple, Union
 
 import pygit2
 import strawberry
 from box import Box
+from fastapi import UploadFile
 
 from softpack_core.spack import Spack
 
@@ -400,7 +401,7 @@ def create_file(
     def create_files(
         self,
         folder_path: Path,
-        files: list[Tuple[str, str]],
+        files: List[Tuple[str, Union[str, UploadFile]]],
         new_folder: bool = False,
         overwrite: bool = False,
     ) -> pygit2.Oid:
@@ -431,7 +432,10 @@ def create_files(
             new_treebuilder = self.repo.TreeBuilder(folder)
 
         for file_name, contents in files:
-            file_oid = self.repo.create_blob(contents.encode())
+            if isinstance(contents, str):
+                file_oid = self.repo.create_blob(contents)
+            else:
+                file_oid = self.repo.create_blob_fromiobase(contents.file)
             new_treebuilder.insert(
                 file_name, file_oid, pygit2.GIT_FILEMODE_BLOB
             )
diff --git a/softpack_core/schemas/environment.py b/softpack_core/schemas/environment.py
index 4113379..cd2b169 100644
--- a/softpack_core/schemas/environment.py
+++ b/softpack_core/schemas/environment.py
@@ -10,8 +10,9 @@
 from typing import Iterable, List, Optional, Tuple, Union, cast
 
 import httpx
+import starlette.datastructures
 import strawberry
-from starlette.datastructures import UploadFile
+from fastapi import UploadFile
 from strawberry.file_uploads import Upload
 
 from softpack_core.artifacts import Artifacts, Package, State
@@ -471,9 +472,15 @@ async def convert_module_file_to_artifacts(
         yml = ToSoftpackYML(env_name, contents)
         readme = GenerateEnvReadme(module_path)
 
-        module_file = UploadFile(file=io.BytesIO(contents))
-        softpack_file = UploadFile(file=io.BytesIO(yml))
-        readme_file = UploadFile(file=io.BytesIO(readme))
+        module_file = UploadFile(
+            filename=cls.artifacts.module_file, file=io.BytesIO(contents)
+        )
+        softpack_file = UploadFile(
+            filename=cls.artifacts.environments_file, file=io.BytesIO(yml)
+        )
+        readme_file = UploadFile(
+            filename=cls.artifacts.readme_file, file=io.BytesIO(readme)
+        )
 
         return await cls.write_module_artifacts(
             module_file=module_file,
@@ -531,7 +538,7 @@ async def write_artifact(
 
     @classmethod
     async def write_artifacts(
-        cls, folder_path: str, files: list[Upload]
+        cls, folder_path: str, files: list[Union[Upload, UploadFile]]
     ) -> WriteArtifactResponse:  # type: ignore
         """Add one or more files to the Artifacts repo.
 
@@ -540,14 +547,16 @@ async def write_artifacts(
             files: the files to add to the repo.
         """
         try:
-            new_files: List[Tuple[str, str]] = []
+            new_files: List[Tuple[str, Union[str, UploadFile]]] = []
             for file in files:
-                contents = cast(str, (await file.read()).decode())
-                try:
-                    name = file.name
-                except BaseException:
-                    name = file.filename
-                new_files.append((name, contents))
+                if isinstance(file, starlette.datastructures.UploadFile):
+                    new_files.append(
+                        (file.filename or "", cast(UploadFile, file))
+                    )
+                else:
+                    new_files.append(
+                        (file.name, cast(str, (await file.read()).decode()))
+                    )
 
             tree_oid = cls.artifacts.create_files(
                 Path(folder_path), new_files, overwrite=True
@@ -625,12 +634,12 @@ class Mutation:
         createEnvironment: CreateResponse = Environment.create  # type: ignore
         updateEnvironment: UpdateResponse = Environment.update  # type: ignore
         deleteEnvironment: DeleteResponse = Environment.delete  # type: ignore
-        writeArtifact: WriteArtifactResponse = (  # type: ignore
-            Environment.write_artifact
-        )
-        writeArtifacts: WriteArtifactResponse = (  # type: ignore
-            Environment.write_artifacts
-        )
+        # writeArtifact: WriteArtifactResponse = (  # type: ignore
+        #     Environment.write_artifact
+        # )
+        # writeArtifacts: WriteArtifactResponse = (  # type: ignore
+        #     Environment.write_artifacts
+        # )
         createFromModule: CreateResponse = (  # type: ignore
             Environment.create_from_module
         )
diff --git a/softpack_core/service.py b/softpack_core/service.py
index 35f21e2..6ea3829 100644
--- a/softpack_core/service.py
+++ b/softpack_core/service.py
@@ -7,7 +7,6 @@
 
 import urllib.parse
 from pathlib import Path
-from typing import List
 
 import typer
 import uvicorn
@@ -19,6 +18,7 @@
     CreateEnvironmentSuccess,
     Environment,
     EnvironmentInput,
+    WriteArtifactSuccess,
 )
 
 from .api import API
@@ -62,14 +62,15 @@ def run(
     @staticmethod
     @router.post("/upload")
     async def upload_artifacts(  # type: ignore[no-untyped-def]
-        file: List[UploadFile], request: Request
+        request: Request,
+        file: list[UploadFile],
     ):
         """upload_artifacts is a POST fn that adds files to an environment.
 
           The environment does not need to exist already.
 
         Args:
-            file (List[MUploadFile]): The files to be uploaded.
+            file (List[UploadFile]): The files to be uploaded.
             request (Request): The POST request which contains the environment
             path in the query.
 
@@ -77,7 +78,6 @@ async def upload_artifacts(  # type: ignore[no-untyped-def]
             WriteArtifactResponse
         """
         env_path = urllib.parse.unquote(request.url.query)
-
         if Environment.check_env_exists(Path(env_path)) is not None:
             create_response = Environment.create_new_env(
                 EnvironmentInput.from_path(env_path),
@@ -87,4 +87,8 @@ async def upload_artifacts(  # type: ignore[no-untyped-def]
             if not isinstance(create_response, CreateEnvironmentSuccess):
                 return create_response
 
-        return await Environment.write_artifacts(env_path, file)
+        resp = await Environment.write_artifacts(env_path, file)
+        if not isinstance(resp, WriteArtifactSuccess):
+            raise Exception(resp)
+
+        return resp
diff --git a/tests/integration/conftest.py b/tests/integration/conftest.py
index 40c7857..56aee99 100644
--- a/tests/integration/conftest.py
+++ b/tests/integration/conftest.py
@@ -7,7 +7,6 @@
 import os
 
 import pytest
-from starlette.datastructures import UploadFile
 
 from softpack_core.artifacts import Artifacts, Package, app
 from softpack_core.schemas.environment import Environment, EnvironmentInput
@@ -43,11 +42,6 @@ def httpx_post(mocker):
     return post_mock
 
 
-@pytest.fixture()
-def upload(mocker):
-    return mocker.Mock(spec=UploadFile)
-
-
 @pytest.fixture
 def testable_env_input(mocker) -> EnvironmentInput:
     ad = new_test_artifacts()
diff --git a/tests/integration/test_environment.py b/tests/integration/test_environment.py
index 060bc59..11678e6 100644
--- a/tests/integration/test_environment.py
+++ b/tests/integration/test_environment.py
@@ -4,11 +4,13 @@
 LICENSE file in the root directory of this source tree.
 """
 
+import io
 from pathlib import Path
 from typing import Optional
 
 import pygit2
 import pytest
+from fastapi import UploadFile
 
 from softpack_core.artifacts import Artifacts
 from softpack_core.schemas.environment import (
@@ -150,10 +152,8 @@ def test_delete(httpx_post, testable_env_input) -> None:
 
 
 @pytest.mark.asyncio
-async def test_write_artifact(httpx_post, testable_env_input, upload):
-    upload.filename = "example.txt"
-    upload.content_type = "text/plain"
-    upload.read.return_value = b"mock data"
+async def test_write_artifact(httpx_post, testable_env_input):
+    upload = UploadFile(filename="example.txt", file=io.BytesIO(b"mock data"))
 
     result = await Environment.write_artifact(
         file=upload,
@@ -195,15 +195,16 @@ def test_iter(testable_env_input):
 
 
 @pytest.mark.asyncio
-async def test_states(httpx_post, testable_env_input, upload):
+async def test_states(httpx_post, testable_env_input):
     result = Environment.create(testable_env_input)
     assert isinstance(result, CreateEnvironmentSuccess)
     httpx_post.assert_called_once()
 
-    upload.filename = Artifacts.environments_file
-    upload.content_type = "text/plain"
-    upload.read.return_value = (
-        b"description: test env\n" b"packages:\n  - zlib@v1.1\n"
+    upload = UploadFile(
+        filename=Artifacts.environments_file,
+        file=io.BytesIO(
+            b"description: test env\n" b"packages:\n  - zlib@v1.1\n"
+        ),
     )
 
     result = await Environment.write_artifact(
@@ -220,9 +221,9 @@ async def test_states(httpx_post, testable_env_input, upload):
     assert env.type == Artifacts.built_by_softpack
     assert env.state == State.queued
 
-    upload.filename = Artifacts.module_file
-    upload.content_type = "text/plain"
-    upload.read.return_value = b"#%Module"
+    upload = UploadFile(
+        filename=Artifacts.module_file, file=io.BytesIO(b"#%Module")
+    )
 
     result = await Environment.write_artifact(
         file=upload,
@@ -243,14 +244,14 @@ def get_env_from_iter(name: str) -> Optional[Environment]:
 
 
 @pytest.mark.asyncio
-async def test_create_from_module(httpx_post, testable_env_input, upload):
+async def test_create_from_module(httpx_post, testable_env_input):
     test_files_dir = Path(__file__).parent.parent / "files" / "modules"
     test_file_path = test_files_dir / "shpc.mod"
 
     with open(test_file_path, "rb") as fh:
-        upload.filename = "shpc.mod"
-        upload.content_type = "text/plain"
-        upload.read.return_value = fh.read()
+        data = fh.read()
+
+    upload = UploadFile(filename="shpc.mod", file=io.BytesIO(data))
 
     env_name = "some-environment"
     name = "groups/hgi/" + env_name
@@ -264,6 +265,8 @@ async def test_create_from_module(httpx_post, testable_env_input, upload):
 
     assert isinstance(result, CreateEnvironmentSuccess)
 
+    upload = UploadFile(filename="shpc.mod", file=io.BytesIO(data))
+
     result = await Environment.create_from_module(
         file=upload,
         module_path=module_path,
@@ -313,9 +316,9 @@ async def test_create_from_module(httpx_post, testable_env_input, upload):
     test_modifiy_file_path = test_files_dir / "all_fields.mod"
 
     with open(test_modifiy_file_path, "rb") as fh:
-        upload.filename = "all_fields.mod"
-        upload.content_type = "text/plain"
-        upload.read.return_value = fh.read()
+        data = fh.read()
+
+    upload = UploadFile(filename="all_fields.mod", file=io.BytesIO(data))
 
     module_path = "HGI/common/all_fields"
 
@@ -339,6 +342,8 @@ async def test_create_from_module(httpx_post, testable_env_input, upload):
     assert env.type == Artifacts.generated_from_module
     assert env.state == State.ready
 
+    upload = UploadFile(filename="all_fields.mod", file=io.BytesIO(data))
+
     result = await Environment.update_from_module(
         file=upload,
         module_path=module_path,
diff --git a/tox.ini b/tox.ini
index bb1214b..14d2cf0 100644
--- a/tox.ini
+++ b/tox.ini
@@ -1,12 +1,10 @@
 [tox]
 isolated_build = true
-envlist = py39, py310, py311, format, lint, build
+envlist = py311, format, lint, build
 
 [gh-actions]
 python =
     3.11: py311, format, lint, build
-    3.10: py310
-    3.9: py39
 
 [flake8]
 max-line-length = 79

From 246f096cdff9a45fa33bd8ecff179a6a878c22d1 Mon Sep 17 00:00:00 2001
From: Michael Woolnough <mw31@sanger.ac.uk>
Date: Tue, 26 Sep 2023 10:34:34 +0100
Subject: [PATCH 127/129] Add python version requirement.

---
 README.md      | 4 +++-
 pyproject.toml | 2 --
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/README.md b/README.md
index 3992145..9a1e8dd 100644
--- a/README.md
+++ b/README.md
@@ -28,7 +28,9 @@ SoftPack Core - GraphQL backend service
 
 ### External dependencies
 
-SoftPack Core relies on Spack. Install that first:
+SoftPack Core requires Python version 3.11 or greater.
+
+This project also relies on Spack. Install that first:
 
 ``` console
 $ git clone -c feature.manyFiles=true --depth 1 https://github.com/spack/spack.git
diff --git a/pyproject.toml b/pyproject.toml
index e44e433..a3cb5d3 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -13,8 +13,6 @@ classifiers=[
     'License :: OSI Approved :: MIT License',
     'Natural Language :: English',
     'Programming Language :: Python :: 3',
-    'Programming Language :: Python :: 3.9',
-    'Programming Language :: Python :: 3.10',
     'Programming Language :: Python :: 3.11',
 ]
 packages = [

From a429fa4bc5771adfd5386028edb76b7a04bd6129 Mon Sep 17 00:00:00 2001
From: Michael Woolnough <mw31@sanger.ac.uk>
Date: Tue, 26 Sep 2023 11:04:22 +0100
Subject: [PATCH 128/129] Update spack version used in CI tests.

---
 .github/workflows/dev.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/dev.yml b/.github/workflows/dev.yml
index 9cc5b82..623b6f9 100644
--- a/.github/workflows/dev.yml
+++ b/.github/workflows/dev.yml
@@ -60,7 +60,7 @@ jobs:
         uses: actions/checkout@v3
         with:
           repository: spack/spack
-          ref: 0707ffd4e466402bf19dff1add59eaf2b6d9154e
+          ref: e8658d6493887ef702dd38f0e9ee5870a1651c1e
           path: spack
 
       - name: Update PATH

From efd268e53fd247497a897c5ca069f25654675342 Mon Sep 17 00:00:00 2001
From: Michael Woolnough <mw31@sanger.ac.uk>
Date: Tue, 26 Sep 2023 11:43:45 +0100
Subject: [PATCH 129/129] Run CI tests on develop branch.

---
 .github/workflows/dev.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/dev.yml b/.github/workflows/dev.yml
index 623b6f9..4013d29 100644
--- a/.github/workflows/dev.yml
+++ b/.github/workflows/dev.yml
@@ -8,7 +8,7 @@ on:
   push:
     branches: [ main ]
   pull_request:
-    branches: [ main ]
+    branches: [ main, develop ]
 
   # Allows you to run this workflow manually from the Actions tab
   workflow_dispatch: