From a3ae5cda3537f43624a17861021581c5b4405ab8 Mon Sep 17 00:00:00 2001 From: Michael Woolnough <130465766+mjkw31@users.noreply.github.com> Date: Mon, 11 Nov 2024 09:52:12 +0000 Subject: [PATCH] Rewrite environment cache so it's persistent (#68) * Environment updates modify the cache in place without regenerating every entry --- schema.graphql | 9 +- softpack_core/artifacts.py | 2 - softpack_core/schemas/environment.py | 154 +++++++++++++--------- softpack_core/service.py | 79 ++++++++--- tests/integration/test_builderupload.py | 2 +- tests/integration/test_environment.py | 109 ++++++--------- tests/integration/test_recipe_requests.py | 20 +-- tests/integration/test_resend_builds.py | 10 +- tests/integration/utils.py | 6 +- tests/unit/test_service.py | 47 +++++++ 10 files changed, 254 insertions(+), 184 deletions(-) diff --git a/schema.graphql b/schema.graphql index befc8f3..2a79b5a 100644 --- a/schema.graphql +++ b/schema.graphql @@ -19,9 +19,6 @@ type CreateEnvironmentSuccess implements Success { union CreateResponse = CreateEnvironmentSuccess | InvalidInputError | EnvironmentAlreadyExistsError | BuilderError -"""Date with time (isoformat)""" -scalar DateTime - type DeleteEnvironmentSuccess implements Success { message: String! } @@ -42,12 +39,8 @@ type Environment { failureReason: String hidden: Boolean! created: Int! - cachedEnvs: [Environment!]! interpreters: Interpreters! - requested: DateTime - buildStart: DateTime - buildDone: DateTime - avgWaitSecs: Float + environments: [Environment!]! } type EnvironmentAlreadyExistsError implements Error { diff --git a/softpack_core/artifacts.py b/softpack_core/artifacts.py index c8be6d5..2ef170c 100644 --- a/softpack_core/artifacts.py +++ b/softpack_core/artifacts.py @@ -94,7 +94,6 @@ class Artifacts: users_folder_name = "users" groups_folder_name = "groups" credentials_callback = None - updated = True @dataclass class Object: @@ -537,7 +536,6 @@ def commit_and_push( ) remote = self.repo.remotes[0] remote.push([self.repo.head.name], callbacks=self.credentials_callback) - self.updated = True return oid def build_tree( diff --git a/softpack_core/schemas/environment.py b/softpack_core/schemas/environment.py index d652810..034a70b 100644 --- a/softpack_core/schemas/environment.py +++ b/softpack_core/schemas/environment.py @@ -4,10 +4,10 @@ LICENSE file in the root directory of this source tree. """ +import bisect import datetime import io import re -import statistics from dataclasses import dataclass, field from pathlib import Path from time import time @@ -87,6 +87,11 @@ class InvalidInputError(Error): """Invalid input data.""" +@strawberry.type +class WriteArtifactFailure(Error): + """Artifact failed to be created.""" + + @strawberry.type class EnvironmentNotFoundError(Error): """Environment not found.""" @@ -338,59 +343,34 @@ class Environment: failure_reason: Optional[str] hidden: bool created: int - cachedEnvs: list["Environment"] = field(default_factory=list) interpreters: Interpreters = field(default_factory=Interpreters) - requested: Optional[datetime.datetime] = None - build_start: Optional[datetime.datetime] = None - build_done: Optional[datetime.datetime] = None - avg_wait_secs: Optional[float] = None + environments: list["Environment"] = field(default_factory=list) @classmethod - def iter(cls) -> list["Environment"]: - """Get an iterator over all Environment objects. - - Returns: - Iterable[Environment]: An iterator of Environment objects. - """ - if not artifacts.updated: - return cls.cachedEnvs - - statuses = BuildStatus.get_all() - if isinstance(statuses, BuilderError): - statuses = [] - - status_map = {s.name: s for s in statuses} - - waits: list[float] = [] - for s in statuses: - if s.build_done is None: - continue - waits.append((s.build_done - s.requested).total_seconds()) - - try: - avg_wait_secs = statistics.mean(waits) - except statistics.StatisticsError: - avg_wait_secs = None + def init(cls, branch: str | None) -> None: + """Initialises Environment.""" + artifacts.clone_repo(branch) + cls.load_initial_environments() + @classmethod + def load_initial_environments(cls) -> None: + """Loads the environments from the repo.""" environment_folders = artifacts.iter() - environment_objects = list( + cls.environments = list( filter(None, map(cls.from_artifact, environment_folders)) ) - for env in environment_objects: - env.avg_wait_secs = avg_wait_secs - status = status_map.get(str(Path(env.path, env.name))) - if not status: - continue - env.requested = status.requested - env.build_start = status.build_start - env.build_done = status.build_done + cls.environments.sort(key=lambda x: x.full_path()) - cls.cachedEnvs = environment_objects - artifacts.updated = False + def full_path(cls) -> Path: + """Return a Path containing the file path and name.""" + return Path(cls.path, cls.name) - return environment_objects + @classmethod + def iter(cls) -> list["Environment"]: + """Return a list of all Enviroments.""" + return cls.environments def has_requested_recipes(self) -> bool: """Do any of the requested packages have an unmade recipe.""" @@ -493,6 +473,12 @@ def create(cls, env: EnvironmentInput) -> CreateResponse: # type: ignore if not isinstance(response, CreateEnvironmentSuccess): return response + environment = Environment.get_env(Path(env.path), env.name) + if environment is not None: + cls.insert_new_env(environment) + else: + return EnvironmentNotFoundError(path=env.path, name=env.name) + response = cls.submit_env_to_builder(env) if response is not None: return response @@ -501,6 +487,13 @@ def create(cls, env: EnvironmentInput) -> CreateResponse: # type: ignore message="Successfully scheduled environment creation" ) + @classmethod + def insert_new_env(cls, env: "Environment") -> None: + """Inserts an enviroment into the correct, sorted position.""" + bisect.insort( + Environment.environments, env, key=lambda x: x.full_path() + ) + @classmethod def submit_env_to_builder( cls, env: EnvironmentInput @@ -647,7 +640,7 @@ def check_env_exists( ) @classmethod - def add_tag( + async def add_tag( cls, name: str, path: str, tag: str ) -> AddTagResponse: # type: ignore """Add a tag to an Environment. @@ -684,9 +677,12 @@ def add_tag( metadata = cls.read_metadata(path, name) metadata.tags = sorted(tags) - cls.store_metadata(environment_path, metadata) + response = await cls.store_metadata(environment_path, metadata) + + if isinstance(response, WriteArtifactSuccess): + return AddTagSuccess(message="Tag successfully added") - return AddTagSuccess(message="Tag successfully added") + return WriteArtifactFailure @classmethod def read_metadata(cls, path: str, name: str) -> Box: @@ -703,22 +699,22 @@ def read_metadata(cls, path: str, name: str) -> Box: return Box() @classmethod - def store_metadata(cls, environment_path: Path, metadata: Box) -> None: + async def store_metadata( + cls, environment_path: Path, metadata: Box + ) -> WriteArtifactResponse: # type: ignore """Store an environments metadata. This method writes the given metadata to the repo for the environment path given. """ - tree_oid = artifacts.create_file( - Path(artifacts.environments_root, environment_path), - artifacts.meta_file, - metadata.to_yaml(), - overwrite=True, + return await Environment.write_artifacts( + str(environment_path), + [(artifacts.meta_file, metadata.to_yaml())], + "update metadata", ) - artifacts.commit_and_push(tree_oid, "update metadata") @classmethod - def set_hidden( + async def set_hidden( cls, name: str, path: str, hidden: bool ) -> HiddenResponse: # type: ignore """This method sets the hidden status for the given environment.""" @@ -734,11 +730,14 @@ def set_hidden( metadata.hidden = hidden - cls.store_metadata(environment_path, metadata) + response = await cls.store_metadata(environment_path, metadata) + + if isinstance(response, WriteArtifactSuccess): + return HiddenSuccess(message="Hidden metadata set") - return HiddenSuccess(message="Hidden metadata set") + return response - def update_metadata(cls, key: str, value: str | None) -> None: + async def update_metadata(cls, key: str, value: str | None) -> None: """Takes a key and sets the value unless value is None.""" metadata = cls.read_metadata(cls.path, cls.name) @@ -747,11 +746,11 @@ def update_metadata(cls, key: str, value: str | None) -> None: else: metadata[key] = value - cls.store_metadata(Path(cls.path, cls.name), metadata) + await cls.store_metadata(Path(cls.path, cls.name), metadata) - def remove_username(cls) -> None: + async def remove_username(cls) -> None: """Remove the username metadata from the meta.yaml file.""" - cls.update_metadata("username", None) + await cls.update_metadata("username", None) @classmethod def delete(cls, name: str, path: str) -> DeleteResponse: # type: ignore @@ -767,6 +766,11 @@ def delete(cls, name: str, path: str) -> DeleteResponse: # type: ignore if artifacts.get(Path(path), name): tree_oid = artifacts.delete_environment(name, path) artifacts.commit_and_push(tree_oid, "delete environment") + + index = cls.env_index_from_path(str(Path(path, name))) + if index is not None: + del Environment.environments[index] + return DeleteEnvironmentSuccess( message="Successfully deleted the environment" ) @@ -915,12 +919,14 @@ async def write_artifacts( cls, folder_path: str, files: list[Union[Upload, UploadFile, Tuple[str, str]]], + commitMsg: str = "write artifact", ) -> WriteArtifactResponse: # type: ignore """Add one or more files to the Artifacts repo. Args: folder_path: the path to the folder that the file will be added to. files: the files to add to the repo. + commitMsg: the msg for the commit. """ try: new_files: List[Tuple[str, Union[str, UploadFile]]] = [] @@ -943,16 +949,40 @@ async def write_artifacts( new_files, overwrite=True, ) - artifacts.commit_and_push(tree_oid, "write artifact") + artifacts.commit_and_push(tree_oid, commitMsg) + + index = cls.env_index_from_path(str(folder_path)) + path = Path(folder_path) + env = Environment.get_env(path.parent, path.name) + + if index is None: + if env: + Environment.insert_new_env(env) + elif env: + Environment.environments[index] = env + else: + del Environment.environments[index] + return WriteArtifactSuccess( message="Successfully written artifact(s)", ) - except Exception as e: return InvalidInputError( message="".join(format_exception_only(type(e), e)) ) + @classmethod + def env_index_from_path(cls, folder_path: str) -> Optional[int]: + """Return the index of a folder_path from the list of environments.""" + return next( + ( + i + for i, env in enumerate(Environment.environments) + if str(env.full_path()) == folder_path + ), + None, + ) + @classmethod async def update_from_module( cls, file: Upload, module_path: str, environment_path: str diff --git a/softpack_core/service.py b/softpack_core/service.py index be56dca..538ce87 100644 --- a/softpack_core/service.py +++ b/softpack_core/service.py @@ -6,6 +6,7 @@ import smtplib +import statistics import urllib.parse from email.mime.text import MIMEText from pathlib import Path @@ -22,6 +23,8 @@ from softpack_core.artifacts import Artifacts, State, artifacts from softpack_core.config.models import EmailConfig from softpack_core.schemas.environment import ( + BuilderError, + BuildStatus, CreateEnvironmentSuccess, Environment, EnvironmentInput, @@ -72,7 +75,7 @@ def run( # FIXME do only when branch does not exist artifacts.create_remote_branch(branch) - artifacts.clone_repo(branch=branch) + Environment.init(branch) uvicorn.run( "softpack_core.app:app.router", @@ -127,10 +130,12 @@ async def upload_artifacts( # type: ignore[no-untyped-def] "concretization failed for the following reasons:" in contents ): - env.update_metadata("failure_reason", "concretization") + await env.update_metadata( + "failure_reason", "concretization" + ) env.failure_reason = "concretization" else: - env.update_metadata("failure_reason", "build") + await env.update_metadata("failure_reason", "build") env.failure_reason = "build" files[i] = (f.filename, contents) @@ -186,7 +191,7 @@ async def upload_artifacts( # type: ignore[no-untyped-def] env.username, newState != State.ready, ) - env.remove_username() + await env.remove_username() resp = await Environment.write_artifacts(env_path, files) if not isinstance(resp, WriteArtifactSuccess): @@ -202,6 +207,7 @@ async def resend_pending_builds( # type: ignore[no-untyped-def] """Resubmit any pending builds to the builder.""" successes = 0 failures = 0 + for env in Environment.iter(): if env.state != State.queued: continue @@ -326,23 +332,27 @@ async def fulfil_recipe( # type: ignore[no-untyped-def] if not changed: continue - artifacts.commit_and_push( - artifacts.create_file( - Path(Artifacts.environments_root, env.path, env.name), - Artifacts.environments_file, - yaml.dump( - dict( - description=env.description, - packages=[ - pkg.name - + ("@" + pkg.version if pkg.version else "") - for pkg in env.packages - ], - ) - ), - False, - True, - ), + await Environment.write_artifacts( + str(Path(env.path, env.name)), + [ + ( + Artifacts.environments_file, + yaml.dump( + dict( + description=env.description, + packages=[ + pkg.name + + ( + "@" + pkg.version + if pkg.version + else "" + ) + for pkg in env.packages + ], + ) + ), + ) + ], "fulfil recipe request for environment", ) @@ -411,6 +421,33 @@ async def recipe_description( # type: ignore[no-untyped-def] return {"description": app.spack.descriptions[data["recipe"]]} + @staticmethod + @router.post("/buildStatus") + async def buildStatus( # type: ignore[no-untyped-def] + request: Request, + ): + """Return the avg wait seconds and a map of names to build status.""" + statuses = BuildStatus.get_all() + if isinstance(statuses, BuilderError): + statuses = [] + + try: + avg_wait_secs = statistics.mean( + (s.build_done - s.requested).total_seconds() + for s in statuses + if s.build_done is not None + ) + except statistics.StatisticsError: + avg_wait_secs = None + + return { + "avg": avg_wait_secs, + "statuses": map( + lambda x: (x.name, x.build_start), + filter(lambda x: x.build_start is not None, statuses), + ), + } + def send_email( emailConfig: EmailConfig, diff --git a/tests/integration/test_builderupload.py b/tests/integration/test_builderupload.py index 4063aac..a24d6a5 100644 --- a/tests/integration/test_builderupload.py +++ b/tests/integration/test_builderupload.py @@ -30,7 +30,7 @@ def test_builder_upload(testable_env_input): softpackYamlContents = b"softpack yaml file" spackLock = "spack.lock" - spackLockContents = b"spack lock file" + spackLockContents = b"{\"spack lock file\":0}" assert Environment.check_env_exists(Path(env_path)) is not None resp = client.post( diff --git a/tests/integration/test_environment.py b/tests/integration/test_environment.py index a32da7e..38dcdfc 100644 --- a/tests/integration/test_environment.py +++ b/tests/integration/test_environment.py @@ -4,7 +4,6 @@ LICENSE file in the root directory of this source tree. """ -import datetime import io import json import time @@ -212,10 +211,18 @@ async def test_create( metadata.force_hidden = True - env.store_metadata( + result = await env.store_metadata( f"{testable_env_input.path}/{testable_env_input.name}-3", metadata ) + assert isinstance(result, WriteArtifactSuccess) + assert ( + Environment.get_env( + testable_env_input.path, testable_env_input.name + "-3" + ) + is None + ) + result = Environment.create(testable_env_input) assert isinstance(result, CreateEnvironmentSuccess) assert testable_env_input.name == "test_env_create-4" @@ -531,61 +538,9 @@ def test_failure_reason_from_build_log( assert env.failure_reason == "concretization" -def test_iter(testable_env_input, mocker): - get_mock = mocker.patch("httpx.get") - get_mock.return_value.json.return_value = [ - { - "Name": "users/test_user/test_environment", - "Requested": "2025-01-02T03:04:00.000000000Z", - "BuildStart": "2025-01-02T03:04:05.000000000Z", - "BuildDone": None, - }, - { - "Name": "groups/test_group/test_environment", - "Requested": "2025-01-02T03:04:00.000000000Z", - "BuildStart": "2025-01-02T03:04:05.000000000Z", - "BuildDone": "2025-01-02T03:04:15.000000000Z", - }, - # only used for average calculations, does not map to an environment in - # the test data - { - "Name": "users/foo/bar", - "Requested": "2025-01-02T03:04:00.000000000Z", - "BuildStart": "2025-01-02T03:04:05.000000000Z", - "BuildDone": "2025-01-02T03:04:25.000000000Z", - }, - ] - - artifacts.updated = True +def test_iter(testable_env_input): envs = list(Environment.iter()) assert len(envs) == 2 - assert envs[0].requested == datetime.datetime( - 2025, 1, 2, 3, 4, 0, tzinfo=datetime.timezone.utc - ) - assert envs[0].build_start == datetime.datetime( - 2025, 1, 2, 3, 4, 5, tzinfo=datetime.timezone.utc - ) - assert envs[0].build_done is None - assert envs[1].requested == datetime.datetime( - 2025, 1, 2, 3, 4, 0, tzinfo=datetime.timezone.utc - ) - assert envs[1].build_start == datetime.datetime( - 2025, 1, 2, 3, 4, 5, tzinfo=datetime.timezone.utc - ) - assert envs[1].build_done == datetime.datetime( - 2025, 1, 2, 3, 4, 15, tzinfo=datetime.timezone.utc - ) - assert envs[0].avg_wait_secs == envs[1].avg_wait_secs == 20 - - -def test_iter_no_statuses(testable_env_input): - artifacts.updated = True - envs = list(Environment.iter()) - assert len(envs) == 2 - assert envs[0].requested is None - assert envs[0].build_start is None - assert envs[0].build_done is None - assert envs[0].avg_wait_secs is None assert envs[0].state == State.queued assert envs[1].state == State.queued @@ -625,6 +580,7 @@ async def test_states(httpx_post, testable_env_input, mocker): ] env = get_env_from_iter(testable_env_input.name + "-1") assert env is not None + assert any(p.name == "zlib" for p in env.packages) assert any(p.version == "v1.1" for p in env.packages) assert env.type == Artifacts.built_by_softpack @@ -796,41 +752,44 @@ def test_environmentinput_from_path(): assert EnvironmentInput.from_path(path).validate() is not None -def test_tagging(httpx_post, testable_env_input: EnvironmentInput) -> None: +@pytest.mark.asyncio +async def test_tagging( + httpx_post, testable_env_input: EnvironmentInput +) -> None: example_env = Environment.iter()[0] assert example_env.tags == [] name, path = example_env.name, example_env.path - result = Environment.add_tag(name, path, tag="test") + result = await Environment.add_tag(name, path, tag="test") assert isinstance(result, AddTagSuccess) assert result.message == "Tag successfully added" - result = Environment.add_tag("foo", "users/xyz", tag="test") + result = await Environment.add_tag("foo", "users/xyz", tag="test") assert isinstance(result, EnvironmentNotFoundError) - result = Environment.add_tag(name, path, tag="../") + result = await Environment.add_tag(name, path, tag="../") assert isinstance(result, InvalidInputError) - result = Environment.add_tag(name, path, tag="") + result = await Environment.add_tag(name, path, tag="") assert isinstance(result, InvalidInputError) - result = Environment.add_tag(name, path, tag=" ") + result = await Environment.add_tag(name, path, tag=" ") assert isinstance(result, InvalidInputError) - result = Environment.add_tag(name, path, tag="foo bar") + result = await Environment.add_tag(name, path, tag="foo bar") assert isinstance(result, InvalidInputError) example_env = Environment.iter()[0] assert len(example_env.tags) == 1 assert example_env.tags[0] == "test" - result = Environment.add_tag(name, path, tag="second test") + result = await Environment.add_tag(name, path, tag="second test") assert isinstance(result, AddTagSuccess) example_env = Environment.iter()[0] assert example_env.tags == ["second test", "test"] - result = Environment.add_tag(name, path, tag="test") + result = await Environment.add_tag(name, path, tag="test") assert isinstance(result, AddTagSuccess) assert result.message == "Tag already present" @@ -838,49 +797,55 @@ def test_tagging(httpx_post, testable_env_input: EnvironmentInput) -> None: assert example_env.tags == ["second test", "test"] -def test_hidden(httpx_post, testable_env_input: EnvironmentInput) -> None: +@pytest.mark.asyncio +async def test_hidden( + httpx_post, testable_env_input: EnvironmentInput +) -> None: example_env = Environment.iter()[0] assert not example_env.hidden name, path = example_env.name, example_env.path - result = Environment.set_hidden(name, path, True) + result = await Environment.set_hidden(name, path, True) assert isinstance(result, HiddenSuccess) assert result.message == "Hidden metadata set" example_env = Environment.iter()[0] assert example_env.hidden - result = Environment.set_hidden(name, path, True) + result = await Environment.set_hidden(name, path, True) assert isinstance(result, HiddenSuccess) assert result.message == "Hidden metadata already set" example_env = Environment.iter()[0] assert example_env.hidden - result = Environment.set_hidden(name, path, False) + result = await Environment.set_hidden(name, path, False) assert isinstance(result, HiddenSuccess) assert result.message == "Hidden metadata set" example_env = Environment.iter()[0] assert not example_env.hidden - result = Environment.set_hidden(name, path, False) + result = await Environment.set_hidden(name, path, False) assert isinstance(result, HiddenSuccess) assert result.message == "Hidden metadata already set" example_env = Environment.iter()[0] assert not example_env.hidden - result = Environment.set_hidden(name, path, True) + result = await Environment.set_hidden(name, path, True) assert isinstance(result, HiddenSuccess) assert result.message == "Hidden metadata set" example_env = Environment.iter()[0] assert example_env.hidden -def test_force_hidden( +@pytest.mark.asyncio +async def test_force_hidden( httpx_post, testable_env_input: EnvironmentInput ) -> None: first_env = Environment.iter()[0] metadata = Environment.read_metadata(first_env.path, first_env.name) metadata.force_hidden = True - Environment.store_metadata(Path(first_env.path, first_env.name), metadata) + await Environment.store_metadata( + Path(first_env.path, first_env.name), metadata + ) new_first = Environment.iter()[0] diff --git a/tests/integration/test_recipe_requests.py b/tests/integration/test_recipe_requests.py index 31c3756..c7f175b 100644 --- a/tests/integration/test_recipe_requests.py +++ b/tests/integration/test_recipe_requests.py @@ -134,11 +134,11 @@ def test_request_recipe(httpx_post, testable_env_input, send_email): envs = Environment.iter() assert len(envs) == existingEnvs + 1 - assert len(envs[0].packages) == 2 - assert envs[0].packages[0].name == "pkg" - assert envs[0].packages[0].version == "1" - assert envs[0].packages[1].name == "*a_recipe" - assert envs[0].packages[1].version == "1.2" + assert len(envs[1].packages) == 2 + assert envs[1].packages[0].name == "pkg" + assert envs[1].packages[0].version == "1" + assert envs[1].packages[1].name == "*a_recipe" + assert envs[1].packages[1].version == "1.2" httpx_post.assert_not_called() @@ -169,11 +169,11 @@ def test_request_recipe(httpx_post, testable_env_input, send_email): envs = Environment.iter() assert len(envs) == existingEnvs + 1 - assert len(envs[0].packages) == 2 - assert envs[0].packages[0].name == "pkg" - assert envs[0].packages[0].version == "1" - assert envs[0].packages[1].name == "finalRecipe" - assert envs[0].packages[1].version == "1.2.1" + assert len(envs[1].packages) == 2 + assert envs[1].packages[0].name == "pkg" + assert envs[1].packages[0].version == "1" + assert envs[1].packages[1].name == "finalRecipe" + assert envs[1].packages[1].version == "1.2.1" resp = client.get(url="/requestedRecipes") diff --git a/tests/integration/test_resend_builds.py b/tests/integration/test_resend_builds.py index e8d2a54..2be7379 100644 --- a/tests/integration/test_resend_builds.py +++ b/tests/integration/test_resend_builds.py @@ -9,7 +9,6 @@ from fastapi.testclient import TestClient from softpack_core.app import app -from softpack_core.artifacts import artifacts from softpack_core.schemas.environment import ( CreateEnvironmentSuccess, Environment, @@ -30,14 +29,13 @@ def test_resend_pending_builds( client = TestClient(app.router) orig_name = testable_env_input.name - testable_env_input.name += "-1" - r = Environment.create_new_env( - testable_env_input, artifacts.built_by_softpack_file - ) + r = Environment.create(testable_env_input) assert isinstance(r, CreateEnvironmentSuccess) + testable_env_input.name = orig_name - httpx_post.assert_not_called() + httpx_post.assert_called_once() + httpx_post.reset_mock() resp = client.post( url="/resend-pending-builds", diff --git a/tests/integration/utils.py b/tests/integration/utils.py index d644b5a..4299e6e 100644 --- a/tests/integration/utils.py +++ b/tests/integration/utils.py @@ -12,7 +12,7 @@ import pytest from softpack_core.artifacts import Artifacts, app, artifacts -from softpack_core.schemas.environment import EnvironmentInput +from softpack_core.schemas.environment import Environment, EnvironmentInput artifacts_dict = dict[ str, @@ -34,12 +34,14 @@ def new_test_artifacts() -> artifacts_dict: temp_dir = tempfile.TemporaryDirectory() app.settings.artifacts.path = Path(temp_dir.name) artifacts.create_remote_branch(branch_name) - artifacts.clone_repo(branch=branch_name) + artifacts.clone_repo(branch_name) dict = reset_test_repo(artifacts) dict["temp_dir"] = temp_dir dict["artifacts"] = artifacts + Environment.load_initial_environments() + return dict diff --git a/tests/unit/test_service.py b/tests/unit/test_service.py index edb3e3d..fe4cc93 100644 --- a/tests/unit/test_service.py +++ b/tests/unit/test_service.py @@ -9,6 +9,7 @@ import httpx from box import Box +from fastapi.testclient import TestClient from softpack_core import __version__ from softpack_core.app import app @@ -91,3 +92,49 @@ def test_send_email(mocker): send_email(emailConfig, "MESSAGE3", "SUBJECT3", "USERNAME3") assert mock_SMTP.return_value.sendmail.call_count == 3 + + +def test_build_status(mocker): + get_mock = mocker.patch("httpx.get") + get_mock.return_value.json.return_value = [ + { + "Name": "users/test_user/test_environment", + "Requested": "2025-01-02T03:04:00.000000000Z", + "BuildStart": "2025-01-02T03:04:05.000000000Z", + "BuildDone": None, + }, + { + "Name": "groups/test_group/test_environment", + "Requested": "2025-01-02T03:04:00.000000000Z", + "BuildStart": "2025-01-02T03:04:05.000000000Z", + "BuildDone": "2025-01-02T03:04:15.000000000Z", + }, + # only used for average calculations, does not map to an environment in + # the test data + { + "Name": "users/foo/bar", + "Requested": "2025-01-02T03:04:00.000000000Z", + "BuildStart": "2025-01-02T03:04:05.000000000Z", + "BuildDone": "2025-01-02T03:04:25.000000000Z", + }, + { + "Name": "users/foo/bar2", + "Requested": "2025-01-02T03:04:00.000000000Z", + "BuildStart": "", + "BuildDone": "", + }, + ] + + client = TestClient(app.router) + resp = client.post("/buildStatus") + + assert resp.status_code == 200 + + status = resp.json() + + assert status.get("avg") == 20 + assert status.get("statuses") == { + "users/test_user/test_environment": "2025-01-02T03:04:05+00:00", + "groups/test_group/test_environment": "2025-01-02T03:04:05+00:00", + "users/foo/bar": "2025-01-02T03:04:05+00:00", + }