From 2c63b6242d730b845cbd073c84c3cc89f99061a8 Mon Sep 17 00:00:00 2001 From: ash Date: Tue, 13 Feb 2024 14:22:23 +0000 Subject: [PATCH] Relay build status information from g-s-b (#45) * Add build status data to environments * Add avg_wait_secs to environments * Commit generated GraphQL schema Generated with: ``` poetry run strawberry export-schema softpack_core.graphql:GraphQL.schema > schema.graphql ``` * Mark queued-but-unknown environments as failed if the builder loses track of a build, it's not coming back (currently) --- schema.graphql | 120 ++++++++++++++++++++++++++ softpack_core/graphql.py | 6 +- softpack_core/schemas/environment.py | 83 +++++++++++++++++- tests/integration/test_environment.py | 73 +++++++++++++++- 4 files changed, 275 insertions(+), 7 deletions(-) create mode 100644 schema.graphql diff --git a/schema.graphql b/schema.graphql new file mode 100644 index 0000000..98f7454 --- /dev/null +++ b/schema.graphql @@ -0,0 +1,120 @@ +schema { + query: SchemaQuery + mutation: SchemaMutation +} + +type BuilderError implements Error { + message: String! +} + +type CreateEnvironmentSuccess implements Success { + message: String! +} + +union CreateResponse = CreateEnvironmentSuccess | InvalidInputError | EnvironmentAlreadyExistsError | BuilderError + +"""Date with time (isoformat)""" +scalar DateTime + +type DeleteEnvironmentSuccess implements Success { + message: String! +} + +union DeleteResponse = DeleteEnvironmentSuccess | EnvironmentNotFoundError + +type Environment { + id: String! + name: String! + path: String! + description: String! + readme: String! + type: Type! + packages: [Package!]! + state: State + requested: DateTime + buildStart: DateTime + buildDone: DateTime + avgWaitSecs: Float +} + +type EnvironmentAlreadyExistsError implements Error { + message: String! + path: String! + name: String! +} + +input EnvironmentInput { + name: String! + path: String! + description: String! + packages: [PackageInput!]! +} + +type EnvironmentNotFoundError implements Error { + message: String! + path: String! + name: String! +} + +interface Error { + message: String! +} + +type Group { + name: String! +} + +type InvalidInputError implements Error { + message: String! +} + +type Package { + name: String! + version: String +} + +input PackageInput { + name: String! + version: String = null +} + +type PackageMultiVersion { + name: String! + versions: [String!]! +} + +type SchemaMutation { + createEnvironment(env: EnvironmentInput!): CreateResponse! + deleteEnvironment(name: String!, path: String!): DeleteResponse! + createFromModule(file: Upload!, modulePath: String!, environmentPath: String!): CreateResponse! + updateFromModule(file: Upload!, modulePath: String!, environmentPath: String!): UpdateResponse! +} + +type SchemaQuery { + environments: [Environment!]! + packageCollections: [PackageMultiVersion!]! + groups(username: String!): [Group!]! +} + +enum State { + ready + queued + failed +} + +interface Success { + message: String! +} + +enum Type { + softpack + module +} + +type UpdateEnvironmentSuccess implements Success { + message: String! +} + +union UpdateResponse = UpdateEnvironmentSuccess | InvalidInputError | EnvironmentNotFoundError + +scalar Upload diff --git a/softpack_core/graphql.py b/softpack_core/graphql.py index ce9cf9f..227af70 100644 --- a/softpack_core/graphql.py +++ b/softpack_core/graphql.py @@ -25,7 +25,11 @@ class GraphQL(API): """GraphQL API.""" prefix = "/graphql" - schemas = [EnvironmentSchema, PackageCollectionSchema, GroupsSchema] + schemas = [ + EnvironmentSchema, + PackageCollectionSchema, + GroupsSchema, + ] commands = Typer(help="GraphQL commands.") @staticmethod diff --git a/softpack_core/schemas/environment.py b/softpack_core/schemas/environment.py index 59d9733..b392abc 100644 --- a/softpack_core/schemas/environment.py +++ b/softpack_core/schemas/environment.py @@ -4,8 +4,10 @@ LICENSE file in the root directory of this source tree. """ +import datetime import io import re +import statistics from dataclasses import dataclass from pathlib import Path from traceback import format_exception_only @@ -200,6 +202,47 @@ def from_path(cls, environment_path: str) -> 'EnvironmentInput': ) +@dataclass +class BuildStatus: + """A class representing the status of a build.""" + + name: str + requested: datetime.datetime + build_start: Optional[datetime.datetime] + build_done: Optional[datetime.datetime] + + @classmethod + def get_all(cls) -> Union[List["BuildStatus"], BuilderError]: + """Get all known environment build statuses.""" + try: + host = app.settings.builder.host + port = app.settings.builder.port + r = httpx.get( + f"http://{host}:{port}/environments/status", + ) + r.raise_for_status() + json = r.json() + except Exception as e: + return BuilderError( + message="Connection to builder failed: " + + "".join(format_exception_only(type(e), e)) + ) + + return [ + BuildStatus( + name=s["Name"], + requested=datetime.datetime.fromisoformat(s["Requested"]), + build_start=datetime.datetime.fromisoformat(s["BuildStart"]) + if s["BuildStart"] + else None, + build_done=datetime.datetime.fromisoformat(s["BuildDone"]) + if s["BuildDone"] + else None, + ) + for s in json + ] + + @strawberry.type class Environment: """A Strawberry model representing a single environment.""" @@ -214,6 +257,11 @@ class Environment: state: Optional[State] artifacts = Artifacts() + requested: Optional[datetime.datetime] = None + build_start: Optional[datetime.datetime] = None + build_done: Optional[datetime.datetime] = None + avg_wait_secs: Optional[float] = None + @classmethod def iter(cls) -> Iterable["Environment"]: """Get an iterator over all Environment objects. @@ -221,9 +269,40 @@ def iter(cls) -> Iterable["Environment"]: Returns: Iterable[Environment]: An iterator of Environment objects. """ + statuses = BuildStatus.get_all() + if isinstance(statuses, BuilderError): + statuses = [] + + status_map = {s.name: s for s in statuses} + + waits: list[float] = [] + for s in statuses: + if s.build_done is None: + continue + waits.append((s.build_done - s.requested).total_seconds()) + + try: + avg_wait_secs = statistics.mean(waits) + except statistics.StatisticsError: + avg_wait_secs = None + environment_folders = cls.artifacts.iter() - environment_objects = map(cls.from_artifact, environment_folders) - return filter(None, environment_objects) + environment_objects = list( + filter(None, map(cls.from_artifact, environment_folders)) + ) + + for env in environment_objects: + status = status_map.get(str(Path(env.path, env.name))) + if not status: + if env.state == State.queued: + env.state = State.failed + continue + env.requested = status.requested + env.build_start = status.build_start + env.build_done = status.build_done + env.avg_wait_secs = avg_wait_secs + + return environment_objects @classmethod def from_artifact(cls, obj: Artifacts.Object) -> Optional["Environment"]: diff --git a/tests/integration/test_environment.py b/tests/integration/test_environment.py index 6eb3a9f..283ed51 100644 --- a/tests/integration/test_environment.py +++ b/tests/integration/test_environment.py @@ -4,6 +4,7 @@ LICENSE file in the root directory of this source tree. """ +import datetime import io from pathlib import Path from typing import Optional @@ -250,13 +251,68 @@ async def test_write_artifact(httpx_post, testable_env_input): assert isinstance(result, InvalidInputError) -def test_iter(testable_env_input): - envs = Environment.iter() - assert len(list(envs)) == 2 +def test_iter(testable_env_input, mocker): + get_mock = mocker.patch("httpx.get") + get_mock.return_value.json.return_value = [ + { + "Name": "users/test_user/test_environment", + "Requested": "2025-01-02T03:04:00.000000000Z", + "BuildStart": "2025-01-02T03:04:05.000000000Z", + "BuildDone": None, + }, + { + "Name": "groups/test_group/test_environment", + "Requested": "2025-01-02T03:04:00.000000000Z", + "BuildStart": "2025-01-02T03:04:05.000000000Z", + "BuildDone": "2025-01-02T03:04:15.000000000Z", + }, + # only used for average calculations, does not map to an environment in + # the test data + { + "Name": "users/foo/bar", + "Requested": "2025-01-02T03:04:00.000000000Z", + "BuildStart": "2025-01-02T03:04:05.000000000Z", + "BuildDone": "2025-01-02T03:04:25.000000000Z", + }, + ] + + envs = list(Environment.iter()) + assert len(envs) == 2 + assert envs[0].requested == datetime.datetime( + 2025, 1, 2, 3, 4, 0, tzinfo=datetime.timezone.utc + ) + assert envs[0].build_start == datetime.datetime( + 2025, 1, 2, 3, 4, 5, tzinfo=datetime.timezone.utc + ) + assert envs[0].build_done is None + assert envs[1].requested == datetime.datetime( + 2025, 1, 2, 3, 4, 0, tzinfo=datetime.timezone.utc + ) + assert envs[1].build_start == datetime.datetime( + 2025, 1, 2, 3, 4, 5, tzinfo=datetime.timezone.utc + ) + assert envs[1].build_done == datetime.datetime( + 2025, 1, 2, 3, 4, 15, tzinfo=datetime.timezone.utc + ) + assert envs[0].avg_wait_secs == envs[1].avg_wait_secs == 20 + + +def test_iter_no_statuses(testable_env_input, mocker): + get_mock = mocker.patch("httpx.get") + get_mock.return_value.json.return_value = [] + + envs = list(Environment.iter()) + assert len(envs) == 2 + assert envs[0].requested is None + assert envs[0].build_start is None + assert envs[0].build_done is None + assert envs[0].avg_wait_secs is None + assert envs[0].state == State.failed + assert envs[1].state == State.failed @pytest.mark.asyncio -async def test_states(httpx_post, testable_env_input): +async def test_states(httpx_post, testable_env_input, mocker): orig_name = testable_env_input.name result = Environment.create(testable_env_input) testable_env_input.name = orig_name @@ -277,6 +333,15 @@ async def test_states(httpx_post, testable_env_input): ) assert isinstance(result, WriteArtifactSuccess) + get_mock = mocker.patch("httpx.get") + get_mock.return_value.json.return_value = [ + { + "Name": "users/test_user/test_env_create-1", + "Requested": "2025-01-02T03:04:00.000000000Z", + "BuildStart": None, + "BuildDone": None, + }, + ] env = get_env_from_iter(testable_env_input.name + "-1") assert env is not None assert any(p.name == "zlib" for p in env.packages)