diff --git a/schema.graphql b/schema.graphql index 98f7454..443d4ed 100644 --- a/schema.graphql +++ b/schema.graphql @@ -3,6 +3,12 @@ schema { mutation: SchemaMutation } +union AddTagResponse = AddTagSuccess | InvalidInputError | EnvironmentNotFoundError + +type AddTagSuccess implements Success { + message: String! +} + type BuilderError implements Error { message: String! } @@ -31,6 +37,7 @@ type Environment { type: Type! packages: [Package!]! state: State + tags: [String!]! requested: DateTime buildStart: DateTime buildDone: DateTime @@ -48,6 +55,7 @@ input EnvironmentInput { path: String! description: String! packages: [PackageInput!]! + tags: [String!] = null } type EnvironmentNotFoundError implements Error { @@ -86,6 +94,7 @@ type PackageMultiVersion { type SchemaMutation { createEnvironment(env: EnvironmentInput!): CreateResponse! deleteEnvironment(name: String!, path: String!): DeleteResponse! + addTag(name: String!, path: String!, tag: String!): AddTagResponse! createFromModule(file: Upload!, modulePath: String!, environmentPath: String!): CreateResponse! updateFromModule(file: Upload!, modulePath: String!, environmentPath: String!): UpdateResponse! } diff --git a/softpack_core/artifacts.py b/softpack_core/artifacts.py index 987f3bb..30dc558 100644 --- a/softpack_core/artifacts.py +++ b/softpack_core/artifacts.py @@ -72,6 +72,7 @@ class Artifacts: builder_out = "builder.out" module_file = "module" readme_file = "README.md" + meta_file = "meta.yml" built_by_softpack_file = ".built_by_softpack" built_by_softpack = Type.softpack.value generated_from_module_file = ".generated_from_module" @@ -154,6 +155,11 @@ def spec(self) -> Box: map(lambda p: Package.from_name(p), info.packages) ) + meta = Box() + if Artifacts.meta_file in self.obj: + meta = Box.from_yaml(self.obj[Artifacts.meta_file].data) + info["tags"] = getattr(meta, "tags", []) + return info def __iter__(self) -> Iterator["Artifacts.Object"]: @@ -322,7 +328,7 @@ def iter(self) -> Iterable: return itertools.chain.from_iterable(map(self.environments, folders)) - def get(self, path: Path, name: str) -> Optional[pygit2.Tree]: + def get(self, path: Path, name: str) -> Optional[Object]: """Return the environment at the specified name and path. Args: @@ -330,10 +336,13 @@ def get(self, path: Path, name: str) -> Optional[pygit2.Tree]: name: the name of the environment folder Returns: - pygit2.Tree: a pygit2.Tree or None + Object: an Object or None """ try: - return self.tree(str(self.environments_folder(str(path), name))) + return self.Object( + Path(path, name), + self.tree(str(self.environments_folder(str(path), name))), + ) except KeyError: return None diff --git a/softpack_core/schemas/environment.py b/softpack_core/schemas/environment.py index 5e0f9ae..8e989d8 100644 --- a/softpack_core/schemas/environment.py +++ b/softpack_core/schemas/environment.py @@ -11,7 +11,7 @@ from dataclasses import dataclass from pathlib import Path from traceback import format_exception_only -from typing import Iterable, List, Optional, Tuple, Union, cast +from typing import List, Optional, Tuple, Union, cast import httpx import starlette.datastructures @@ -52,6 +52,11 @@ class UpdateEnvironmentSuccess(Success): """Environment successfully updated.""" +@strawberry.type +class AddTagSuccess(Success): + """Successfully added tag to environment.""" + + @strawberry.type class DeleteEnvironmentSuccess(Success): """Environment successfully deleted.""" @@ -74,6 +79,7 @@ class EnvironmentNotFoundError(Error): path: str name: str + message: str = "No environment with this path and name found." @strawberry.type @@ -109,6 +115,15 @@ class BuilderError(Error): ], ) +AddTagResponse = strawberry.union( + "AddTagResponse", + [ + AddTagSuccess, + InvalidInputError, + EnvironmentNotFoundError, + ], +) + DeleteResponse = strawberry.union( "DeleteResponse", [ @@ -126,6 +141,32 @@ class BuilderError(Error): ) +def validate_tag(tag: str) -> Union[None, InvalidInputError]: + """If the given tag is invalid, return an error describing why, else None. + + Tags must be composed solely of alphanumerics, dots, underscores, + dashes, and spaces, and not contain runs of multiple spaces or + leading/trailing whitespace. + """ + if tag != tag.strip(): + return InvalidInputError( + message="Tags must not contain leading or trailing whitespace" + ) + + if re.fullmatch(r"[a-zA-Z0-9 ._-]+", tag) is None: + return InvalidInputError( + message="Tags must contain only alphanumerics, dots, " + "underscores, dashes, and spaces" + ) + + if re.search(r"\s\s", tag) is not None: + return InvalidInputError( + message="Tags must not contain runs of multiple spaces" + ) + + return None + + @strawberry.input class PackageInput(Package): """A Strawberry input model representing a package.""" @@ -146,6 +187,7 @@ class EnvironmentInput: path: str description: str packages: list[PackageInput] + tags: Optional[list[str]] = None def validate(self) -> Union[None, InvalidInputError]: """Validate all values. @@ -156,7 +198,11 @@ def validate(self) -> Union[None, InvalidInputError]: Returns: None if good, or InvalidInputError if not all values supplied. """ - if any(len(value) == 0 for value in vars(self).values()): + if any( + len(value) == 0 + for key, value in vars(self).items() + if key != "tags" + ): return InvalidInputError(message="all fields must be filled in") if not re.fullmatch("^[a-zA-Z0-9_-][a-zA-Z0-9_.-]*$", self.name): @@ -178,6 +224,10 @@ def validate(self) -> Union[None, InvalidInputError]: "alphanumerics, dash, and underscore" ) + for tag in self.tags or []: + if (response := validate_tag(tag)) is not None: + return response + return None @classmethod @@ -255,6 +305,7 @@ class Environment: type: Type packages: list[Package] state: Optional[State] + tags: list[str] artifacts = Artifacts() requested: Optional[datetime.datetime] = None @@ -263,7 +314,7 @@ class Environment: avg_wait_secs: Optional[float] = None @classmethod - def iter(cls) -> Iterable["Environment"]: + def iter(cls) -> list["Environment"]: """Get an iterator over all Environment objects. Returns: @@ -323,6 +374,7 @@ def from_artifact(cls, obj: Artifacts.Object) -> Optional["Environment"]: state=spec.state, readme=spec.get("readme", ""), type=spec.get("type", ""), + tags=spec.tags, ) except KeyError: return None @@ -443,7 +495,7 @@ def create_new_env( name=env.name, ) - # Create folder with place-holder file + # Create folder with initial files new_folder_path = Path(env.path, env.name) try: softpack_definition = dict( @@ -453,13 +505,20 @@ def create_new_env( for pkg in env.packages ], ) - ymlData = yaml.dump(softpack_definition) + definitionData = yaml.dump(softpack_definition) + + meta = dict(tags=sorted(set(env.tags or []))) + metaData = yaml.dump(meta) tree_oid = cls.artifacts.create_files( new_folder_path, [ (env_type, ""), # e.g. .built_by_softpack - (cls.artifacts.environments_file, ymlData), # softpack.yml + ( + cls.artifacts.environments_file, + definitionData, + ), # softpack.yml + (cls.artifacts.meta_file, metaData), ], True, ) @@ -491,11 +550,52 @@ def check_env_exists( return None return EnvironmentNotFoundError( - message="No environment with this path and name found.", path=str(path.parent), name=path.name, ) + @classmethod + def add_tag( + cls, name: str, path: str, tag: str + ) -> AddTagResponse: # type: ignore + """Add a tag to an Environment. + + Tags must be valid as defined by validate_tag(). + + Adding a tag that already exists is not an error. + + Args: + name: the name of of environment + path: the path of the environment + tag: the tag to add + + Returns: + A message confirming the success or failure of the operation. + """ + environment_path = Path(path, name) + response: Optional[Error] = cls.check_env_exists(environment_path) + if response is not None: + return response + + if (response := validate_tag(tag)) is not None: + return response + + tree = cls.artifacts.get(Path(path), name) + if tree is None: + return EnvironmentNotFoundError(path=path, name=name) + box = tree.spec() + tags = set(box.tags) + if tag in tags: + return AddTagSuccess(message="Tag already present") + tags.add(tag) + + metadata = yaml.dump({"tags": sorted(tags)}) + tree_oid = cls.artifacts.create_file( + environment_path, cls.artifacts.meta_file, metadata, overwrite=True + ) + cls.artifacts.commit_and_push(tree_oid, "create environment folder") + return AddTagSuccess(message="Tag successfully added") + @classmethod def delete(cls, name: str, path: str) -> DeleteResponse: # type: ignore """Delete an Environment. @@ -749,6 +849,7 @@ class Mutation: createEnvironment: CreateResponse = Environment.create # type: ignore deleteEnvironment: DeleteResponse = Environment.delete # type: ignore + addTag: AddTagResponse = Environment.add_tag # type: ignore # writeArtifact: WriteArtifactResponse = ( # type: ignore # Environment.write_artifact # ) diff --git a/softpack_core/service.py b/softpack_core/service.py index e9f11ab..ff38101 100644 --- a/softpack_core/service.py +++ b/softpack_core/service.py @@ -125,4 +125,8 @@ async def resend_pending_builds( # type: ignore[no-untyped-def] response.status_code = 500 message = "Failed to trigger all resends" - return {"message": message, "successes": successes, "failures": failures} + return { + "message": message, + "successes": successes, + "failures": failures, + } diff --git a/tests/integration/test_builderupload.py b/tests/integration/test_builderupload.py index b250177..a752b43 100644 --- a/tests/integration/test_builderupload.py +++ b/tests/integration/test_builderupload.py @@ -54,5 +54,5 @@ def test_builder_upload(testable_env_input): tree = Environment.artifacts.get(env_parent, env_name) assert tree is not None - assert tree[softpackYaml].data == softpackYamlContents - assert tree[spackLock].data == spackLockContents + assert tree.get(softpackYaml).data == softpackYamlContents + assert tree.get(spackLock).data == spackLockContents diff --git a/tests/integration/test_environment.py b/tests/integration/test_environment.py index 478fd9c..d59fb5d 100644 --- a/tests/integration/test_environment.py +++ b/tests/integration/test_environment.py @@ -1,4 +1,4 @@ -"""Copyright (c) 2023 Genome Research Ltd. +"""Copyright (c) 2023, 2024 Genome Research Ltd. This source code is licensed under the MIT license found in the LICENSE file in the root directory of this source tree. @@ -16,6 +16,7 @@ from softpack_core.artifacts import Artifacts from softpack_core.schemas.environment import ( + AddTagSuccess, BuilderError, CreateEnvironmentSuccess, DeleteEnvironmentSuccess, @@ -51,6 +52,7 @@ def test_create(httpx_post, testable_env_input: EnvironmentInput) -> None: Package(name="pkg_test2"), Package(name="pkg_test3", version="3.1"), ], + tags=["foo", "foo", "bar"], ) ) assert isinstance(result, CreateEnvironmentSuccess) @@ -70,6 +72,10 @@ def test_create(httpx_post, testable_env_input: EnvironmentInput) -> None: "packages": ["pkg_test"], } assert yaml.safe_load(ymlFile.data.decode()) == expected_yaml + meta_yml = file_in_remote(dir / Environment.artifacts.meta_file) + expected_meta_yml = {"tags": []} + actual_meta_yml = yaml.safe_load(meta_yml.data.decode()) + assert actual_meta_yml == expected_meta_yml dir = Path( Environment.artifacts.environments_root, @@ -86,6 +92,11 @@ def test_create(httpx_post, testable_env_input: EnvironmentInput) -> None: } assert yaml.safe_load(ymlFile.data.decode()) == expected_yaml + meta_yml = file_in_remote(dir / Environment.artifacts.meta_file) + expected_meta_yml = {"tags": ["bar", "foo"]} + actual_meta_yml = yaml.safe_load(meta_yml.data.decode()) + assert actual_meta_yml == expected_meta_yml + result = Environment.create(testable_env_input) testable_env_input.name = orig_input_name assert isinstance(result, CreateEnvironmentSuccess) @@ -99,6 +110,19 @@ def test_create(httpx_post, testable_env_input: EnvironmentInput) -> None: assert file_in_remote(path) +def test_create_no_tags(httpx_post, testable_env_input): + testable_env_input.tags = None + result = Environment.create(testable_env_input) + assert isinstance(result, CreateEnvironmentSuccess) + + +def test_create_illegal_tags(httpx_post, testable_env_input): + for tag in [" ", " ", " leading whitespace", "trailing whitespace "]: + testable_env_input.tags = [tag] + result = Environment.create(testable_env_input) + assert isinstance(result, InvalidInputError) + + def test_create_name_empty_disallowed(httpx_post, testable_env_input): testable_env_input.name = "" result = Environment.create(testable_env_input) @@ -469,6 +493,7 @@ async def test_create_from_module(httpx_post, testable_env_input): ) assert isinstance(result, EnvironmentNotFoundError) + def test_environmentinput_from_path(): for path in ( "users/any1/envName", @@ -485,4 +510,46 @@ def test_environmentinput_from_path(): "users/any1/..envName-1", "users/any1/../envName-1.1", ]: - assert EnvironmentInput.from_path(path).validate() is not None \ No newline at end of file + assert EnvironmentInput.from_path(path).validate() is not None + + +def test_tagging(httpx_post, testable_env_input: EnvironmentInput) -> None: + example_env = Environment.iter()[0] + assert example_env.tags == [] + + name, path = example_env.name, example_env.path + result = Environment.add_tag(name, path, tag="test") + assert isinstance(result, AddTagSuccess) + assert result.message == "Tag successfully added" + + result = Environment.add_tag("foo", "users/xyz", tag="test") + assert isinstance(result, EnvironmentNotFoundError) + + result = Environment.add_tag(name, path, tag="../") + assert isinstance(result, InvalidInputError) + + result = Environment.add_tag(name, path, tag="") + assert isinstance(result, InvalidInputError) + + result = Environment.add_tag(name, path, tag=" ") + assert isinstance(result, InvalidInputError) + + result = Environment.add_tag(name, path, tag="foo bar") + assert isinstance(result, InvalidInputError) + + example_env = Environment.iter()[0] + assert len(example_env.tags) == 1 + assert example_env.tags[0] == "test" + + result = Environment.add_tag(name, path, tag="second test") + assert isinstance(result, AddTagSuccess) + + example_env = Environment.iter()[0] + assert example_env.tags == ["second test", "test"] + + result = Environment.add_tag(name, path, tag="test") + assert isinstance(result, AddTagSuccess) + assert result.message == "Tag already present" + + example_env = Environment.iter()[0] + assert example_env.tags == ["second test", "test"]