chore: release v1.1.1 (#2748)

CHANGES.rst

@@ -18,6 +18,17 @@
 Changes
 =======
 
+`1.1.1 <https://github.com/SwissDataScienceCenter/renku-python/compare/v1.1.0...v1.1.1>`__ (2022-03-10)
+-------------------------------------------------------------------------------------------------------
+
+This is a hotfix release fixing an issue with id generation for activities.
+
+Bug Fixes
+~~~~~~~~~
+
+-  **core:** Add doctor fix and on-the-fly migration for wrong activity ids
+   (`#2747 <https://github.com/SwissDataScienceCenter/renku-python/issues/2747>`__)
+
 `1.1.0 <https://github.com/SwissDataScienceCenter/renku-python/compare/v1.0.6...v1.1.0>`__ (2022-03-04)
 -------------------------------------------------------------------------------------------------------
 

Makefile

@@ -22,7 +22,7 @@ DOCKER_PREFIX:=${DOCKER_REGISTRY}$(DOCKER_REPOSITORY)
 GIT_MASTER_HEAD_SHA:=$(shell git rev-parse --short --verify HEAD)
 
 TEMPLATE_URL:=https://github.com/SwissDataScienceCenter/renku-project-template
-TEMPLATE_REFERENCE:=0.2.1
+TEMPLATE_REFERENCE:=0.3.0
 TEMPLATE_DIR:=renku/templates/
 
 .PHONY: service cli docker-tag docker-push docker-login

docs/spelling_wordlist.txt

@@ -81,6 +81,7 @@ gitlabClientSecret
 GraphQL
 Homebrew
 hostname
+hotfix
 https
 hyperkit
 hypervisor

helm-chart/renku-core/Chart.yaml

@@ -3,4 +3,4 @@ appVersion: "1.0"
 description: A Helm chart for Kubernetes
 name: renku-core
 icon: https://avatars0.githubusercontent.com/u/53332360?s=400&u=a4311d22842343604ef61a8c8a1e5793209a67e9&v=4
-version: 1.1.0
+version: 1.1.1

helm-chart/renku-core/values.yaml

@@ -16,7 +16,7 @@ global:
       tag: '0.0.1'
     customCAs: []
       # - secret:
-  
+
   ## Redis configuration. This is where renku-core expects to find
   ## a functioning redis instance and credentials to connect to it.
   redis:
@@ -27,7 +27,7 @@ global:
       coreService: "1"
     host: renku-redis
     port: 26379
-    clientLabel: 
+    clientLabel:
       renku-redis-host: "true"
     existingSecret: redis-secret
     existingSecretPasswordKey: redis-password
@@ -109,7 +109,7 @@ versions:
     fullnameOverride: ""
     image:
       repository: renku/renku-core
-      tag: "v1.0.6"
+      tag: "v1.1.1"
       pullPolicy: IfNotPresent
   v8:
     name: v8

renku/core/commands/checks/__init__.py

@@ -17,6 +17,7 @@
 # limitations under the License.
 """Define repository checks for :program:`renku doctor`."""
 
+from .activities import check_migrated_activity_ids
 from .datasets import check_dataset_old_metadata_location, check_invalid_datasets_derivation, check_missing_files
 from .external import check_missing_external_files
 from .githooks import check_git_hooks_installed
@@ -33,6 +34,7 @@
     "check_git_hooks_installed",
     "check_invalid_datasets_derivation",
     "check_lfs_info",
+    "check_migrated_activity_ids",
     "check_migration",
     "check_missing_external_files",
     "check_missing_files",

renku/core/commands/checks/activities.py

@@ -0,0 +1,97 @@
+# -*- coding: utf-8 -*-
+#
+# Copyright 2020 - Swiss Data Science Center (SDSC)
+# A partnership between École Polytechnique Fédérale de Lausanne (EPFL) and
+# Eidgenössische Technische Hochschule Zürich (ETHZ).
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Checks needed to determine integrity of datasets."""
+
+from itertools import chain
+
+import click
+
+from renku.core.commands.echo import WARNING
+from renku.core.management.command_builder import inject
+from renku.core.management.interface.activity_gateway import IActivityGateway
+from renku.core.management.interface.database_dispatcher import IDatabaseDispatcher
+from renku.core.utils import communication
+
+
+@inject.autoparams()
+def check_migrated_activity_ids(
+    client, fix, activity_gateway: IActivityGateway, database_dispatcher: IDatabaseDispatcher
+):
+    """Check that activity ids were correctly migrated in the past."""
+    activities = activity_gateway.get_all_activities()
+
+    wrong_activities = [a for a in activities if not a.id.startswith("/activities/")]
+
+    if fix:
+        current_database = database_dispatcher.current_database
+        for activity in wrong_activities:
+            communication.info(f"Fixing activity '{activity.id}'")
+
+            old_id = activity.id
+
+            # NOTE: Remove activity relations
+            tok = current_database["activity-catalog"].tokenizeQuery
+            relations = chain(
+                list(current_database["activity-catalog"].findRelationChains(tok(downstream=activity))),
+                list(current_database["activity-catalog"].findRelationChains(tok(upstream=activity))),
+            )
+            for rel_collection in relations:
+                for r in list(rel_collection):
+                    current_database["activity-catalog"].unindex(r)
+
+            current_database["activities"].pop(old_id)
+
+            # NOTE: Modify id on activity and children
+            activity.unfreeze()
+            activity.id = f"/activities/{activity.id}"
+            activity._p_oid = current_database.hash_id(activity.id)
+            activity.freeze()
+
+            for usage in activity.usages:
+                current_database["activities-by-usage"][usage.entity.path] = [
+                    a for a in current_database["activities-by-usage"][usage.entity.path] if a != activity
+                ]
+                object.__setattr__(usage, "id", f"/activities/{usage.id}")
+
+            for generation in activity.generations:
+                current_database["activities-by-generation"][generation.entity.path] = [
+                    a for a in current_database["activities-by-generation"][generation.entity.path] if a != activity
+                ]
+                object.__setattr__(generation, "id", f"/activities/{generation.id}")
+
+            for parameter in activity.parameters:
+                object.__setattr__(parameter, "id", f"/activities/{parameter.id}")
+
+            activity.association.id = f"/activities/{activity.association.id}"
+
+            activity_gateway.add(activity)
+
+        wrong_activities = []
+
+    if not wrong_activities:
+        return True, None
+
+    problems = (
+        WARNING
+        + "There are invalid activity ids in the project (use 'renku doctor --fix' to fix them):"
+        + "\n\n\t"
+        + "\n\t".join(click.style(a.id, fg="yellow") for a in wrong_activities)
+        + "\n"
+    )
+
+    return False, problems

renku/core/commands/schema/activity.py

@@ -17,19 +17,42 @@
 # limitations under the License.
 """Activity JSON-LD schema."""
 
-from marshmallow import EXCLUDE
+from marshmallow import EXCLUDE, pre_dump
 
 from renku.core.commands.schema.agent import PersonSchema, SoftwareAgentSchema
 from renku.core.commands.schema.annotation import AnnotationSchema
-from renku.core.commands.schema.calamus import JsonLDSchema, Nested, fields, oa, prov, renku
+from renku.core.commands.schema.calamus import JsonLDSchema, Nested, fields, oa, prov, renku, schema
 from renku.core.commands.schema.entity import CollectionSchema, EntitySchema
-from renku.core.commands.schema.parameter import ParameterValueSchema
 from renku.core.commands.schema.plan import PlanSchema
 from renku.core.models.provenance.activity import Activity, Association, Generation, Usage
+from renku.core.models.provenance.parameter import ParameterValue
 
 NON_EXISTING_ENTITY_CHECKSUM = "0" * 40
 
 
+class _ObjectWrapper:
+    """Object wrapper that allows temporarily overriding fields of immutable objects."""
+
+    def __init__(self, wrapped, **override):
+        self.__wrapped = wrapped
+        self.__override = override
+
+    def __getattr__(self, name):
+        if name in self.__override:
+            return self.__override[name]
+
+        return getattr(self.__wrapped, name)
+
+
+def _fix_id(obj):
+    """Fix ids under an activity that were wrong due to a bug."""
+
+    if not obj.id.startswith("/activities/"):
+        obj = _ObjectWrapper(obj, id=f"/activities/{obj.id}")
+
+    return obj
+
+
 class AssociationSchema(JsonLDSchema):
     """Association schema."""
 
@@ -44,6 +67,11 @@ class Meta:
     id = fields.Id()
     plan = Nested(prov.hadPlan, PlanSchema)
 
+    @pre_dump
+    def _pre_dump(self, obj, **kwargs):
+        """Pre-dump hook."""
+        return _fix_id(obj)
+
 
 class UsageSchema(JsonLDSchema):
     """Usage schema."""
@@ -59,6 +87,11 @@ class Meta:
     # TODO: DatasetSchema, DatasetFileSchema
     entity = Nested(prov.entity, [EntitySchema, CollectionSchema])
 
+    @pre_dump
+    def _pre_dump(self, obj, **kwargs):
+        """Pre-dump hook."""
+        return _fix_id(obj)
+
 
 class GenerationSchema(JsonLDSchema):
     """Generation schema."""
@@ -74,6 +107,32 @@ class Meta:
     # TODO: DatasetSchema, DatasetFileSchema
     entity = Nested(prov.qualifiedGeneration, [EntitySchema, CollectionSchema], reverse=True)
 
+    @pre_dump
+    def _pre_dump(self, obj, **kwargs):
+        """Pre-dump hook."""
+        return _fix_id(obj)
+
+
+class ParameterValueSchema(JsonLDSchema):
+    """ParameterValue schema."""
+
+    class Meta:
+        """Meta class."""
+
+        rdf_type = [renku.ParameterValue, schema.PropertyValue]
+        model = ParameterValue
+        unknown = EXCLUDE
+
+    id = fields.Id()
+
+    parameter = fields.IRI(schema.valueReference, attribute="parameter_id")
+    value = fields.Raw(schema.value)
+
+    @pre_dump
+    def _pre_dump(self, obj, **kwargs):
+        """Pre-dump hook."""
+        return _fix_id(obj)
+
 
 class ActivitySchema(JsonLDSchema):
     """Activity schema."""
@@ -102,3 +161,8 @@ class Meta:
     project_id = fields.IRI(renku.hasActivity, reverse=True)
     started_at_time = fields.DateTime(prov.startedAtTime, add_value_types=True)
     usages = Nested(prov.qualifiedUsage, UsageSchema, many=True)
+
+    @pre_dump
+    def _pre_dump(self, obj, **kwargs):
+        """Pre-dump hook."""
+        return _fix_id(obj)

renku/core/commands/schema/parameter.py

@@ -20,7 +20,6 @@
 from marshmallow import EXCLUDE
 
 from renku.core.commands.schema.calamus import JsonLDSchema, Nested, fields, prov, renku, schema
-from renku.core.models.provenance.parameter import ParameterValue
 from renku.core.models.workflow.parameter import (
     CommandInput,
     CommandOutput,
@@ -32,22 +31,6 @@
 )
 
 
-class ParameterValueSchema(JsonLDSchema):
-    """ParameterValue schema."""
-
-    class Meta:
-        """Meta class."""
-
-        rdf_type = [renku.ParameterValue, schema.PropertyValue]
-        model = ParameterValue
-        unknown = EXCLUDE
-
-    id = fields.Id()
-
-    parameter = fields.IRI(schema.valueReference, attribute="parameter_id")
-    value = fields.Raw(schema.value)
-
-
 class MappedIOStreamSchema(JsonLDSchema):
     """MappedIOStream schema."""
 

renku/core/management/migrations/m_0009__new_metadata_storage.py

@@ -389,7 +389,7 @@ def generate_activity_id(process_run_id: str, suffix: str = None) -> str:
         if suffix:
             original_id += suffix
 
-        return sha1(original_id.encode("utf-8")).hexdigest()
+        return Activity.generate_id(sha1(original_id.encode("utf-8")).hexdigest())
 
     assert not isinstance(process_run, old_schema.WorkflowRun)
 
@@ -405,7 +405,6 @@ def generate_activity_id(process_run_id: str, suffix: str = None) -> str:
     activities = []
     for i, run in enumerate(runs):
         activity_id = generate_activity_id(process_run._id, suffix=str(i) if i else None)
-
         plan = _convert_run_to_plan(run, project_id=project_id)
 
         agents = [_old_agent_to_new_agent(a) for a in process_run.agents or []]

renku/core/models/provenance/activity.py

@@ -19,7 +19,7 @@
 
 from datetime import datetime
 from itertools import chain
-from typing import List, Union
+from typing import List, Optional, Union
 from uuid import uuid4
 
 from werkzeug.utils import cached_property
@@ -118,6 +118,9 @@ def __init__(
         self.started_at_time: datetime = started_at_time
         self.usages: List[Usage] = usages or []
 
+        if not self.id.startswith("/activities/"):
+            self.id = f"/activities/{self.id}"
+
         # TODO: _was_informed_by = attr.ib(kw_only=True)
         # TODO: influenced = attr.ib(kw_only=True)
 
@@ -220,10 +223,11 @@ def plan_with_values(self) -> Plan:
         return plan
 
     @staticmethod
-    def generate_id() -> str:
+    def generate_id(uuid: Optional[str] = None) -> str:
         """Generate an identifier for an activity."""
-        # TODO: make id generation idempotent
-        return f"/activities/{uuid4().hex}"
+        if uuid is None:
+            uuid = uuid4().hex
+        return f"/activities/{uuid}"
 
     def has_identical_inputs_and_outputs_as(self, other: "Activity"):
         """Return true if all input and outputs paths are identical regardless of the order."""

renku/core/models/workflow/parameter.py

@@ -87,7 +87,7 @@ def _generate_id(plan_id: str, parameter_type: str, position: Optional[int], pos
         """Generate an id for parameters."""
         # /plans/723fd784-9347-4081-84de-a6dbb067545b/inputs/1
         # /plans/723fd784-9347-4081-84de-a6dbb067545b/inputs/stdin
-        # /plans/723fd784-9347-4081-84de-a6dbb067545b/inputs/dda5fcbf-0098-4917-be46-dc12f5f7b675
+        # /plans/723fd784-9347-4081-84de-a6dbb067545b/inputs/dda5fcbf00984917be46dc12f5f7b675
         position = str(position) if position is not None else uuid4().hex
         postfix = urllib.parse.quote(postfix) if postfix else position
         return f"{plan_id}/{parameter_type}/{postfix}"