feature(argus): Release Planner

This commit adds a new way to manage releases inside Argus, called
Release Planner. Release Planner replaces current Release Scheduling
mechanism (but not the Duty Calendar for now). Main features include:

* Plans are now separate entities (replacing one global release
  schedule)
* Each plan contains tests, groups and people it will be using for
  assignments
* Plans can target a specific scylla version to scope the plan to a
  point or RC release.
* Plans can be edited, deleted and copied.
* On copy, plan will offer to replace missing tests/groups if it is
  being copied to another release and that release is missing required
  tests.
* Each plan automatically creates and maintains a view dashboard for
  said plan and can be viewed from plan overview page or inside views
  overview
* The new assignment logic is now applied to all releases not marked as
  "endless" (perpetual in the admin panel). Currently the only one that
  is marked that way is scylla-master.

Fixes scylladb#377
Task: scylladb/qa-tasks#1717

argus/backend/controller/api.py

@@ -13,6 +13,7 @@
 from argus.backend.controller.testrun_api import bp as testrun_bp
 from argus.backend.controller.team import bp as team_bp
 from argus.backend.controller.view_api import bp as view_bp
+from argus.backend.controller.planner_api import bp as planner_bp
 from argus.backend.service.argus_service import ArgusService, ScheduleUpdateRequest
 from argus.backend.service.results_service import ResultsService
 from argus.backend.service.user import UserService, api_login_required
@@ -26,6 +27,7 @@
 bp.register_blueprint(testrun_bp)
 bp.register_blueprint(team_bp)
 bp.register_blueprint(view_bp)
+bp.register_blueprint(planner_bp)
 bp.register_error_handler(Exception, handle_api_exception)
 LOGGER = logging.getLogger(__name__)
 

argus/backend/controller/main.py

@@ -9,6 +9,7 @@
 from argus.backend.controller.team_ui import bp as teams_bp
 from argus.backend.service.argus_service import ArgusService
 from argus.backend.models.web import WebFileStorage
+from argus.backend.service.planner_service import PlanningService
 from argus.backend.service.user import UserService, login_required
 from argus.backend.service.views import UserViewService
 
@@ -113,6 +114,14 @@ def release_scheduler(name: str):
     return render_template("release_schedule.html.j2", release_name=name, data=data_json)
 
 
+@bp.route("/release/<string:name>/planner")
+@login_required
+def release_planner(name: str):
+    service = PlanningService()
+    planner_data = service.release_planner(name)
+    return render_template("release_planner.html.j2", release_name=planner_data["release"]["name"], planner_data=planner_data)
+
+
 @bp.route("/release/<string:name>/duty")
 @login_required
 def duty_planner(name: str):

argus/backend/controller/planner_api.py

@@ -0,0 +1,178 @@
+import logging
+from uuid import UUID
+from flask import (
+    Blueprint,
+    request
+)
+from argus.backend.error_handlers import handle_api_exception
+from argus.backend.service.planner_service import CopyPlanPayload, PlanningService, TempPlanPayload
+from argus.backend.service.test_lookup import TestLookup
+from argus.backend.service.user import api_login_required
+from argus.backend.util.common import get_payload
+
+bp = Blueprint('planning_api', __name__, url_prefix='/planning')
+LOGGER = logging.getLogger(__name__)
+bp.register_error_handler(Exception, handle_api_exception)
+
+
+@bp.route("/", methods=["GET"])
+@api_login_required
+def version():
+
+    result = PlanningService().version()
+
+    return {
+        "status": "ok",
+        "response": result
+    }
+
+
+@bp.route("/plan/<string:plan_id>/copy/check", methods=["GET"])
+@api_login_required
+def is_plan_eligible_for_copy(plan_id: str):
+    release_id = request.args.get("releaseId")
+    if not release_id:
+        raise Exception("Missing release id.")
+
+    result = PlanningService().check_plan_copy_eligibility(plan_id=UUID(plan_id), target_release_id=UUID(release_id))
+
+    return {
+        "status": "ok",
+        "response": result
+    }
+
+
+@bp.route("/release/<string:release_id>/gridview", methods=["GET"])
+@api_login_required
+def grid_view_for_release(release_id: str):
+
+    result = PlanningService().get_gridview_for_release(release_id=UUID(release_id))
+
+    return {
+        "status": "ok",
+        "response": result
+    }
+
+
+@bp.route("/search", methods=["GET"])
+@api_login_required
+def search_tests():
+    query = request.args.get("query")
+    release_id = request.args.get('releaseId')
+    service = TestLookup
+    if query:
+        res = service.test_lookup(query, release_id=release_id)
+    else:
+        res = []
+    return {
+        "status": "ok",
+        "response": {
+            "hits": res,
+            "total": len(res)
+        }
+    }
+
+@bp.route("/group/<string:group_id>/explode", methods=["GET"])
+@api_login_required
+def explode_group(group_id: str):
+    service = TestLookup
+    res = service.explode_group(group_id=group_id)
+    return {
+        "status": "ok",
+        "response": res
+    }
+
+
+@bp.route("/plan/<string:plan_id>/get", methods=["GET"])
+@api_login_required
+def get_plan(plan_id: str):
+    result = PlanningService().get_plan(plan_id)
+
+    return {
+        "status": "ok",
+        "response": result
+    }
+
+
+@bp.route("/release/<string:release_id>/all", methods=["GET"])
+@api_login_required
+def get_plans_for_release(release_id: str):
+    result = PlanningService().get_plans_for_release(release_id)
+
+    return {
+        "status": "ok",
+        "response": result
+    }
+
+
+@bp.route("/plan/create", methods=["POST"])
+@api_login_required
+def create_plan():
+    payload = get_payload(request)
+    result = PlanningService().create_plan(payload)
+
+    return {
+        "status": "ok",
+        "response": result
+    }
+
+
+@bp.route("/plan/update", methods=["POST"])
+@api_login_required
+def update_plan():
+    payload = get_payload(request)
+    result = PlanningService().update_plan(payload)
+
+    return {
+        "status": "ok",
+        "response": result
+    }
+
+@bp.route("/plan/copy", methods=["POST"])
+@api_login_required
+def copy_plan():
+    payload = get_payload(request)
+    payload["plan"] = TempPlanPayload(**payload["plan"])
+    result = PlanningService().copy_plan(CopyPlanPayload(**payload))
+
+    return {
+        "status": "ok",
+        "response": result
+    }
+
+
+
+@bp.route("/plan/<string:plan_id>/delete", methods=["POST"])
+@api_login_required
+def delete_plan(plan_id: str):
+    result = PlanningService().delete_plan(plan_id)
+
+    return {
+        "status": "ok",
+        "response": result
+    }
+
+
+@bp.route("/plan/<string:plan_id>/owner/set", methods=["POST"])
+@api_login_required
+def change_plan_owner(plan_id: str):
+    payload = get_payload(request)
+    result = PlanningService().change_plan_owner(plan_id=plan_id, new_owner=payload["newOwner"])
+
+    return {
+        "status": "ok",
+        "response": result
+    }
+
+
+@bp.route("/plan/<string:plan_id>/resolve_entities", methods=["GET"])
+@api_login_required
+def resolve_plan_entities(plan_id: str):
+
+    service = PlanningService()
+    result = service.resolve_plan(plan_id)
+
+    return {
+        "status": "ok",
+        "response": result,
+    }

argus/backend/models/plan.py

@@ -0,0 +1,25 @@
+import datetime
+from cassandra.cqlengine import columns
+from cassandra.cqlengine.models import Model
+from cassandra.cqlengine.usertype import UserType
+from cassandra.util import uuid_from_time
+
+
+class ArgusReleasePlan(Model):
+    id = columns.TimeUUID(partition_key=True, default=lambda: uuid_from_time(datetime.datetime.now(tz=datetime.UTC)))
+    name = columns.Text(required=True)
+    complete = columns.Boolean(default=lambda: False)
+    description = columns.Text()
+    owner = columns.UUID(required=True)
+    participants = columns.List(value_type=columns.UUID)
+    target_version = columns.Ascii()
+    assignee_mapping = columns.Map(key_type=columns.UUID, value_type=columns.UUID)
+    assignee_rotation = columns.List(value_type=columns.List(value_type=columns.UUID))
+    rotation_rule = columns.Ascii()
+    release_id = columns.UUID(index=True)
+    tests = columns.List(value_type=columns.UUID)
+    groups = columns.List(value_type=columns.UUID)
+    view_id = columns.UUID(index=True)
+    creation_time = columns.DateTime(default=lambda: datetime.datetime.now(tz=datetime.UTC))
+    last_updated = columns.DateTime(default=lambda: datetime.datetime.now(tz=datetime.UTC))
+    ends_at = columns.DateTime()

argus/backend/models/web.py

@@ -6,6 +6,7 @@
 from cassandra.cqlengine import columns
 from cassandra.util import uuid_from_time, unix_time_from_uuid1  # pylint: disable=no-name-in-module
 
+from argus.backend.models.plan import ArgusReleasePlan
 from argus.backend.models.result import ArgusGenericResultMetadata, ArgusGenericResultData
 
 
@@ -154,6 +155,7 @@ class ArgusUserView(Model):
     display_name = columns.Text()
     description = columns.Text()
     user_id = columns.UUID(required=True, index=True)
+    plan_id = columns.UUID(index=True)
     tests = columns.List(value_type=columns.UUID, default=lambda: [])
     release_ids = columns.List(value_type=columns.UUID, default=lambda: [])
     group_ids = columns.List(value_type=columns.UUID, default=lambda: [])
@@ -381,6 +383,7 @@ class WebFileStorage(Model):
     ArgusScheduleTest,
     ArgusGenericResultMetadata,
     ArgusGenericResultData,
+    ArgusReleasePlan,
 ]
 
 USED_TYPES: list[UserType] = [

argus/backend/plugins/core.py

@@ -9,6 +9,7 @@
 from cassandra.cqlengine.usertype import UserType
 from flask import Blueprint
 from argus.backend.db import ScyllaCluster
+from argus.backend.models.plan import ArgusReleasePlan
 from argus.backend.models.web import (
     ArgusTest,
     ArgusGroup,
@@ -67,13 +68,38 @@ def assign_categories(self):
         except ArgusTest.DoesNotExist:
             LOGGER.warning("Test entity missing for key \"%s\", run won't be visible until this is corrected", key)
 
+    def get_assignment(self, version: str | None = None) -> UUID | None:
+        associated_test: ArgusTest = ArgusTest.get(build_system_id=self.build_id)
+        associated_release: ArgusRelease = ArgusRelease.get(id=associated_test.release_id)
+        if associated_release.perpetual:
+            return self._legacy_get_scheduled_assignee()
+
+        plans: list[ArgusReleasePlan] = list(ArgusReleasePlan.filter(release_id=associated_release.id))
+
+        if version:
+            plans = [plan for plan in plans if plan.target_version == version]
+
+        for plan in plans:
+            if associated_test.group_id in plan.groups:
+                return plan.assignee_mapping.get(associated_test.group_id, plan.owner)
+            if associated_test.id in plan.tests:
+                return plan.assignee_mapping.get(associated_test.id, plan.owner)
+
+        return None
+
+
     def get_scheduled_assignee(self) -> UUID:
+        return self._legacy_get_scheduled_assignee()
+
+    def _legacy_get_scheduled_assignee(self) -> UUID:
         """
             Iterate over all schedules (groups and tests) and return first available assignee
         """
         associated_test = ArgusTest.get(build_system_id=self.build_id)
         associated_group = ArgusGroup.get(id=associated_test.group_id)
         associated_release = ArgusRelease.get(id=associated_test.release_id)
+        if not associated_release.perpetual:
+            return self.get_assignment()
 
         scheduled_groups = ArgusScheduleGroup.filter(
             release_id=associated_release.id,

argus/backend/plugins/driver_matrix_tests/model.py

@@ -412,6 +412,9 @@ def get_events(self) -> list:
 
     def submit_product_version(self, version: str):
         self.scylla_version = version
+        new_assignee = self.get_assignment(version)
+        if new_assignee:
+            self.assignee = new_assignee
 
     def finish_run(self, payload: dict = None):
         self.end_time = datetime.utcnow()

argus/backend/plugins/generic/model.py

@@ -41,6 +41,9 @@ def submit_product_version(self, version: str):
         pattern = re.compile(r"((?P<short>[\w.~]+)-(?P<build>(0\.)?(?P<date>[0-9]{8,8})\.(?P<commit>\w+).*))")
         if match := pattern.search(version):
             self.scylla_version = match.group("short")
+            new_assignee = self.get_assignment(match.group("short"))
+            if new_assignee:
+                self.assignee = new_assignee
             self.set_full_version(version)
 
     @classmethod

argus/backend/plugins/sct/testrun.py

@@ -216,6 +216,9 @@ def get_events(self) -> list[EventsBySeverity]:
 
     def submit_product_version(self, version: str):
         self.scylla_version = version
+        new_assignee = self.get_assignment(version)
+        if new_assignee:
+            self.assignee = new_assignee
 
     def finish_run(self, payload: dict = None):
         self.end_time = datetime.utcnow()

argus/backend/plugins/sirenada/model.py

@@ -61,6 +61,9 @@ def get_distinct_product_versions(cls, release: ArgusRelease, cluster: ScyllaClu
 
     def submit_product_version(self, version: str):
         self.scylla_version = version
+        new_assignee = self.get_assignment(version)
+        if new_assignee:
+            self.assignee = new_assignee
 
     def submit_logs(self, logs: dict[str, str]):
         raise SirenadaPluginException("Log submission is not supported for Sirenada")

argus/backend/service/argus_service.py

@@ -26,6 +26,7 @@
     User,
 )
 from argus.backend.events.event_processors import EVENT_PROCESSORS
+from argus.backend.service.planner_service import PlanningService
 from argus.backend.service.testrun import TestRunService
 from argus.backend.util.common import chunk
 
@@ -580,6 +581,8 @@ def _batch_get_schedules_from_ids(self, release_id: UUID, schedule_ids: list[UUI
     def get_groups_assignees(self, release_id: UUID | str):
         release_id = UUID(release_id) if isinstance(release_id, str) else release_id
         release = ArgusRelease.get(id=release_id)
+        if not release.perpetual:
+            return PlanningService().get_assignments_for_groups(release_id)
 
         groups = ArgusGroup.filter(release_id=release_id).all()
         group_ids = [group.id for group in groups if group.enabled]
@@ -618,6 +621,9 @@ def get_tests_assignees(self, group_id: UUID | str):
         group = ArgusGroup.get(id=group_id)
 
         release = ArgusRelease.get(id=group.release_id)
+        if not release.perpetual:
+            return PlanningService().get_assignments_for_tests(group_id)
+
         tests = ArgusTest.filter(group_id=group_id).all()
 
         test_ids = [test.id for test in tests if test.enabled]