General: Add cleanup service for admins (#9296)

docs/admin/cleanup-service.rst

@@ -0,0 +1,47 @@
+.. _cleanup:
+
+Cleanup Service
+===============
+
+Artemis provides a feature to delete data from older courses.
+
+.. _cleanup-menu:
+
+.. figure:: cleanup/cleanup-menu.png
+    :align: center
+    :alt: Cleanup view
+
+As shown in the image, administrators can delete the following data types:
+
+* Plagiarism results with an undecided outcome
+* Orphaned data
+* Non-rated results from older courses
+* Rated results from older courses
+
+Since orphaned data has no connections to other data by nature, it is deleted without considering specific dates.
+For other types, administrators can track the related exercises and courses.
+When a cleanup operation is performed with specified "from" and "to" dates, all data associated with that type and related to courses that started after the "from" date and ended before the "to" date is deleted.
+
+Data Deletion by Operation Type
+------------------------------------------
+
+1. **Orphaned Data**:
+      - Long Feedback Text with feedback that has no results
+      - Text Block with feedback that has no results
+      - Feedback records without results
+      - Student and team scores where either a student or a team is specified
+      - Long Feedback Text where both participation and submission are missing
+      - Text Block where the referenced feedback has no associated participation or submission
+      - Feedback with no associated participation or submission
+      - All Ratings where the related result has no associated participation or submission
+      - Results without associated participation or submission
+
+2. **Plagiarism Results with an Undecided Outcome**:
+      - All plagiarism comparisons related to courses within the specified dates and marked as undecided.
+
+3. **Rated and Non-rated Results**:
+      - Both types follow the same logic, except for the rating status of results (rated vs. non-rated).
+      - For each type, only the latest valid result within a participation is retained, while all others are deleted.
+      - Because direct result deletion is restricted due to data integrity reasons, Artemis first removes associated data for results scheduled for deletion, including Long Feedback Text, Text Block, Feedback, Student Score, and Team Score.
+
+Artemis also records the date of the last cleanup operation, as seen in the last column of the table shown in the image.

docs/index.rst

@@ -79,6 +79,7 @@ All these exercises are supposed to be run either live in the lecture with insta
    admin/knownIssues
    admin/benchmarking-tool
    admin/telemetry
+   admin/cleanup-service
 
 
 .. toctree::

src/main/java/de/tum/cit/aet/artemis/assessment/repository/FeedbackRepository.java

@@ -32,8 +32,8 @@ public interface FeedbackRepository extends ArtemisJpaRepository<Feedback, Long>
 
     @Query("""
             SELECT feedback
-            FROM Feedback feedback
-            WHERE feedback.gradingInstruction.id IN :gradingInstructionsIds
+                FROM Feedback feedback
+                WHERE feedback.gradingInstruction.id IN :gradingInstructionsIds
             """)
     List<Feedback> findFeedbackByGradingInstructionIds(@Param("gradingInstructionsIds") List<Long> gradingInstructionsIds);
 

src/main/java/de/tum/cit/aet/artemis/assessment/repository/StudentScoreRepository.java

@@ -69,4 +69,5 @@ public interface StudentScoreRepository extends ArtemisJpaRepository<StudentScor
     @Transactional // ok because of delete
     @Modifying
     void deleteByExerciseAndUser(Exercise exercise, User user);
+
 }

src/main/java/de/tum/cit/aet/artemis/assessment/repository/cleanup/FeedbackCleanupRepository.java

@@ -0,0 +1,117 @@
+package de.tum.cit.aet.artemis.assessment.repository.cleanup;
+
+import static de.tum.cit.aet.artemis.core.config.Constants.PROFILE_CORE;
+
+import java.time.ZonedDateTime;
+
+import org.springframework.context.annotation.Profile;
+import org.springframework.data.jpa.repository.Modifying;
+import org.springframework.data.jpa.repository.Query;
+import org.springframework.data.repository.query.Param;
+import org.springframework.stereotype.Repository;
+import org.springframework.transaction.annotation.Transactional;
+
+import de.tum.cit.aet.artemis.assessment.domain.Feedback;
+import de.tum.cit.aet.artemis.assessment.domain.Result;
+import de.tum.cit.aet.artemis.core.repository.base.ArtemisJpaRepository;
+import de.tum.cit.aet.artemis.exercise.domain.participation.Participation;
+
+/**
+ * Spring Data JPA repository for cleaning up old and orphaned feedback entries.
+ * THE FOLLOWING METHODS ARE USED FOR CLEANUP PURPOSES AND SHOULD NOT BE USED IN OTHER CASES
+ */
+@Profile(PROFILE_CORE)
+@Repository
+public interface FeedbackCleanupRepository extends ArtemisJpaRepository<Feedback, Long> {
+
+    /**
+     * Deletes {@link Feedback} entries where the associated {@link Result} has no submission and no participation.
+     */
+    @Modifying
+    @Transactional // ok because of delete
+    // Subquery ok
+    @Query("""
+            DELETE FROM Feedback f
+            WHERE f.result IN (
+                SELECT r
+                FROM Result r
+                WHERE r.submission IS NULL
+                    AND r.participation IS NULL
+                )
+            """)
+    void deleteFeedbackForOrphanResults();
+
+    /**
+     * Deletes {@link Feedback} entries with a {@code null} result.
+     */
+    @Modifying
+    @Transactional // ok because of delete
+    @Query("""
+            DELETE FROM Feedback f
+            WHERE f.result IS NULL
+            """)
+    void deleteOrphanFeedback();
+
+    /**
+     * Deletes {@link Feedback} entries associated with rated {@link Result} that are not the latest rated result
+     * for a {@link Participation}, within courses conducted between the specified date range.
+     * This query removes old feedback entries that are not part of the latest rated results, for courses whose
+     * end date is before {@code deleteTo} and start date is after {@code deleteFrom}.
+     *
+     * @param deleteFrom the start date for selecting courses
+     * @param deleteTo   the end date for selecting courses
+     */
+    @Modifying
+    @Transactional // ok because of delete
+    @Query("""
+            DELETE FROM Feedback f
+            WHERE f.result IN (
+                SELECT r
+                FROM Result r
+                    LEFT JOIN r.participation p
+                    LEFT JOIN p.exercise e
+                    LEFT JOIN e.course c
+                WHERE r.id NOT IN (
+                    SELECT MAX(r2.id)
+                    FROM Result r2
+                    WHERE r2.participation.id = p.id
+                        AND r2.rated = TRUE
+                    )
+                    AND r.rated = TRUE
+                    AND c.endDate < :deleteTo
+                    AND c.startDate > :deleteFrom
+                )
+            """)
+    void deleteOldFeedbackThatAreNotLatestRatedResultsWhereCourseDateBetween(@Param("deleteFrom") ZonedDateTime deleteFrom, @Param("deleteTo") ZonedDateTime deleteTo);
+
+    /**
+     * Deletes non-rated {@link Feedback} entries that are not the latest non-rated result, where the associated course's start and end dates
+     * are between the specified date range.
+     * This query removes old feedback entries that are not part of the latest non-rated result within courses whose end date is before
+     * {@code deleteTo} and start date is after {@code deleteFrom}.
+     *
+     * @param deleteFrom the start date for selecting courses
+     * @param deleteTo   the end date for selecting courses
+     */
+    @Modifying
+    @Transactional // ok because of delete
+    @Query("""
+            DELETE FROM Feedback f
+            WHERE f.result IN (
+                SELECT r
+                FROM Result r
+                    LEFT JOIN r.participation p
+                    LEFT JOIN p.exercise e
+                    LEFT JOIN e.course c
+                WHERE r.id NOT IN (
+                    SELECT MAX(r2.id)
+                    FROM Result r2
+                    WHERE r2.participation.id = p.id
+                    )
+                    AND r.rated = FALSE
+                    AND c.endDate < :deleteTo
+                    AND c.startDate > :deleteFrom
+                )
+            """)
+    void deleteOldNonRatedFeedbackWhereCourseDateBetween(@Param("deleteFrom") ZonedDateTime deleteFrom, @Param("deleteTo") ZonedDateTime deleteTo);
+}

src/main/java/de/tum/cit/aet/artemis/assessment/repository/cleanup/LongFeedbackTextCleanupRepository.java

@@ -0,0 +1,125 @@
+package de.tum.cit.aet.artemis.assessment.repository.cleanup;
+
+import static de.tum.cit.aet.artemis.core.config.Constants.PROFILE_CORE;
+
+import java.time.ZonedDateTime;
+
+import org.springframework.context.annotation.Profile;
+import org.springframework.data.jpa.repository.Modifying;
+import org.springframework.data.jpa.repository.Query;
+import org.springframework.data.repository.query.Param;
+import org.springframework.stereotype.Repository;
+import org.springframework.transaction.annotation.Transactional;
+
+import de.tum.cit.aet.artemis.assessment.domain.Feedback;
+import de.tum.cit.aet.artemis.assessment.domain.LongFeedbackText;
+import de.tum.cit.aet.artemis.assessment.domain.Result;
+import de.tum.cit.aet.artemis.core.repository.base.ArtemisJpaRepository;
+import de.tum.cit.aet.artemis.exercise.domain.participation.Participation;
+
+/**
+ * Spring Data JPA repository for cleaning up old and orphaned long feedback text entries.
+ * THE FOLLOWING METHODS ARE USED FOR CLEANUP PURPOSES AND SHOULD NOT BE USED IN OTHER CASES
+ */
+@Profile(PROFILE_CORE)
+@Repository
+public interface LongFeedbackTextCleanupRepository extends ArtemisJpaRepository<LongFeedbackText, Long> {
+
+    /**
+     * Deletes {@link LongFeedbackText} entries linked to {@link Feedback} where the associated
+     * {@link Result} has no participation and no submission.
+     */
+    @Modifying
+    @Transactional // ok because of delete
+    @Query("""
+            DELETE FROM LongFeedbackText lft
+            WHERE lft.feedback.id IN (
+                SELECT f.id
+                FROM Feedback f
+                WHERE f.result.participation IS NULL
+                    AND f.result.submission IS NULL
+                )
+            """)
+    void deleteLongFeedbackTextForOrphanResult();
+
+    /**
+     * Deletes {@link LongFeedbackText} linked to {@link Feedback} with a {@code null} result.
+     */
+    @Modifying
+    @Transactional // ok because of delete
+    @Query("""
+            DELETE FROM LongFeedbackText lft
+            WHERE lft.feedback IN (
+                SELECT f
+                FROM Feedback f
+                WHERE f.result IS NULL
+                )
+            """)
+    void deleteLongFeedbackTextForOrphanedFeedback();
+
+    /**
+     * Deletes {@link LongFeedbackText} entries associated with rated {@link Result} that are not the latest rated result
+     * for a {@link Participation}, within courses conducted between the specified date range.
+     * This query removes old long feedback text that is not part of the latest rated results, for courses whose
+     * end date is before {@code deleteTo} and start date is after {@code deleteFrom}.
+     *
+     * @param deleteFrom the start date for selecting courses
+     * @param deleteTo   the end date for selecting courses
+     */
+    @Modifying
+    @Transactional // ok because of delete
+    @Query("""
+            DELETE FROM LongFeedbackText lft
+            WHERE lft.feedback IN (
+                SELECT f
+                FROM Feedback f
+                    LEFT JOIN f.result r
+                    LEFT JOIN r.participation p
+                    LEFT JOIN p.exercise e
+                    LEFT JOIN e.course c
+                WHERE f.result.id NOT IN (
+                    SELECT MAX(r2.id)
+                    FROM Result r2
+                    WHERE r2.participation.id = p.id
+                        AND r2.rated = TRUE
+                    )
+                    AND c.endDate < :deleteTo
+                    AND c.startDate > :deleteFrom
+                    AND r.rated = TRUE
+                )
+            """)
+    void deleteLongFeedbackTextForRatedResultsWhereCourseDateBetween(@Param("deleteFrom") ZonedDateTime deleteFrom, @Param("deleteTo") ZonedDateTime deleteTo);
+
+    /**
+     * Deletes {@link LongFeedbackText} entries linked to non-rated {@link Feedback} that are not the latest non-rated result where the associated course's start
+     * and end dates are between the specified date range.
+     * This query deletes long feedback text for feedback associated with non-rated results, within courses whose
+     * end date is before {@code deleteTo} and start date is after {@code deleteFrom}.
+     *
+     * @param deleteFrom the start date for selecting courses
+     * @param deleteTo   the end date for selecting courses
+     */
+    @Modifying
+    @Transactional // ok because of delete
+    @Query("""
+            DELETE FROM LongFeedbackText lft
+            WHERE lft.feedback IN (
+                SELECT f
+                FROM Feedback f
+                    LEFT JOIN f.result r
+                    LEFT JOIN r.participation p
+                    LEFT JOIN p.exercise e
+                    LEFT JOIN e.course c
+                WHERE f.result.id NOT IN (
+                    SELECT MAX(r2.id)
+                    FROM Result r2
+                    WHERE r2.participation.id = p.id
+                        AND r2.rated = FALSE
+                    )
+                    AND r.rated = FALSE
+                    AND c.endDate < :deleteTo
+                    AND c.startDate > :deleteFrom
+                )
+            """)
+    void deleteLongFeedbackTextForNonRatedResultsWhereCourseDateBetween(@Param("deleteFrom") ZonedDateTime deleteFrom, @Param("deleteTo") ZonedDateTime deleteTo);
+}