CONCD-824 error in counting

concordia/models.py

@@ -6,12 +6,12 @@
 
 import pytesseract
 from django.conf import settings
-from django.contrib.auth.models import User
+from django.contrib.auth.models import BaseUserManager, User
 from django.core import signing
 from django.core.cache import cache
 from django.core.exceptions import ValidationError
 from django.core.validators import RegexValidator
-from django.db import connection, models
+from django.db import models
 from django.db.models import Count, ExpressionWrapper, F, JSONField, Q
 from django.db.models.functions import Round
 from django.db.models.signals import post_save
@@ -27,8 +27,10 @@
 
 User._meta.get_field("email").__dict__["_unique"] = True
 
+ONE_MINUTE = datetime.timedelta(minutes=1)
 ONE_DAY = datetime.timedelta(days=1)
 ONE_DAY_AGO = timezone.now() - ONE_DAY
+THRESHOLD = 3
 
 
 def resource_file_upload_path(instance, filename):
@@ -38,6 +40,28 @@ def resource_file_upload_path(instance, filename):
     return time.strftime(path)
 
 
+class ConcordiaUserManager(BaseUserManager):
+    def review_incidents(self):
+        user_incident_count = {}
+
+        for user in self.get_queryset().filter(is_superuser=False, is_staff=False):
+            incident_count = user.review_incidents()
+            if incident_count > 0:
+                user_incident_count[user.id] = incident_count
+
+        return user_incident_count
+
+    def transcribe_incidents(self):
+        user_incident_count = {}
+
+        for user in self.get_queryset().filter(is_superuser=False, is_staff=False):
+            incident_count = user.transcribe_incidents()
+            if incident_count > 0:
+                user_incident_count[user.id] = incident_count
+
+        return user_incident_count
+
+
 class ConcordiaUser(User):
     # This class is a simple proxy model to add
     # additional user functionality to, without changing
@@ -72,6 +96,48 @@ def get_email_reconfirmation_key(self):
     def validate_reconfirmation_email(self, email):
         return email == self.get_email_for_reconfirmation()
 
+    def review_incidents(self, start=ONE_DAY_AGO, threshold=THRESHOLD):
+        recent_accepts = Transcription.objects.filter(
+            accepted__gte=start, reviewed_by=self
+        ).values_list("accepted", flat=True)
+        recent_rejects = Transcription.objects.filter(
+            rejected__gte=start, reviewed_by=self
+        ).values_list("rejected", flat=True)
+        timestamps = list(recent_accepts) + list(recent_rejects)
+        timestamps.sort()
+        incidents = 0
+        for i in range(len(timestamps)):
+            count = 1
+            for j in range(i + 1, len(timestamps)):
+                if (timestamps[j] - timestamps[i]).seconds <= 60:
+                    count += 1
+                    if count == threshold:
+                        incidents += 1
+                        break
+                else:
+                    break
+        return incidents
+
+    def transcribe_incidents(self, start=ONE_DAY_AGO, threshold=THRESHOLD):
+        recent_transcriptions = Transcription.objects.filter(
+            submitted__gte=start, user=self
+        ).order_by("submitted")
+        timestamps = recent_transcriptions.values_list("submitted", flat=True)
+        incidents = 0
+        for i in range(len(timestamps)):
+            count = 1
+            for j in range(i + 1, len(timestamps)):
+                if (timestamps[j] - timestamps[i]).seconds <= 60:
+                    count += 1
+                    if count == threshold:
+                        incidents += 1
+                        break
+                else:
+                    break
+        return incidents
+
+    objects = ConcordiaUserManager()
+
 
 class UserProfile(MetricsModelMixin("userprofile"), models.Model):
     user = models.OneToOneField(User, on_delete=models.CASCADE, related_name="profile")
@@ -798,64 +864,6 @@ def recent_review_actions(self, days=1):
         START = timezone.now() - datetime.timedelta(days=days)
         return self.review_actions(START)
 
-    def reviewing_too_quickly(self, start=ONE_DAY_AGO):
-        with connection.cursor() as cursor:
-            cursor.execute(
-                f"""SELECT u.id, u.username, COUNT(*)
-                FROM concordia_transcription t1
-                JOIN concordia_transcription t2
-                ON t1.id < t2.id
-                JOIN concordia_transcription t3
-                ON t2.id < t3.id
-                AND t1.reviewed_by_id = t2.reviewed_by_id
-                AND t2.reviewed_by_id = t3.reviewed_by_id
-                AND t1.accepted >= '{start}'
-                AND t2.accepted >= '{start}'
-                AND t3.accepted >= '{start}'
-                AND ABS(
-                    EXTRACT(EPOCH FROM (t1.updated_on - t2.updated_on))
-                ) < 60
-                AND ABS(
-                    EXTRACT(EPOCH FROM (t1.updated_on - t3.updated_on))
-                ) < 60
-                AND ABS(EXTRACT(
-                    EPOCH FROM (t2.updated_on - t3.updated_on))
-                ) < 60
-                JOIN auth_user u on t1.reviewed_by_id = u.id
-                WHERE u.is_superuser = FALSE and u.is_staff = False
-                GROUP BY u.id, u.username"""  # nosec B608
-            )
-            return cursor.fetchall()
-
-    def transcribing_too_quickly(self, start=ONE_DAY_AGO):
-        with connection.cursor() as cursor:
-            cursor.execute(
-                f"""SELECT u.id, u.username, COUNT(*)
-                FROM concordia_transcription t1
-                JOIN concordia_transcription t2
-                ON t1.id < t2.id
-                JOIN concordia_transcription t3
-                ON t2.id < t3.id
-                AND t1.user_id = t2.user_id
-                AND t2.user_id = t3.user_id
-                AND t1.submitted >= '{start}'
-                AND t2.submitted >= '{start}'
-                AND t3.submitted >= '{start}'
-                AND ABS(
-                    EXTRACT(EPOCH FROM (t1.created_on - t2.created_on))
-                ) < 60
-                AND ABS(
-                    EXTRACT(EPOCH FROM (t1.created_on - t3.created_on))
-                ) < 60
-                AND ABS(
-                    EXTRACT(EPOCH FROM (t2.created_on - t3.created_on))
-                ) < 60
-                JOIN auth_user u on t1.user_id = u.id
-                WHERE u.is_superuser = FALSE and u.is_staff = False
-                GROUP BY u.id, u.username"""  # nosec B608
-            )
-            return cursor.fetchall()
-
 
 class Transcription(MetricsModelMixin("transcription"), models.Model):
     asset = models.ForeignKey(Asset, on_delete=models.CASCADE)

concordia/tasks.py

@@ -24,6 +24,7 @@
     AssetTranscriptionReservation,
     Campaign,
     CampaignRetirementProgress,
+    ConcordiaUser,
     Item,
     Project,
     ResourceFile,
@@ -1067,8 +1068,8 @@ def unusual_activity():
         "title": "Unusual User Activity Report for "
         + timezone.now().strftime("%b %d %Y, %I:%M %p"),
         "domain": "https://" + site.domain,
-        "transcriptions": transcribing_too_quickly(),
-        "reviews": reviewing_too_quickly(),
+        "transcriptions": ConcordiaUser.objects.transcribe_incidents(),
+        "reviews": ConcordiaUser.objects.review_incidents(),
     }
 
     text_body_template = loader.get_template("emails/unusual_activity.txt")

concordia/tests/test_models.py

@@ -11,6 +11,7 @@
     AssetTranscriptionReservation,
     Campaign,
     CardFamily,
+    ConcordiaUser,
     Resource,
     Transcription,
     TranscriptionStatus,
@@ -40,6 +41,101 @@
 )
 
 
+class ConcordiaUserTestCase(CreateTestUsers, TestCase):
+    def setUp(self):
+        self.transcription1 = create_transcription(
+            user=self.create_user(username="tester1"),
+            rejected=timezone.now() - timedelta(days=2),
+        )
+        self.transcription2 = create_transcription(
+            asset=self.transcription1.asset, user=get_anonymous_user()
+        )
+
+    def test_review_incidents(self):
+        self.transcription1.accepted = timezone.now()
+        self.transcription1.reviewed_by = self.create_user(username="tester2")
+        self.transcription1.save()
+        self.transcription2.accepted = self.transcription1.accepted + timedelta(
+            seconds=29
+        )
+        self.transcription2.reviewed_by = self.transcription1.reviewed_by
+        self.transcription2.save()
+        users = ConcordiaUser.objects.review_incidents()
+        self.assertNotIn(self.transcription1.user.id, users)
+
+        transcription3 = create_transcription(
+            asset=self.transcription1.asset,
+            user=self.transcription1.user,
+            reviewed_by=self.transcription1.reviewed_by,
+            accepted=self.transcription1.accepted + timedelta(seconds=58),
+        )
+        transcription4 = create_transcription(
+            asset=self.transcription1.asset,
+            user=self.transcription1.user,
+            reviewed_by=self.transcription1.reviewed_by,
+            accepted=transcription3.accepted + timedelta(minutes=1, seconds=1),
+        )
+        users = ConcordiaUser.objects.review_incidents()
+        self.assertEqual(len(users), 1)
+        self.assertEqual(users[self.transcription1.reviewed_by.id], 1)
+
+        create_transcription(
+            asset=self.transcription1.asset,
+            user=self.transcription1.user,
+            reviewed_by=self.transcription1.reviewed_by,
+            accepted=transcription4.accepted + timedelta(seconds=29),
+        )
+        create_transcription(
+            asset=self.transcription1.asset,
+            user=self.transcription1.user,
+            reviewed_by=self.transcription1.reviewed_by,
+            accepted=transcription4.accepted + timedelta(seconds=58),
+        )
+        users = ConcordiaUser.objects.review_incidents()
+        self.assertEqual(len(users), 1)
+        self.assertEqual(users[self.transcription1.reviewed_by.id], 2)
+
+    def test_transcribe_incidents(self):
+        self.transcription1.submitted = timezone.now()
+        self.transcription1.save()
+        self.transcription2.submitted = self.transcription1.submitted + timedelta(
+            seconds=29
+        )
+        self.transcription2.user = self.transcription1.user
+        self.transcription2.save()
+        users = ConcordiaUser.objects.transcribe_incidents()
+        self.assertEqual(len(users), 0)
+        self.assertNotIn(self.transcription1.user.id, users)
+
+        transcription3 = create_transcription(
+            asset=self.transcription1.asset,
+            user=self.transcription1.user,
+            submitted=self.transcription1.submitted + timedelta(seconds=58),
+        )
+        transcription4 = create_transcription(
+            asset=self.transcription1.asset,
+            user=self.transcription1.user,
+            submitted=transcription3.submitted + timedelta(minutes=1, seconds=1),
+        )
+        create_transcription(
+            asset=self.transcription1.asset,
+            user=self.transcription1.user,
+            submitted=transcription4.submitted + timedelta(seconds=59),
+        )
+        users = ConcordiaUser.objects.transcribe_incidents()
+        self.assertEqual(len(users), 1)
+        self.assertEqual(users[self.transcription1.user.id], 1)
+
+        create_transcription(
+            asset=self.transcription1.asset,
+            user=self.transcription1.user,
+            submitted=self.transcription1.submitted + timedelta(minutes=1, seconds=59),
+        )
+        users = ConcordiaUser.objects.transcribe_incidents()
+        self.assertEqual(len(users), 1)
+        self.assertEqual(users[self.transcription1.user.id], 2)
+
+
 class AssetTestCase(CreateTestUsers, TestCase):
     def setUp(self):
         self.asset = create_asset()
@@ -176,44 +272,6 @@ def test_status(self):
             TranscriptionStatus.CHOICE_MAP[TranscriptionStatus.COMPLETED],
         )
 
-    def test_reviewing_too_quickly(self):
-        self.transcription1.accepted = timezone.now()
-        self.transcription1.reviewed_by = self.create_user(username="tester2")
-        self.transcription1.save()
-        self.transcription2.accepted = self.transcription1.accepted
-        self.transcription2.reviewed_by = self.transcription1.reviewed_by
-        self.transcription2.save()
-        transcriptions = Transcription.objects.reviewing_too_quickly()
-        self.assertEqual(len(transcriptions), 0)
-
-        transcription3 = create_transcription(
-            asset=self.transcription1.asset,
-            user=self.transcription1.user,
-            reviewed_by=self.transcription1.reviewed_by,
-            accepted=self.transcription1.accepted,
-        )
-        transcriptions = Transcription.objects.reviewing_too_quickly()
-        self.assertEqual(len(transcriptions), 1)
-        self.assertEqual(transcriptions[0][0], transcription3.reviewed_by.id)
-
-    def test_transcribing_too_quickly(self):
-        self.transcription1.submitted = timezone.now()
-        self.transcription1.save()
-        self.transcription2.submitted = self.transcription1.submitted
-        self.transcription2.user = self.transcription1.user
-        self.transcription2.save()
-        transcriptions = Transcription.objects.transcribing_too_quickly()
-        self.assertEqual(len(transcriptions), 0)
-
-        transcription3 = create_transcription(
-            asset=self.transcription1.asset,
-            user=self.transcription1.user,
-            submitted=self.transcription1.submitted,
-        )
-        transcriptions = Transcription.objects.transcribing_too_quickly()
-        self.assertEqual(len(transcriptions), 1)
-        self.assertEqual(transcriptions[0][0], transcription3.user.id)
-
 
 class AssetTranscriptionReservationTest(CreateTestUsers, TestCase):
     def setUp(self):