Modified daily data backfill functions to split them using tasks and …

…allow date data to be passed in from the admin (#2246) (#2247)

* Modified daily data backfill functions to split them using tasks and allow date data to be psssed in from the admin

* Updated black version in pre-commit config

* Updated black version to lint files

* Linted one other file

Co-authored-by: Josh Stegmaier <104993387+joshuastegmaier@users.noreply.github.com>

.pre-commit-config.yaml

@@ -15,7 +15,7 @@ repos:
       hooks:
           - id: djhtml
     - repo: https://github.com/ambv/black
-      rev: 23.10.0
+      rev: 24.1.0
       hooks:
           - id: black
     - repo: https://github.com/pre-commit/pre-commit-hooks

Pipfile

@@ -44,7 +44,7 @@ django-admin-multiple-choice-list-filter = "*"
 django-npm = "*"
 fpdf2 = "==2.5.5"
 pymemcache = "*"
-black = "*"
+black = ">=24"
 weasyprint = "*"
 tesseract = "==0.1.3"
 pytesseract = "*"

concordia/api_views.py

@@ -13,6 +13,7 @@
 The base APIViewMixin implements a base implementation of serialize_object which
 uses the generic django.forms.models.model_to_dict and can be overridden as needed.
 """
+
 from time import time
 
 from django.core.serializers.json import DjangoJSONEncoder

concordia/tasks.py

@@ -430,12 +430,21 @@ def retired_total_report():
 ONE_DAY = datetime.timedelta(days=1)
 
 
-def _backfill_by_date(day):
-    start = day - ONE_DAY
-    q_accepted = Q(transcription__accepted__gte=start, transcription__accepted__lte=day)
-    q_rejected = Q(transcription__rejected__gte=start, transcription__rejected__lte=day)
+@celery_app.task
+def backfill_by_date(date, days):
+    logger.debug("Backfilling daily data for %s ", date)
+    start = date - ONE_DAY
+
+    q_accepted = Q(
+        transcription__accepted__gte=start, transcription__accepted__lte=date
+    )
+    q_rejected = Q(
+        transcription__rejected__gte=start, transcription__rejected__lte=date
+    )
     assets = Asset.objects.filter(q_accepted | q_rejected)
-    site_reports = SiteReport.objects.filter(created_on__gte=start, created_on__lte=day)
+    site_reports = SiteReport.objects.filter(
+        created_on__gte=start, created_on__lte=date
+    )
     topic_assets = assets.filter(item__project__topics=OuterRef("topic__pk"))
     subquery = Subquery(
         topic_assets.annotate(cnt=Count("transcription")).values("cnt")[:1]
@@ -452,18 +461,15 @@ def _backfill_by_date(day):
         )
     )
 
-
-def _backfill_data(start, days=0):
-    for n in range(days - 1):
-        day = start - datetime.timedelta(days=n)
-        _backfill_by_date(day)
+    if days >= 0:
+        backfill_by_date.delay(start, days - 1)
 
 
 @celery_app.task
-def backfill_daily_data():
-    _backfill_data(
-        timezone.make_aware(datetime.datetime(year=2023, month=9, day=17)), 10
-    )
+def backfill_daily_data(start, days):
+    date = timezone.make_aware(datetime.datetime(**start))
+    logger.debug("Backfilling daily data for the %s days before %s", days, date)
+    backfill_by_date.delay(date - ONE_DAY, days - 1)
 
 
 @celery_app.task

concordia/tests/test_account_views.py

@@ -1,6 +1,7 @@
 """
 Tests for user account-related views
 """
+
 from django.core import mail
 from django.test import TestCase, override_settings
 from django.urls import reverse

concordia/tests/test_registration_views.py

@@ -1,6 +1,7 @@
 """
 Tests for user registration-related views
 """
+
 from logging import getLogger
 
 from django.contrib.auth import get_user_model

importer/models.py

@@ -1,6 +1,7 @@
 """
 See the module-level docstring for implementation details
 """
+
 from django.core.validators import MinValueValidator
 from django.db import models