Reworked daily data backfill process to update one report at a time a…

…nd split out into separate tasks to reduce the database load when backfilling. (#2252)

concordia/admin/__init__.py

@@ -833,7 +833,43 @@ class SimplePageAdmin(admin.ModelAdmin):
 
 @admin.register(SiteReport)
 class SiteReportAdmin(admin.ModelAdmin):
- list_display = ("created_on", "report_name", "campaign", "topic")
+ list_display = ("created_on", "report_type")
+ readonly_fields = ("created_on", "report_type")
+ fieldsets = (
+ ("Summary", {"fields": ("created_on", "report_type")}),
+ (
+ "Data",
+ {
+ "fields": (
+ "report_name",
+ "campaign",
+ "topic",
+ "assets_total",
+ "assets_published",
+ "assets_not_started",
+ "assets_in_progress",
+ "assets_waiting_review",
+ "assets_completed",
+ "assets_unpublished",
+ "items_published",
+ "items_unpublished",
+ "projects_published",
+ "projects_unpublished",
+ "anonymous_transcriptions",
+ "transcriptions_saved",
+ "daily_review_actions",
+ "distinct_tags",
+ "tag_uses",
+ "campaigns_published",
+ "campaigns_unpublished",
+ "users_registered",
+ "users_activated",
+ "registered_contributors",
+ "daily_active_users",
+ )
+ },
+ ),
+ )
 
  list_filter = (
  "report_name",
@@ -842,6 +878,15 @@ class SiteReportAdmin(admin.ModelAdmin):
  "topic",
  )
 
+ @admin.display(description="Report type")
+ def report_type(self, obj):
+ if obj.report_name:
+ return f"Report name: {obj.report_name}"
+ elif obj.campaign:
+ return f"Campaign: {obj.campaign}"
+ elif obj.topic:
+ return f"Topic: {obj.topic}"
+
  def export_to_csv(self, request, queryset):
  return export_to_csv_action(
  self, request, queryset, field_names=SiteReport.DEFAULT_EXPORT_FIELDNAMES

concordia/models.py

@@ -803,7 +803,7 @@ class ReportName(models.TextChoices):
  TOTAL = "Active and completed campaigns", "Active and completed campaigns"
  RETIRED_TOTAL = "Retired campaigns", "Retired campaigns"
 
- created_on = models.DateTimeField(editable=False, auto_now_add=True)
+ created_on = models.DateTimeField(auto_now_add=True)
  report_name = models.CharField(
  max_length=80, blank=True, default="", choices=ReportName.choices
  )

concordia/tasks.py

@@ -10,7 +10,7 @@
 from django.contrib.auth.models import User
 from django.core.management import call_command
 from django.db import transaction
-from django.db.models import Count, F, OuterRef, Q, Subquery
+from django.db.models import Count, F, Q
 from django.utils import timezone
 from more_itertools.more import chunked
 
@@ -37,6 +37,8 @@
 
 logger = getLogger(__name__)
 
+ONE_DAY = datetime.timedelta(days=1)
+
 
 @celery_app.task
 def expire_inactive_asset_reservations():
@@ -427,51 +429,145 @@ def retired_total_report():
  total_site_report.save()
 
 
-ONE_DAY = datetime.timedelta(days=1)
+def site_reports_for_date(date):
+ start = date - ONE_DAY
+ return SiteReport.objects.filter(created_on__gte=start, created_on__lte=date)
 
 
-@celery_app.task
-def backfill_by_date(date, days):
- logger.info("Backfilling daily data for %s ", date)
+def assets_for_date(date):
  start = date - ONE_DAY
-
  q_accepted = Q(
  transcription__accepted__gte=start, transcription__accepted__lte=date
  )
  q_rejected = Q(
  transcription__rejected__gte=start, transcription__rejected__lte=date
  )
- assets = Asset.objects.filter(q_accepted | q_rejected)
- site_reports = SiteReport.objects.filter(
- created_on__gte=start, created_on__lte=date
+ return Asset.objects.filter(q_accepted | q_rejected)
+
+
+@celery_app.task(ignore_result=True)
+def backfill_total(date, days):
+ logger.info(
+ "STARTING: Backfilling daily data for %s on %s",
+ SiteReport.ReportName.TOTAL,
+ date,
  )
- topic_assets = assets.filter(item__project__topics=OuterRef("topic__pk"))
- subquery = Subquery(
- topic_assets.annotate(cnt=Count("transcription")).values("cnt")[:1]
+ site_report = site_reports_for_date(date).filter(
+ report_name=SiteReport.ReportName.TOTAL
+ )[0]
+ logger.info(
+ "STARTING: Backfilling daily data for report %s (%s)", site_report.id, date
  )
- site_reports.filter(topic__isnull=False).update(daily_review_actions=subquery)
- campaign_assets = assets.filter(item__project__campaign=OuterRef("campaign__pk"))
- subquery = Subquery(
- campaign_assets.annotate(cnt=Count("transcription")).values("cnt")[:1]
+ daily_review_actions = assets_for_date(date).count()
+ logger.debug(
+ "%s daily review actions for report %s (%s)",
+ daily_review_actions,
+ site_report.id,
+ date,
  )
- site_reports.filter(campaign__isnull=False).update(daily_review_actions=subquery)
- site_reports.filter(topic__isnull=True, campaign__isnull=True).update(
- daily_review_actions=Subquery(
- assets.annotate(cnt=Count("transcription")).values("cnt")[:1]
- )
+ site_report.daily_review_actions = daily_review_actions
+ site_report.save()
+ logger.info(
+ "FINISHED: Backfilling daily data for %s on %s",
+ SiteReport.ReportName.TOTAL,
+ date,
  )
+ logger.info("FINISHED: Backfilling daily data for all reports on %s", date)
 
- if days >= 0:
- backfill_by_date.delay(start, days - 1)
+ if days > 0:
+ return backfill_topics.delay(date - ONE_DAY, days - 1)
  else:
- logger.info("Backfilling daily data finished")
+ logger.info("Backfilling daily data complete")
 
 
-@celery_app.task
+@celery_app.task(ignore_result=True)
+def backfill_next_campaign_report(date, days, site_report_ids):
+ try:
+ site_report_id = site_report_ids.pop()
+ except IndexError:
+ logger.info("FINISHED: Backfilling daily data for campaigns on %s", date)
+ backfill_total.delay(date, days)
+ return
+ site_report = SiteReport.objects.get(id=site_report_id)
+ logger.info(
+ "STARTING: Backfilling daily data for report %s (%s)", site_report.id, date
+ )
+ daily_review_actions = (
+ assets_for_date(date)
+ .filter(item__project__campaign=site_report.campaign)
+ .count()
+ )
+ logger.debug(
+ "%s daily review actions for report %s (%s)",
+ daily_review_actions,
+ site_report.id,
+ date,
+ )
+ site_report.daily_review_actions = daily_review_actions
+ site_report.save()
+ logger.info(
+ "FINISHED: Backfilling daily data for report %s (%s)", site_report.id, date
+ )
+ return backfill_next_campaign_report.delay(date, days, site_report_ids)
+
+
+@celery_app.task(ignore_result=True)
+def backfill_campaigns(date, days):
+ site_report_ids = list(
+ site_reports_for_date(date)
+ .filter(campaign__isnull=False)
+ .values_list("id", flat=True)
+ )
+ logger.info("STARTING: Backfilling daily data for campaigns on %s", date)
+ return backfill_next_campaign_report.delay(date, days, site_report_ids)
+
+
+@celery_app.task(ignore_result=True)
+def backfill_next_topic_report(date, days, site_report_ids):
+ try:
+ site_report_id = site_report_ids.pop()
+ except IndexError:
+ logger.info("FINISHED: Backfilling daily data for topics on %s", date)
+ backfill_campaigns.delay(date, days)
+ return
+ site_report = SiteReport.objects.get(id=site_report_id)
+ logger.info(
+ "STARTING: Backfilling daily data for report %s (%s)", site_report.id, date
+ )
+ daily_review_actions = (
+ assets_for_date(date).filter(item__project__topics=site_report.topic).count()
+ )
+ logger.debug(
+ "%s daily review actions for report %s (%s)",
+ daily_review_actions,
+ site_report.id,
+ date,
+ )
+ site_report.daily_review_actions = daily_review_actions
+ site_report.save()
+ logger.info(
+ "FINISHED: Backfilling daily data for report %s (%s)", site_report.id, date
+ )
+ return backfill_next_topic_report.delay(date, days, site_report_ids)
+
+
+@celery_app.task(ignore_result=True)
+def backfill_topics(date, days):
+ site_report_ids = list(
+ site_reports_for_date(date)
+ .filter(topic__isnull=False)
+ .values_list("id", flat=True)
+ )
+ logger.info("STARTING: Backfilling daily data for topics on %s", date)
+ return backfill_next_topic_report.delay(date, days, site_report_ids)
+
+
+@celery_app.task(ignore_result=True)
 def backfill_daily_data(start, days):
  date = timezone.make_aware(datetime.datetime(**start))
  logger.info("Backfilling daily data for the %s days before %s", days, date)
- backfill_by_date.delay(date - ONE_DAY, days - 1)
+ logger.info("STARTED: Backfilling daily data for all reports on %s", date)
+ return backfill_topics.delay(date, days - 1)
 
 
 @celery_app.task