feat: Add a publisher model, use for traceability page initially

This speeds up load times from seconds to ~150ms.

dashboard/templates/traceability.html

@@ -23,49 +23,48 @@
               <th>Percentage Traceable Spend</th>
             </tr></thead>
             <tbody>
-              {% for publisher_title,publisher in publishers_ordered_by_title %}
-              {% set publisher_stats = func.get_publisher_stats(publisher) %}
+              {% for publisher in publishers %}
                   <tr>
-                      <td><a href="{% if publisher %}{{ url("dash-headlines-publisher-detail", args=[publisher]) }}{% endif %}">{{ publisher_title }}</a></td>
+                      <td><a href="{% if publisher %}{{ url("dash-headlines-publisher-detail", args=[publisher.slug]) }}{% endif %}">{{ publisher.title }}</a></td>
                       <td>
-                        {%- if publisher_stats.traceable_activities_by_publisher_id -%}
-                          {{ '{:,}'.format(publisher_stats.traceable_activities_by_publisher_id) }}
+                        {%- if publisher.traceable_activities_by_publisher_id -%}
+                          {{ '{:,}'.format(publisher.traceable_activities_by_publisher_id) }}
                         {%- else -%}
                           0
                         {%- endif -%}
                       </td>
                       <td>
-                        {%- if publisher_stats.traceable_activities_by_publisher_id_denominator -%}
-                          {{ '{:,}'.format(publisher_stats.traceable_activities_by_publisher_id_denominator) }}
+                        {%- if publisher.traceable_activities_by_publisher_id_denominator -%}
+                          {{ '{:,}'.format(publisher.traceable_activities_by_publisher_id_denominator) }}
                         {%- else -%}
                           0
                         {%- endif -%}
                       </td>
                       <td>
-                        {%- if publisher_stats.traceable_activities_by_publisher_id and publisher_stats.traceable_activities_by_publisher_id_denominator -%}
-                          {{ (publisher_stats.traceable_activities_by_publisher_id / publisher_stats.traceable_activities_by_publisher_id_denominator * 100) | round_nicely }}
-                        {%- elif publisher_stats.traceable_activities_by_publisher_id_denominator -%}
+                        {%- if publisher.traceable_activities_by_publisher_id and publisher.traceable_activities_by_publisher_id_denominator -%}
+                          {{ (publisher.traceable_activities_by_publisher_id / publisher.traceable_activities_by_publisher_id_denominator * 100) | round_nicely }}
+                        {%- elif publisher.traceable_activities_by_publisher_id_denominator -%}
                           0
                         {%- endif -%}
                       </td>
                       <td>
-                        {%- if publisher_stats.traceable_sum_commitments_and_disbursements_by_publisher_id -%}
-                          {{ '{:,.2f}'.format(publisher_stats.traceable_sum_commitments_and_disbursements_by_publisher_id) }}
+                        {%- if publisher.traceable_sum_commitments_and_disbursements_by_publisher_id -%}
+                          {{ '{:,.2f}'.format(publisher.traceable_sum_commitments_and_disbursements_by_publisher_id) }}
                         {%- else -%}
                           0.00
                         {%- endif -%}
                       </td>
                       <td>
-                        {%- if publisher_stats.traceable_sum_commitments_and_disbursements_by_publisher_id_denominator -%}
-                          {{ '{:,.2f}'.format(publisher_stats.traceable_sum_commitments_and_disbursements_by_publisher_id_denominator) }}
+                        {%- if publisher.traceable_sum_commitments_and_disbursements_by_publisher_id_denominator -%}
+                          {{ '{:,.2f}'.format(publisher.traceable_sum_commitments_and_disbursements_by_publisher_id_denominator) }}
                         {%- else -%}
                           0.00
                         {%- endif -%}
                       </td>
                       <td>
-                        {%- if publisher_stats.traceable_sum_commitments_and_disbursements_by_publisher_id and publisher_stats.traceable_sum_commitments_and_disbursements_by_publisher_id_denominator -%}
-                          {{ (publisher_stats.traceable_sum_commitments_and_disbursements_by_publisher_id / publisher_stats.traceable_sum_commitments_and_disbursements_by_publisher_id_denominator * 100) | round_nicely }}
-                        {%- elif publisher_stats.traceable_sum_commitments_and_disbursements_by_publisher_id_denominator -%}
+                        {%- if publisher.traceable_sum_commitments_and_disbursements_by_publisher_id and publisher.traceable_sum_commitments_and_disbursements_by_publisher_id_denominator -%}
+                          {{ (publisher.traceable_sum_commitments_and_disbursements_by_publisher_id / publisher.traceable_sum_commitments_and_disbursements_by_publisher_id_denominator * 100) | round_nicely }}
+                        {%- elif publisher.traceable_sum_commitments_and_disbursements_by_publisher_id_denominator -%}
                           0
                         {%- endif -%}
                       </td>

dashboard/ui/management/commands/dashboard_import.py

@@ -0,0 +1,15 @@
+from django.core.management.base import BaseCommand
+from django.db import transaction
+
+from data import get_publisher_stats, publishers_ordered_by_title
+from ui.models import Publisher
+
+
+class Command(BaseCommand):
+    @transaction.atomic
+    def handle(self, *args, **options):
+        Publisher.objects.all().delete()
+        for publisher_title, publisher_slug in publishers_ordered_by_title:
+            stats_json = dict(get_publisher_stats(publisher_slug))
+            publisher = Publisher(title=publisher_title, slug=publisher_slug, stats_json=stats_json)
+            publisher.save()

dashboard/ui/migrations/0001_initial.py

@@ -0,0 +1,62 @@
+# Generated by Django 5.1.4 on 2024-12-20 13:31
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    initial = True
+
+    dependencies = []
+
+    operations = [
+        migrations.CreateModel(
+            name="Publisher",
+            fields=[
+                ("id", models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name="ID")),
+                ("slug", models.CharField()),
+                ("title", models.CharField()),
+                ("stats_json", models.JSONField()),
+                (
+                    "traceable_sum_commitments_and_disbursements_by_publisher_id_den",
+                    models.GeneratedField(
+                        db_persist=True,
+                        expression=models.F(
+                            "stats_json__traceable_sum_commitments_and_disbursements_by_publisher_id_denominator"
+                        ),
+                        output_field=models.JSONField(),
+                    ),
+                ),
+                (
+                    "activities",
+                    models.GeneratedField(
+                        db_persist=True, expression=models.F("stats_json__activities"), output_field=models.JSONField()
+                    ),
+                ),
+                (
+                    "traceable_activities_by_publisher_id",
+                    models.GeneratedField(
+                        db_persist=True,
+                        expression=models.F("stats_json__traceable_activities_by_publisher_id"),
+                        output_field=models.JSONField(),
+                    ),
+                ),
+                (
+                    "traceable_activities_by_publisher_id_denominator",
+                    models.GeneratedField(
+                        db_persist=True,
+                        expression=models.F("stats_json__traceable_activities_by_publisher_id_denominator"),
+                        output_field=models.JSONField(),
+                    ),
+                ),
+                (
+                    "traceable_sum_commitments_and_disbursements_by_publisher_id",
+                    models.GeneratedField(
+                        db_persist=True,
+                        expression=models.F("stats_json__traceable_sum_commitments_and_disbursements_by_publisher_id"),
+                        output_field=models.JSONField(),
+                    ),
+                ),
+            ],
+        ),
+    ]

dashboard/ui/models.py

@@ -0,0 +1,33 @@
+import hashlib
+
+from django.db import models
+
+
+class Publisher(models.Model):
+    slug = models.CharField()
+    title = models.CharField()
+    stats_json = models.JSONField()
+    # too long
+    traceable_sum_commitments_and_disbursements_by_publisher_id_den = models.GeneratedField(
+        expression=models.F("stats_json__traceable_sum_commitments_and_disbursements_by_publisher_id_denominator"),
+        output_field=models.JSONField(),
+        db_persist=True,
+    )
+
+    @property
+    def traceable_sum_commitments_and_disbursements_by_publisher_id_denominator(self):
+        return self.traceable_sum_commitments_and_disbursements_by_publisher_id_den
+
+
+for key in [
+    "activities",
+    "traceable_activities_by_publisher_id",
+    "traceable_activities_by_publisher_id_denominator",
+    "traceable_sum_commitments_and_disbursements_by_publisher_id",
+]:
+    Publisher.add_to_class(
+        key,
+        models.GeneratedField(
+            expression=models.F(f"stats_json__{key}"), output_field=models.JSONField(), db_persist=True
+        ),
+    )

dashboard/ui/settings.py

@@ -37,6 +37,7 @@
 # Application definition
 
 INSTALLED_APPS = [
+    "ui",
     "django.contrib.admin",
     "django.contrib.auth",
     "django.contrib.contenttypes",
@@ -93,12 +94,7 @@
 # Database
 # https://docs.djangoproject.com/en/5.1/ref/settings/#databases
 
-DATABASES = {
-    "default": {
-        "ENGINE": "django.db.backends.sqlite3",
-        "NAME": BASE_DIR / "db.sqlite3",
-    }
-}
+DATABASES = {"default": env.db()}
 
 
 # Password validation

dashboard/ui/views.py

@@ -16,6 +16,7 @@
 import summary_stats
 import text
 import timeliness
+import ui.models
 import ui.template_funcs
 import vars
 from data import (
@@ -161,6 +162,7 @@ def _make_context(page_name: str, include_large_dicts: bool = True):
         },
         publisher_name=publisher_name,
         publishers_ordered_by_title=publishers_ordered_by_title,
+        publishers=ui.models.Publisher.objects.all().defer("stats_json"),
         github_issues=github_issues,
         MAJOR_VERSIONS=MAJOR_VERSIONS,
         expected_versions=vars.expected_versions,
@@ -493,7 +495,7 @@ def exploringdata_dates(request):
 
 def exploringdata_traceability(request):
     template = loader.get_template("traceability.html")
-    return HttpResponse(template.render(_make_context("traceability"), request))
+    return HttpResponse(template.render(_make_context("traceability", include_large_dicts=False), request))
 
 
 #

requirements.in

@@ -1,5 +1,6 @@
 django
 django-environ
+psycopg
 gunicorn
 flask
 frozen-flask

requirements.txt

@@ -65,6 +65,8 @@ packaging==24.1
     #   matplotlib
 pillow==11.0.0
     # via matplotlib
+psycopg==3.2.3
+    # via -r requirements.in
 pyparsing==3.2.0
     # via matplotlib
 python-dateutil==2.9.0.post0
@@ -81,6 +83,8 @@ sqlparse==0.5.1
     # via django
 tqdm==4.67.0
     # via -r requirements.in
+typing-extensions==4.12.2
+    # via psycopg
 urllib3==2.2.3
     # via requests
 werkzeug==3.1.2

requirements_dev.txt

@@ -124,6 +124,8 @@ platformdirs==4.3.6
     # via black
 pluggy==1.5.0
     # via pytest
+psycopg==3.2.3
+    # via -r requirements.txt
 pycodestyle==2.12.1
     # via flake8
 pyflakes==3.2.0
@@ -158,6 +160,10 @@ sqlparse==0.5.1
     #   django
 tqdm==4.67.0
     # via -r requirements.txt
+typing-extensions==4.12.2
+    # via
+    #   -r requirements.txt
+    #   psycopg
 urllib3==2.2.3
     # via
     #   -r requirements.txt