refactor: Created views to render the top level pages.

This commit implements a URL structure for the Dashboard and includes
views for the top level navigation of the Dashboard as well as the
main pages in the Headlines section.  This required changes in the
templates to remove Flask-specific template functions and minor
syntax changes that prevented rendering.  The views themselves will
eventually benefit from some optimisation.  Note, this commit is a
breaking change for make_html.py and now the repo will not generate
a Flask website.

src/dashboard/jinja2.py

@@ -0,0 +1,17 @@
+"""Jinja2 template configuration
+"""
+from django.templatetags.static import static
+from django.urls import reverse
+
+from jinja2 import Environment
+
+
+def environment(**options):
+    env = Environment(**options)
+    env.globals.update(
+        {
+            "static": static,
+            "url": reverse,
+        }
+    )
+    return env

src/dashboard/settings.py

@@ -65,6 +65,20 @@
             ],
         },
     },
+    {
+        'BACKEND': 'django.template.backends.jinja2.Jinja2',
+        'DIRS': ["templates/"],
+        'APP_DIRS': True,
+        'OPTIONS': {
+            'context_processors': [
+                'django.template.context_processors.debug',
+                'django.template.context_processors.request',
+                'django.contrib.auth.context_processors.auth',
+                'django.contrib.messages.context_processors.messages',
+            ],
+            'environment': 'dashboard.jinja2.environment'
+        },
+    },
 ]
 
 WSGI_APPLICATION = 'dashboard.wsgi.application'
@@ -116,6 +130,7 @@
 # https://docs.djangoproject.com/en/5.1/howto/static-files/
 
 STATIC_URL = 'static/'
+STATICFILES_DIRS = [BASE_DIR / 'static',]
 
 # Default primary key field type
 # https://docs.djangoproject.com/en/5.1/ref/settings/#default-auto-field

src/dashboard/urls.py

@@ -16,7 +16,63 @@
 """
 from django.contrib import admin
 from django.urls import path
+# from django.shortcuts import redirect
+
+import dashboard.views
+
 
 urlpatterns = [
-    path('admin/', admin.site.urls)
+    path('admin/', admin.site.urls),
+
+    # Top level dashboard pages.
+    path('', dashboard.views.index, name="dash-index"),
+    path('headlines', dashboard.views.headlines, name="dash-headlines"),
+    path('data-quality', dashboard.views.data_quality, name="dash-dataquality"),
+    path('publishing-statistics', dashboard.views.publishing_stats, name="dash-publishingstats"),
+    path('exploring-data', dashboard.views.exploring_data, name="dash-exploringdata"),
+    path('faq', dashboard.views.faq, name="dash-faq"),
+
+    # Headlines pages and detail pages - placeholders for now.
+    path('headlines/publishers', dashboard.views.headlines_publishers, name="dash-headlines-publishers"),
+    path('headlines/files', dashboard.views.headlines_files, name="dash-headlines-files"),
+    path('headlines/activities', dashboard.views.headlines_activities, name="dash-headlines-activities"),
+    path('headlines/publishers/<slug:publisher>', dashboard.views.headlines_publisher_detail, name='dash-headlines-publisher-detail'),
+
+    # Data quality pages.
+    path('data-quality/download-errors', lambda x: None, name="dash-dataquality-download"),
+    path('data-quality/xml-errors', lambda x: None, name="dash-dataquality-xml"),
+    path('data-quality/validation', lambda x: None, name="dash-dataquality-validation"),
+    path('data-quality/versions', lambda x: None, name="dash-dataquality-versions"),
+    path('data-quality/organisation-xml', lambda x: None, name="dash-dataquality-organisation"),
+    path('data-quality/licenses', lambda x: None, name="dash-dataquality-licenses"),
+    path('data-quality/identifiers', lambda x: None, name="dash-dataquality-identifiers"),
+    path('data-quality/reporting-orgs', lambda x: None, name="dash-dataquality-reportingorgs"),
+
+    # Exploring data pages.
+    path('exploring-data/elements', lambda x: None, name="dash-exploringdata-elements"),
+    path('exploring-data/codelists', lambda x: None, name="dash-exploringdata-codelists"),
+    path('exploring-data/booleans', lambda x: None, name="dash-exploringdata-booleans"),
+    path('exploring-data/dates', lambda x: None, name="dash-exploringdata-dates"),
+    path('exploring-data/traceability', lambda x: None, name="dash-exploringdata-traceability"),
+    path('exploring-data/organisation-identifiers', lambda x: None, name="dash-exploringdata-orgids"),
+
+    # Publishing statistics pages.
+    path('publishing-statistics/timeliness', lambda x: None, name="dash-publishingstats-timeliness"),
+    path('publishing-statistics/forward-looking', lambda x: None, name="dash-publishingstats-forwardlooking"),
+    path('publishing-statistics/comprehensiveness', lambda x: None, name="dash-publishingstats-comprehensiveness"),
+    path('publishing-statistics/coverage', lambda x: None, name="dash-publishingstats-coverage"),
+    path('publishing-statistics/summary-statistics', lambda x: None, name="dash-publishingstats-summarystats"),
+    path('publishing-statistics/humanitarian-reporting', lambda x: None, name="dash-publishingstats-humanitarian"),
+
+    # Licenses
+    path('licenses/<slug:licence_id>', lambda x: None, name="dash-licence-detail"),
+    path('licenses', lambda x: None, name="dash-licences")
+
+    # Redirects to support any users with bookmarks to pages on the old Dashboard.
+    # path('timeliness.html', redirect("dash-publishingstats-timeliness")),
+    # path('index.html', redirect("dash-index")),
+    # path('summary_stats.html', redirect("dash-publishingstats-summarystats")),
+    # path('exploring_data.html', redirect("dash-exploringdata"))
+
 ]
+# Unsure where "rulesets" and "registration_agencies" should belong - can't find the route to these in make_html.py

src/dashboard/views.py

@@ -0,0 +1,174 @@
+"""Views for the IATI Dashboard"""
+
+# Note: in the page views I am unsure where "rulesets" and "registration_agencies" should
+# belong - they exist in text.page_tiles but I can't find the route to these in make_html.py
+# so not sure where they should fit.  I've not included them in the page_view_names so hopefully
+# an exception will be raised if they are referenced somewhere.
+
+import dateutil.parser
+import subprocess
+
+from django.http import HttpResponse
+from django.template import loader
+
+import config
+import text
+import dashboard.template_funcs
+
+from data import (
+    ckan,
+    ckan_publishers,
+    codelist_mapping,
+    codelist_sets,
+    codelist_lookup,
+    current_stats,
+    dataset_to_publisher_dict,
+    github_issues,
+    get_publisher_stats,
+    MAJOR_VERSIONS,
+    metadata,
+    publisher_name,
+    publishers_ordered_by_title,
+    is_valid_element,
+    slugs)
+
+
+COMMIT_HASH = subprocess.run('git show --format=%H --no-patch'.split(),
+                             cwd=config.join_base_path(""),
+                             capture_output=True).stdout.decode().strip()
+STATS_COMMIT_HASH = subprocess.run('git -C stats-calculated show --format=%H --no-patch'.split(),
+                                   cwd=config.join_base_path(""),
+                                   capture_output=True).stdout.decode().strip()
+STATS_GH_URL = 'https://github.com/codeforIATI/IATI-Stats-public/tree/' + STATS_COMMIT_HASH
+
+
+def _make_context(page_name: str):
+    """Make a basic context dictionary for a given page
+    """
+    context = dict(
+        page=page_name,
+        top_titles=text.top_titles,
+        page_titles=text.page_titles,
+        short_page_titles=text.short_page_titles,
+        page_leads=text.page_leads,
+        page_sub_leads=text.page_sub_leads,
+        top_navigation=text.top_navigation,
+        navigation=text.navigation,
+        navigation_reverse={page: k for k, pages in text.navigation.items() for page in pages},
+        page_view_names={"index": "dash-index",
+                         "headlines": "dash-headlines",
+                         "data_quality": "dash-dataquality",
+                         "publishing_stats": "dash-publishingstats",
+                         "exploring_data": "dash-exploringdata",
+                         "faq": "dash-faq",
+
+                         "publishers": "dash-headlines-publishers",
+                         "files": "dash-headlines-files",
+                         "activities": "dash-headlines-activities",
+                         "publisher": "dash-headlines-publisher-detail",
+
+                         "download": "dash-dataquality-download",
+                         "xml": "dash-dataquality-xml",
+                         "validation": "dash-dataquality-validation",
+                         "versions": "dash-dataquality-versions",
+                         "organisation": "dash-dataquality-organisation",
+                         "licenses": "dash-dataquality-licenses",
+                         "identifiers": "dash-dataquality-identifiers",
+                         "reporting_orgs": "dash-dataquality-reportingorgs",
+
+                         "elements": "dash-exploringdata-elements",
+                         "codelists": "dash-exploringdata-codelists",
+                         "booleans": "dash-exploringdata-booleans",
+                         "dates": "dash-exploringdata-dates",
+                         "traceability": "dash-exploringdata-traceability",
+                         "org_ids": "dash-exploringdata-orgids",
+
+                         "timeliness": "dash-publishingstats-timeliness",
+                         "forwardlooking": "dash-publishingstats-forwardlooking",
+                         "comprehensiveness": "dash-publishingstats-comprehensiveness",
+                         "coverage": "dash-publishingstats-coverage",
+                         "summary_stats": "dash-publishingstats-summarystats",
+                         "humanitarian": "dash-publishingstats-humanitarian"
+                         },
+        current_stats=current_stats,
+        publisher_name=publisher_name,
+        publishers_ordered_by_title=publishers_ordered_by_title,
+        ckan_publishers=ckan_publishers,
+        ckan=ckan,
+        codelist_lookup=codelist_lookup,
+        codelist_mapping=codelist_mapping,
+        codelist_sets=codelist_sets,
+        github_issues=github_issues,
+        MAJOR_VERSIONS=MAJOR_VERSIONS,
+        metadata=metadata,
+        slugs=slugs,
+        datetime_data=dateutil.parser.parse(metadata['created_at']).strftime('%-d %B %Y (at %H:%M %Z)'),
+        stats_url='https://stats.codeforiati.org',
+        stats_gh_url=STATS_GH_URL,
+        commit_hash=COMMIT_HASH,
+        stats_commit_hash=STATS_COMMIT_HASH,
+        func={"sorted": sorted,
+              "firstint": dashboard.template_funcs.firstint,
+              "dataset_to_publisher": lambda x: dataset_to_publisher_dict.get(x, ""),
+              "get_publisher_stats": get_publisher_stats,
+              "is_valid_element": is_valid_element}
+    )
+    context["navigation_reverse"].update({k: k for k in text.navigation})
+
+    return context
+
+
+#
+# Top level navigation pages.
+#
+def index(request):
+    template = loader.get_template("index.html")
+    return HttpResponse(template.render(_make_context("index"), request))
+
+
+def headlines(request):
+    template = loader.get_template("headlines.html")
+    return HttpResponse(template.render(_make_context("headlines"), request))
+
+
+def data_quality(request):
+    template = loader.get_template("data_quality.html")
+    return HttpResponse(template.render(_make_context("data_quality"), request))
+
+
+def publishing_stats(request):
+    template = loader.get_template("publishing_stats.html")
+    return HttpResponse(template.render(_make_context("publishing_stats"), request))
+
+
+def exploring_data(request):
+    template = loader.get_template("exploring_data.html")
+    return HttpResponse(template.render(_make_context("exploring_data"), request))
+
+
+def faq(request):
+    template = loader.get_template("faq.html")
+    return HttpResponse(template.render(_make_context("faq"), request))
+
+
+#
+# Headline pages.
+#
+def headlines_publishers(request):
+    template = loader.get_template("publishers.html")
+    return HttpResponse(template.render(_make_context("publishers"), request))
+
+
+def headlines_activities(request):
+    template = loader.get_template("activities.html")
+    return HttpResponse(template.render(_make_context("activities"), request))
+
+
+def headlines_files(request):
+    template = loader.get_template("files.html")
+    return HttpResponse(template.render(_make_context("files"), request))
+
+
+def headlines_publisher_detail(request, publisher=None):
+    # Not implemented yet.
+    return None

src/templates/_partials/boxes.html

@@ -8,15 +8,19 @@ <h3 class="panel-title">
               <span class="title-text">{{ title }}</span>
             </h3>
             {% if json %}
-            <a href="{{ stats_url }}/gitaggregate{{ folderextra }}-dated/{{ json }}" style="float:right">(J)</a>
+              {% if folderextra %}
+                <a href="{{ stats_url }}/gitaggregate{{ folderextra }}-dated/{{ json }}" style="float:right">(J)</a>
+              {% else %}
+                <a href="{{ stats_url }}/gitaggregate-dated/{{ json }}" style="float:right">(J)</a>
+              {% endif %}
             {% endif %}
             <div class="clearfix"></div>
           </div>
           <div class="panel-body">
             <p>{{ description|safe }}</p>
-            <img src="{{ image }}" width="100%" />
+            <img src="{{ static(image) }}" width="100%" />
             {% if legend %}
-            <img src="{{ legend }}" width="100%" />
+            <img src="{{ static(legend) }}" width="100%" />
             {% endif %}
           </div>
         </div>

src/templates/_partials/tablesorter_instructions.html

@@ -1 +1 @@
-<p>Click the <img src="/img/tablesorter-icons.gif" style="width: 21px; height: 9px"/> icons to sort the table by a column. Selecting further columns whilst holding the shift key will enable secondary (tertiary etc) sorting by the desired column/s.</p>
+<p>Click the <img src="{{ static('img/tablesorter-icons.gif') }}" style="width: 21px; height: 9px"/> icons to sort the table by a column. Selecting further columns whilst holding the shift key will enable secondary (tertiary etc) sorting by the desired column/s.</p>

src/templates/activities.html

@@ -1,14 +1,14 @@
 {% extends 'base.html' %}
-{% import '_partials/boxes.html' as boxes %}
+{% import '_partials/boxes.html' as boxes with context %} 
 {% block content %}
     <div class="row">
-      {{ boxes.box('Total activities', current_stats.aggregated.activities, 'activities.png', 'activities.json',
+      {{ boxes.box('Total activities', current_stats.aggregated.activities, 'img/aggregate/activities.png', 'activities.json',
         description='Total count of activities across all publishers, over time.
         Note: this includes activities with duplicate <code>iati-identifier</code>') }}
-      {{ boxes.box('Unique Activities', current_stats.aggregated.unique_identifiers, 'unique_identifiers.png', 'unique_identifiers.json',
+      {{ boxes.box('Unique Activities', current_stats.aggregated.unique_identifiers, 'img/aggregate/unique_identifiers.png', 'unique_identifiers.json',
         description='Total count of unique activities across all publishers, over time
         Note: this excludes counts of duplicate <code>iati-identifier</code>') }}
-      {{ boxes.box('Activities by publisher type', '', 'activities_per_publisher_type.png', None, 'activities_per_publisher_type_legend.png',
+      {{ boxes.box('Activities by publisher type', '', 'img/aggregate/activities_per_publisher_type.png', None, 'img/aggregate/activities_per_publisher_type_legend.png',
         description='Count of all activities, aggregated by publisher type, over time.') }}
     </div>
 {% endblock %}

src/templates/base.html

@@ -5,8 +5,8 @@
   <meta http-equiv="X-UA-Compatible" content="IE=edge">
   <meta name="viewport" content="width=device-width, initial-scale=1.0">
   <link rel="stylesheet" type="text/css" href="//netdna.bootstrapcdn.com/bootstrap/3.4.1/css/bootstrap.min.css" />
-  <link rel="stylesheet" type="text/css" href="{{ url_for('static', filename='style.css') }}" />
-  <link rel="icon" href="{{ url_for('static', filename='img/favicon-32x32.png') }}" sizes="32x32" type="image/png">
+  <link rel="stylesheet" type="text/css" href="{{ static('style.css') }}" />
+  <link rel="icon" href="{{ static('img/favicon-32x32.png') }}" sizes="32x32" type="image/png">
   <title>IATI Dashboard – {% block title %}{{ page_titles[page] }}{% endblock %}</title>
 
   <script defer data-domain="dashboard.iatistandard.org" src="https://plausible.io/js/script.file-downloads.outbound-links.js"></script>
@@ -30,16 +30,16 @@
     <div class="container">
       <div class="collapse navbar-collapse">
         <ul class="nav navbar-nav">
-          <li {% if page=='index' %}class="active"{% endif %}><a href="{{ url_for('homepage') }}">{{ top_titles['index'] }}</a></li>
+          <li {% if page=='index' %}class="active"{% endif %}><a href="{{ url(page_view_names['index']) }}">{{ top_titles['index'] }}</a></li>
           {% for item in top_navigation %}
-          <li {% if item==page or page in navigation[item] %}class="active"{% endif %}><a href="{{ url_for('basic_page', page_name=item) }}">{{ top_titles[item] }}</a></li>
+          <li {% if item==page or page in navigation[item] %}class="active"{% endif %}><a href="{{ url(page_view_names[item]) }}">{{ top_titles[item] }}</a></li>
           {% endfor %}
         </ul>
       </div>
       <div class="collapse navbar-collapse">
         <ul class="nav navbar-nav">
         {% for item in navigation[navigation_reverse[page]] %}
-            <li{% if item==page %} class="active"{% endif %}><a href="{{ url_for('basic_page', page_name=item) }}">{{ short_page_titles[item] }}</a></li>
+            <li{% if item==page %} class="active"{% endif %}><a href="{{ url(page_view_names[item]) }}">{{ short_page_titles[item] }}</a></li>
         {% endfor %}
         </ul>
       </div><!--/.nav-collapse -->
@@ -77,11 +77,11 @@ <h1>{{ page_titles[page] }}</h1>
 
           <a href="https://github.com/IATI/IATI-Dashboard/issues?state=open">Report bugs, and request features using Github issues</a>.<br/>
 
-          Generated on <em>{{ datetime_generated() }}</em> from data downloaded on <em>{{ datetime_data }}</em>.<br/>
+          Generated from data downloaded on <em>{{ datetime_data }}</em>.<br/>
 
           (NB This is the time the download task started. Any changes made after this time may not be reflected).<br />
 
-          For details on how often these updates are applied, see the <a href="{{ url_for('basic_page', page_name='faq') }}">IATI Dashboard FAQ</a>.
+          For details on how often these updates are applied, see the <a href="{{ url(page_view_names[page]) }}">IATI Dashboard FAQ</a>.
 
           <a href="https://iatistandard.org/en/privacy-policy/">Privacy Policy</a></p>
     </div>