Added a new JSON editor widget and attached it to the admin pages for…

… Specification Templates and Evaluation Definitions

python/lib/evaluations/dmod/test/specification/test_deserialization.py

@@ -1,11 +1,8 @@
 import os.path
 import unittest
 import json
-import pathlib
-import typing
 
 from ...evaluations import specification
-from ..common import ConstructionTest
 from ..common import RESOURCE_DIRECTORY
 
 
@@ -53,6 +50,9 @@ def test_multitemplate(self):
         self.assertEqual(single_instance, pure_template_instance)
 
     def test_evaluation_deserialization(self):
+        from pprint import pprint
+
+        pprint(dir(specification.EvaluationSpecification))
         normal_specification = self.template_manager.get_template(
             specification_type=specification.EvaluationSpecification,
             name="no-template"

python/services/evaluationservice/dmod/evaluationservice/evaluation_service/admin.py

@@ -3,11 +3,148 @@
 
 from . import models
 
+from evaluation_service.forms import EvaluationDefinitionForm
+from evaluation_service.forms import SpecificationTemplateForm
 
+@admin.register(models.EvaluationDefinition)
+class EvaluationDefinitionAdmin(admin.ModelAdmin):
+    """
+    A dedicated view to edit a Specification class instance
+    """
+    form = EvaluationDefinitionForm
+
+    # Display these fields on the list screen
+    list_display = ('name', 'author', 'description', "last_edited")
+
+    # Regardless of what fields are displayed on the screen,
+    # we want to be able to enter the editor by clicking on the template_name
+    list_display_links = ('name',)
+
+    def get_list_display(self, request: HttpRequest) -> (str,):
+        """
+        Gets the fields that may be shown as columns on the list screen
+
+        :param HttpRequest request: The request that asked for the columns
+        :rtype: (str,)
+        :return: The fields to display on the list screen
+        """
+        # Get the master list of columns to show
+        list_display = super().get_list_display(request)
+
+        # If the user is a superuser, they'll see DataSources belonging to all users;
+        # tack the author's name to it so that superusers may know whose DataSources they are editing
+        if request.user.is_superuser:
+            list_display = ["author"] + list(list_display)
+
+        return list_display
+
+    def get_list_filter(self, request: HttpRequest) -> (str,):
+        """
+        Gets the list of fields that elements may be filtered by. This will show a box on the side of the
+        screen where a user may click a link that might limit all elements displayed on the screen to those
+        whose variables are 'flow'
+
+        :param HttpRequest request: The request asking for which fields to allow a user to filter by
+        :rtype: (str,)
+        :return: A collection of all fields to filter by
+        """
+        list_filter = super().get_list_filter(request)
+
+        # If the user is a superuser, they will also be able to see the authors of the datasources.
+        # Allow the user to limit displayed items to those owned by specific users
+        if request.user.is_superuser:
+            list_filter = ["author"] + list(list_filter)
+
+        return list_filter
+
+    def get_readonly_fields(self, request: HttpRequest, obj: models.SpecificationTemplate = None) -> (str,):
+        """
+        Determines which fields should not be available for editing
+
+        :param HttpRequest request: The request asking which fields displayed on the screen should be read only
+        :param models.DataSource obj: The datasource whose fields may be displayed on the screen
+        :rtype: (str,)
+        :return: A collection of all fields that cannot be modified
+        """
+        readonly_fields = super().get_readonly_fields(request, obj)
+
+        # If a user is a superuser, they will see the name of the author.
+        # In order to prevent the user from changing that, we set it as read-only
+        if request.user.is_superuser:
+            readonly_fields = tuple(['author'] + list(readonly_fields))
+
+        return readonly_fields
+
+    def save_model(self, request, obj, form, change):
+        """
+        Attaches the user to the object and calls the parent save method
+
+        :type request: HttpRequest
+        :param request: The web request
+        :type obj: DataSource
+        :param obj: The object that is to be saved
+        :param form: The form that called the save function
+        :param change: The change to the object
+        :return: The result of the parent class's save_model function
+        """
+
+        # If the object doesn't have an author, make the current user
+        if obj.author is None:
+            if request.user.first_name and request.user.last_name:
+                name = f"{request.user.first_name} {request.user.last_name}"
+            elif request.user.first_name:
+                name = request.user.first_name
+            elif request.user.last_name:
+                name = request.user.last_name
+            else:
+                name = request.user.username
+            obj.author = name
+
+        super().save_model(request, obj, form, change)
+
+    def get_form(self, request, obj=None, **kwargs):
+        """
+        Gets the custom form used for validation for the data source being edited
+
+        :param request: The request that asked for the form
+        :param obj: The object to edit
+        :param kwargs: Keyword arguments being passed down the line
+        :return: The form that will be used to validate the values configured for the data source
+        """
+
+        # First perform what it would have done prior. It's complicated logic and we can't do better
+        form = super().get_form(request, obj, **kwargs)
+
+        if form and hasattr(form, "editor"):
+            # All we want to do is attach the user, so we go ahead and do that now
+            form.editor = request.user
+
+        return form
+
+    def get_queryset(self, request):
+        """
+        Obtains a set of the appropriate data source configurations to load into the list view
+
+        :type request: HttpRequest
+        :param request: The http request that told the application to load data source configurations into the list
+        :rtype: QuerySet[DataSource]
+        :return: A QuerySet containing all DataSource objects that may be edited.
+        """
+        qs = super().get_queryset(request)
+
+        # If the user isn't a superuser, only the user's DataSource objects will be returned
+        if not request.user.is_superuser:
+            qs = qs.filter(author=request.user)
+
+        return qs
+
+
+@admin.register(models.SpecificationTemplate)
 class SpecificationTemplateAdmin(admin.ModelAdmin):
     """
     A dedicated view to edit a Specification class instance
     """
+    form = SpecificationTemplateForm
 
     fields = (
         "template_specification_type",
@@ -138,6 +275,4 @@ def get_queryset(self, request):
 
 
 # Register your models here.
-admin.site.register(models.StoredDataset)
-admin.site.register(models.EvaluationDefinition)
-admin.site.register(models.SpecificationTemplate, SpecificationTemplateAdmin)
+admin.site.register(models.StoredDataset)

python/services/evaluationservice/dmod/evaluationservice/evaluation_service/forms/__init__.py

@@ -0,0 +1,6 @@
+"""
+All custom forms
+"""
+
+from .json import EvaluationDefinitionForm
+from .json import SpecificationTemplateForm

python/services/evaluationservice/dmod/evaluationservice/evaluation_service/forms/json.py

@@ -0,0 +1,96 @@
+"""
+Definitions for forms that provide advanced handling for json data
+"""
+from __future__ import annotations
+
+import typing
+import json
+import re
+
+from pydantic import BaseModel
+
+from django import forms
+
+from widgets import JSONArea
+
+from dmod.evaluations import specification
+from dmod.evaluations.specification.base import get_subclasses
+
+from evaluation_service import models
+
+BINARY_TYPES = re.compile(
+    r'((?<!,)\s*\{\s*"type": "string",\s*"format": "binary"\s*},\s*|,\s*\{\s*"type": "string",\s*"format": "binary"\s*}\s*)'
+)
+"""
+A regular expression that finds all type definitions that are strings formatted as bytes
+"""
+
+def get_editor_friendly_model_schema(model: typing.Type[BaseModel]) -> typing.Optional[dict]:
+    """
+    Get the schema for a model and scrub any editor unfriendly type from it
+
+    An example of an editor unfriendly type is a string in a binary format
+
+    Args:
+        model: The model whose schema to retrieve
+
+    Returns:
+        A schema for the model if it is available
+    """
+    if hasattr(model, "schema_json"):
+        json_data = model.schema_json()
+
+        json_data = BINARY_TYPES.sub("", json_data)
+
+        return json.loads(json_data)
+    return None
+
+def get_specification_schema_map() -> typing.Dict[str, typing.Any]:
+    """
+    Generate a dictionary mapping specification types to their schemas
+
+    Return:
+        A dictionary mapping the value of a specification type that will be on the template type selector to schema data that is compatible with the client side editor
+    """
+    return {
+        specification_type.get_specification_type(): get_editor_friendly_model_schema(specification_type)
+        for specification_type in get_subclasses(specification.TemplatedSpecification)
+    }
+
+
+class EvaluationDefinitionForm(forms.ModelForm):
+    """
+    A specialized form for EvaluationDefinition that allows its JSON data to be manipulated within a JSONArea
+    """
+    class Meta:
+        model = models.EvaluationDefinition
+        fields = "__all__"
+
+    definition = forms.JSONField(
+        widget=JSONArea(
+            schema=get_editor_friendly_model_schema(specification.EvaluationSpecification)
+        )
+    )
+
+class SpecificationTemplateForm(forms.ModelForm):
+    """
+    A specialized form for SpecificationTemplate that allows its JSON data to be manipulated within a JSONArea
+    """
+    class Meta:
+        model = models.SpecificationTemplate
+        fields = "__all__"
+
+    class Media:
+        # Include a script to add functionality that will update the json area's
+        # schema when the template type is changed
+        js = [
+            "evaluation_service/js/template_specification.js"
+        ]
+
+    template_configuration = forms.JSONField(
+        widget=JSONArea(
+            extra_data={
+                "schemas": get_specification_schema_map()
+            }
+        )
+    )

python/services/evaluationservice/dmod/evaluationservice/evaluation_service/migrations/0004_alter_specificationtemplate_template_configuration.py

@@ -0,0 +1,18 @@
+# Generated by Django 4.1.3 on 2023-07-24 19:28
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('evaluation_service', '0003_specificationtemplate_and_more'),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name='specificationtemplate',
+            name='template_configuration',
+            field=models.JSONField(help_text='The configuration that should be applied to a given specification type'),
+        ),
+    ]

python/services/evaluationservice/dmod/evaluationservice/evaluation_service/models.py

@@ -29,8 +29,7 @@ class Meta:
         choices=get_specification_options(),
         help_text="The type of specification that this template pertains to"
     )
-    template_configuration = models.CharField(
-        max_length=30000,
+    template_configuration = models.JSONField(
         help_text="The configuration that should be applied to a given specification type"
     )
     template_description = models.CharField(

python/services/evaluationservice/dmod/evaluationservice/evaluation_service/static/evaluation_service/js/template_specification.js

@@ -0,0 +1,77 @@
+/**
+ * Supplies custom functions and handling for the SpecificationTemplateForm
+ */
+
+/**
+ * Handler for when the template type selector has changed values
+ */
+function templateTypeChanged() {
+    changeConfigurationSchema(this);
+}
+
+/**
+ * Update the JSON editor with the selected schema
+ *
+ * @param {HTMLElement?} selector A select element stating what schema to use
+ */
+function changeConfigurationSchema(selector) {
+    let newSchema = null;
+    const editorData = getEditorData("template_configuration");
+
+    if (selector === null || selector === undefined) {
+        selector = django.jQuery("select[name=template_specification_type]")[0];
+    } else if (selector instanceof django.jQuery) {
+        selector = selector[0];
+    }
+
+    if (!(selector.hasOwnProperty("value") || "value" in selector)) {
+        return;
+    }
+
+    if (editorData !== null && "schemas" in editorData && selector.value in editorData.schemas) {
+        newSchema = editorData.schemas[selector.value];
+    }
+
+    if (newSchema === null || newSchema === undefined) {
+        return;
+    }
+
+    const editor = getEditor("template_configuration");
+
+    if (editor) {
+        editor.setSchema(newSchema);
+
+        let currentText = editor.getText();
+
+        if (currentText === null) {
+            currentText = "";
+        } else {
+            currentText = currentText.trim();
+        }
+
+        if (currentText.length === 0 || currentText.match(/^\{?\s*}?$/)) {
+            const newData = buildObjectFromSchema(newSchema);
+            editor.set(newData);
+        }
+    }
+}
+
+/**
+ * Attach the `templateTypeChanged` function to the 'change' event for the template specification type selector
+ */
+function attachSpecificationTypeChanged() {
+    const selector = django.jQuery("select[name=template_specification_type]");
+    selector.on("change", templateTypeChanged);
+}
+
+/**
+ * Make sure that the change handler for the template type selector is attached and
+ * that the proper schema is attached to the editor once the page is done loading
+ */
+document.addEventListener(
+    "DOMContentLoaded",
+    function() {
+        attachSpecificationTypeChanged();
+        changeConfigurationSchema();
+    }
+);