diff --git a/cove/cove_360/fixtures/additional_fields.json b/cove/cove_360/fixtures/additional_fields.json new file mode 100644 index 0000000..c4dbb9f --- /dev/null +++ b/cove/cove_360/fixtures/additional_fields.json @@ -0,0 +1,27 @@ +{ + "grants": [ + { + "id": "360G-sampletrust-105177/Z/14/Z", + "awardDate": "2024-12-30", + "amountAwarded": 10, + "url": "http://example.com", + "title": "test", + "currency": "GBP", + "description": "test", + "AdditionalField": "2", + "CheckMicOne": "two", + "recipientOrganization": [ + { + "id": "GB-323242-test", + "name": "Example Project Limited" + } + ], + "fundingOrganization": [ + { + "id": "GB-323242-test", + "name": "Example Project Limited" + } + ] + } + ] +} diff --git a/cove/cove_360/fixtures/duration_usefulness.json b/cove/cove_360/fixtures/duration_usefulness.json index 87507d1..9543e8f 100644 --- a/cove/cove_360/fixtures/duration_usefulness.json +++ b/cove/cove_360/fixtures/duration_usefulness.json @@ -14,7 +14,6 @@ "name": "Example Project Limited" } ], - "plannedDates": [], "fundingOrganization": [ { "id": "GB-323242-test", diff --git a/cove/cove_360/templates/cove_360/components/explore/accuracy.html b/cove/cove_360/templates/cove_360/components/explore/accuracy.html index eab70be..1ff3560 100644 --- a/cove/cove_360/templates/cove_360/components/explore/accuracy.html +++ b/cove/cove_360/templates/cove_360/components/explore/accuracy.html @@ -4,6 +4,9 @@

Data Accuracy

+ {% if validation_errors or additional_closed_codelist_values %} +

The data provided did not pass validation therefore the accuracy of the data could not be determined. + {% else %} {% if quality_accuracy_checks_passed %}

What is working well

@@ -85,4 +88,5 @@

{{category}}

{% endwith %} {% endfor %} -{% endif %} {# Quality accuracy errored #} \ No newline at end of file +{% endif %} {# Quality accuracy errored #} +{% endif %} {# validation failed #} \ No newline at end of file diff --git a/cove/cove_360/templates/cove_360/components/explore/summary.html b/cove/cove_360/templates/cove_360/components/explore/summary.html index 9f2e538..79491ea 100644 --- a/cove/cove_360/templates/cove_360/components/explore/summary.html +++ b/cove/cove_360/templates/cove_360/components/explore/summary.html @@ -25,6 +25,24 @@

{% trans "This data uses the 360Giving Data Standard co

{% endif %} + + {% if conversion_warning_messages or conversion_error %} +
+

This data could not be converted ({% blocktrans count n_warnings=conversion_warning_messages|length %}{{n_warnings}} Error) {% plural %}{{n_warnings}} Errors){% endblocktrans %} + +

+

We could not converting the data for checking. See Data conversion errors for details. +

+
+ {% endif %} + + {% if conversion == 'flatten' %} +
+

File conversion successful

+

We have converted your JSON data into spreadsheet format. See Download and share for more details.

+
+ {% endif %} +

The file "{{file_name}}" was @@ -36,9 +54,56 @@

{% trans "This data uses the 360Giving Data Standard co {% trans "on " %} {{created_date}} at {{created_time}}.

+

Explore your data:

+
    + {% if additional_fields_count %} +
  1. Additional fields ({{additional_fields_count}})
  2. + {% endif %} + {% if conversion_warning_messages %} +
  3. Conversion errors ({{conversion_warning_messages|length}})
  4. + {% endif %} + + {% if metadata %} +
  5. Metadata
  6. + {% endif %} +
  7. File contents
  8. +
  9. Identifiers
  10. +
  11. Data convertor
  12. +
  13. Download and share
  14. +
  15. Further help
  16. +
+ + {% if conversion_warning_messages or conversion_error %} +
+

+ {% trans "Data conversion unsuccessful - " %} + {% blocktrans count n_warnings=conversion_warning_messages|length %}{{n_warnings}} Error has been found{% plural %}{{n_warnings}} Errors have been found{% endblocktrans %} +

+ +

{% trans "Before checking your data we needed to convert it to JSON" %}{% if conversion_error or conversion_warning_messages %}{% blocktrans %} but we were not able to do this successfully{% endblocktrans %}{% endif %}.

+ {% blocktrans %}This tool converts data into JSON because the 360Giving Data Standard uses a JSON Schema to describe the standard in a technical way.{% endblocktrans %}
+ {% blocktrans %}If a file cannot be converted to JSON it indicates that it cannot be correctly mapped to the standard and needs to be reviewed. If this is the case you should check the file and re-upload it once you’ve fixed the problem. The conversion errors below will give an indication of where the issue is.{% endblocktrans %} + {% if conversion_error %} +

{% blocktrans %}The JSON data could not be converted to spreadsheet due to the following error: {{conversion_error}}{% endblocktrans %}

+ {% include 'error_extra.html' %} + {% endif %} + + {% if conversion_warning_messages %} +

{% trans "Conversion errors:" %}

+ +

{% trans "Please resolve this error, as well as any others found, and test the data again." %} Load new file.

+ {% endif %} +{% endif %} {# / if conversion errors #} + + + {% if metadata %} -

Metadata

+

Metadata

{% if metadata.publisher %} @@ -85,7 +150,8 @@

{% endif %} -

This file contains

+
+

This file contains

{% trans "Use this section to get an overview of the data and check if this was what you were expecting to see." %}

-

Identifiers

- + {% if grants_aggregates.distinct_recipient_org_identifier|length %} .{% endif %} -

{% blocktrans %}Do these results look correct?{% endblocktrans %}

+

{% blocktrans %}Do these results look correct?{% endblocktrans %}

{% blocktrans %}If any of this information appears incorrect, the feedback below will help you to investigate what happened. {% endblocktrans %}

-

{% blocktrans %}About the feedback{% endblocktrans %}

-

{% blocktrans %}The feedback is split into several sections. Use the small arrow icon on the far right side to display or hide the details for each section.{% endblocktrans %}

- -

A maximum of 10 additional checks can be displayed at once.

-

The check your data section allows you to review the data itself to help identify any issues.

-

If you make changes to the data prompted by this feedback, return here to upload the updated file for a new check. It may take several iterations to get the data to a state you are happy with and are ready to publish as open data.

- -

Getting further help

-

You can read about common data errors and what causes them in the Common Errors section.

-

Visit our Data Quality Dashboard to see the data quality of 360Giving data as a whole and for each individual publisher. If you have already published data, you can view your own publisher page there too. It provides insights into the key features that make the data useful for analysis to help publishers to identify opportunities for their data to be improved.

- +
-

Data Conversion

- {% if conversion == 'unflatten' or conversion == 'flatten'%} -

- {% if conversion_warning_messages or conversion_error %} - {% trans "Data conversion unsuccessful - " %} - {% else %} - {% trans "Data conversion successful" %} - {% endif %} - {% if conversion_warning_messages %} - {% blocktrans count n_warnings=conversion_warning_messages|length %}{{n_warnings}} Error has been found{% plural %}{{n_warnings}} Errors have been found{% endblocktrans %} - {% endif %} -

+

Data conversion

+ {% if conversion == 'flattenable' %} +

Convert the data from JSON format to spreadsheet.

+
+ + {% csrf_token %} +
+ {% endif %} + - {% if conversion == 'unflatten' %} -

{% trans "Before checking your data we needed to convert it to JSON" %}{% if conversion_error or conversion_warning_messages %}{% blocktrans %} but we were not able to do this successfully{% endblocktrans %}{% endif %}.

- {% blocktrans %}This tool converts data into JSON because the 360Giving Data Standard uses a JSON Schema to describe the standard in a technical way.{% endblocktrans %}
- {% if conversion_error %} - {% blocktrans %}If a file cannot be converted to JSON it indicates that it cannot be correctly mapped to the standard and needs to be reviewed. If this is the case you should check the file and re-upload it once you’ve fixed the problem. The conversion errors below will give an indication of where the issue is.{% endblocktrans %} - {% endif %} - {% else %} -

We have converted your JSON data into spreadsheet format.

- {% if conversion_error %} -

{% blocktrans %}The JSON data could not be converted to spreadsheet due to the following error: {{conversion_error}}{% endblocktrans %}

- {% include 'error_extra.html' %} - {% endif %} - {% endif %} - {% if conversion_warning_messages %} -
-

{% trans "Conversion errors:" %}

-
    - {% for warning_message in conversion_warning_messages %} -
  • {{warning_message}}
  • - {% endfor %} -
-

{% trans "Please resolve this error, as well as any others found, and test the data again." %}

- {% endif %} -{% endif %} {% cove_360_modal_list className="unique-ids" modalTitle="Unique IDs" itemList=grants_aggregates.unique_ids %} {% cove_360_modal_list className="distinct-funding-org-identifier" modalTitle="Funder Organisation IDs" itemList=grants_aggregates.distinct_funding_org_identifier %} {% cove_360_modal_list className="distinct-recipient-org-identifier" modalTitle="Recipient Organisation IDs" itemList=grants_aggregates.distinct_recipient_org_identifier %} - -

+

{% trans "Download data and share these results" %}

Share

@@ -246,13 +286,6 @@

Download

{% trans "This application converts data in Excel and CSV format into JSON format, allowing you download the converted version." %}

{% trans "If your file is originally in JSON format select ‘Convert to Spreadsheet’ in the summary section to create an Excel version of the file." %}

- {% if conversion == 'flattenable' %} -
- - {% csrf_token %} -
- {% endif %} - {% if not conversion_error and conversion != 'flattenable' %}

{% trans "We provide the following formats to download:" %}

@@ -281,4 +314,11 @@

Original file download for admin users

{% endif %} -
\ No newline at end of file +
+ +
+ +

Getting further help

+

You can read about common data errors and what causes them in the Common Errors section.

+

Visit our Data Quality Dashboard to see the data quality of 360Giving data as a whole and for each individual publisher. If you have already published data, you can view your own publisher page there too. It provides insights into the key features that make the data useful for analysis to help publishers to identify opportunities for their data to be improved.

+ diff --git a/cove/cove_360/templates/cove_360/components/explore/usefulness.html b/cove/cove_360/templates/cove_360/components/explore/usefulness.html index b586102..91a3fc5 100644 --- a/cove/cove_360/templates/cove_360/components/explore/usefulness.html +++ b/cove/cove_360/templates/cove_360/components/explore/usefulness.html @@ -4,6 +4,10 @@

Usefulness Opportunities

+ {% if validation_errors or additional_closed_codelist_values %} +

The data provided did not pass validation therefore the usefulness of the data could not be determined. + {% else %} + {% if usefulness_checks_passed %}

What is working well

checkCongratulations, {{usefulness_checks_passed|length}} of our usefulness checks passed.

@@ -99,3 +103,4 @@

{{category}}

{% else %}

No usefulness opportunities detected.

{% endif %} +{% endif %} {# validation failed #} \ No newline at end of file diff --git a/cove/cove_360/templates/cove_360/components/explore_checking.html b/cove/cove_360/templates/cove_360/components/explore_checking.html index 0f1e9dc..248c85d 100644 --- a/cove/cove_360/templates/cove_360/components/explore_checking.html +++ b/cove/cove_360/templates/cove_360/components/explore_checking.html @@ -40,7 +40,13 @@

-

{{quality_accuracy_checks_count}}

+

+ {% if validation_and_closed_codelist_errors_count %} + 0 + {% else %} + {{quality_accuracy_checks_count}} + {% endif %} +

Potential accuracy issue

@@ -49,7 +55,13 @@

{{quality_accuracy_checks_count}}

-

{{usefulness_checks_count }}

+

+ {% if validation_and_closed_codelist_errors_count %} + 0 + {% else %} + {{usefulness_checks_count }} + {% endif %} +

Usefulness opportunities

@@ -86,7 +98,7 @@

{{usefulness_checks_count }}

href="#accuracy" id="accuracy-tab-link" > - {% trans "Accuracy" %} ({{quality_accuracy_checks_count}}) + {% trans "Accuracy" %} ({% if validation_and_closed_codelist_errors_count %}0{% else %}{{quality_accuracy_checks_count}}{% endif %}) @@ -96,8 +108,8 @@

{{usefulness_checks_count }}

href="#usefulness" id="usefulness-tab-link" > - {% trans "Usefulness" %} ({{usefulness_checks_count}}) - + {% trans "Usefulness" %} ({% if validation_and_closed_codelist_errors_count %}0{% else %}{{usefulness_checks_count }}{% endif %}) +
diff --git a/cove/cove_360/templates/cove_360/error.html b/cove/cove_360/templates/cove_360/error.html index 899f0ed..062677b 100644 --- a/cove/cove_360/templates/cove_360/error.html +++ b/cove/cove_360/templates/cove_360/error.html @@ -1,23 +1,14 @@ {% extends request.current_app_base_template %} {% load i18n %} {% block content %} +{# fixme this template isn't getting used #} +

{{ sub_title }}

-
-
- - {{ sub_title }} -
-
{{ msg | linebreaks }} {% include 'error_extra.html' %} -
-
- - + {{ link_text }} {% endblock %} diff --git a/cove/cove_360/tests/test_browser.py b/cove/cove_360/tests/test_browser.py index 9a8d883..c0f78aa 100644 --- a/cove/cove_360/tests/test_browser.py +++ b/cove/cove_360/tests/test_browser.py @@ -343,7 +343,7 @@ def mockflatten(input_name, output_name, *args, **kwargs): warning_heading = "Data conversion unsuccessful - 1 Error has been found" - conversion_title = browser.find_element(By.ID, "conversion-title") + conversion_title = browser.find_element(By.ID, "conversion-errors") conversion_title_text = conversion_title.text if iserror: @@ -541,11 +541,11 @@ def test_oneof_validation(server_url, browser, httpserver): ("duration_usefulness.json", [ "1 grant does not contain plannedDates/0/duration or (plannedDates/startDate and plannedDates/endDate)", ], []), - ("duration_usefulness.json", [ - "1 grant does not contain plannedDates/0/duration or (plannedDates/startDate and plannedDates/endDate)", + ("additional_fields.json", [ + "Additional fields which do not use 360Giving Data Standard titles were found in your data.", ], []), ("multiple_fundiner_names_org_ids.json", [ - "added an additional name for an existing Funding Org" + "introduced an additional name for an existing Funding Org" ], []), ]) def test_quality_checks(server_url, browser, httpserver, source_filename, expected_texts, unexpected_texts): diff --git a/lib360dataquality/cove/threesixtygiving.py b/lib360dataquality/cove/threesixtygiving.py index f0eee9f..eefe3d2 100644 --- a/lib360dataquality/cove/threesixtygiving.py +++ b/lib360dataquality/cove/threesixtygiving.py @@ -1769,7 +1769,7 @@ class MultiFundingOrgIdsForName(AdditionalTest): """Check for Funding org ids with multiple names.""" check_text = { - "heading": mark_safe("added an additional name for an existing Funding Org"), + "heading": mark_safe("introduced an additional name for an existing Funding Org"), "message": RangeDict(), } check_text["message"][(0, 100)] = mark_safe( @@ -1894,10 +1894,13 @@ def create_grant_dates_dict(grant): grant_dates = {} award_date = grant.grant.get("awardDate") - planned_start_date = grant.grant.get("plannedDates", [{}])[0].get("startDate") - planned_end_date = grant.grant.get("plannedDates", [{}])[0].get("endDate") - actual_start_date = grant.grant.get("actualDates", [{}])[0].get("startDate") - actual_end_date = grant.grant.get("actualDates", [{}])[0].get("endDate") + try: + planned_start_date = grant.grant.get("plannedDates", [{}])[0].get("startDate") + planned_end_date = grant.grant.get("plannedDates", [{}])[0].get("endDate") + actual_start_date = grant.grant.get("actualDates", [{}])[0].get("startDate") + actual_end_date = grant.grant.get("actualDates", [{}])[0].get("endDate") + except IndexError: + return {} for date_type, input_date in [ ["award_date", award_date],