Merge branch 'Health-RI-contact_fixes'

.github/workflows/test.yml

@@ -22,10 +22,13 @@ jobs:
         include:
           - ckan-version: "2.11"
             ckan-image: "ckan/ckan-dev:2.11-py3.10"
+            solr-version: "9"
           - ckan-version: "2.10"
             ckan-image: "ckan/ckan-dev:2.10-py3.10"
+            solr-version: "9"
           - ckan-version: "2.9"
             ckan-image: "ckan/ckan-dev:2.9-py3.9"
+            solr-version: "8"
       fail-fast: false
 
     name: CKAN ${{ matrix.ckan-version }}
@@ -35,7 +38,7 @@ jobs:
       options: --user root
     services:
       solr:
-        image: ckan/ckan-solr:${{ matrix.ckan-version }}-solr9
+        image: ckan/ckan-solr:${{ matrix.ckan-version }}-solr${{ matrix.solr-version }}
       postgres:
         image: ckan/ckan-postgres-dev:${{ matrix.ckan-version }}
         env:

ckanext/dcat/harvesters/base.py

@@ -159,17 +159,21 @@ def _read_datasets_from_db(self, guid):
         '''
         Returns a database result of datasets matching the given guid.
         '''
-        if toolkit.check_ckan_version(max_version="2.11"):
-            datasets = model.Session.query(model.Package.id) \
-                                    .join(model.PackageExtra) \
-                                    .filter(model.PackageExtra.key == 'guid') \
-                                    .filter(model.PackageExtra.value == guid) \
-                                    .filter(model.Package.state == 'active') \
-                                    .all()
+        if toolkit.check_ckan_version(max_version="2.11.99"):
+            datasets = (
+                model.Session.query(model.Package.id)
+                .join(model.PackageExtra)
+                .filter(model.PackageExtra.key == "guid")
+                .filter(model.PackageExtra.value == guid)
+                .filter(model.Package.state == "active")
+                .all()
+            )
         else:
-            datasets = model.Session.query(model.Package.id) \
-                                    .filter(model.Package.extras['guid'] == f'"{guid}"') \
-                                    .all()
+            datasets = (
+                model.Session.query(model.Package.id)
+                .filter(model.Package.extras["guid"] == f'"{guid}"')
+                .all()
+            )
 
         return datasets
 

ckanext/dcat/profiles/base.py

@@ -536,7 +536,9 @@ def _agents_details(self, subject, predicate):
             agent_details = {}
             agent_details["uri"] = str(agent) if isinstance(agent, term.URIRef) else ""
             agent_details["name"] = self._object_value(agent, FOAF.name)
-            agent_details["email"] = self._object_value(agent, FOAF.mbox)
+            agent_details["email"] = self._without_mailto(
+                self._object_value(agent, FOAF.mbox)
+            )
             if not agent_details["email"]:
                 agent_details["email"] = self._without_mailto(
                     self._object_value(agent, VCARD.hasEmail)
@@ -573,6 +575,9 @@ def _contact_details(self, subject, predicate):
             )
 
             contact["identifier"] = self._get_vcard_property_value(agent, VCARD.hasUID)
+
+            contact["url"] = self._get_vcard_property_value(agent, VCARD.hasURL)
+
             contacts.append(contact)
 
         return contacts
@@ -818,9 +823,7 @@ def _add_spatial_value_to_graph(self, spatial_ref, predicate, value):
         or object.
         """
         spatial_formats = aslist(
-            config.get(
-                "ckanext.dcat.output_spatial_format", DEFAULT_SPATIAL_FORMATS
-            )
+            config.get("ckanext.dcat.output_spatial_format", DEFAULT_SPATIAL_FORMATS)
         )
 
         if isinstance(value, str):

ckanext/dcat/profiles/euro_dcat_ap_base.py

@@ -144,12 +144,12 @@ def _parse_dataset_base(self, dataset_dict, dataset_ref):
                 contact = self._contact_details(dataset_ref, ADMS.contactPoint)
             if contact:
                 contact = contact[0]
-                for key in ("uri", "name", "email", "identifier"):
+                for key in ("uri", "name", "email", "identifier", "url"):
                     if contact.get(key):
                         dataset_dict["extras"].append(
                             {
                                 "key": "contact_{0}".format(key),
-                                "value": contact.get(key)
+                                "value": contact.get(key),
                             }
                         )
 

ckanext/dcat/profiles/euro_dcat_ap_scheming.py

@@ -168,6 +168,13 @@ def _graph_from_dataset_v2_scheming(self, dataset_dict, dataset_ref):
                     "identifier",
                     _type=URIRefOrLiteral,
                 )
+                self._add_triple_from_dict(
+                    item,
+                    contact_details,
+                    VCARD.hasURL,
+                    "url",
+                    _type=URIRef,
+                )
 
         self._add_agents(dataset_ref, dataset_dict, "publisher", DCT.publisher)
         self._add_agents(dataset_ref, dataset_dict, "creator", DCT.creator)

ckanext/dcat/schemas/dcat_ap_full.yaml

@@ -46,7 +46,10 @@ dataset_fields:
     - field_name: identifier
       label: Identifier
       help_text: Unique identifier for the contact point. Such as a ROR ID.
-
+
+    - field_name: url
+      label: URL
+      help_text: A URL associated with the contact
   help_text: Contact information for enquiries about the dataset.
 
 - field_name: publisher

ckanext/dcat/schemas/dcat_ap_multilingual.yaml

@@ -104,7 +104,10 @@ dataset_fields:
     - field_name: identifier
       label: Identifier
       help_text: Unique identifier for the contact point. Such as a ROR ID.
-
+
+    - field_name: url
+      label: URL
+      help_text: A URL associated with the contact
   help_text: Contact information for enquiries about the dataset.
 
 - field_name: license_id

ckanext/dcat/schemas/dcat_ap_recommended.yaml

@@ -42,6 +42,10 @@ dataset_fields:
     - field_name: email
       label: Email
       display_snippet: email.html
+
+    - field_name: url
+      label: URL
+      help_text: A URL associated with the contact
   help_text: Contact information for enquiries about the dataset.
 
 - field_name: publisher

ckanext/dcat/schemas/dcat_us_full.yaml

@@ -42,6 +42,10 @@ dataset_fields:
     - field_name: email
       label: Email
       display_snippet: email.html
+
+    - field_name: url
+      label: URL
+      help_text: A URL associated with the contact
   help_text: Contact information for enquiries about the dataset.
 
 - field_name: publisher

ckanext/dcat/schemas/dcat_us_recommended.yaml

@@ -42,6 +42,10 @@ dataset_fields:
     - field_name: email
       label: Email
       display_snippet: email.html
+
+    - field_name: url
+      label: URL
+      help_text: A URL associated with the contact
   help_text: Contact information for enquiries about the dataset.
 
 - field_name: publisher

ckanext/dcat/tests/profiles/dcat_ap_3/test_euro_dcatap_3_profile_parse.py

@@ -91,6 +91,7 @@ def test_e2e_dcat_to_ckan(self):
 
         assert dataset["contact"][0]["name"] == "Point of Contact"
         assert dataset["contact"][0]["email"] == "contact@some.org"
+        assert dataset["contact"][0]["url"] == "https://example.org"
 
         assert (
             dataset["publisher"][0]["name"] == "Publishing Organization for dataset 1"

ckanext/dcat/tests/profiles/dcat_ap_3/test_euro_dcatap_3_profile_serialize.py

@@ -148,6 +148,9 @@ def test_e2e_ckan_to_dcat(self):
             VCARD.hasUID,
             dataset_dict["contact"][0]["identifier"],
         )
+        assert self._triple(
+            g, contact_details[0][2], VCARD.hasURL, URIRef(dataset_dict["contact"][0]["url"])
+        )
         assert self._triple(
             g, contact_details[1][2], VCARD.fn, dataset_dict["contact"][1]["name"]
         )

ckanext/dcat/tests/shacl/test_shacl.py

@@ -199,9 +199,6 @@ def test_validate_dcat_us_3_graph():
 
     graph = graph_from_dataset("ckan_full_dataset_dcat_us_vocabularies.json")
 
-    graph.serialize(destination="graph.ttl")
-
-    graph.serialize(destination="graph.xml")
     path = _get_shacl_file_path("dcat-us_3.0_shacl_shapes.ttl")
     r = validate(graph, shacl_graph=path)
     conforms, results_graph, results_text = r

examples/ckan/ckan_full_dataset_dcat_ap.json

@@ -53,7 +53,8 @@
         {
             "name": "Contact 1",
             "email": "contact1@example.org",
-            "identifier": "123"
+            "identifier": "123",
+            "url": "https://example.org"
         },
         {
             "name": "Contact 2",

examples/dcat/dataset.rdf

@@ -68,6 +68,7 @@
       <vcard:Organization>
         <vcard:fn>Point of Contact</vcard:fn>
         <vcard:hasEmail rdf:resource="mailto:contact@some.org"/>
+        <vcard:hasURL rdf:resource="https://example.org"/>
       </vcard:Organization>
     </dcat:contactPoint>
     <dct:publisher>