[ENG-5493][ENG-5412] more helpful trove responses (#820)

[ENG-5493]
- fill in `trove/exceptions.py` with situation-specific exception classes
- use those specific exceptions thruout trove code
- give an appropriate response when those specific exceptions happen while handling a request (currently always in jsonapi)
- reduce dead links and other errors when browsing the search api
  - allow resolving static vocab terms (including "new" static vocabs `skos`, `foaf`, `dc`, `dctype`, `dcam`)
  - display datatypes for literals
- add better search-api tests (with actual search indexes running)
- fix broken `next` links with field-specific search ([ENG-5412])
- code cleanup
  - add `trove.trovesearch` with modules previously at `trove.trovesearch_gathering`, `share.search.search_params`, and `share.search.search_response`
  - render derived indexcard metadata as rdf literal (except html rendering) -- avoid errors from non-standard "quoted triple" or "quoted graph"
  - remove `trove.render.osfmap_jsonld` (subsumed by `trove.derive.osfmap_json`)
  - remove `ingest_rdf_vocabs` management command (unnecessary with "static vocab" update)
  - rename `...VOCAB` to `...THESAURUS` (not just a set of terms, but a whole rdf graph describing those terms)
  - remove `trove.util.iri_labeler` (completely replaced by `primitive_rdf.IriShorthand`
  - consolidate mediatypes as constants in `trove.vocab.mediatypes`
  - move all "static vocabs" into `trove.vocab.static_vocab`
  - add `functools.cache` to things that need be computed only once

requirements.txt

@@ -31,7 +31,7 @@ psycogreen==1.0.2 # BSD
 psycopg2==2.9.5 # LGPL with exceptions or ZPL
 python-dateutil==2.8.1 # Apache 2.0
 PyJWE==1.0.0 # Apache 2.0
-pyshacl==0.22.0 # Apache 2.0
+rdflib==7.0.0
 pyyaml==6.0 # MIT
 requests==2.25.1 # Apache 2.0
 sentry-sdk[django]==1.22.2 # MIT
@@ -43,4 +43,4 @@ xmltodict==0.12.0 # MIT
 # Allows custom-rendered IDs, hiding null values, and including data in error responses
 git+https://github.com/cos-forks/django-rest-framework-json-api.git@v4.2.1+cos0
 
-git+https://github.com/aaxelb/primitive_metadata.git@0.2023.57
+git+https://github.com/aaxelb/primitive_metadata.git@0.2024.09

share/search/exceptions.py

@@ -15,15 +15,3 @@ class DaemonIndexingError(ShareException):
 
 class IndexStrategyError(ShareException):
  pass
-
-
-class SearchApiError(ShareException):
- pass
-
-
-class InvalidSearchParam(SearchApiError):
- pass
-
-
-class UnsupportedSearchParam(IndexStrategyError):
- pass

share/search/index_strategy/_base.py

@@ -10,15 +10,15 @@
 from share.models.index_backfill import IndexBackfill
 from share.search.exceptions import IndexStrategyError
 from share.search.index_status import IndexStatus
-from share.search.search_params import (
+from share.util.checksum_iri import ChecksumIri
+from trove.trovesearch.search_params import (
  CardsearchParams,
  ValuesearchParams,
 )
-from share.search.search_response import (
+from trove.trovesearch.search_response import (
  CardsearchResponse,
  ValuesearchResponse,
 )
-from share.util.checksum_iri import ChecksumIri
 
 
 logger = logging.getLogger(__name__)

share/search/index_strategy/trove_indexcard_flats.py

@@ -7,7 +7,7 @@
 import logging
 import re
 import uuid
-from typing import Iterable, ClassVar, Optional
+from typing import Iterable, ClassVar, Optional, Iterator
 
 from django.conf import settings
 from django.db.models import Exists, OuterRef
@@ -18,7 +18,9 @@
 from share.search import messages
 from share.search.index_strategy.elastic8 import Elastic8IndexStrategy
 from share.search.index_strategy._util import encode_cursor_dataclass, decode_cursor_dataclass
-from share.search.search_params import (
+from share.util.checksum_iri import ChecksumIri
+from trove import models as trove_db
+from trove.trovesearch.search_params import (
  CardsearchParams,
  ValuesearchParams,
  SearchFilter,
@@ -27,16 +29,14 @@
  PageParam,
  GLOB_PATHSTEP,
 )
-from share.search.search_response import (
+from trove.trovesearch.search_response import (
  CardsearchResponse,
  ValuesearchResponse,
  TextMatchEvidence,
  CardsearchResult,
  ValuesearchResult,
  PropertypathUsage,
 )
-from share.util.checksum_iri import ChecksumIri
-from trove import models as trove_db
 from trove.util.iris import get_sufficiently_unique_iri, is_worthwhile_iri, iri_path_as_keyword
 from trove.vocab.osfmap import is_date_property
 from trove.vocab.namespaces import TROVE, FOAF, RDF, RDFS, DCTERMS, OWL, SKOS, OSFMAP
@@ -689,7 +689,7 @@ def _cardsearch_date_filter(self, search_filter):
  'query': {'bool': {'filter': list(self._iter_nested_date_filters(search_filter))}},
  }}
 
- def _iter_nested_date_filters(self, search_filter) -> dict:
+ def _iter_nested_date_filters(self, search_filter) -> Iterator[dict]:
  # filter by requested paths
  yield _pathset_as_nestedvalue_filter(search_filter.propertypath_set, 'nested_date')
  # filter by requested value/operator
@@ -1065,6 +1065,7 @@ def cardsearch_start_index(self) -> int:
 
 class _PredicatePathWalker:
  WalkYield = tuple[tuple[str, ...], primitive_rdf.RdfObject]
+ _visiting: set[str | frozenset]
 
  def __init__(self, tripledict: primitive_rdf.RdfTripleDictionary):
  self.tripledict = tripledict

tests/share/search/index_strategy/_with_real_services.py

@@ -19,8 +19,8 @@ class RealElasticTestCase(TransactionTestCase):
  serialized_rollback = True # for TransactionTestCase; restore db after
 
  # required for subclasses
- strategy_name_for_real = None
- strategy_name_for_test = None
+ strategy_name_for_real: str
+ strategy_name_for_test: str
 
  @classmethod
  def setUpClass(cls):
@@ -38,7 +38,7 @@ def setUp(self):
  )
  self.current_index = self.index_strategy.for_current_index()
  self.current_index.pls_delete() # in case it already exists
- self._assert_happypath_until_ingest()
+ self._assert_setup_happypath()
 
  def tearDown(self):
  super().tearDown()
@@ -120,7 +120,7 @@ def _assert_happypath_with_daemon(self, messages_chunk, expected_doc_count):
  else:
  assert False, 'checked and waited but the daemon did not do the thing'
 
- def _assert_happypath_until_ingest(self):
+ def _assert_setup_happypath(self):
  # initial
  assert not self.current_index.pls_check_exists()
  index_status = self.current_index.pls_get_status()