Search improvements: release to user-facing interface (#417)

Merge this when you want to deploy the search improvements to prod
==============================================

Makes the search improvements underpinning the explore page join up with
and connect in to the main user facing search
also updates tracking version number for easier analytics segregation


NB. May need to re-index and/or run page save operation for each page in
order to re-populate index

---------

Co-authored-by: Cameron Lamb <cameron.lamb@digital.trade.gov.uk>

src/content/models.py

@@ -162,7 +162,7 @@ class ContentPageIndexManager(ModelIndexManager):
             explicit=True,
         ),
         IndexedField("is_creatable", filter=True),
-        IndexedField("last_published_at", proximity=True),
+        IndexedField("published_date", proximity=True),
     ]
 
 
@@ -258,6 +258,10 @@ class ContentPage(BasePage):
 
     search_fields = BasePage.search_fields + ContentPageIndexManager()
 
+    @property
+    def published_date(self):
+        return self.last_published_at
+
     def _generate_search_field_content(self):
         self.search_title = self.title
         self.search_headings = ""

src/extended_search/index.py

@@ -25,6 +25,31 @@ def _check_search_fields(cls, **kwargs):
                         id="wagtailsearch.W004",
                     )
                 )
+
+        parent_fields = []
+        for parent_cls in cls.__bases__:
+            parent_fields += getattr(parent_cls, "search_fields", [])
+        model_fields = []
+        for field in cls.get_search_fields():
+            model_field_name = getattr(field, "model_field_name", None)
+            if not model_field_name:
+                model_field_name = field.field_name
+            if field not in parent_fields and model_field_name not in model_fields:
+                message = "indexed field '{name}' is defined in {model} and {parent}"
+                definition_model = field.get_definition_model(cls)
+                if definition_model != cls:
+                    errors.append(
+                        checks.Warning(
+                            message.format(
+                                model=cls.__name__,
+                                name=field.field_name,
+                                parent=definition_model.__name__,
+                            ),
+                            obj=cls,
+                            id="extended_search.E001",
+                        )
+                    )
+                    model_fields.append(model_field_name)
         return errors
 
     search_fields = []

src/search/search.py

@@ -1,8 +1,8 @@
 from content.models import ContentPage, ContentPageIndexManager
 from extended_search.managers import get_search_query
-from news.models import NewsPage
+from news.models import NewsPage, NewsPageIndexManager
 from peoplefinder.models import Person, PersonIndexManager, Team, TeamIndexManager
-from tools.models import Tool
+from tools.models import Tool, ToolIndexManager
 from working_at_dit.models import PoliciesAndGuidanceHome
 
 
@@ -35,24 +35,35 @@ def pinned(self, query):
 
 class PagesSearchVector(SearchVector):
     page_model = None
+    page_index_manager = None
 
     def get_queryset(self):
         return self.page_model.objects.public_or_login().live()
 
+    def get_query(self, query_str):
+        return get_search_query(
+            self.page_index_manager,
+            query_str,
+            self.page_model,
+        )
+
     def pinned(self, query):
         return self.get_queryset().pinned(query)
 
     def search(self, query, *args, **kwargs):
         queryset = self.get_queryset().not_pinned(query)
+        query = self.get_query(query)
         return self._wagtail_search(queryset, query, *args, **kwargs)
 
 
 class AllPagesSearchVector(PagesSearchVector):
     page_model = ContentPage
+    page_index_manager = ContentPageIndexManager
 
 
 class GuidanceSearchVector(PagesSearchVector):
     page_model = ContentPage
+    page_index_manager = ContentPageIndexManager
 
     def get_queryset(self):
         policies_and_guidance_home = PoliciesAndGuidanceHome.objects.first()
@@ -62,10 +73,12 @@ def get_queryset(self):
 
 class NewsSearchVector(PagesSearchVector):
     page_model = NewsPage
+    page_index_manager = NewsPageIndexManager
 
 
 class ToolsSearchVector(PagesSearchVector):
     page_model = Tool
+    page_index_manager = ToolIndexManager
 
 
 class PeopleSearchVector(SearchVector):
@@ -81,63 +94,20 @@ def get_queryset(self):
 
     def search(self, query, *args, **kwargs):
         queryset = self.get_queryset()
+        query = get_search_query(
+            PersonIndexManager,
+            query,
+            Person,
+            *args,
+            **kwargs,
+        )
         return self._wagtail_search(queryset, query, *args, **kwargs)
 
 
 class TeamsSearchVector(SearchVector):
     def get_queryset(self):
         return Team.objects.all().with_all_parents()
 
-
-#
-# New vectors for complex search alongside v2 - should get rolled in at end of
-# the indexing improvements workstream. Need to be alongside to run v2 and v2.5
-# queries side by side, e.g. for "explore" page
-#
-
-
-class NewAllPagesSearchVector(AllPagesSearchVector):
-    def _wagtail_search(self, queryset, query, *args, **kwargs):
-        return queryset.search(query, *args, **kwargs).annotate_score("_score")
-
-    def search(self, query, *args, **kwargs):
-        query = get_search_query(
-            ContentPageIndexManager, query, ContentPage, *args, **kwargs
-        )
-        return self._wagtail_search(self.get_queryset(), query, *args, **kwargs)
-
-
-class NewGuidanceSearchVector(GuidanceSearchVector):
-    ...
-
-
-class NewNewsSearchVector(NewsSearchVector):
-    ...
-
-
-class NewToolsSearchVector(ToolsSearchVector):
-    ...
-
-
-class NewPeopleSearchVector(PeopleSearchVector):
-    def _wagtail_search(self, queryset, query, *args, **kwargs):
-        return queryset.search(query, *args, **kwargs).annotate_score("_score")
-
-    def search(self, query, *args, **kwargs):
-        queryset = self.get_queryset()
-        query_obj = get_search_query(PersonIndexManager, query, Person, *args, **kwargs)
-        results = set(self._wagtail_search(queryset, query_obj, *args, **kwargs))
-        autocomplete_results = set(
-            self.get_queryset().autocomplete(query).annotate_score("_score")
-        )
-        all_results = results | autocomplete_results
-        return sorted(all_results, key=lambda x: x._score, reverse=True)
-
-
-class NewTeamsSearchVector(TeamsSearchVector):
-    def _wagtail_search(self, queryset, query, *args, **kwargs):
-        return queryset.search(query, *args, **kwargs).annotate_score("_score")
-
     def search(self, query, *args, **kwargs):
         queryset = self.get_queryset()
         query = get_search_query(TeamIndexManager, query, Team, *args, **kwargs)

src/search/templates/search/search.html

@@ -17,7 +17,7 @@
     <script lang="javascript">
         document.onload = dataLayer.push({
             'event': 'search',
-            'search_version': 'v2-2023.04-1',
+            'search_version': 'v2-2023.08-01',
             'search_query': '{{ search_query }}',
             'search_results_page_type': '{{ search_category }}',
             'search_result_selected_page': '{{ page }}'

src/search/templatetags/search_explore.py

@@ -11,12 +11,12 @@
 
 
 SEARCH_VECTORS: dict[str, search_vectors.SearchVector] = {
-    "all_pages": search_vectors.NewAllPagesSearchVector,
-    "people": search_vectors.NewPeopleSearchVector,
-    "teams": search_vectors.NewTeamsSearchVector,
-    "guidance": search_vectors.NewGuidanceSearchVector,
-    "tools": search_vectors.NewToolsSearchVector,
-    "news": search_vectors.NewNewsSearchVector,
+    "all_pages": search_vectors.AllPagesSearchVector,
+    "people": search_vectors.PeopleSearchVector,
+    "teams": search_vectors.TeamsSearchVector,
+    "guidance": search_vectors.GuidanceSearchVector,
+    "tools": search_vectors.ToolsSearchVector,
+    "news": search_vectors.NewsSearchVector,
 }
 
 

src/tools/models.py

@@ -5,8 +5,8 @@
 from wagtail.admin.panels import FieldPanel
 
 from content.models import ContentPage
-from extended_search.fields import IndexedField
 from extended_search.managers.index import ModelIndexManager
+from extended_search.fields import IndexedField
 from working_at_dit.models import PageWithTopics
 
 
@@ -74,7 +74,7 @@ def __str__(self):
 class ToolIndexManager(ModelIndexManager):
     fields = [
         IndexedField(
-            "search_title",
+            "search_tool_name",
             fuzzy=True,
             tokenized=True,
             explicit=True,
@@ -107,6 +107,10 @@ class Tool(PageWithTopics):
 
     search_fields = PageWithTopics.search_fields + ToolIndexManager()
 
+    @property
+    def search_tool_name(self):
+        return self.title
+
     parent_page_types = ["tools.ToolsHome"]
     subpage_types = []  # Should not be able to create children