Fixed some issues with admin notes

doajtest/testbook/administrative_search/journals.yml

@@ -153,3 +153,47 @@ tests:
   - step: Search for the journal from api search with same words given in the note
     results:
       - The journal record is not displayed in the search results
+
+- title: Test notes not searchable for public
+  context:
+    role: anonymous
+  steps:
+  - step: Search for the keyword which is given in a notes.
+    results: No results displayed
+  - step: Modify the search url to add the notes keyword and the default field 'admin.notes.note' and then hit the url
+    results: Error message displayed
+
+- title: Test notes searchable for Editor
+  context:
+    role: Editor
+  steps:
+  - step: Make sure the editor group is assigned for the journal / application which is under test and assigned to the user
+  - step: Go to the tab 'Your group’s journals'
+    results: Group's journals page displayed
+  - step: Search for the keyword which is given in a notes in a journal
+    results: The journal record is displayed in the search results
+  - step: Go to the tab 'Your group’s applications'
+    results: Group's applications page displayed
+  - step: Search for the keyword which is given in a notes in an application
+    results: The application is displayed in the search results
+  - step: Go to the tab 'Journals assigned to you'
+    results: Journals assigned to you page displayed
+  - step: Search for the keyword which is given in a notes in a journal
+    results: The journal record is displayed in the search results
+  - step: Go to the tab 'Applications assigned to you'
+    results: Applications assigned to you page displayed
+  - step: Search for the keyword which is given in a notes in an application
+    results: The application is displayed in the search results
+
+- title: Test notes searchable for Associate Editor
+  context:
+    role: AssociateEditor
+  steps:
+  - step: Go to the tab 'Journals assigned to you'
+    results: 'Journals assigned to you page' displayed
+  - step: Search for the keyword which is given in a notes in a journal
+    results: The journal record is displayed in the search results
+  - step: Go to the tab 'Applications assigned to you'
+    results: 'Applications assigned to you' page displayed
+  - step: Search for the keyword which is given in a notes in an application
+    results: The application is displayed in the search results

portality/bll/services/query.py

@@ -43,18 +43,20 @@ def _get_config_for_search(self, domain, index_type, account):
         return cfg
 
     def _validate_query(self, cfg, query):
-        validator = cfg.get("query_validator")
-        if validator is None:
-            return True
+        validators = cfg.get("query_validators")
+        if validators:
+            for validator in validators:
+                filters = app.config.get("QUERY_FILTERS", {})
+                validator_path = filters.get(validator)
+                fn = plugin.load_function(validator_path)
+                if fn is None:
+                    msg = "Unable to load query validator for {x}".format(x=validator)
+                    raise exceptions.ConfigurationException(msg)
 
-        filters = app.config.get("QUERY_FILTERS", {})
-        validator_path = filters.get(validator)
-        fn = plugin.load_function(validator_path)
-        if fn is None:
-            msg = "Unable to load query validator for {x}".format(x=validator)
-            raise exceptions.ConfigurationException(msg)
+                if not fn(query):
+                    return False
+        return True
 
-        return fn(query)
 
     def _pre_filter_search_query(self, cfg, query):
         # now run the query through the filters
@@ -197,15 +199,20 @@ def add_must(self, filter):
             context["must"] = []
         context["must"].append(filter)
 
-    def add_default_field(self, value: str):
-        """ Add a default field to the query string, if one is not already present"""
+    def get_field_context(self):
+        """Get query string context"""
         context = None
         if "query_string" in self.q["query"]:
             context = self.q["query"]["query_string"]
 
         elif "bool" in self.q["query"]:
             if "must" in self.q["query"]["bool"]:
                 context = self.q["query"]["bool"]["must"]
+        return context
+
+    def add_default_field(self, value: str):
+        """ Add a default field to the query string, if one is not already present"""
+        context = self.get_field_context()
 
         if context:
             if isinstance(context, dict):
@@ -216,6 +223,7 @@ def add_default_field(self, value: str):
                     if "query_string" in item:
                         if "default_field" not in item["query_string"]:
                             item["query_string"]["default_field"] = value
+                            break
 
     def add_must_filter(self, filter):
         self.convert_to_bool()

portality/lib/query_filters.py

@@ -29,22 +29,35 @@ def public_query_validator(q):
     return True
 
 
+def non_public_fields_validator(q):
+    exclude_fields = app.config.get("EXCLUDED_FIELDS", {})
+    context = q.get_field_context()
+    if context:
+        if "default_field" in context:
+            field_value = context["default_field"]
+            if field_value in exclude_fields:
+                return False
+    return True
+
 # query filters
 ###############
 
 def remove_search_limits(query: dict):
     return remove_fields(query, ['size', 'from'])
 
+
 def only_in_doaj(q):
     q.clear_match_all()
     q.add_must_filter({"term": {"admin.in_doaj": True}})
     return q
 
+
 def search_all_meta(q):
     """Search by all_meta field, which is a concatenation of all the fields in the record"""
     q.add_default_field("all_meta")
     return q
 
+
 def owner(q):
     q.clear_match_all()
     q.add_must_filter({"term" : {"admin.owner.exact" : current_user.id}})

portality/models/v1/journal.py

@@ -1282,32 +1282,10 @@ def get_preferred_issn(self):
     "dynamic": None,
     "coerces": app.config["DATAOBJ_TO_MAPPING_DEFAULTS"],
     "exceptions": {
-        "admin.notes.id": {
-                "type": "text",
-                "fields": {
-                    "exact": {
-                        "type": "keyword",
-                        "store": True
-                    }
-                }
-        },
         "admin.notes.note": {
-                "type": "text",
-                "fields": {
-                    "exact": {
-                        "type": "keyword",
-                        "store": True
-                    }
-                }
-        },
-        "admin.notes.author_id": {
-                "type": "text",
-                "fields": {
-                    "exact": {
-                        "type": "keyword",
-                        "store": True
-                    }
-                }
+            "type": "text",
+            "index": False,
+            #"include_in_all": False        # Removed in es6 fixme: do we need to look at copy_to for the mapping?
         }
     }
 }

portality/models/v2/application.py

@@ -244,36 +244,8 @@ class AllPublisherApplications(DomainObject):
 
 MAPPING_OPTS = {
     "dynamic": None,
-    "coerces": app.config["DATAOBJ_TO_MAPPING_DEFAULTS"],
-    "exceptions": {
-        "admin.notes.id": {
-                "type": "text",
-                "fields": {
-                    "exact": {
-                        "type": "keyword",
-                        "store": True
-                    }
-                }
-        },
-        "admin.notes.note": {
-                "type": "text",
-                "fields": {
-                    "exact": {
-                        "type": "keyword",
-                        "store": True
-                    }
-                }
-        },
-        "admin.notes.author_id": {
-                "type": "text",
-                "fields": {
-                    "exact": {
-                        "type": "keyword",
-                        "store": True
-                    }
-                }
-        }
-    }
+    "coerces": Journal.add_mapping_extensions(app.config["DATAOBJ_TO_MAPPING_DEFAULTS"]),
+    "exceptions": app.config["ADMIN_NOTES_SEARCH_MAPPING"]
 }
 
 

portality/models/v2/journal.py

@@ -564,6 +564,15 @@ def delete_selected(cls, query, articles=False, snapshot_journals=True, snapshot
         # finally issue a delete request against the journals
         cls.delete_by_query(query)
 
+    @classmethod
+    def add_mapping_extensions(cls, default_mappings: dict):
+        default_mappings_copy = deepcopy(default_mappings)
+        mapping_extensions = app.config.get("DATAOBJ_TO_MAPPING_EXTENSIONS")
+        for key, value in mapping_extensions.items():
+            if key in default_mappings_copy:
+                default_mappings_copy[key] = {**default_mappings_copy[key], **value}
+        return default_mappings_copy
+
     def all_articles(self):
         from portality.models import Article
         return Article.find_by_issns(self.known_issns())
@@ -891,36 +900,8 @@ def _calculate_has_apc(self):
 
 MAPPING_OPTS = {
     "dynamic": None,
-    "coerces": app.config["DATAOBJ_TO_MAPPING_DEFAULTS"],
-    "exceptions": {
-        "admin.notes.id": {
-                "type": "text",
-                "fields": {
-                    "exact": {
-                        "type": "keyword",
-                        "store": True
-                    }
-                }
-        },
-        "admin.notes.note": {
-                "type": "text",
-                "fields": {
-                    "exact": {
-                        "type": "keyword",
-                        "store": True
-                    }
-                }
-        },
-        "admin.notes.author_id": {
-                "type": "text",
-                "fields": {
-                    "exact": {
-                        "type": "keyword",
-                        "store": True
-                    }
-                }
-        }
-    }
+    "coerces": Journal.add_mapping_extensions(app.config["DATAOBJ_TO_MAPPING_DEFAULTS"]),
+    "exceptions": app.config["ADMIN_NOTES_SEARCH_MAPPING"]
 }
 
 

portality/models/v2/shared_structs.py

@@ -175,8 +175,7 @@
         "created_date" : {"coerce" : "utcdatetime"},
         "last_updated" : {"coerce" : "utcdatetime"},
         "last_manual_update" : {"coerce" : "utcdatetime"},
-        "es_type": {"coerce": "unicode"},
-        "all_meta" : {"coerce" : "unicode"}
+        "es_type": {"coerce": "unicode"}
     },
     "objects" : [
         "admin",