Improve Search Relevancy for tier, usage and use stemming to generate…

… ngrams
open-metadata · Sep 17, 2024 · 157963e · 157963e
1 parent a1a8d93
commit 157963e
Show file tree

Hide file tree

Showing 45 changed files with 824 additions and 433 deletions.
diff --git a/...adata-service/src/main/java/org/openmetadata/service/resources/search/SearchResource.java b/...adata-service/src/main/java/org/openmetadata/service/resources/search/SearchResource.java
@@ -171,7 +171,13 @@ public Response search(
                   "Fetch search results in hierarchical order of children elements. By default hierarchy is not fetched. Currently only supported for glossary_term_search_index.")
           @DefaultValue("false")
           @QueryParam("getHierarchy")
-          boolean getHierarchy)
+          boolean getHierarchy,
+      @Parameter(
+              description =
+                  "Explain the results of the query. Defaults to false. Only for debugging purposes.")
+          @DefaultValue("false")
+          @QueryParam("explain")
+          boolean explain)
       throws IOException {
 
     if (nullOrEmpty(query)) {
@@ -202,6 +208,7 @@ public Response search(
             .applyDomainFilter(
                 !subjectContext.isAdmin() && subjectContext.hasAnyRole(DOMAIN_ONLY_ACCESS_ROLE))
             .searchAfter(searchAfter)
+            .explain(explain)
             .build();
     return searchRepository.search(request, subjectContext);
   }

diff --git a/...metadata-service/src/main/java/org/openmetadata/service/search/EntityBuilderConstant.java b/...metadata-service/src/main/java/org/openmetadata/service/search/EntityBuilderConstant.java
@@ -9,6 +9,9 @@ public class EntityBuilderConstant {
   public static final String ES_MESSAGE_SCHEMA_FIELD = "messageSchema.schemaFields.name";
   public static final String ES_MESSAGE_SCHEMA_FIELD_KEYWORD =
       "messageSchema.schemaFields.name.keyword";
+  public static final String API_RESPONSE_SCHEMA_FIELD = "responseSchema.schemaFields.name";
+  public static final String API_RESPONSE_SCHEMA_FIELD_KEYWORD =
+      "responseSchema.schemaFields.name.keyword";
   public static final String ES_TAG_FQN_FIELD = "tags.tagFQN";
 
   public static final String COLUMNS_NAME_KEYWORD = "columns.name.keyword";

diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/search/SearchRequest.java b/openmetadata-service/src/main/java/org/openmetadata/service/search/SearchRequest.java
@@ -29,6 +29,7 @@ public class SearchRequest {
   private final List<String> domains;
   private final boolean getHierarchy;
   private final Object[] searchAfter;
+  private final boolean explain;
 
   public SearchRequest(ElasticSearchRequestBuilder builder) {
     this.query = builder.query;
@@ -48,6 +49,7 @@ public SearchRequest(ElasticSearchRequestBuilder builder) {
     this.domains = builder.domains;
     this.applyDomainFilter = builder.applyDomainFilter;
     this.searchAfter = builder.searchAfter;
+    this.explain = builder.explain;
   }
 
   // Builder class for ElasticSearchRequest
@@ -70,6 +72,7 @@ public static class ElasticSearchRequestBuilder {
     private boolean applyDomainFilter;
     private List<String> domains;
     private Object[] searchAfter;
+    private boolean explain;
 
     public ElasticSearchRequestBuilder(String query, int size, String index) {
       this.query = query;
@@ -153,6 +156,11 @@ public ElasticSearchRequestBuilder searchAfter(String searchAfter) {
       return this;
     }
 
+    public ElasticSearchRequestBuilder explain(boolean explain) {
+      this.explain = explain;
+      return this;
+    }
+
     public SearchRequest build() {
       return new SearchRequest(this);
     }

diff --git a/...vice/src/main/java/org/openmetadata/service/search/elasticsearch/ElasticSearchClient.java b/...vice/src/main/java/org/openmetadata/service/search/elasticsearch/ElasticSearchClient.java
diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/SearchIndex.java b/openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/SearchIndex.java
@@ -5,7 +5,6 @@
 import static org.openmetadata.service.Entity.FIELD_DISPLAY_NAME;
 import static org.openmetadata.service.Entity.FIELD_NAME;
 import static org.openmetadata.service.jdbi3.LineageRepository.buildRelationshipDetailsMap;
-import static org.openmetadata.service.search.EntityBuilderConstant.DISPLAY_NAME_KEYWORD;
 import static org.openmetadata.service.search.EntityBuilderConstant.FIELD_DISPLAY_NAME_NGRAM;
 import static org.openmetadata.service.search.EntityBuilderConstant.FIELD_NAME_NGRAM;
 import static org.openmetadata.service.search.EntityBuilderConstant.FULLY_QUALIFIED_NAME;
@@ -167,9 +166,8 @@ static Map<String, Float> getDefaultFields() {
     fields.put(FIELD_DISPLAY_NAME, 10.0f);
     fields.put(FIELD_DISPLAY_NAME_NGRAM, 1.0f);
     fields.put(FIELD_NAME, 10.0f);
+    fields.put(NAME_KEYWORD, 10.0f);
     fields.put(FIELD_NAME_NGRAM, 1.0f);
-    fields.put(DISPLAY_NAME_KEYWORD, 8.0f);
-    fields.put(NAME_KEYWORD, 8.0f);
     fields.put(FIELD_DESCRIPTION, 2.0f);
     fields.put(FULLY_QUALIFIED_NAME, 5.0f);
     fields.put(FULLY_QUALIFIED_NAME_PARTS, 5.0f);

diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/TableIndex.java b/openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/TableIndex.java
@@ -91,8 +91,7 @@ public Map<String, Object> buildSearchIndexDocInternal(Map<String, Object> doc)
             .collect(ArrayList::new, ArrayList::add, ArrayList::addAll);
     Map<String, Object> commonAttributes = getCommonAttributesMap(table, Entity.TABLE);
     doc.putAll(commonAttributes);
-    doc.put(
-        "displayName", table.getDisplayName() != null ? table.getDisplayName() : table.getName());
+    doc.put("displayName", table.getDisplayName());
     doc.put("tags", flattenedTagList);
     doc.put("tier", parseTags.getTierTag());
     doc.put("service_suggest", serviceSuggest);
@@ -113,7 +112,7 @@ public static Map<String, Float> getFields() {
     fields.put("columns.name", 5.0f);
     fields.put("columns.displayName", 5.0f);
     fields.put("columns.description", 2.0f);
-    fields.put("columns.children.name", 5.0f);
+    fields.put("columns.children.name", 3.0f);
     return fields;
   }
 }