Fix es name match (#13346)

* Fix exact match in search results * Fix pyyaml version * Failing EntityRepo fix --------- Co-authored-by: Pere Miquel Brull <peremiquelbrull@gmail.com> Co-authored-by: mohitdeuex <mohit.y@deuexsolutions.com>
open-metadata · Sep 27, 2023 · f2c88ff · f2c88ff
1 parent 81f273e
commit f2c88ff
Show file tree

Hide file tree

Showing 51 changed files with 228 additions and 100 deletions.
diff --git a/...ta-service/src/main/java/org/openmetadata/service/migration/utils/V114/MigrationUtil.java b/...ta-service/src/main/java/org/openmetadata/service/migration/utils/V114/MigrationUtil.java
@@ -142,7 +142,7 @@ private static TestSuite getOrCreateExecutableTestSuite(
                   "ingestion-bot")
               .withExecutable(true)
               .withFullyQualifiedName(executableTestSuiteFQN);
-      testSuiteRepository.prepareInternal(newExecutableTestSuite, false);
+      testSuiteRepository.prepareInternal(newExecutableTestSuite);
       testSuiteRepository
           .getDao()
           .insert("fqnHash", newExecutableTestSuite, newExecutableTestSuite.getFullyQualifiedName());

diff --git a/...metadata-service/src/main/java/org/openmetadata/service/search/EntityBuilderConstant.java b/...metadata-service/src/main/java/org/openmetadata/service/search/EntityBuilderConstant.java
@@ -17,6 +17,8 @@ public class EntityBuilderConstant {
   public static final String DISPLAY_NAME_KEYWORD = "displayName.keyword";
   public static final String FIELD_DISPLAY_NAME_NGRAM = "displayName.ngram";
   public static final String FIELD_DESCRIPTION_NGRAM = "description.ngram";
+
+  public static final String FULLY_QUALIFIED_NAME_PARTS = "fqnParts";
   public static final String PRE_TAG = "<span class=\"text-highlighter\">";
   public static final String POST_TAG = "</span>";
   public static final Integer MAX_AGGREGATE_SIZE = 50;

diff --git a/.../src/main/java/org/openmetadata/service/search/elasticSearch/ElasticSearchClientImpl.java b/.../src/main/java/org/openmetadata/service/search/elasticSearch/ElasticSearchClientImpl.java
@@ -16,9 +16,9 @@
 import static org.openmetadata.service.search.EntityBuilderConstant.DISPLAY_NAME_KEYWORD;
 import static org.openmetadata.service.search.EntityBuilderConstant.ES_MESSAGE_SCHEMA_FIELD;
 import static org.openmetadata.service.search.EntityBuilderConstant.ES_TAG_FQN_FIELD;
-import static org.openmetadata.service.search.EntityBuilderConstant.FIELD_DESCRIPTION_NGRAM;
 import static org.openmetadata.service.search.EntityBuilderConstant.FIELD_DISPLAY_NAME_NGRAM;
 import static org.openmetadata.service.search.EntityBuilderConstant.FIELD_NAME_NGRAM;
+import static org.openmetadata.service.search.EntityBuilderConstant.FULLY_QUALIFIED_NAME_PARTS;
 import static org.openmetadata.service.search.EntityBuilderConstant.MAX_AGGREGATE_SIZE;
 import static org.openmetadata.service.search.EntityBuilderConstant.MAX_RESULT_HITS;
 import static org.openmetadata.service.search.EntityBuilderConstant.NAME_KEYWORD;
@@ -479,9 +479,9 @@ private static SearchSourceBuilder buildPipelineSearchBuilder(String query, int
             .field(FIELD_DISPLAY_NAME, 15.0f)
             .field(FIELD_DISPLAY_NAME_NGRAM)
             .field(FIELD_NAME, 15.0f)
-            .field(FIELD_DESCRIPTION_NGRAM, 1.0f)
             .field(DISPLAY_NAME_KEYWORD, 25.0f)
             .field(NAME_KEYWORD, 25.0f)
+            .field(FULLY_QUALIFIED_NAME_PARTS, 10.0f)
             .field(FIELD_DESCRIPTION, 1.0f)
             .field("tasks.name", 2.0f)
             .field("tasks.description", 1.0f)
@@ -512,8 +512,8 @@ private static SearchSourceBuilder buildMlModelSearchBuilder(String query, int f
             .field(FIELD_DISPLAY_NAME, 15.0f)
             .field(FIELD_DISPLAY_NAME_NGRAM)
             .field(FIELD_NAME, 15.0f)
-            .field(FIELD_DESCRIPTION_NGRAM, 1.0f)
             .field(DISPLAY_NAME_KEYWORD, 25.0f)
+            .field(FULLY_QUALIFIED_NAME_PARTS, 10.0f)
             .field(NAME_KEYWORD, 25.0f)
             .field(FIELD_DESCRIPTION, 1.0f)
             .field("mlFeatures.name", 2.0f)
@@ -544,7 +544,7 @@ private static SearchSourceBuilder buildTopicSearchBuilder(String query, int fro
             .field(FIELD_DISPLAY_NAME_NGRAM)
             .field(FIELD_NAME, 15.0f)
             .field(FIELD_NAME_NGRAM)
-            .field(FIELD_DESCRIPTION_NGRAM, 1.0f)
+            .field(FULLY_QUALIFIED_NAME_PARTS, 10.0f)
             .field(DISPLAY_NAME_KEYWORD, 25.0f)
             .field(NAME_KEYWORD, 25.0f)
             .field(FIELD_DESCRIPTION, 1.0f)
@@ -574,8 +574,8 @@ private static SearchSourceBuilder buildDashboardSearchBuilder(String query, int
             .field(FIELD_DISPLAY_NAME_NGRAM)
             .field(FIELD_NAME, 15.0f)
             .field(FIELD_NAME_NGRAM)
-            .field(FIELD_DESCRIPTION_NGRAM, 1.0f)
             .field(DISPLAY_NAME_KEYWORD, 25.0f)
+            .field(FULLY_QUALIFIED_NAME_PARTS, 10.0f)
             .field(NAME_KEYWORD, 25.0f)
             .field(FIELD_DESCRIPTION, 1.0f)
             .field("charts.name", 2.0f)
@@ -614,8 +614,8 @@ private static SearchSourceBuilder buildTableSearchBuilder(String query, int fro
             .field(FIELD_NAME_NGRAM)
             .field(DISPLAY_NAME_KEYWORD, 25.0f)
             .field(NAME_KEYWORD, 25.0f)
+            .field(FULLY_QUALIFIED_NAME_PARTS, 10.0f)
             .field(FIELD_DESCRIPTION, 1.0f)
-            .field(FIELD_DESCRIPTION_NGRAM, 1.0f)
             .field(COLUMNS_NAME_KEYWORD, 10.0f)
             .field("columns.name", 2.0f)
             .field("columns.name.ngram")
@@ -667,6 +667,7 @@ private static SearchSourceBuilder buildUserOrTeamSearchBuilder(String query, in
             .field(FIELD_DISPLAY_NAME, 3.0f)
             .field(DISPLAY_NAME_KEYWORD, 5.0f)
             .field(FIELD_DISPLAY_NAME_NGRAM)
+            .field(FULLY_QUALIFIED_NAME_PARTS, 10.0f)
             .field(FIELD_NAME, 2.0f)
             .field(NAME_KEYWORD, 3.0f)
             .defaultOperator(Operator.AND)
@@ -682,8 +683,7 @@ private static SearchSourceBuilder buildGlossaryTermSearchBuilder(String query,
             .field(FIELD_NAME, 10.0f)
             .field(NAME_KEYWORD, 10.0f)
             .field(DISPLAY_NAME_KEYWORD, 10.0f)
-            .field(FIELD_DISPLAY_NAME, 10.0f)
-            .field(FIELD_DISPLAY_NAME_NGRAM)
+            .field(FULLY_QUALIFIED_NAME_PARTS, 10.0f)
             .field("synonyms", 5.0f)
             .field("synonyms.ngram")
             .field(FIELD_DESCRIPTION, 3.0f)
@@ -724,6 +724,7 @@ private static SearchSourceBuilder buildTagSearchBuilder(String query, int from,
             .field(FIELD_NAME, 10.0f)
             .field(FIELD_DISPLAY_NAME, 10.0f)
             .field(FIELD_NAME_NGRAM, 1.0f)
+            .field(FULLY_QUALIFIED_NAME_PARTS, 10.0f)
             .field("classification.name", 1.0f)
             .field(FIELD_DESCRIPTION, 3.0f)
             .defaultOperator(Operator.AND)
@@ -752,8 +753,8 @@ private static SearchSourceBuilder buildContainerSearchBuilder(String query, int
             .field(FIELD_DISPLAY_NAME_NGRAM)
             .field(FIELD_NAME, 15.0f)
             .field(FIELD_DESCRIPTION, 1.0f)
-            .field(FIELD_DESCRIPTION_NGRAM, 1.0f)
             .field(DISPLAY_NAME_KEYWORD, 25.0f)
+            .field(FULLY_QUALIFIED_NAME_PARTS, 10.0f)
             .field(NAME_KEYWORD, 25.0f)
             .field("dataModel.columns.name", 2.0f)
             .field(DATA_MODEL_COLUMNS_NAME_KEYWORD, 10.0f)
@@ -797,7 +798,6 @@ private static SearchSourceBuilder buildQuerySearchBuilder(String query, int fro
             .field(QUERY, 10.0f)
             .field(QUERY_NGRAM)
             .field(FIELD_DESCRIPTION, 1.0f)
-            .field(FIELD_DESCRIPTION_NGRAM, 1.0f)
             .defaultOperator(Operator.AND)
             .fuzziness(Fuzziness.AUTO);
 

diff --git a/...etadata-service/src/main/java/org/openmetadata/service/search/indexes/ContainerIndex.java b/...etadata-service/src/main/java/org/openmetadata/service/search/indexes/ContainerIndex.java
@@ -3,6 +3,7 @@
 import java.util.ArrayList;
 import java.util.List;
 import java.util.Map;
+import java.util.stream.Collectors;
 import org.openmetadata.common.utils.CommonUtil;
 import org.openmetadata.schema.entity.data.Container;
 import org.openmetadata.schema.type.EntityReference;
@@ -55,6 +56,11 @@ public Map<String, Object> buildESDoc() {
     doc.put("column_suggest", columnSuggest);
     doc.put("entityType", Entity.CONTAINER);
     doc.put("serviceType", container.getServiceType());
+    doc.put(
+        "fqnParts",
+        getFQNParts(
+            container.getFullyQualifiedName(),
+            suggest.stream().map(SearchSuggest::getInput).collect(Collectors.toList())));
     return doc;
   }
 }
diff --git a/...etadata-service/src/main/java/org/openmetadata/service/search/indexes/DashboardIndex.java b/...etadata-service/src/main/java/org/openmetadata/service/search/indexes/DashboardIndex.java
@@ -5,6 +5,7 @@
 import java.util.ArrayList;
 import java.util.List;
 import java.util.Map;
+import java.util.stream.Collectors;
 import org.openmetadata.common.utils.CommonUtil;
 import org.openmetadata.schema.entity.data.Dashboard;
 import org.openmetadata.schema.type.EntityReference;
@@ -58,6 +59,7 @@ public Map<String, Object> buildESDoc() {
     doc.put("service_suggest", serviceSuggest);
     doc.put("entityType", Entity.DASHBOARD);
     doc.put("serviceType", dashboard.getServiceType());
+    doc.put("fqnParts", suggest.stream().map(SearchSuggest::getInput).collect(Collectors.toList()));
     return doc;
   }
 }
diff --git a/...ata-service/src/main/java/org/openmetadata/service/search/indexes/ElasticSearchIndex.java b/...ata-service/src/main/java/org/openmetadata/service/search/indexes/ElasticSearchIndex.java
@@ -1,7 +1,23 @@
 package org.openmetadata.service.search.indexes;
 
+import java.util.HashSet;
+import java.util.List;
 import java.util.Map;
+import java.util.Set;
+import org.openmetadata.service.util.FullyQualifiedName;
 
 public interface ElasticSearchIndex {
   Map<String, Object> buildESDoc();
+
+  default Set<String> getFQNParts(String fqn, List<String> fqnSplits) {
+    Set<String> fqnParts = new HashSet<>();
+    fqnParts.add(fqn);
+    String parent = FullyQualifiedName.getParentFQN(fqn);
+    while (parent != null) {
+      fqnParts.add(parent);
+      parent = FullyQualifiedName.getParentFQN(parent);
+    }
+    fqnParts.addAll(fqnSplits);
+    return fqnParts;
+  }
 }
diff --git a/...data-service/src/main/java/org/openmetadata/service/search/indexes/GlossaryTermIndex.java b/...data-service/src/main/java/org/openmetadata/service/search/indexes/GlossaryTermIndex.java
@@ -3,6 +3,7 @@
 import java.util.ArrayList;
 import java.util.List;
 import java.util.Map;
+import java.util.stream.Collectors;
 import org.openmetadata.common.utils.CommonUtil;
 import org.openmetadata.schema.entity.data.GlossaryTerm;
 import org.openmetadata.schema.type.EntityReference;
@@ -32,6 +33,11 @@ public Map<String, Object> buildESDoc() {
     suggest.add(SearchSuggest.builder().input(glossaryTerm.getDisplayName()).weight(10).build());
     doc.put("suggest", suggest);
     doc.put("entityType", Entity.GLOSSARY_TERM);
+    doc.put(
+        "fqnParts",
+        getFQNParts(
+            glossaryTerm.getFullyQualifiedName(),
+            suggest.stream().map(SearchSuggest::getInput).collect(Collectors.toList())));
     return doc;
   }
 }
diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/MlModelIndex.java b/openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/MlModelIndex.java
@@ -3,6 +3,7 @@
 import java.util.ArrayList;
 import java.util.List;
 import java.util.Map;
+import java.util.stream.Collectors;
 import org.openmetadata.common.utils.CommonUtil;
 import org.openmetadata.schema.entity.data.MlModel;
 import org.openmetadata.schema.type.EntityReference;
@@ -40,6 +41,11 @@ public Map<String, Object> buildESDoc() {
     doc.put("suggest", suggest);
     doc.put("entityType", Entity.MLMODEL);
     doc.put("serviceType", mlModel.getServiceType());
+    doc.put(
+        "fqnParts",
+        getFQNParts(
+            mlModel.getFullyQualifiedName(),
+            suggest.stream().map(SearchSuggest::getInput).collect(Collectors.toList())));
     return doc;
   }
 }
diff --git a/...metadata-service/src/main/java/org/openmetadata/service/search/indexes/PipelineIndex.java b/...metadata-service/src/main/java/org/openmetadata/service/search/indexes/PipelineIndex.java
@@ -3,6 +3,7 @@
 import java.util.ArrayList;
 import java.util.List;
 import java.util.Map;
+import java.util.stream.Collectors;
 import org.openmetadata.common.utils.CommonUtil;
 import org.openmetadata.schema.entity.data.Pipeline;
 import org.openmetadata.schema.type.EntityReference;
@@ -51,6 +52,11 @@ public Map<String, Object> buildESDoc() {
     doc.put("service_suggest", serviceSuggest);
     doc.put("entityType", Entity.PIPELINE);
     doc.put("serviceType", pipeline.getServiceType());
+    doc.put(
+        "fqnParts",
+        getFQNParts(
+            pipeline.getFullyQualifiedName(),
+            suggest.stream().map(SearchSuggest::getInput).collect(Collectors.toList())));
     return doc;
   }
 }
diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/QueryIndex.java b/openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/QueryIndex.java
@@ -3,6 +3,7 @@
 import java.util.ArrayList;
 import java.util.List;
 import java.util.Map;
+import java.util.stream.Collectors;
 import org.openmetadata.schema.entity.data.Query;
 import org.openmetadata.service.Entity;
 import org.openmetadata.service.search.ParseTags;
@@ -33,6 +34,10 @@ public Map<String, Object> buildESDoc() {
     doc.put("followers", SearchIndexUtils.parseFollowers(query.getFollowers()));
     doc.put("suggest", suggest);
     doc.put("entityType", Entity.QUERY);
+    doc.put(
+        "fqnParts",
+        getFQNParts(
+            query.getFullyQualifiedName(), suggest.stream().map(SearchSuggest::getInput).collect(Collectors.toList())));
     return doc;
   }
 }
diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/TableIndex.java b/openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/TableIndex.java
@@ -4,6 +4,7 @@
 import java.util.List;
 import java.util.Map;
 import java.util.regex.Pattern;
+import java.util.stream.Collectors;
 import org.openmetadata.common.utils.CommonUtil;
 import org.openmetadata.schema.entity.data.Table;
 import org.openmetadata.schema.type.EntityReference;
@@ -60,6 +61,10 @@ public Map<String, Object> buildESDoc() {
     doc.put("tags", parseTags.getTags());
     doc.put("tier", parseTags.getTierTag());
     doc.put("followers", SearchIndexUtils.parseFollowers(table.getFollowers()));
+    doc.put(
+        "fqnParts",
+        getFQNParts(
+            table.getFullyQualifiedName(), suggest.stream().map(SearchSuggest::getInput).collect(Collectors.toList())));
     doc.put("suggest", suggest);
     doc.put("service_suggest", serviceSuggest);
     doc.put("column_suggest", columnSuggest);
@@ -82,5 +87,7 @@ private void parseTableSuggest(List<SearchSuggest> suggest) {
         suggest.add(SearchSuggest.builder().input(fqnPartsWithoutDB[1]).weight(5).build());
       }
     }
+    suggest.add(SearchSuggest.builder().input(table.getDatabaseSchema().getName()).weight(5).build());
+    suggest.add(SearchSuggest.builder().input(table.getDatabase().getName()).weight(5).build());
   }
 }
diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/TagIndex.java b/openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/TagIndex.java
@@ -3,6 +3,7 @@
 import java.util.ArrayList;
 import java.util.List;
 import java.util.Map;
+import java.util.stream.Collectors;
 import org.openmetadata.schema.entity.classification.Tag;
 import org.openmetadata.service.Entity;
 import org.openmetadata.service.search.SearchIndexUtils;
@@ -30,6 +31,10 @@ public Map<String, Object> buildESDoc() {
     }
     doc.put("suggest", suggest);
     doc.put("entityType", Entity.TAG);
+    doc.put(
+        "fqnParts",
+        getFQNParts(
+            tag.getFullyQualifiedName(), suggest.stream().map(SearchSuggest::getInput).collect(Collectors.toList())));
     return doc;
   }
 }
diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/TeamIndex.java b/openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/TeamIndex.java
@@ -3,6 +3,7 @@
 import java.util.ArrayList;
 import java.util.List;
 import java.util.Map;
+import java.util.stream.Collectors;
 import org.openmetadata.common.utils.CommonUtil;
 import org.openmetadata.schema.entity.teams.Team;
 import org.openmetadata.service.Entity;
@@ -29,6 +30,10 @@ public Map<String, Object> buildESDoc() {
     suggest.add(SearchSuggest.builder().input(team.getDisplayName()).weight(10).build());
     doc.put("suggest", suggest);
     doc.put("entityType", Entity.TEAM);
+    doc.put(
+        "fqnParts",
+        getFQNParts(
+            team.getFullyQualifiedName(), suggest.stream().map(SearchSuggest::getInput).collect(Collectors.toList())));
     return doc;
   }
 }
diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/TopicIndex.java b/openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/TopicIndex.java
@@ -5,6 +5,7 @@
 import java.util.Map;
 import java.util.Optional;
 import java.util.function.Predicate;
+import java.util.stream.Collectors;
 import org.openmetadata.common.utils.CommonUtil;
 import org.openmetadata.schema.entity.data.Topic;
 import org.openmetadata.schema.type.EntityReference;
@@ -63,6 +64,11 @@ public Map<String, Object> buildESDoc() {
     doc.put("entityType", Entity.TOPIC);
     doc.put("serviceType", topic.getServiceType());
     doc.put("messageSchema", topic.getMessageSchema() != null ? topic.getMessageSchema() : null);
+    doc.put(
+        "fqnParts",
+        getFQNParts(
+            topic.getFullyQualifiedName(), suggest.stream().map(SearchSuggest::getInput).collect(Collectors.toList())));
+
     return doc;
   }
 

diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/UserIndex.java b/openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/UserIndex.java
@@ -3,6 +3,7 @@
 import java.util.ArrayList;
 import java.util.List;
 import java.util.Map;
+import java.util.stream.Collectors;
 import org.openmetadata.common.utils.CommonUtil;
 import org.openmetadata.schema.entity.teams.User;
 import org.openmetadata.service.Entity;
@@ -33,6 +34,10 @@ public Map<String, Object> buildESDoc() {
 
     doc.put("suggest", suggest);
     doc.put("entityType", Entity.USER);
+    doc.put(
+        "fqnParts",
+        getFQNParts(
+            user.getFullyQualifiedName(), suggest.stream().map(SearchSuggest::getInput).collect(Collectors.toList())));
     return doc;
   }
 }