Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Improve Search Relevancy for tier, usage and use stemming to generate ngrams #17871

Merged
merged 4 commits into from
Sep 18, 2024
Merged

Improve Search Relevancy for tier, usage and use stemming to generate ngrams #17871

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
157963e
Improve Search Relevancy for tier, usage and use stemming to generate…
harshach Sep 17, 2024
a8bc9f7
Improve Search Relevancy for tier, usage and use stemming to generate…
harshach Sep 17, 2024
0c217ef
Merge branch 'main' into improve_search_ranking
Sachin-chaurasiya Sep 17, 2024
27f5b07
add displayName using name
harshach Sep 17, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
9 changes: 8 additions & 1 deletion ...adata-service/src/main/java/org/openmetadata/service/resources/search/SearchResource.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -171,7 +171,13 @@ public Response search(
"Fetch search results in hierarchical order of children elements. By default hierarchy is not fetched. Currently only supported for glossary_term_search_index.")
@DefaultValue("false")
@QueryParam("getHierarchy")
boolean getHierarchy)
boolean getHierarchy,
@Parameter(
description =
"Explain the results of the query. Defaults to false. Only for debugging purposes.")
@DefaultValue("false")
@QueryParam("explain")
boolean explain)
throws IOException {

if (nullOrEmpty(query)) {
Expand Down Expand Up @@ -202,6 +208,7 @@ public Response search(
.applyDomainFilter(
!subjectContext.isAdmin() && subjectContext.hasAnyRole(DOMAIN_ONLY_ACCESS_ROLE))
.searchAfter(searchAfter)
.explain(explain)
.build();
return searchRepository.search(request, subjectContext);
}
Expand Down
3 changes: 3 additions & 0 deletions ...metadata-service/src/main/java/org/openmetadata/service/search/EntityBuilderConstant.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,9 @@ public class EntityBuilderConstant {
public static final String ES_MESSAGE_SCHEMA_FIELD = "messageSchema.schemaFields.name";
public static final String ES_MESSAGE_SCHEMA_FIELD_KEYWORD =
"messageSchema.schemaFields.name.keyword";
public static final String API_RESPONSE_SCHEMA_FIELD = "responseSchema.schemaFields.name";
public static final String API_RESPONSE_SCHEMA_FIELD_KEYWORD =
"responseSchema.schemaFields.name.keyword";
public static final String ES_TAG_FQN_FIELD = "tags.tagFQN";

public static final String COLUMNS_NAME_KEYWORD = "columns.name.keyword";
Expand Down
8 changes: 8 additions & 0 deletions openmetadata-service/src/main/java/org/openmetadata/service/search/SearchRequest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -29,6 +29,7 @@ public class SearchRequest {
private final List<String> domains;
private final boolean getHierarchy;
private final Object[] searchAfter;
private final boolean explain;

public SearchRequest(ElasticSearchRequestBuilder builder) {
this.query = builder.query;
Expand All @@ -48,6 +49,7 @@ public SearchRequest(ElasticSearchRequestBuilder builder) {
this.domains = builder.domains;
this.applyDomainFilter = builder.applyDomainFilter;
this.searchAfter = builder.searchAfter;
this.explain = builder.explain;
}

// Builder class for ElasticSearchRequest
Expand All @@ -70,6 +72,7 @@ public static class ElasticSearchRequestBuilder {
private boolean applyDomainFilter;
private List<String> domains;
private Object[] searchAfter;
private boolean explain;

public ElasticSearchRequestBuilder(String query, int size, String index) {
this.query = query;
Expand Down Expand Up @@ -153,6 +156,11 @@ public ElasticSearchRequestBuilder searchAfter(String searchAfter) {
return this;
}

public ElasticSearchRequestBuilder explain(boolean explain) {
this.explain = explain;
return this;
}

public SearchRequest build() {
return new SearchRequest(this);
}
Expand Down
93 changes: 44 additions & 49 deletions ...vice/src/main/java/org/openmetadata/service/search/elasticsearch/ElasticSearchClient.java
View file
Open in desktop

Large diffs are not rendered by default.

3 changes: 0 additions & 3 deletions ...etadata-service/src/main/java/org/openmetadata/service/search/indexes/ContainerIndex.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -57,9 +57,6 @@ public Map<String, Object> buildSearchIndexDocInternal(Map<String, Object> doc)

Map<String, Object> commonAttributes = getCommonAttributesMap(container, Entity.CONTAINER);
doc.putAll(commonAttributes);
doc.put(
"displayName",
container.getDisplayName() != null ? container.getDisplayName() : container.getName());
doc.put("tags", flattenedTagList);
doc.put("tier", parseTags.getTierTag());
doc.put("service_suggest", serviceSuggest);
Expand Down
4 changes: 0 additions & 4 deletions ...etadata-service/src/main/java/org/openmetadata/service/search/indexes/DashboardIndex.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -48,10 +48,6 @@ public Map<String, Object> buildSearchIndexDocInternal(Map<String, Object> doc)
}
Map<String, Object> commonAttributes = getCommonAttributesMap(dashboard, Entity.DASHBOARD);
doc.putAll(commonAttributes);

doc.put(
"displayName",
dashboard.getDisplayName() != null ? dashboard.getDisplayName() : dashboard.getName());
doc.put("tags", parseTags.getTags());
doc.put("tier", parseTags.getTierTag());
doc.put("chart_suggest", chartSuggest);
Expand Down
5 changes: 0 additions & 5 deletions ...service/src/main/java/org/openmetadata/service/search/indexes/IngestionPipelineIndex.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -60,11 +60,6 @@ public Map<String, Object> buildSearchIndexDocInternal(Map<String, Object> doc)
ingestionPipeline.getName() != null
? ingestionPipeline.getName()
: ingestionPipeline.getDisplayName());
doc.put(
"displayName",
ingestionPipeline.getDisplayName() != null
? ingestionPipeline.getDisplayName()
: ingestionPipeline.getName());
doc.put("tags", parseTags.getTags());
doc.put("tier", parseTags.getTierTag());
doc.put("service_suggest", serviceSuggest);
Expand Down
3 changes: 0 additions & 3 deletions openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/MlModelIndex.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -32,9 +32,6 @@ public Map<String, Object> buildSearchIndexDocInternal(Map<String, Object> doc)
ParseTags parseTags = new ParseTags(Entity.getEntityTags(Entity.MLMODEL, mlModel));
Map<String, Object> commonAttributes = getCommonAttributesMap(mlModel, Entity.MLMODEL);
doc.putAll(commonAttributes);
doc.put(
sonika-shah marked this conversation as resolved.
Show resolved Hide resolved
"displayName",
mlModel.getDisplayName() != null ? mlModel.getDisplayName() : mlModel.getName());
doc.put("tags", parseTags.getTags());
doc.put("tier", parseTags.getTierTag());
doc.put("serviceType", mlModel.getServiceType());
Expand Down
3 changes: 0 additions & 3 deletions ...metadata-service/src/main/java/org/openmetadata/service/search/indexes/PipelineIndex.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -46,9 +46,6 @@ public Map<String, Object> buildSearchIndexDocInternal(Map<String, Object> doc)
Map<String, Object> commonAttributes = getCommonAttributesMap(pipeline, Entity.PIPELINE);
doc.putAll(commonAttributes);
doc.put("name", pipeline.getName() != null ? pipeline.getName() : pipeline.getDisplayName());
doc.put(
"displayName",
pipeline.getDisplayName() != null ? pipeline.getDisplayName() : pipeline.getName());
doc.put("tags", parseTags.getTags());
doc.put("tier", parseTags.getTierTag());
doc.put("task_suggest", taskSuggest);
Expand Down
1 change: 0 additions & 1 deletion openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/QueryIndex.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -36,7 +36,6 @@ public Map<String, Object> buildSearchIndexDocInternal(Map<String, Object> doc)
ParseTags parseTags = new ParseTags(Entity.getEntityTags(Entity.QUERY, query));
Map<String, Object> commonAttributes = getCommonAttributesMap(query, Entity.QUERY);
doc.putAll(commonAttributes);
doc.put("displayName", query.getDisplayName() != null ? query.getDisplayName() : "");
doc.put("tags", parseTags.getTags());
doc.put("tier", parseTags.getTierTag());
return doc;
Expand Down
14 changes: 6 additions & 8 deletions openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/SearchIndex.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,14 +3,11 @@
import static org.openmetadata.common.utils.CommonUtil.nullOrEmpty;
import static org.openmetadata.service.Entity.FIELD_DESCRIPTION;
import static org.openmetadata.service.Entity.FIELD_DISPLAY_NAME;
import static org.openmetadata.service.Entity.FIELD_NAME;
import static org.openmetadata.service.jdbi3.LineageRepository.buildRelationshipDetailsMap;
import static org.openmetadata.service.search.EntityBuilderConstant.DISPLAY_NAME_KEYWORD;
import static org.openmetadata.service.search.EntityBuilderConstant.FIELD_DISPLAY_NAME_NGRAM;
import static org.openmetadata.service.search.EntityBuilderConstant.FIELD_NAME_NGRAM;
import static org.openmetadata.service.search.EntityBuilderConstant.FULLY_QUALIFIED_NAME;
import static org.openmetadata.service.search.EntityBuilderConstant.FULLY_QUALIFIED_NAME_PARTS;
import static org.openmetadata.service.search.EntityBuilderConstant.NAME_KEYWORD;

import java.util.ArrayList;
import java.util.Collections;
Expand Down Expand Up @@ -68,6 +65,9 @@ default List<SearchSuggest> getSuggest() {
default Map<String, Object> getCommonAttributesMap(EntityInterface entity, String entityType) {
Map<String, Object> map = new HashMap<>();
List<SearchSuggest> suggest = getSuggest();
map.put(
"displayName",
entity.getDisplayName() != null ? entity.getDisplayName() : entity.getName());
map.put("entityType", entityType);
map.put("owners", getEntitiesWithDisplayName(entity.getOwners()));
map.put("domain", getEntityWithDisplayName(entity.getDomain()));
Expand Down Expand Up @@ -164,12 +164,9 @@ static List<Map<String, Object>> getLineageData(EntityReference entity) {

static Map<String, Float> getDefaultFields() {
Map<String, Float> fields = new HashMap<>();
fields.put(FIELD_DISPLAY_NAME, 10.0f);
fields.put(DISPLAY_NAME_KEYWORD, 10.0f);
fields.put(FIELD_DISPLAY_NAME_NGRAM, 1.0f);
fields.put(FIELD_NAME, 10.0f);
fields.put(FIELD_NAME_NGRAM, 1.0f);
fields.put(DISPLAY_NAME_KEYWORD, 8.0f);
fields.put(NAME_KEYWORD, 8.0f);
fields.put(FIELD_DISPLAY_NAME, 10.0f);
fields.put(FIELD_DESCRIPTION, 2.0f);
fields.put(FULLY_QUALIFIED_NAME, 5.0f);
fields.put(FULLY_QUALIFIED_NAME_PARTS, 5.0f);
Expand All @@ -179,6 +176,7 @@ static Map<String, Float> getDefaultFields() {
static Map<String, Float> getAllFields() {
Map<String, Float> fields = getDefaultFields();
fields.putAll(TableIndex.getFields());
fields.putAll(StoredProcedureIndex.getFields());
fields.putAll(DashboardIndex.getFields());
fields.putAll(DashboardDataModelIndex.getFields());
fields.putAll(PipelineIndex.getFields());
Expand Down
4 changes: 1 addition & 3 deletions openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/TableIndex.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -91,8 +91,6 @@ public Map<String, Object> buildSearchIndexDocInternal(Map<String, Object> doc)
.collect(ArrayList::new, ArrayList::add, ArrayList::addAll);
Map<String, Object> commonAttributes = getCommonAttributesMap(table, Entity.TABLE);
doc.putAll(commonAttributes);
doc.put(
"displayName", table.getDisplayName() != null ? table.getDisplayName() : table.getName());
doc.put("tags", flattenedTagList);
doc.put("tier", parseTags.getTierTag());
doc.put("service_suggest", serviceSuggest);
Expand All @@ -113,7 +111,7 @@ public static Map<String, Float> getFields() {
fields.put("columns.name", 5.0f);
fields.put("columns.displayName", 5.0f);
fields.put("columns.description", 2.0f);
fields.put("columns.children.name", 5.0f);
fields.put("columns.children.name", 3.0f);
return fields;
}
}
4 changes: 0 additions & 4 deletions openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/TeamIndex.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,6 @@
import java.util.List;
import java.util.Map;
import java.util.Set;
import org.openmetadata.common.utils.CommonUtil;
import org.openmetadata.schema.entity.teams.Team;
import org.openmetadata.service.Entity;
import org.openmetadata.service.search.models.SearchSuggest;
Expand Down Expand Up @@ -39,9 +38,6 @@ public Map<String, Object> buildSearchIndexDocInternal(Map<String, Object> doc)
Map<String, Object> commonAttributes = getCommonAttributesMap(team, Entity.TEAM);
doc.putAll(commonAttributes);
doc.put("isBot", false);
doc.put(
"displayName",
CommonUtil.nullOrEmpty(team.getDisplayName()) ? team.getName() : team.getDisplayName());
return doc;
}
}
2 changes: 0 additions & 2 deletions openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/TopicIndex.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -76,8 +76,6 @@ public Map<String, Object> buildSearchIndexDocInternal(Map<String, Object> doc)
.collect(ArrayList::new, ArrayList::add, ArrayList::addAll);
Map<String, Object> commonAttributes = getCommonAttributesMap(topic, Entity.TOPIC);
doc.putAll(commonAttributes);
doc.put(
"displayName", topic.getDisplayName() != null ? topic.getDisplayName() : topic.getName());
doc.put("tags", flattenedTagList);
doc.put("tier", parseTags.getTierTag());
doc.put("field_suggest", fieldSuggest);
Expand Down
4 changes: 0 additions & 4 deletions openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/UserIndex.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,6 @@
import java.util.List;
import java.util.Map;
import java.util.Set;
import org.openmetadata.common.utils.CommonUtil;
import org.openmetadata.schema.entity.teams.User;
import org.openmetadata.service.Entity;
import org.openmetadata.service.search.models.SearchSuggest;
Expand Down Expand Up @@ -38,9 +37,6 @@ public Set<String> getExcludedFields() {
public Map<String, Object> buildSearchIndexDocInternal(Map<String, Object> doc) {
Map<String, Object> commonAttributes = getCommonAttributesMap(user, Entity.USER);
doc.putAll(commonAttributes);
doc.put(
"displayName",
CommonUtil.nullOrEmpty(user.getDisplayName()) ? user.getName() : user.getDisplayName());
if (user.getIsBot() == null) {
doc.put("isBot", false);
}
Expand Down
Loading
Loading