Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Improve Search Relevancy for tier, usage and use stemming to generate ngrams #17871

Merged
merged 4 commits into from
Sep 18, 2024
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -171,7 +171,13 @@ public Response search(
"Fetch search results in hierarchical order of children elements. By default hierarchy is not fetched. Currently only supported for glossary_term_search_index.")
@DefaultValue("false")
@QueryParam("getHierarchy")
boolean getHierarchy)
boolean getHierarchy,
@Parameter(
description =
"Explain the results of the query. Defaults to false. Only for debugging purposes.")
@DefaultValue("false")
@QueryParam("explain")
boolean explain)
throws IOException {

if (nullOrEmpty(query)) {
Expand Down Expand Up @@ -202,6 +208,7 @@ public Response search(
.applyDomainFilter(
!subjectContext.isAdmin() && subjectContext.hasAnyRole(DOMAIN_ONLY_ACCESS_ROLE))
.searchAfter(searchAfter)
.explain(explain)
.build();
return searchRepository.search(request, subjectContext);
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,9 @@ public class EntityBuilderConstant {
public static final String ES_MESSAGE_SCHEMA_FIELD = "messageSchema.schemaFields.name";
public static final String ES_MESSAGE_SCHEMA_FIELD_KEYWORD =
"messageSchema.schemaFields.name.keyword";
public static final String API_RESPONSE_SCHEMA_FIELD = "responseSchema.schemaFields.name";
public static final String API_RESPONSE_SCHEMA_FIELD_KEYWORD =
"responseSchema.schemaFields.name.keyword";
public static final String ES_TAG_FQN_FIELD = "tags.tagFQN";

public static final String COLUMNS_NAME_KEYWORD = "columns.name.keyword";
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -29,6 +29,7 @@ public class SearchRequest {
private final List<String> domains;
private final boolean getHierarchy;
private final Object[] searchAfter;
private final boolean explain;

public SearchRequest(ElasticSearchRequestBuilder builder) {
this.query = builder.query;
Expand All @@ -48,6 +49,7 @@ public SearchRequest(ElasticSearchRequestBuilder builder) {
this.domains = builder.domains;
this.applyDomainFilter = builder.applyDomainFilter;
this.searchAfter = builder.searchAfter;
this.explain = builder.explain;
}

// Builder class for ElasticSearchRequest
Expand All @@ -70,6 +72,7 @@ public static class ElasticSearchRequestBuilder {
private boolean applyDomainFilter;
private List<String> domains;
private Object[] searchAfter;
private boolean explain;

public ElasticSearchRequestBuilder(String query, int size, String index) {
this.query = query;
Expand Down Expand Up @@ -153,6 +156,11 @@ public ElasticSearchRequestBuilder searchAfter(String searchAfter) {
return this;
}

public ElasticSearchRequestBuilder explain(boolean explain) {
this.explain = explain;
return this;
}

public SearchRequest build() {
return new SearchRequest(this);
}
Expand Down

Large diffs are not rendered by default.

Original file line number Diff line number Diff line change
Expand Up @@ -57,9 +57,6 @@ public Map<String, Object> buildSearchIndexDocInternal(Map<String, Object> doc)

Map<String, Object> commonAttributes = getCommonAttributesMap(container, Entity.CONTAINER);
doc.putAll(commonAttributes);
doc.put(
"displayName",
container.getDisplayName() != null ? container.getDisplayName() : container.getName());
doc.put("tags", flattenedTagList);
doc.put("tier", parseTags.getTierTag());
doc.put("service_suggest", serviceSuggest);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -48,10 +48,6 @@ public Map<String, Object> buildSearchIndexDocInternal(Map<String, Object> doc)
}
Map<String, Object> commonAttributes = getCommonAttributesMap(dashboard, Entity.DASHBOARD);
doc.putAll(commonAttributes);

doc.put(
"displayName",
dashboard.getDisplayName() != null ? dashboard.getDisplayName() : dashboard.getName());
doc.put("tags", parseTags.getTags());
doc.put("tier", parseTags.getTierTag());
doc.put("chart_suggest", chartSuggest);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -60,11 +60,6 @@ public Map<String, Object> buildSearchIndexDocInternal(Map<String, Object> doc)
ingestionPipeline.getName() != null
? ingestionPipeline.getName()
: ingestionPipeline.getDisplayName());
doc.put(
"displayName",
ingestionPipeline.getDisplayName() != null
? ingestionPipeline.getDisplayName()
: ingestionPipeline.getName());
doc.put("tags", parseTags.getTags());
doc.put("tier", parseTags.getTierTag());
doc.put("service_suggest", serviceSuggest);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -32,9 +32,6 @@ public Map<String, Object> buildSearchIndexDocInternal(Map<String, Object> doc)
ParseTags parseTags = new ParseTags(Entity.getEntityTags(Entity.MLMODEL, mlModel));
Map<String, Object> commonAttributes = getCommonAttributesMap(mlModel, Entity.MLMODEL);
doc.putAll(commonAttributes);
doc.put(
sonika-shah marked this conversation as resolved.
Show resolved Hide resolved
"displayName",
mlModel.getDisplayName() != null ? mlModel.getDisplayName() : mlModel.getName());
doc.put("tags", parseTags.getTags());
doc.put("tier", parseTags.getTierTag());
doc.put("serviceType", mlModel.getServiceType());
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -46,9 +46,6 @@ public Map<String, Object> buildSearchIndexDocInternal(Map<String, Object> doc)
Map<String, Object> commonAttributes = getCommonAttributesMap(pipeline, Entity.PIPELINE);
doc.putAll(commonAttributes);
doc.put("name", pipeline.getName() != null ? pipeline.getName() : pipeline.getDisplayName());
doc.put(
"displayName",
pipeline.getDisplayName() != null ? pipeline.getDisplayName() : pipeline.getName());
doc.put("tags", parseTags.getTags());
doc.put("tier", parseTags.getTierTag());
doc.put("task_suggest", taskSuggest);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -36,7 +36,6 @@ public Map<String, Object> buildSearchIndexDocInternal(Map<String, Object> doc)
ParseTags parseTags = new ParseTags(Entity.getEntityTags(Entity.QUERY, query));
Map<String, Object> commonAttributes = getCommonAttributesMap(query, Entity.QUERY);
doc.putAll(commonAttributes);
doc.put("displayName", query.getDisplayName() != null ? query.getDisplayName() : "");
doc.put("tags", parseTags.getTags());
doc.put("tier", parseTags.getTierTag());
return doc;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -3,14 +3,11 @@
import static org.openmetadata.common.utils.CommonUtil.nullOrEmpty;
import static org.openmetadata.service.Entity.FIELD_DESCRIPTION;
import static org.openmetadata.service.Entity.FIELD_DISPLAY_NAME;
import static org.openmetadata.service.Entity.FIELD_NAME;
import static org.openmetadata.service.jdbi3.LineageRepository.buildRelationshipDetailsMap;
import static org.openmetadata.service.search.EntityBuilderConstant.DISPLAY_NAME_KEYWORD;
import static org.openmetadata.service.search.EntityBuilderConstant.FIELD_DISPLAY_NAME_NGRAM;
import static org.openmetadata.service.search.EntityBuilderConstant.FIELD_NAME_NGRAM;
import static org.openmetadata.service.search.EntityBuilderConstant.FULLY_QUALIFIED_NAME;
import static org.openmetadata.service.search.EntityBuilderConstant.FULLY_QUALIFIED_NAME_PARTS;
import static org.openmetadata.service.search.EntityBuilderConstant.NAME_KEYWORD;

import java.util.ArrayList;
import java.util.Collections;
Expand Down Expand Up @@ -68,6 +65,9 @@ default List<SearchSuggest> getSuggest() {
default Map<String, Object> getCommonAttributesMap(EntityInterface entity, String entityType) {
Map<String, Object> map = new HashMap<>();
List<SearchSuggest> suggest = getSuggest();
map.put(
"displayName",
entity.getDisplayName() != null ? entity.getDisplayName() : entity.getName());
map.put("entityType", entityType);
map.put("owners", getEntitiesWithDisplayName(entity.getOwners()));
map.put("domain", getEntityWithDisplayName(entity.getDomain()));
Expand Down Expand Up @@ -164,12 +164,9 @@ static List<Map<String, Object>> getLineageData(EntityReference entity) {

static Map<String, Float> getDefaultFields() {
Map<String, Float> fields = new HashMap<>();
fields.put(FIELD_DISPLAY_NAME, 10.0f);
fields.put(DISPLAY_NAME_KEYWORD, 10.0f);
fields.put(FIELD_DISPLAY_NAME_NGRAM, 1.0f);
fields.put(FIELD_NAME, 10.0f);
fields.put(FIELD_NAME_NGRAM, 1.0f);
fields.put(DISPLAY_NAME_KEYWORD, 8.0f);
fields.put(NAME_KEYWORD, 8.0f);
fields.put(FIELD_DISPLAY_NAME, 10.0f);
fields.put(FIELD_DESCRIPTION, 2.0f);
fields.put(FULLY_QUALIFIED_NAME, 5.0f);
fields.put(FULLY_QUALIFIED_NAME_PARTS, 5.0f);
Expand All @@ -179,6 +176,7 @@ static Map<String, Float> getDefaultFields() {
static Map<String, Float> getAllFields() {
Map<String, Float> fields = getDefaultFields();
fields.putAll(TableIndex.getFields());
fields.putAll(StoredProcedureIndex.getFields());
fields.putAll(DashboardIndex.getFields());
fields.putAll(DashboardDataModelIndex.getFields());
fields.putAll(PipelineIndex.getFields());
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -91,8 +91,6 @@ public Map<String, Object> buildSearchIndexDocInternal(Map<String, Object> doc)
.collect(ArrayList::new, ArrayList::add, ArrayList::addAll);
Map<String, Object> commonAttributes = getCommonAttributesMap(table, Entity.TABLE);
doc.putAll(commonAttributes);
doc.put(
"displayName", table.getDisplayName() != null ? table.getDisplayName() : table.getName());
doc.put("tags", flattenedTagList);
doc.put("tier", parseTags.getTierTag());
doc.put("service_suggest", serviceSuggest);
Expand All @@ -113,7 +111,7 @@ public static Map<String, Float> getFields() {
fields.put("columns.name", 5.0f);
fields.put("columns.displayName", 5.0f);
fields.put("columns.description", 2.0f);
fields.put("columns.children.name", 5.0f);
fields.put("columns.children.name", 3.0f);
return fields;
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,6 @@
import java.util.List;
import java.util.Map;
import java.util.Set;
import org.openmetadata.common.utils.CommonUtil;
import org.openmetadata.schema.entity.teams.Team;
import org.openmetadata.service.Entity;
import org.openmetadata.service.search.models.SearchSuggest;
Expand Down Expand Up @@ -39,9 +38,6 @@ public Map<String, Object> buildSearchIndexDocInternal(Map<String, Object> doc)
Map<String, Object> commonAttributes = getCommonAttributesMap(team, Entity.TEAM);
doc.putAll(commonAttributes);
doc.put("isBot", false);
doc.put(
"displayName",
CommonUtil.nullOrEmpty(team.getDisplayName()) ? team.getName() : team.getDisplayName());
return doc;
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -76,8 +76,6 @@ public Map<String, Object> buildSearchIndexDocInternal(Map<String, Object> doc)
.collect(ArrayList::new, ArrayList::add, ArrayList::addAll);
Map<String, Object> commonAttributes = getCommonAttributesMap(topic, Entity.TOPIC);
doc.putAll(commonAttributes);
doc.put(
"displayName", topic.getDisplayName() != null ? topic.getDisplayName() : topic.getName());
doc.put("tags", flattenedTagList);
doc.put("tier", parseTags.getTierTag());
doc.put("field_suggest", fieldSuggest);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,6 @@
import java.util.List;
import java.util.Map;
import java.util.Set;
import org.openmetadata.common.utils.CommonUtil;
import org.openmetadata.schema.entity.teams.User;
import org.openmetadata.service.Entity;
import org.openmetadata.service.search.models.SearchSuggest;
Expand Down Expand Up @@ -38,9 +37,6 @@ public Set<String> getExcludedFields() {
public Map<String, Object> buildSearchIndexDocInternal(Map<String, Object> doc) {
Map<String, Object> commonAttributes = getCommonAttributesMap(user, Entity.USER);
doc.putAll(commonAttributes);
doc.put(
"displayName",
CommonUtil.nullOrEmpty(user.getDisplayName()) ? user.getName() : user.getDisplayName());
if (user.getIsBot() == null) {
doc.put("isBot", false);
}
Expand Down
Loading
Loading