Skip to content

Commit

Permalink
Fix #issue-13914-: The persona JSON schema is named Team
Browse files Browse the repository at this point in the history
  • Loading branch information
@harshach
harshach committed Dec 16, 2024
1 parent 26a6e92 commit f7b65eb
Show file tree
Hide file tree
Showing 6 changed files with 713 additions and 119 deletions.
64 changes: 64 additions & 0 deletions openmetadata-service/src/main/java/org/openmetadata/service/search/SearchQueryBuilder.java
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,64 @@
package org.openmetadata.service.search;

import es.org.elasticsearch.search.builder.SearchSourceBuilder;
import org.openmetadata.schema.api.search.SearchSettings;
import org.openmetadata.schema.settings.SettingsType;
import org.openmetadata.service.Entity;
import org.openmetadata.service.resources.settings.SettingsCache;

import java.io.IOException;

public interface SearchQueryBuilder {
SearchSourceBuilder getSearchSourceBuilder(String index, String q, int from, int size) throws IOException;
void applyDeletedLogic(SearchRequest request, SearchSourceBuilder searchSourceBuilder);
void applyGlossaryHierarchyLogic(SearchRequest request, SearchSourceBuilder searchSourceBuilder);
default String getAssetTypeFromIndex(String index) {
return switch (index) {
case "topic_search_index", "topic" -> "topic";
case "dashboard_search_index", "dashboard" -> "dashboard";
case "pipeline_search_index", "pipeline" -> "pipeline";
case "mlmodel_search_index", "mlmodel" -> "mlmodel";
case "table_search_index", "table" -> "table";
case "database_schema_search_index",
"databaseSchema",
"database_search_index",
"database" -> "dataAsset";
case "user_search_index", "user", "team_search_index", "team" -> "user";
case "glossary_term_search_index", "glossaryTerm" -> "glossaryTerm";
case "tag_search_index", "tag" -> "tag";
case "container_search_index", "container" -> "container";
case "query_search_index", "query" -> "query";
case "test_case_search_index",
"testCase",
"test_suite_search_index",
"testSuite" -> "testCase";
case "stored_procedure_search_index", "storedProcedure" -> "storedProcedure";
case "dashboard_data_model_search_index",
"dashboardDataModel" -> "dashboardDataModel";
case "search_entity_search_index", "searchIndex" -> "searchIndex";
case "domain_search_index", "domain" -> "domain";
case "raw_cost_analysis_report_data_index" -> "rawCostAnalysisReportData";
case "aggregated_cost_analysis_report_data_index" -> "aggregatedCostAnalysisReportData";
case "data_product_search_index" -> "dataProduct";
case "test_case_resolution_status_search_index" -> "test_case_resolution_status_search_index";
case "test_case_result_search_index" -> "testCase";
case "api_endpoint_search_index", "apiEndpoint" -> "apiEndpoint";
case "api_service_search_index",
"mlmodel_service_search_index",
"database_service_search_index",
"messaging_service_index",
"dashboard_service_index",
"pipeline_service_index",
"storage_service_index",
"search_service_index",
"metadata_service_index" -> "default";
case "dataAsset" -> "dataAsset";
case "all" -> "all";
default -> "dataAsset";
};
}

default SearchSettings getSearchSettings() {
return SettingsCache.getSetting(SettingsType.SEARCH_SETTINGS, SearchSettings.class);
}
}
126 changes: 12 additions & 114 deletions ...vice/src/main/java/org/openmetadata/service/search/elasticsearch/ElasticSearchClient.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -166,6 +166,7 @@
import org.openmetadata.service.search.SearchAggregation;
import org.openmetadata.service.search.SearchClient;
import org.openmetadata.service.search.SearchIndexUtils;
import org.openmetadata.service.search.SearchQueryBuilder;
import org.openmetadata.service.search.SearchRequest;
import org.openmetadata.service.search.SearchSortFilter;
import org.openmetadata.service.search.UpdateSearchEventsConstant;
Expand Down Expand Up @@ -244,6 +245,8 @@ public class ElasticSearchClient implements SearchClient {
Stream.concat(FIELDS_TO_REMOVE.stream(), Stream.of("schemaDefinition", "customMetrics"))
.toList();

private final SearchQueryBuilder searchQueryBuilder = new ElasticSearchQueryBuilder();

static {
SearchModule searchModule = new SearchModule(Settings.EMPTY, false, List.of());
xContentRegistry = new NamedXContentRegistry(searchModule.getNamedXContents());
Expand Down Expand Up @@ -366,9 +369,9 @@ public void deleteIndex(IndexMapping indexMapping) {

@Override
public Response search(SearchRequest request, SubjectContext subjectContext) throws IOException {
// Instead of using the old large switch-case, we now rely on SearchQueryBuilder to use searchSettings
SearchSourceBuilder searchSourceBuilder =
getSearchSourceBuilder(
request.getIndex(), request.getQuery(), request.getFrom(), request.getSize());
searchQueryBuilder.getSearchSourceBuilder(request.getIndex(), request.getQuery(), request.getFrom(), request.getSize());

buildSearchRBACQuery(subjectContext, searchSourceBuilder);

Expand All @@ -393,62 +396,10 @@ public Response search(SearchRequest request, SubjectContext subjectContext) thr
searchSourceBuilder.searchAfter(request.getSearchAfter());
}

/* For backward-compatibility we continue supporting the deleted argument, this should be removed in future versions */
if (request
.getIndex()
.equalsIgnoreCase(Entity.getSearchRepository().getIndexOrAliasName(GLOBAL_SEARCH_ALIAS))
|| request
.getIndex()
.equalsIgnoreCase(Entity.getSearchRepository().getIndexOrAliasName("dataAsset"))) {
es.org.elasticsearch.index.query.BoolQueryBuilder boolQueryBuilder =
QueryBuilders.boolQuery();
boolQueryBuilder.should(
QueryBuilders.boolQuery()
.must(searchSourceBuilder.query())
.must(QueryBuilders.existsQuery("deleted"))
.must(QueryBuilders.termQuery("deleted", request.isDeleted())));
boolQueryBuilder.should(
QueryBuilders.boolQuery()
.must(searchSourceBuilder.query())
.mustNot(QueryBuilders.existsQuery("deleted")));
searchSourceBuilder.query(boolQueryBuilder);
} else if (request
.getIndex()
.equalsIgnoreCase(
Entity.getSearchRepository().getIndexMapping(DOMAIN).getIndexName(clusterAlias))
|| request
.getIndex()
.equalsIgnoreCase(
Entity.getSearchRepository()
.getIndexMapping(DATA_PRODUCT)
.getIndexName(clusterAlias))
|| request
.getIndex()
.equalsIgnoreCase(
Entity.getSearchRepository().getIndexMapping(QUERY).getIndexName(clusterAlias))
|| request
.getIndex()
.equalsIgnoreCase(
Entity.getSearchRepository().getIndexOrAliasName("knowledge_page_search_index"))
|| request
.getIndex()
.equalsIgnoreCase(
Entity.getSearchRepository()
.getIndexMapping(RAW_COST_ANALYSIS_REPORT_DATA)
.getIndexName(clusterAlias))
|| request
.getIndex()
.equalsIgnoreCase(
Entity.getSearchRepository()
.getIndexMapping(AGGREGATED_COST_ANALYSIS_REPORT_DATA)
.getIndexName(clusterAlias))) {
searchSourceBuilder.query(QueryBuilders.boolQuery().must(searchSourceBuilder.query()));
} else {
searchSourceBuilder.query(
QueryBuilders.boolQuery()
.must(searchSourceBuilder.query())
.must(QueryBuilders.termQuery("deleted", request.isDeleted())));
}
// Deleted filtering logic can be integrated into SearchQueryBuilder if needed,
// or remain here for backward compatibility:
// Apply deleted logic for backward-compatibility
searchQueryBuilder.applyDeletedLogic(request, searchSourceBuilder);

if (!nullOrEmpty(request.getSortFieldParam()) && !request.isGetHierarchy()) {
FieldSortBuilder fieldSortBuilder =
Expand All @@ -461,62 +412,9 @@ public Response search(SearchRequest request, SubjectContext subjectContext) thr
searchSourceBuilder.sort(fieldSortBuilder);
}

if (request
.getIndex()
.equalsIgnoreCase(
Entity.getSearchRepository()
.getIndexMapping(GLOSSARY_TERM)
.getIndexName(clusterAlias))) {
searchSourceBuilder.query(QueryBuilders.boolQuery().must(searchSourceBuilder.query()));

if (request.isGetHierarchy()) {
QueryBuilder baseQuery =
QueryBuilders.boolQuery()
.should(searchSourceBuilder.query())
.should(QueryBuilders.matchPhraseQuery("fullyQualifiedName", request.getQuery()))
.should(QueryBuilders.matchPhraseQuery("name", request.getQuery()))
.should(QueryBuilders.matchPhraseQuery("displayName", request.getQuery()))
.should(
QueryBuilders.matchPhraseQuery(
"glossary.fullyQualifiedName", request.getQuery()))
.should(QueryBuilders.matchPhraseQuery("glossary.displayName", request.getQuery()))
.must(QueryBuilders.matchQuery("status", "Approved"))
.minimumShouldMatch(1);
searchSourceBuilder.query(baseQuery);

SearchResponse searchResponse =
client.search(
new es.org.elasticsearch.action.search.SearchRequest(request.getIndex())
.source(searchSourceBuilder),
RequestOptions.DEFAULT);

// Extract parent terms from aggregation
BoolQueryBuilder parentTermQueryBuilder = QueryBuilders.boolQuery();
Terms parentTerms = searchResponse.getAggregations().get("fqnParts_agg");

// Build es query to get parent terms for the user input query , to build correct hierarchy
if (!parentTerms.getBuckets().isEmpty() && !request.getQuery().equals("*")) {
parentTerms.getBuckets().stream()
.map(Terms.Bucket::getKeyAsString)
.forEach(
parentTerm ->
parentTermQueryBuilder.should(
QueryBuilders.matchQuery("fullyQualifiedName", parentTerm)));

searchSourceBuilder.query(
parentTermQueryBuilder
.minimumShouldMatch(1)
.must(QueryBuilders.matchQuery("status", "Approved")));
}
searchSourceBuilder.sort(SortBuilders.fieldSort("fullyQualifiedName").order(SortOrder.ASC));
}
}
searchQueryBuilder.applyGlossaryHierarchyLogic(request, searchSourceBuilder);

/* for performance reasons ElasticSearch doesn't provide accurate hits
if we enable trackTotalHits parameter it will try to match every result, count and return hits
however in most cases for search results an approximate value is good enough.
we are displaying total entity counts in landing page and explore page where we need the total count
https://github.com/elastic/elasticsearch/issues/33028 */
// fetch source and track hits
searchSourceBuilder.fetchSource(
new FetchSourceContext(
request.isFetchSource(),
Expand All @@ -526,7 +424,7 @@ public Response search(SearchRequest request, SubjectContext subjectContext) thr
if (request.isTrackTotalHits()) {
searchSourceBuilder.trackTotalHits(true);
} else {
searchSourceBuilder.trackTotalHitsUpTo(MAX_RESULT_HITS);
searchSourceBuilder.trackTotalHitsUpTo(searchQueryBuilder.getSearchSettings().getGlobalSettings().getMaxResultHits());
}

searchSourceBuilder.timeout(new TimeValue(30, TimeUnit.SECONDS));
Expand Down
Loading

0 comments on commit f7b65eb

Please sign in to comment.