Update the indexes to not have changeDescription

openmetadata-service/src/main/java/org/openmetadata/service/search/SearchIndexUtils.java

@@ -3,7 +3,7 @@
 import java.util.Collections;
 import java.util.List;
 import java.util.Map;
-import java.util.stream.Collectors;
+import java.util.Set;
 import org.openmetadata.schema.type.EntityReference;
 import org.openmetadata.schema.type.TagLabel;
 
@@ -15,15 +15,39 @@ public static List<String> parseFollowers(List<EntityReference> followersRef) {
  if (followersRef == null) {
  return Collections.emptyList();
  }
- return followersRef.stream().map(item -> item.getId().toString()).collect(Collectors.toList());
+ return followersRef.stream().map(item -> item.getId().toString()).toList();
  }
 
- public static void removeNonIndexableFields(Map<String, Object> doc, List<String> fields) {
+ public static void removeNonIndexableFields(Map<String, Object> doc, Set<String> fields) {
  for (String key : fields) {
- doc.remove(key);
+ if (key.contains(".")) {
+ removeFieldByPath(doc, key);
+ } else {
+ doc.remove(key);
+ }
  }
  }
 
+ public static void removeFieldByPath(Map<String, Object> jsonMap, String path) {
+ String[] pathElements = path.split("\\.");
+ Map<String, Object> currentMap = jsonMap;
+
+ for (int i = 0; i < pathElements.length - 1; i++) {
+ String key = pathElements[i];
+ Object value = currentMap.get(key);
+ if (value instanceof Map) {
+ currentMap = (Map<String, Object>) value;
+ } else {
+ // Path Not Found
+ return;
+ }
+ }
+
+ // Remove the field at the last path element
+ String lastKey = pathElements[pathElements.length - 1];
+ currentMap.remove(lastKey);
+ }
+
  public static List<TagLabel> parseTags(List<TagLabel> tags) {
  if (tags == null) {
  return Collections.emptyList();

openmetadata-service/src/main/java/org/openmetadata/service/search/SearchRepository.java

@@ -252,7 +252,7 @@ public void createEntity(EntityInterface entity) {
  try {
  IndexMapping indexMapping = entityIndexMap.get(entityType);
  SearchIndex index = searchIndexFactory.buildIndex(entityType, entity);
- String doc = JsonUtils.pojoToJson(index.buildESDoc());
+ String doc = JsonUtils.pojoToJson(index.buildSearchIndexDoc());
  searchClient.createEntity(indexMapping.getIndexName(clusterAlias), entityId, doc);
  } catch (Exception ie) {
  LOG.error(
@@ -280,7 +280,7 @@ public void createTimeSeriesEntity(EntityTimeSeriesInterface entity) {
  try {
  IndexMapping indexMapping = entityIndexMap.get(entityType);
  SearchIndex index = searchIndexFactory.buildIndex(entityType, entity);
- String doc = JsonUtils.pojoToJson(index.buildESDoc());
+ String doc = JsonUtils.pojoToJson(index.buildSearchIndexDoc());
  searchClient.createTimeSeriesEntity(indexMapping.getIndexName(clusterAlias), entityId, doc);
  } catch (Exception ie) {
  LOG.error(
@@ -309,7 +309,7 @@ public void updateEntity(EntityInterface entity) {
  scriptTxt = getScriptWithParams(entity, doc);
  } else {
  SearchIndex elasticSearchIndex = searchIndexFactory.buildIndex(entityType, entity);
- doc = elasticSearchIndex.buildESDoc();
+ doc = elasticSearchIndex.buildSearchIndexDoc();
  }
  searchClient.updateEntity(
  indexMapping.getIndexName(clusterAlias), entityId, doc, scriptTxt);

openmetadata-service/src/main/java/org/openmetadata/service/search/elasticsearch/ElasticSearchDataInsightProcessor.java

@@ -86,7 +86,8 @@ private UpdateRequest getUpdateRequest(String entityType, ReportData reportData)
  indexMapping.getIndexName(Entity.getSearchRepository().getClusterAlias()),
  reportData.getId().toString());
  updateRequest.doc(
- JsonUtils.pojoToJson(new ReportDataIndexes(reportData).buildESDoc()), XContentType.JSON);
+ JsonUtils.pojoToJson(new ReportDataIndexes(reportData).buildSearchIndexDoc()),
+ XContentType.JSON);
  updateRequest.docAsUpsert(true);
  return updateRequest;
  }

openmetadata-service/src/main/java/org/openmetadata/service/search/elasticsearch/ElasticSearchEntitiesProcessor.java

@@ -89,7 +89,8 @@ public static UpdateRequest getUpdateRequest(String entityType, EntityInterface
  entity.getId().toString());
  updateRequest.doc(
  JsonUtils.pojoToJson(
- Objects.requireNonNull(Entity.buildSearchIndex(entityType, entity)).buildESDoc()),
+ Objects.requireNonNull(Entity.buildSearchIndex(entityType, entity))
+ .buildSearchIndexDoc()),
  XContentType.JSON);
  updateRequest.docAsUpsert(true);
  return updateRequest;

openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/AggregatedCostAnalysisReportDataIndex.java

@@ -2,13 +2,16 @@
 
 import java.util.Map;
 import org.openmetadata.schema.analytics.ReportData;
-import org.openmetadata.service.util.JsonUtils;
 
 public record AggregatedCostAnalysisReportDataIndex(ReportData reportData) implements SearchIndex {
 
  @Override
- public Map<String, Object> buildESDoc() {
- Map<String, Object> doc = JsonUtils.getMap(reportData);
+ public Object getEntity() {
+ return reportData;
+ }
+
+ @Override
+ public Map<String, Object> buildSearchIndexDocInternal(Map<String, Object> doc) {
  doc.put("entityType", "aggregatedCostAnalysisReportData");
  return doc;
  }

openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/ChartIndex.java

@@ -8,14 +8,11 @@
 import org.openmetadata.service.Entity;
 import org.openmetadata.service.search.SearchIndexUtils;
 import org.openmetadata.service.search.models.SearchSuggest;
-import org.openmetadata.service.util.JsonUtils;
 
 public record ChartIndex(Chart chart) implements SearchIndex {
- private static final List<String> excludeFields = List.of("changeDescription");
 
- public Map<String, Object> buildESDoc() {
- Map<String, Object> doc = JsonUtils.getMap(chart);
- SearchIndexUtils.removeNonIndexableFields(doc, excludeFields);
+ @Override
+ public Map<String, Object> buildSearchIndexDocInternal(Map<String, Object> doc) {
  List<SearchSuggest> suggest = new ArrayList<>();
  suggest.add(SearchSuggest.builder().input(chart.getName()).weight(10).build());
  suggest.add(SearchSuggest.builder().input(chart.getFullyQualifiedName()).weight(5).build());
@@ -35,4 +32,9 @@ public Map<String, Object> buildESDoc() {
  : chart.getVotes().getUpVotes() - chart.getVotes().getDownVotes());
  return doc;
  }
+
+ @Override
+ public Object getEntity() {
+ return chart;
+ }
 }

openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/ClassificationIndex.java

@@ -7,14 +7,11 @@
 import org.openmetadata.service.Entity;
 import org.openmetadata.service.search.SearchIndexUtils;
 import org.openmetadata.service.search.models.SearchSuggest;
-import org.openmetadata.service.util.JsonUtils;
 
 public record ClassificationIndex(Classification classification) implements SearchIndex {
- private static final List<String> excludeFields = List.of("changeDescription");
 
- public Map<String, Object> buildESDoc() {
- Map<String, Object> doc = JsonUtils.getMap(classification);
- SearchIndexUtils.removeNonIndexableFields(doc, excludeFields);
+ @Override
+ public Map<String, Object> buildSearchIndexDocInternal(Map<String, Object> doc) {
  List<SearchSuggest> suggest = new ArrayList<>();
  suggest.add(SearchSuggest.builder().input(classification.getName()).weight(10).build());
  suggest.add(
@@ -30,4 +27,9 @@ public Map<String, Object> buildESDoc() {
  doc.put("followers", SearchIndexUtils.parseFollowers(classification.getFollowers()));
  return doc;
  }
+
+ @Override
+ public Object getEntity() {
+ return classification;
+ }
 }

openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/ContainerIndex.java

@@ -15,19 +15,21 @@
 import org.openmetadata.service.search.SearchIndexUtils;
 import org.openmetadata.service.search.models.FlattenColumn;
 import org.openmetadata.service.search.models.SearchSuggest;
-import org.openmetadata.service.util.JsonUtils;
 
 public record ContainerIndex(Container container) implements ColumnIndex {
- private static final List<String> excludeFields = List.of("changeDescription");
 
- public Map<String, Object> buildESDoc() {
- Map<String, Object> doc = JsonUtils.getMap(container);
+ @Override
+ public Object getEntity() {
+ return container;
+ }
+
+ @Override
+ public Map<String, Object> buildSearchIndexDocInternal(Map<String, Object> doc) {
  List<SearchSuggest> suggest = new ArrayList<>();
  List<SearchSuggest> columnSuggest = new ArrayList<>();
  List<SearchSuggest> serviceSuggest = new ArrayList<>();
  Set<List<TagLabel>> tagsWithChildren = new HashSet<>();
  List<String> columnsWithChildrenName = new ArrayList<>();
- SearchIndexUtils.removeNonIndexableFields(doc, excludeFields);
  suggest.add(SearchSuggest.builder().input(container.getFullyQualifiedName()).weight(5).build());
  suggest.add(SearchSuggest.builder().input(container.getName()).weight(10).build());
  if (container.getDataModel() != null && container.getDataModel().getColumns() != null) {

openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/DashboardDataModelIndex.java

@@ -15,15 +15,17 @@
 import org.openmetadata.service.search.SearchIndexUtils;
 import org.openmetadata.service.search.models.FlattenColumn;
 import org.openmetadata.service.search.models.SearchSuggest;
-import org.openmetadata.service.util.JsonUtils;
 
 public record DashboardDataModelIndex(DashboardDataModel dashboardDataModel)
  implements ColumnIndex {
- private static final List<String> excludeFields = List.of("changeDescription");
 
- public Map<String, Object> buildESDoc() {
- Map<String, Object> doc = JsonUtils.getMap(dashboardDataModel);
- SearchIndexUtils.removeNonIndexableFields(doc, excludeFields);
+ @Override
+ public Object getEntity() {
+ return dashboardDataModel;
+ }
+
+ @Override
+ public Map<String, Object> buildSearchIndexDocInternal(Map<String, Object> doc) {
  List<SearchSuggest> suggest = new ArrayList<>();
  List<SearchSuggest> columnSuggest = new ArrayList<>();
  suggest.add(SearchSuggest.builder().input(dashboardDataModel.getName()).weight(10).build());
@@ -34,7 +36,6 @@ public Map<String, Object> buildESDoc() {
  .build());
  Set<List<TagLabel>> tagsWithChildren = new HashSet<>();
  List<String> columnsWithChildrenName = new ArrayList<>();
- SearchIndexUtils.removeNonIndexableFields(doc, excludeFields);
  if (dashboardDataModel.getColumns() != null) {
  List<FlattenColumn> cols = new ArrayList<>();
  parseColumns(dashboardDataModel.getColumns(), cols, null);

openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/DashboardIndex.java

@@ -12,19 +12,21 @@
 import org.openmetadata.service.search.ParseTags;
 import org.openmetadata.service.search.SearchIndexUtils;
 import org.openmetadata.service.search.models.SearchSuggest;
-import org.openmetadata.service.util.JsonUtils;
 
 public class DashboardIndex implements SearchIndex {
  final Dashboard dashboard;
- final List<String> excludeFields = List.of("changeDescription");
 
  public DashboardIndex(Dashboard dashboard) {
  this.dashboard = dashboard;
  }
 
- public Map<String, Object> buildESDoc() {
- Map<String, Object> doc = JsonUtils.getMap(dashboard);
- SearchIndexUtils.removeNonIndexableFields(doc, excludeFields);
+ @Override
+ public Object getEntity() {
+ return dashboard;
+ }
+
+ @Override
+ public Map<String, Object> buildSearchIndexDocInternal(Map<String, Object> doc) {
  List<SearchSuggest> suggest = new ArrayList<>();
  List<SearchSuggest> serviceSuggest = new ArrayList<>();
  List<SearchSuggest> chartSuggest = new ArrayList<>();

openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/DashboardServiceIndex.java

@@ -7,15 +7,16 @@
 import org.openmetadata.service.Entity;
 import org.openmetadata.service.search.SearchIndexUtils;
 import org.openmetadata.service.search.models.SearchSuggest;
-import org.openmetadata.service.util.JsonUtils;
 
 public record DashboardServiceIndex(DashboardService dashboardService) implements SearchIndex {
 
- private static final List<String> excludeFields = List.of("changeDescription");
+ @Override
+ public Object getEntity() {
+ return dashboardService;
+ }
 
- public Map<String, Object> buildESDoc() {
- Map<String, Object> doc = JsonUtils.getMap(dashboardService);
- SearchIndexUtils.removeNonIndexableFields(doc, excludeFields);
+ @Override
+ public Map<String, Object> buildSearchIndexDocInternal(Map<String, Object> doc) {
  List<SearchSuggest> suggest = new ArrayList<>();
  suggest.add(SearchSuggest.builder().input(dashboardService.getName()).weight(5).build());
  suggest.add(

openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/DataProductIndex.java

@@ -7,14 +7,16 @@
 import org.openmetadata.service.Entity;
 import org.openmetadata.service.search.SearchIndexUtils;
 import org.openmetadata.service.search.models.SearchSuggest;
-import org.openmetadata.service.util.JsonUtils;
 
 public record DataProductIndex(DataProduct dataProduct) implements SearchIndex {
- private static final List<String> excludeFields = List.of("changeDescription");
 
- public Map<String, Object> buildESDoc() {
- Map<String, Object> doc = JsonUtils.getMap(dataProduct);
- SearchIndexUtils.removeNonIndexableFields(doc, excludeFields);
+ @Override
+ public Object getEntity() {
+ return dataProduct;
+ }
+
+ @Override
+ public Map<String, Object> buildSearchIndexDocInternal(Map<String, Object> doc) {
  List<SearchSuggest> suggest = new ArrayList<>();
  suggest.add(SearchSuggest.builder().input(dataProduct.getName()).weight(5).build());
  suggest.add(

openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/DatabaseIndex.java

@@ -8,14 +8,16 @@
 import org.openmetadata.service.Entity;
 import org.openmetadata.service.search.SearchIndexUtils;
 import org.openmetadata.service.search.models.SearchSuggest;
-import org.openmetadata.service.util.JsonUtils;
 
 public record DatabaseIndex(Database database) implements SearchIndex {
- private static final List<String> excludeFields = List.of("changeDescription");
 
- public Map<String, Object> buildESDoc() {
- Map<String, Object> doc = JsonUtils.getMap(database);
- SearchIndexUtils.removeNonIndexableFields(doc, excludeFields);
+ @Override
+ public Object getEntity() {
+ return database;
+ }
+
+ @Override
+ public Map<String, Object> buildSearchIndexDocInternal(Map<String, Object> doc) {
  List<SearchSuggest> suggest = new ArrayList<>();
  suggest.add(SearchSuggest.builder().input(database.getName()).weight(5).build());
  suggest.add(SearchSuggest.builder().input(database.getFullyQualifiedName()).weight(5).build());

openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/DatabaseSchemaIndex.java

@@ -8,14 +8,16 @@
 import org.openmetadata.service.Entity;
 import org.openmetadata.service.search.SearchIndexUtils;
 import org.openmetadata.service.search.models.SearchSuggest;
-import org.openmetadata.service.util.JsonUtils;
 
 public record DatabaseSchemaIndex(DatabaseSchema databaseSchema) implements SearchIndex {
- private static final List<String> excludeFields = List.of("changeDescription");
 
- public Map<String, Object> buildESDoc() {
- Map<String, Object> doc = JsonUtils.getMap(databaseSchema);
- SearchIndexUtils.removeNonIndexableFields(doc, excludeFields);
+ @Override
+ public Object getEntity() {
+ return databaseSchema;
+ }
+
+ @Override
+ public Map<String, Object> buildSearchIndexDocInternal(Map<String, Object> doc) {
  List<SearchSuggest> suggest = new ArrayList<>();
  suggest.add(SearchSuggest.builder().input(databaseSchema.getName()).weight(5).build());
  suggest.add(

openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/DatabaseServiceIndex.java

@@ -7,14 +7,16 @@
 import org.openmetadata.service.Entity;
 import org.openmetadata.service.search.SearchIndexUtils;
 import org.openmetadata.service.search.models.SearchSuggest;
-import org.openmetadata.service.util.JsonUtils;
 
 public record DatabaseServiceIndex(DatabaseService databaseService) implements SearchIndex {
- private static final List<String> excludeFields = List.of("changeDescription");
 
- public Map<String, Object> buildESDoc() {
- Map<String, Object> doc = JsonUtils.getMap(databaseService);
- SearchIndexUtils.removeNonIndexableFields(doc, excludeFields);
+ @Override
+ public Object getEntity() {
+ return databaseService;
+ }
+
+ @Override
+ public Map<String, Object> buildSearchIndexDocInternal(Map<String, Object> doc) {
  List<SearchSuggest> suggest = new ArrayList<>();
  suggest.add(SearchSuggest.builder().input(databaseService.getName()).weight(5).build());
  suggest.add(

openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/DomainIndex.java

@@ -7,14 +7,16 @@
 import org.openmetadata.service.Entity;
 import org.openmetadata.service.search.SearchIndexUtils;
 import org.openmetadata.service.search.models.SearchSuggest;
-import org.openmetadata.service.util.JsonUtils;
 
 public record DomainIndex(Domain domain) implements SearchIndex {
- private static final List<String> excludeFields = List.of("changeDescription");
 
- public Map<String, Object> buildESDoc() {
- Map<String, Object> doc = JsonUtils.getMap(domain);
- SearchIndexUtils.removeNonIndexableFields(doc, excludeFields);
+ @Override
+ public Object getEntity() {
+ return domain;
+ }
+
+ @Override
+ public Map<String, Object> buildSearchIndexDocInternal(Map<String, Object> doc) {
  List<SearchSuggest> suggest = new ArrayList<>();
  suggest.add(SearchSuggest.builder().input(domain.getName()).weight(5).build());
  suggest.add(SearchSuggest.builder().input(domain.getFullyQualifiedName()).weight(5).build());