FIX: test summary reindexing (open-metadata#18667)

* fix: test summary reindexing * fix: clean up test case result from index on test case deletion
SumanMaharana · Nov 18, 2024 · 8391150 · 8391150
1 parent f0b0a47
commit 8391150
Show file tree

Hide file tree

Showing 3 changed files with 82 additions and 15 deletions.
diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/TestCaseRepository.java b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/TestCaseRepository.java
@@ -36,7 +36,6 @@
 import org.openmetadata.schema.api.tests.CreateTestCaseResult;
 import org.openmetadata.schema.entity.data.Table;
 import org.openmetadata.schema.entity.teams.User;
-import org.openmetadata.schema.tests.ResultSummary;
 import org.openmetadata.schema.tests.TestCase;
 import org.openmetadata.schema.tests.TestCaseParameter;
 import org.openmetadata.schema.tests.TestCaseParameterValidationRule;
@@ -48,7 +47,6 @@
 import org.openmetadata.schema.tests.type.TestCaseResolutionStatus;
 import org.openmetadata.schema.tests.type.TestCaseResolutionStatusTypes;
 import org.openmetadata.schema.tests.type.TestCaseResult;
-import org.openmetadata.schema.tests.type.TestCaseStatus;
 import org.openmetadata.schema.type.ChangeDescription;
 import org.openmetadata.schema.type.EntityReference;
 import org.openmetadata.schema.type.FieldChange;
@@ -79,7 +77,6 @@ public class TestCaseRepository extends EntityRepository<TestCase> {
       "owners,entityLink,testSuite,testSuites,testDefinition";
   private static final String PATCH_FIELDS =
       "owners,entityLink,testSuite,testDefinition,computePassedFailedRowCount,useDynamicAssertion";
-  public static final String TESTCASE_RESULT_EXTENSION = "testCase.testCaseResult";
   public static final String FAILED_ROWS_SAMPLE_EXTENSION = "testCase.failedRowsSample";
 
   public TestCaseRepository() {
@@ -370,14 +367,6 @@ private void deleteAllTestCaseResults(String fqn) {
     testCaseResultRepository.deleteAllTestCaseResults(fqn);
   }
 
-  private ResultSummary getResultSummary(
-      TestCase testCase, Long timestamp, TestCaseStatus testCaseStatus) {
-    return new ResultSummary()
-        .withTestCaseName(testCase.getFullyQualifiedName())
-        .withStatus(testCaseStatus)
-        .withTimestamp(timestamp);
-  }
-
   @SneakyThrows
   private TestCaseResult getTestCaseResult(TestCase testCase) {
     TestCaseResult testCaseResult;
@@ -395,14 +384,16 @@ private TestCaseResult getTestCaseResult(TestCase testCase) {
       testCaseResult =
           timeSeriesRepository.latestFromSearch(Fields.EMPTY_FIELDS, searchListFilter, null);
     } catch (Exception e) {
-      // Index may not exist in the search index (e.g. reindexing with recreate index on). Fall back
-      // to database
       LOG.debug(
           "Error fetching test case result from search. Fetching from test case results from database",
           e);
       testCaseResult =
           timeSeriesRepository.listLastTestCaseResult(testCase.getFullyQualifiedName());
     }
+    if (nullOrEmpty(testCaseResult)) {
+      testCaseResult =
+          timeSeriesRepository.listLastTestCaseResult(testCase.getFullyQualifiedName());
+    }
     return testCaseResult;
   }
 

diff --git a/...tadata-service/src/main/java/org/openmetadata/service/jdbi3/TestCaseResultRepository.java b/...tadata-service/src/main/java/org/openmetadata/service/jdbi3/TestCaseResultRepository.java
@@ -2,11 +2,13 @@
 
 import static org.openmetadata.schema.type.EventType.ENTITY_DELETED;
 import static org.openmetadata.service.Entity.TEST_CASE;
+import static org.openmetadata.service.Entity.TEST_CASE_RESULT;
 import static org.openmetadata.service.Entity.TEST_DEFINITION;
 
 import java.io.IOException;
 import java.util.Arrays;
 import java.util.List;
+import java.util.Map;
 import java.util.Optional;
 import java.util.UUID;
 import javax.json.JsonPatch;
@@ -237,6 +239,11 @@ protected List<String> getIncludeSearchFields() {
   protected void deleteAllTestCaseResults(String fqn) {
     // Delete all the test case results
     daoCollection.dataQualityDataTimeSeriesDao().deleteAll(fqn);
+    Map<String, Object> params = Map.of("fqn", fqn);
+    searchRepository.deleteByScript(
+        TEST_CASE_RESULT,
+        "if (!(doc['testCaseFQN.keyword'].empty)) { doc['testCaseFQN.keyword'].value == params.fqn}",
+        params);
   }
 
   public boolean hasTestCaseFailure(String fqn) throws IOException {

diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/TestSuiteRepository.java b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/TestSuiteRepository.java
@@ -1,6 +1,7 @@
 package org.openmetadata.service.jdbi3;
 
 import static org.openmetadata.common.utils.CommonUtil.listOrEmpty;
+import static org.openmetadata.common.utils.CommonUtil.nullOrEmpty;
 import static org.openmetadata.schema.type.EventType.ENTITY_DELETED;
 import static org.openmetadata.schema.type.EventType.ENTITY_SOFT_DELETED;
 import static org.openmetadata.schema.type.Include.ALL;
@@ -14,10 +15,12 @@
 
 import java.io.IOException;
 import java.util.ArrayList;
+import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
 import java.util.Optional;
 import java.util.UUID;
+import java.util.stream.Collectors;
 import javax.json.JsonArray;
 import javax.json.JsonObject;
 import javax.json.JsonValue;
@@ -29,6 +32,7 @@
 import org.openmetadata.schema.entity.data.Table;
 import org.openmetadata.schema.tests.DataQualityReport;
 import org.openmetadata.schema.tests.ResultSummary;
+import org.openmetadata.schema.tests.TestCase;
 import org.openmetadata.schema.tests.TestSuite;
 import org.openmetadata.schema.tests.type.ColumnTestSummaryDefinition;
 import org.openmetadata.schema.tests.type.TestCaseResult;
@@ -111,10 +115,12 @@ public TestSuiteRepository() {
   public void setFields(TestSuite entity, EntityUtil.Fields fields) {
     entity.setPipelines(
         fields.contains("pipelines") ? getIngestionPipelines(entity) : entity.getPipelines());
-    entity.setSummary(
-        fields.contains("summary") ? getTestSummary(entity.getId()) : entity.getSummary());
     entity.withTests(fields.contains(UPDATE_FIELDS) ? getTestCases(entity) : entity.getTests());
     entity.withTestCaseResultSummary(getResultSummary(entity.getId()));
+    entity.setSummary(
+        fields.contains("summary")
+            ? getTestSummary(entity.getTestCaseResultSummary())
+            : entity.getSummary());
   }
 
   @Override
@@ -218,8 +224,66 @@ public DataQualityReport getDataQualityReport(String q, String aggQuery, String
     return searchRepository.genericAggregation(q, index, searchAggregation);
   }
 
+  public TestSummary getTestSummary(List<ResultSummary> testCaseResults) {
+    Map<String, Map<String, Integer>> summaries =
+        testCaseResults.stream()
+            .collect(
+                Collectors.groupingBy(
+                    result -> {
+                      TestCase testCase =
+                          Entity.getEntityByName(TEST_CASE, result.getTestCaseName(), "", ALL);
+                      MessageParser.EntityLink entityLink =
+                          MessageParser.EntityLink.parse(testCase.getEntityLink());
+                      return entityLink.getFieldName() == null
+                          ? "table"
+                          : entityLink.getLinkString();
+                    },
+                    Collectors.groupingBy(
+                        result -> result.getStatus().toString(),
+                        Collectors.collectingAndThen(Collectors.counting(), Long::intValue))));
+
+    Map<String, Integer> testSummaryMap = summaries.getOrDefault("table", new HashMap<>());
+    TestSummary testSummary = createTestSummary(testSummaryMap);
+    testSummary.setTotal(testCaseResults.size());
+
+    List<ColumnTestSummaryDefinition> columnTestSummaryDefinitions =
+        summaries.entrySet().stream()
+            .filter(entry -> !entry.getKey().equals("table"))
+            .map(
+                entry -> {
+                  Map<String, Integer> columnSummaryMap = entry.getValue();
+                  ColumnTestSummaryDefinition columnTestSummaryDefinition =
+                      createColumnSummary(columnSummaryMap);
+                  columnTestSummaryDefinition.setEntityLink(entry.getKey());
+                  return columnTestSummaryDefinition;
+                })
+            .toList();
+    testSummary.setColumnTestSummary(columnTestSummaryDefinitions);
+    return testSummary;
+  }
+
+  private TestSummary createTestSummary(Map<String, Integer> summaryMap) {
+    TestSummary summary = new TestSummary();
+    summary.setSuccess(summaryMap.getOrDefault("Success", 0));
+    summary.setFailed(summaryMap.getOrDefault("Failed", 0));
+    summary.setAborted(summaryMap.getOrDefault("Aborted", 0));
+    summary.setQueued(summaryMap.getOrDefault("Queued", 0));
+    return summary;
+  }
+
+  private ColumnTestSummaryDefinition createColumnSummary(Map<String, Integer> summaryMap) {
+    ColumnTestSummaryDefinition summary = new ColumnTestSummaryDefinition();
+    summary.setSuccess(summaryMap.getOrDefault("Success", 0));
+    summary.setFailed(summaryMap.getOrDefault("Failed", 0));
+    summary.setAborted(summaryMap.getOrDefault("Aborted", 0));
+    summary.setQueued(summaryMap.getOrDefault("Queued", 0));
+    summary.setTotal(summaryMap.values().stream().mapToInt(Integer::intValue).sum());
+    return summary;
+  }
+
   public TestSummary getTestSummary(UUID testSuiteId) {
     try {
+      // TODO: Delete with https://github.com/open-metadata/OpenMetadata/pull/18323
       TestSummary testSummary;
       if (testSuiteId == null) {
         String aggregationStr =
@@ -293,6 +357,11 @@ private List<ResultSummary> getResultSummary(UUID testSuiteId) {
           entityTimeSeriesRepository.listLastTestCaseResultsForTestSuite(testSuiteId);
     }
 
+    if (nullOrEmpty(latestTestCaseResultResults.getData())) {
+      latestTestCaseResultResults =
+          entityTimeSeriesRepository.listLastTestCaseResultsForTestSuite(testSuiteId);
+    }
+
     latestTestCaseResultResults
         .getData()
         .forEach(