diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/EntityTimeSeriesRepository.java b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/EntityTimeSeriesRepository.java index 208d833415c8..539b182f73dc 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/EntityTimeSeriesRepository.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/EntityTimeSeriesRepository.java @@ -2,6 +2,7 @@ import static org.openmetadata.schema.type.EventType.ENTITY_UPDATED; import static org.openmetadata.schema.type.Include.ALL; +import static org.openmetadata.service.Entity.getEntityFields; import java.beans.IntrospectionException; import java.io.IOException; @@ -11,9 +12,9 @@ import java.util.List; import java.util.Map; import java.util.Optional; +import java.util.Set; import java.util.UUID; import javax.json.Json; -import javax.json.JsonArray; import javax.json.JsonObject; import javax.json.JsonObjectBuilder; import javax.json.JsonPatch; @@ -46,6 +47,7 @@ public abstract class EntityTimeSeriesRepository entityClass; protected final CollectionDAO daoCollection; + protected final Set allowedFields; public EntityTimeSeriesRepository( String collectionPath, @@ -58,21 +60,19 @@ public EntityTimeSeriesRepository( this.entityType = entityType; this.searchRepository = Entity.getSearchRepository(); this.daoCollection = Entity.getCollectionDAO(); + this.allowedFields = getEntityFields(entityClass); Entity.registerEntity(entityClass, entityType, this); } @Transaction - public T createNewRecord(T recordEntity, String extension, String recordFQN) { - recordEntity.setId(UUID.randomUUID()); - storeInternal(recordEntity, recordFQN, extension); - storeRelationshipInternal(recordEntity); - postCreate(recordEntity); - return recordEntity; + public T createNewRecord(T recordEntity, String recordFQN) { + return createNewRecord(recordEntity, null, recordFQN); } - public T createNewRecord(T recordEntity, String recordFQN) { + @Transaction + public T createNewRecord(T recordEntity, String extension, String recordFQN) { recordEntity.setId(UUID.randomUUID()); - storeInternal(recordEntity, recordFQN); + storeInternal(recordEntity, recordFQN, extension); storeRelationshipInternal(recordEntity); postCreate(recordEntity); return recordEntity; @@ -80,12 +80,23 @@ public T createNewRecord(T recordEntity, String recordFQN) { @Transaction protected void storeInternal(T recordEntity, String recordFQN) { - timeSeriesDao.insert(recordFQN, entityType, JsonUtils.pojoToJson(recordEntity)); + storeInternal(recordEntity, recordFQN, null); } @Transaction protected void storeInternal(T recordEntity, String recordFQN, String extension) { - timeSeriesDao.insert(recordFQN, extension, entityType, JsonUtils.pojoToJson(recordEntity)); + if (extension != null) { + timeSeriesDao.insert(recordFQN, extension, entityType, JsonUtils.pojoToJson(recordEntity)); + } else { + timeSeriesDao.insert(recordFQN, entityType, JsonUtils.pojoToJson(recordEntity)); + } + } + + public final EntityUtil.Fields getFields(String fields) { + if ("*".equals(fields)) { + return new EntityUtil.Fields(allowedFields, String.join(",", allowedFields)); + } + return new EntityUtil.Fields(allowedFields, fields); } protected void storeRelationshipInternal(T recordEntity) { @@ -124,7 +135,6 @@ protected void setUpdatedFields(T updated, String user) { protected void validatePatchFields(T updated, T original) { // Nothing to do in the default implementation } - ; @Transaction public final void addRelationship( @@ -214,29 +224,25 @@ public ResultList listWithOffset( boolean latest, boolean skipErrors) { int total = timeSeriesDao.listCount(filter, startTs, endTs, latest); - List entityList = new ArrayList<>(); - List errors = null; - int offsetInt = getOffset(offset); - String afterOffset = getAfterOffset(offsetInt, limitParam, total); - String beforeOffset = getBeforeOffset(offsetInt, limitParam); - - if (limitParam > 0) { - List jsons = - timeSeriesDao.listWithOffset(filter, limitParam, offsetInt, startTs, endTs, latest); - Map> entityListMap = getEntityList(jsons, skipErrors); - entityList = (List) entityListMap.get("entityList"); - if (skipErrors) { - errors = (List) entityListMap.get("errors"); - } - return getResultList(entityList, beforeOffset, afterOffset, total, errors); - } else { - return getResultList(entityList, null, null, total); - } + return listWithOffsetInternal( + offset, filter, limitParam, startTs, endTs, latest, skipErrors, total); } public ResultList listWithOffset( String offset, ListFilter filter, int limitParam, boolean skipErrors) { int total = timeSeriesDao.listCount(filter); + return listWithOffsetInternal(offset, filter, limitParam, null, null, false, skipErrors, total); + } + + private ResultList listWithOffsetInternal( + String offset, + ListFilter filter, + int limitParam, + Long startTs, + Long endTs, + boolean latest, + boolean skipErrors, + int total) { List entityList = new ArrayList<>(); List errors = null; @@ -244,7 +250,10 @@ public ResultList listWithOffset( String afterOffset = getAfterOffset(offsetInt, limitParam, total); String beforeOffset = getBeforeOffset(offsetInt, limitParam); if (limitParam > 0) { - List jsons = timeSeriesDao.listWithOffset(filter, limitParam, offsetInt); + List jsons = + (startTs != null && endTs != null) + ? timeSeriesDao.listWithOffset(filter, limitParam, offsetInt, startTs, endTs, latest) + : timeSeriesDao.listWithOffset(filter, limitParam, offsetInt); Map> entityListMap = getEntityList(jsons, skipErrors); entityList = (List) entityListMap.get("entityList"); if (skipErrors) { @@ -397,21 +406,40 @@ public ResultList listLatestFromSearch( List entityList = new ArrayList<>(); setIncludeSearchFields(searchListFilter); setExcludeSearchFields(searchListFilter); + String aggregationPath = "$.sterms#byTerms.buckets"; String aggregationStr = - "{\"aggregations\": {\"byTerms\": {\"terms\": {\"field\": \"%s\", \"size\":100},\"aggs\": {\"latest\": " - + "{\"top_hits\": {\"size\": 1, \"sort_field\":\"timestamp\",\"sort_order\":\"desc\"}}}}}}"; + "{\"aggregations\":{\"byTerms\":{\"terms\": {\"field\":\"%s\",\"size\":100},\"aggs\":{\"latest\":" + + "{\"top_hits\":{\"size\":1,\"sort_field\":\"timestamp\",\"sort_order\":\"desc\"}}}}}}"; aggregationStr = String.format(aggregationStr, groupBy); JsonObject aggregation = JsonUtils.readJson(aggregationStr).asJsonObject(); JsonObject jsonObjResults = searchRepository.aggregate(q, entityType, aggregation, searchListFilter); - List jsonTestCaseResults = parseListLatestAggregation(jsonObjResults); - for (JsonObject json : jsonTestCaseResults) { - T entity = setFieldsInternal(JsonUtils.readOrConvertValue(json, entityClass), fields); - setInheritedFields(entity); - clearFieldsInternal(entity, fields); - entityList.add(entity); - } + Optional jsonObjects = + JsonUtils.readJsonAtPath(jsonObjResults.toString(), aggregationPath, List.class); + jsonObjects.ifPresent( + jsonObjectList -> { + for (Map json : (List>) jsonObjectList) { + String bucketAggregationPath = "top_hits#latest.hits.hits"; + Optional hits = + JsonUtils.readJsonAtPath( + JsonUtils.pojoToJson(json), bucketAggregationPath, List.class); + hits.ifPresent( + hitList -> { + for (Map hit : (List>) hitList) { + JsonObject source = getSourceDocument(JsonUtils.pojoToJson(hit)); + T entity = + setFieldsInternal( + JsonUtils.readOrConvertValue(source, entityClass), fields); + if (entity != null) { + setInheritedFields(entity); + clearFieldsInternal(entity, fields); + entityList.add(entity); + } + } + }); + } + }); return new ResultList<>(entityList, null, null, entityList.size()); } @@ -447,48 +475,27 @@ protected List getExcludeSearchFields() { return new ArrayList<>(); } - private List parseListLatestAggregation(JsonObject jsonObjResults) { - JsonObject jsonByTerms = jsonObjResults.getJsonObject("sterms#byTerms"); - List jsonTestCaseResults = new ArrayList<>(); + private JsonObject getSourceDocument(String hit) { List includeSearchFields = getIncludeSearchFields(); List excludeSearchFields = getExcludeSearchFields(); - Optional.ofNullable(jsonByTerms) - .map(jbt -> jbt.getJsonArray("buckets")) - .ifPresent( - termsBucket -> { - for (JsonValue bucket : termsBucket) { - JsonObject hitsBucket = bucket.asJsonObject().getJsonObject("top_hits#latest"); - if (hitsBucket != null) { - JsonObject hitsTwo = hitsBucket.getJsonObject("hits"); - if (hitsTwo != null) { - JsonArray hits = hitsTwo.getJsonArray("hits"); - if (hits != null) { - for (JsonValue hit : hits) { - JsonObject source = hit.asJsonObject().getJsonObject("_source"); - // Aggregation results will return all fields by default, so we need to - // filter out the fields - // that are not included in the search fields - if (source != null - && (!CommonUtil.nullOrEmpty(includeSearchFields) - || !CommonUtil.nullOrEmpty(excludeSearchFields))) { - JsonObjectBuilder sourceCopy = Json.createObjectBuilder(); - for (Map.Entry entry : source.entrySet()) { - if (includeSearchFields.contains(entry.getKey()) - || (CommonUtil.nullOrEmpty(includeSearchFields) - && !excludeSearchFields.contains(entry.getKey()))) { - sourceCopy.add(entry.getKey(), entry.getValue()); - } - } - jsonTestCaseResults.add(sourceCopy.build()); - } else { - if (source != null) jsonTestCaseResults.add(source); - } - } - } - } - } - } - }); - return jsonTestCaseResults; + JsonObject hitJson = JsonUtils.readJson(hit).asJsonObject(); + JsonObject source = hitJson.asJsonObject().getJsonObject("_source"); + // Aggregation results will return all fields by default, + // so we need to filter out the fields that are not included + // in the search fields + if (source != null + && (!CommonUtil.nullOrEmpty(includeSearchFields) + || !CommonUtil.nullOrEmpty(excludeSearchFields))) { + JsonObjectBuilder sourceCopy = Json.createObjectBuilder(); + for (Map.Entry entry : source.entrySet()) { + if (includeSearchFields.contains(entry.getKey()) + || (CommonUtil.nullOrEmpty(includeSearchFields) + && !excludeSearchFields.contains(entry.getKey()))) { + sourceCopy.add(entry.getKey(), entry.getValue()); + } + } + return sourceCopy.build(); + } + return source; } } diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/TestCaseResolutionStatusRepository.java b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/TestCaseResolutionStatusRepository.java index ecec0d8f41f5..628e2a287159 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/TestCaseResolutionStatusRepository.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/TestCaseResolutionStatusRepository.java @@ -165,7 +165,8 @@ private void validateStatus( @Override @Transaction - public void storeInternal(TestCaseResolutionStatus recordEntity, String recordFQN) { + public void storeInternal( + TestCaseResolutionStatus recordEntity, String recordFQN, String extension) { TestCaseResolutionStatus lastIncident = getLatestRecord(recordFQN); @@ -212,7 +213,7 @@ public void storeInternal(TestCaseResolutionStatus recordEntity, String recordFQ } EntityReference testCaseReference = recordEntity.getTestCaseReference(); recordEntity.withTestCaseReference(null); // we don't want to store the reference in the record - super.storeInternal(recordEntity, recordFQN); + timeSeriesDao.insert(recordFQN, entityType, JsonUtils.pojoToJson(recordEntity)); recordEntity.withTestCaseReference(testCaseReference); } @@ -302,7 +303,10 @@ private void resolveTask( EntityReference testCaseReference = newIncidentStatus.getTestCaseReference(); newIncidentStatus.setTestCaseReference( null); // we don't want to store the reference in the record - super.storeInternal(newIncidentStatus, testCase.getFullyQualifiedName()); + timeSeriesDao.insert( + testCaseReference.getFullyQualifiedName(), + entityType, + JsonUtils.pojoToJson(newIncidentStatus)); newIncidentStatus.setTestCaseReference(testCaseReference); } diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/resources/dqtests/TestCaseResultResource.java b/openmetadata-service/src/main/java/org/openmetadata/service/resources/dqtests/TestCaseResultResource.java index 63b5cf49bcf1..30d8b444eb40 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/resources/dqtests/TestCaseResultResource.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/resources/dqtests/TestCaseResultResource.java @@ -222,10 +222,41 @@ public ResultList listTestCaseResultsFromSearch( @Parameter( description = "Get the latest test case result for each test case -- requires `testSuiteId`. Offset and limit are ignored", - schema = @Schema(type = "boolean", example = "true|false")) + schema = + @Schema( + type = "boolean", + example = "false", + allowableValues = {"true", "false"})) @QueryParam("latest") @DefaultValue("false") String latest, + @Parameter( + description = "Filter for test case result by type (e.g. column, table, all)", + schema = + @Schema( + type = "string", + example = "all", + allowableValues = {"column", "table", "all"})) + @QueryParam("testCaseType") + @DefaultValue("all") + String type, + @Parameter( + description = + "Filter for test case by data quality dimension (e.g. OpenMetadata, dbt, etc.)", + schema = + @Schema( + type = "string", + allowableValues = { + "Completeness", + "Accuracy", + "Consistency", + "Validity", + "Uniqueness", + "Integrity", + "SQL" + })) + @QueryParam("dataQualityDimension") + String dataQualityDimension, @Parameter( description = "search query term to use in list", schema = @Schema(type = "string")) @@ -235,7 +266,7 @@ public ResultList listTestCaseResultsFromSearch( if (latest.equals("true") && testSuiteId == null) { throw new IllegalArgumentException("latest=true requires testSuiteId"); } - EntityUtil.Fields fields = new EntityUtil.Fields(Set.of(""), fieldParams); + EntityUtil.Fields fields = repository.getFields(fieldParams); SearchListFilter searchListFilter = new SearchListFilter(); Optional.ofNullable(startTimestamp) .ifPresent(ts -> searchListFilter.addQueryParam("startTimestamp", ts.toString())); @@ -247,6 +278,9 @@ public ResultList listTestCaseResultsFromSearch( .ifPresent(tcf -> searchListFilter.addQueryParam("testCaseFQN", tcf)); Optional.ofNullable(testSuiteId) .ifPresent(tsi -> searchListFilter.addQueryParam("testSuiteId", tsi)); + Optional.ofNullable(type).ifPresent(t -> searchListFilter.addQueryParam("testCaseType", t)); + Optional.ofNullable(dataQualityDimension) + .ifPresent(dqd -> searchListFilter.addQueryParam("dataQualityDimension", dqd)); ResourceContextInterface resourceContextInterface = getResourceContext(testCaseFQN); // Override OperationContext to change the entity to table @@ -259,7 +293,7 @@ public ResultList listTestCaseResultsFromSearch( securityContext, fields, searchListFilter, - "testSuites.id", + "testCaseFQN.keyword", q, operationContext, resourceContextInterface); diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/search/SearchListFilter.java b/openmetadata-service/src/main/java/org/openmetadata/service/search/SearchListFilter.java index caea29b57b95..c3e6bb22d880 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/search/SearchListFilter.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/search/SearchListFilter.java @@ -193,15 +193,7 @@ private String getTestCaseCondition() { String.format("{\"term\": {\"testCaseResult.testCaseStatus\": \"%s\"}}", status)); } - if (type != null) { - conditions.add( - switch (type) { - case Entity - .TABLE -> "{\"bool\": {\"must_not\": [{\"regexp\": {\"entityLink\": \".*::columns::.*\"}}]}}"; - case "column" -> "{\"regexp\": {\"entityLink\": \".*::columns::.*\"}}"; - default -> ""; - }); - } + if (type != null) conditions.add(getTestCaseTypeCondition(type, "entityLink")); if (testPlatform != null) { String platforms = @@ -216,10 +208,9 @@ private String getTestCaseCondition() { getTimestampFilter("testCaseResult.timestamp", "lte", Long.parseLong(endTimestamp))); } - if (dataQualityDimension != null) { + if (dataQualityDimension != null) conditions.add( - String.format("{\"term\": {\"dataQualityDimension\": \"%s\"}}", dataQualityDimension)); - } + getDataQualityDimensionCondition(dataQualityDimension, "dataQualityDimension")); return addCondition(conditions); } @@ -227,6 +218,8 @@ private String getTestCaseCondition() { private String getTestCaseResultCondition() { ArrayList conditions = new ArrayList<>(); + String dataQualityDimension = getQueryParam("dataQualityDimension"); + String type = getQueryParam("testCaseType"); String startTimestamp = getQueryParam("startTimestamp"); String endTimestamp = getQueryParam("endTimestamp"); String testCaseFQN = getQueryParam("testCaseFQN"); @@ -245,10 +238,14 @@ private String getTestCaseResultCondition() { + "{\"term\": {\"testCase.fullyQualifiedName\": \"%1$s\"}}]}}", escapeDoubleQuotes(testCaseFQN))); } - if (testCaseStatus != null) { + if (testCaseStatus != null) conditions.add(String.format("{\"term\": {\"testCaseStatus\": \"%s\"}}", testCaseStatus)); - } + if (type != null) conditions.add(getTestCaseTypeCondition(type, "testCase.entityLink")); if (testSuiteId != null) conditions.add(getTestSuiteIdCondition(testSuiteId)); + if (dataQualityDimension != null) + conditions.add( + getDataQualityDimensionCondition( + dataQualityDimension, "testDefinition.dataQualityDimension")); return addCondition(conditions); } @@ -287,4 +284,17 @@ private String getTestSuiteIdCondition(String testSuiteId) { "{\"nested\":{\"path\":\"testSuites\",\"query\":{\"term\":{\"testSuites.id\":\"%s\"}}}}", testSuiteId); } + + private String getTestCaseTypeCondition(String type, String field) { + return switch (type) { + case Entity.TABLE -> String.format( + "{\"bool\": {\"must_not\": [{\"regexp\": {\"%s\": \".*::columns::.*\"}}]}}", field); + case "column" -> String.format("{\"regexp\": {\"%s\": \".*::columns::.*\"}}", field); + default -> ""; + }; + } + + private String getDataQualityDimensionCondition(String dataQualityDimension, String field) { + return String.format("{\"term\": {\"%s\": \"%s\"}}", field, dataQualityDimension); + } } diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/util/JsonUtils.java b/openmetadata-service/src/main/java/org/openmetadata/service/util/JsonUtils.java index bacda0c6cce1..0840d45bf361 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/util/JsonUtils.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/util/JsonUtils.java @@ -27,6 +27,8 @@ import com.fasterxml.jackson.databind.type.TypeFactory; import com.fasterxml.jackson.datatype.jsr353.JSR353Module; import com.github.fge.jsonpatch.diff.JsonDiff; +import com.jayway.jsonpath.DocumentContext; +import com.jayway.jsonpath.JsonPath; import com.networknt.schema.JsonSchema; import com.networknt.schema.JsonSchemaFactory; import com.networknt.schema.SpecVersion.VersionFlag; @@ -40,6 +42,7 @@ import java.util.Map; import java.util.Map.Entry; import java.util.Objects; +import java.util.Optional; import java.util.Set; import java.util.TreeMap; import java.util.stream.Collectors; @@ -145,6 +148,16 @@ public static T readValue(String json, String clazzName) { } } + public static Optional readJsonAtPath(String json, String path, Class clazz) { + try { + DocumentContext documentContext = JsonPath.parse(json); + return Optional.ofNullable(documentContext.read(path, clazz)); + } catch (Exception e) { + LOG.error("Failed to read value at path {}", path, e); + return Optional.empty(); + } + } + public static T readValue(String json, Class clz) { if (json == null) { return null; diff --git a/openmetadata-service/src/main/resources/elasticsearch/en/test_case_result_index_mapping.json b/openmetadata-service/src/main/resources/elasticsearch/en/test_case_result_index_mapping.json index b62c711ad536..f2ebdd3f32b8 100644 --- a/openmetadata-service/src/main/resources/elasticsearch/en/test_case_result_index_mapping.json +++ b/openmetadata-service/src/main/resources/elasticsearch/en/test_case_result_index_mapping.json @@ -127,6 +127,15 @@ "type": "keyword", "normalizer": "lowercase_normalizer" }, + "entityLink": { + "type": "keyword", + "normalizer": "lowercase_normalizer", + "fields": { + "nonNormalized": { + "type": "keyword" + } + } + }, "parameterValues": { "properties": { "name": { diff --git a/openmetadata-service/src/main/resources/elasticsearch/jp/test_case_result_index_mapping.json b/openmetadata-service/src/main/resources/elasticsearch/jp/test_case_result_index_mapping.json index b44162fff29f..b21893d1341f 100644 --- a/openmetadata-service/src/main/resources/elasticsearch/jp/test_case_result_index_mapping.json +++ b/openmetadata-service/src/main/resources/elasticsearch/jp/test_case_result_index_mapping.json @@ -121,6 +121,15 @@ "type": "keyword", "normalizer": "lowercase_normalizer" }, + "entityLink": { + "type": "keyword", + "normalizer": "lowercase_normalizer", + "fields": { + "nonNormalized": { + "type": "keyword" + } + } + }, "parameterValues": { "properties": { "name": { diff --git a/openmetadata-service/src/main/resources/elasticsearch/zh/test_case_result_index_mapping.json b/openmetadata-service/src/main/resources/elasticsearch/zh/test_case_result_index_mapping.json index 169e9cdb726c..dbf33d89ed4a 100644 --- a/openmetadata-service/src/main/resources/elasticsearch/zh/test_case_result_index_mapping.json +++ b/openmetadata-service/src/main/resources/elasticsearch/zh/test_case_result_index_mapping.json @@ -118,6 +118,15 @@ "type": "keyword", "normalizer": "lowercase_normalizer" }, + "entityLink": { + "type": "keyword", + "normalizer": "lowercase_normalizer", + "fields": { + "nonNormalized": { + "type": "keyword" + } + } + }, "parameterValues": { "properties": { "name": { diff --git a/openmetadata-service/src/test/java/org/openmetadata/service/resources/dqtests/TestCaseResourceTest.java b/openmetadata-service/src/test/java/org/openmetadata/service/resources/dqtests/TestCaseResourceTest.java index 81c606a86e05..26e2cff21a73 100644 --- a/openmetadata-service/src/test/java/org/openmetadata/service/resources/dqtests/TestCaseResourceTest.java +++ b/openmetadata-service/src/test/java/org/openmetadata/service/resources/dqtests/TestCaseResourceTest.java @@ -28,6 +28,8 @@ import static org.openmetadata.schema.type.ColumnDataType.BIGINT; import static org.openmetadata.schema.type.MetadataOperation.EDIT_TESTS; import static org.openmetadata.service.Entity.ADMIN_USER_NAME; +import static org.openmetadata.service.Entity.TEST_CASE; +import static org.openmetadata.service.Entity.TEST_DEFINITION; import static org.openmetadata.service.exception.CatalogExceptionMessage.permissionNotAllowed; import static org.openmetadata.service.jdbi3.TestCaseRepository.FAILED_ROWS_SAMPLE_EXTENSION; import static org.openmetadata.service.security.SecurityUtil.authHeaders; @@ -80,6 +82,7 @@ import org.openmetadata.schema.tests.ResultSummary; import org.openmetadata.schema.tests.TestCase; import org.openmetadata.schema.tests.TestCaseParameterValue; +import org.openmetadata.schema.tests.TestDefinition; import org.openmetadata.schema.tests.TestPlatform; import org.openmetadata.schema.tests.TestSuite; import org.openmetadata.schema.tests.type.Assigned; @@ -95,6 +98,7 @@ import org.openmetadata.schema.type.ChangeDescription; import org.openmetadata.schema.type.Column; import org.openmetadata.schema.type.ColumnDataType; +import org.openmetadata.schema.type.DataQualityDimensions; import org.openmetadata.schema.type.EntityReference; import org.openmetadata.schema.type.Include; import org.openmetadata.schema.type.TableData; @@ -3094,6 +3098,92 @@ void createTestCaseResults_wrongTs(TestInfo testInfo) throws IOException, HttpRe "Timestamp 1725521153 is not valid, it should be in milliseconds since epoch"); } + @Test + void test_listTestCaseFromSearch(TestInfo testInfo) throws HttpResponseException, ParseException { + CreateTestCase create = createRequest(testInfo); + create + .withEntityLink(TABLE_COLUMN_LINK) + .withTestSuite(TEST_SUITE1.getFullyQualifiedName()) + .withTestDefinition(TEST_DEFINITION3.getFullyQualifiedName()) + .withParameterValues( + List.of(new TestCaseParameterValue().withValue("100").withName("missingCountValue"))); + TestCase testCase = createEntity(create, ADMIN_AUTH_HEADERS); + for (int i = 1; i < 10; i++) { + CreateTestCaseResult createTestCaseResult = + new CreateTestCaseResult() + .withResult("tested") + .withTestCaseStatus(TestCaseStatus.Success) + .withTimestamp(TestUtils.dateToTimestamp("2021-09-0%s".formatted(i))); + postTestCaseResult( + testCase.getFullyQualifiedName(), createTestCaseResult, ADMIN_AUTH_HEADERS); + } + + Map queryParams = new HashMap<>(); + + queryParams.put("fields", "testCase,testDefinition"); + ResultList testCaseResultResultList = + listTestCaseResultsFromSearch( + queryParams, 10, 0, "/testCaseResults/search/list", ADMIN_AUTH_HEADERS); + assertNotEquals(testCaseResultResultList.getData().size(), 0); + testCaseResultResultList + .getData() + .forEach( + testCaseResult -> { + assertNotNull(testCaseResult.getTestCase()); + assertNotNull(testCaseResult.getTestDefinition()); + }); + + queryParams.clear(); + Long ts = TestUtils.dateToTimestamp("2021-09-01"); + queryParams.put("startTimestamp", ts.toString()); + queryParams.put("endTimestamp", TestUtils.dateToTimestamp("2021-09-01").toString()); + queryParams.put("latest", "true"); + queryParams.put("testSuiteId", TEST_SUITE1.getId().toString()); + + testCaseResultResultList = + listTestCaseResultsFromSearch( + queryParams, 10, 0, "/testCaseResults/search/list", ADMIN_AUTH_HEADERS); + assertNotEquals(testCaseResultResultList.getData().size(), 0); + testCaseResultResultList + .getData() + .forEach( + testCaseResult -> { + assertEquals(testCaseResult.getTimestamp(), ts); + }); + + queryParams.clear(); + queryParams.put("dataQualityDimension", "Completeness"); + queryParams.put("fields", "testDefinition"); + testCaseResultResultList = + listTestCaseResultsFromSearch( + queryParams, 10, 0, "/testCaseResults/search/list", ADMIN_AUTH_HEADERS); + assertNotEquals(testCaseResultResultList.getData().size(), 0); + testCaseResultResultList + .getData() + .forEach( + testCaseResult -> { + EntityReference testDefinition = testCaseResult.getTestDefinition(); + TestDefinition td = + Entity.getEntity(TEST_DEFINITION, testDefinition.getId(), "", Include.ALL); + assertEquals(td.getDataQualityDimension(), DataQualityDimensions.COMPLETENESS); + }); + + queryParams.clear(); + queryParams.put("testCaseType", "column"); + testCaseResultResultList = + listTestCaseResultsFromSearch( + queryParams, 10, 0, "/testCaseResults/search/list", ADMIN_AUTH_HEADERS); + assertNotEquals(testCaseResultResultList.getData().size(), 0); + testCaseResultResultList + .getData() + .forEach( + testCaseResult -> { + EntityReference testDefinition = testCaseResult.getTestCase(); + TestCase tc = Entity.getEntity(TEST_CASE, testCase.getId(), "", Include.ALL); + assertTrue(tc.getEntityLink().contains("columns")); + }); + } + private void putInspectionQuery(TestCase testCase, String sql) throws IOException { TestCase putResponse = putInspectionQuery(testCase.getId(), sql, ADMIN_AUTH_HEADERS); assertEquals(sql, putResponse.getInspectionQuery());