Skip to content

Commit

Permalink
Add backend tests , and error msg improvements
Browse files Browse the repository at this point in the history
  • Loading branch information
@sonika-shah
sonika-shah committed Sep 27, 2024
1 parent 9fbde3c commit e5176f9
Show file tree
Hide file tree
Showing 6 changed files with 344 additions and 41 deletions.
5 changes: 3 additions & 2 deletions openmetadata-service/src/main/java/org/openmetadata/csv/CsvUtil.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -273,10 +273,10 @@ private static String quoteCsvField(String str) {
return str;
}

public static void addExtension(List<String> csvRecord, Object extension) {
public static List<String> addExtension(List<String> csvRecord, Object extension) {
if (extension == null) {
csvRecord.add(null);
return;
return csvRecord;
}

ObjectMapper objectMapper = new ObjectMapper();
Expand All @@ -293,6 +293,7 @@ public static void addExtension(List<String> csvRecord, Object extension) {
.collect(Collectors.joining(FIELD_SEPARATOR));

csvRecord.add(extensionString);
return csvRecord;
}

private static String formatValue(Object value) {
Expand Down
94 changes: 63 additions & 31 deletions openmetadata-service/src/main/java/org/openmetadata/csv/EntityCsv.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -370,7 +370,7 @@ private void validateExtension(
// Fetch the JSON schema and property type for the given field name
JsonSchema jsonSchema = TypeRegistry.instance().getSchema(entityType, fieldName);
if (jsonSchema == null) {
importFailure(printer, "Unknown custom field: " + fieldName, csvRecord);
importFailure(printer, invalidCustomPropertyKey(fieldNumber, fieldName), csvRecord);
return;
}
String customPropertyType = TypeRegistry.getCustomPropertyType(entityType, fieldName);
Expand All @@ -382,34 +382,52 @@ private void validateExtension(
case "entityReference", "entityReferenceList" -> {
boolean isList = "entityReferenceList".equals(customPropertyType);
fieldValue =
parseEntityReferences(fieldValue.toString(), printer, csvRecord, fieldNumber, isList);
parseEntityReferences(printer, csvRecord, fieldNumber, fieldValue.toString(), isList);
}
case "date-cp", "dateTime-cp", "time-cp" -> fieldValue =
getFormattedDateTimeField(
customPropertyType,
fieldValue.toString(),
propertyConfig,
fieldName,
printer,
csvRecord);
csvRecord,
fieldNumber,
fieldName,
fieldValue.toString(),
customPropertyType,
propertyConfig);
case "enum", "enumList" -> {
List<String> enumKeys = listOrEmpty(fieldToInternalArray(fieldValue.toString()));
fieldValue = enumKeys.isEmpty() ? null : enumKeys;
}
case "timeInterval" -> fieldValue =
handleTimeInterval(
printer, csvRecord, fieldNumber, fieldName, fieldValue, extensionMap, jsonSchema);
case "number", "integer", "timestamp" -> fieldValue = Long.parseLong(fieldValue.toString());
handleTimeInterval(printer, csvRecord, fieldNumber, fieldName, fieldValue);
case "number", "integer", "timestamp" -> {
try {
fieldValue = Long.parseLong(fieldValue.toString());
} catch (NumberFormatException e) {
importFailure(
printer,
invalidCustomPropertyValue(
fieldNumber, fieldName, customPropertyType, fieldValue.toString()),
csvRecord);
fieldValue = null;
}
}
default -> {}
}
// Validate the field against the JSON schema
validateAndUpdateExtension(
printer, csvRecord, fieldNumber, fieldName, fieldValue, extensionMap, jsonSchema);
printer,
csvRecord,
fieldNumber,
fieldName,
fieldValue,
customPropertyType,
extensionMap,
jsonSchema);
}
}

private Object parseEntityReferences(
String fieldValue, CSVPrinter printer, CSVRecord csvRecord, int fieldNumber, boolean isList)
CSVPrinter printer, CSVRecord csvRecord, int fieldNumber, String fieldValue, boolean isList)
throws IOException {
List<EntityReference> entityReferences = new ArrayList<>();

Expand Down Expand Up @@ -439,12 +457,13 @@ private Object parseEntityReferences(
}

protected String getFormattedDateTimeField(
String fieldType,
String fieldValue,
String propertyConfig,
String fieldName,
CSVPrinter printer,
CSVRecord csvRecord)
CSVRecord csvRecord,
int fieldNumber,
String fieldName,
String fieldValue,
String fieldType,
String propertyConfig)
throws IOException {
try {
DateTimeFormatter formatter = DateTimeFormatter.ofPattern(propertyConfig, Locale.ENGLISH);
Expand All @@ -470,21 +489,14 @@ protected String getFormattedDateTimeField(
} catch (DateTimeParseException e) {
importFailure(
printer,
String.format(
"Custom field %s value is not as per defined format %s", fieldName, propertyConfig),
invalidCustomPropertyFieldFormat(fieldNumber, fieldName, fieldType, propertyConfig),
csvRecord);
return null;
}
}

private Map<String, Long> handleTimeInterval(
CSVPrinter printer,
CSVRecord csvRecord,
int fieldNumber,
String fieldName,
Object fieldValue,
Map<String, Object> extensionMap,
JsonSchema jsonSchema)
CSVPrinter printer, CSVRecord csvRecord, int fieldNumber, String fieldName, Object fieldValue)
throws IOException {
List<String> timestampValues = fieldToEntities(fieldValue.toString());
Map<String, Long> timestampMap = new HashMap<>();
Expand All @@ -494,7 +506,10 @@ private Map<String, Long> handleTimeInterval(
} else {
importFailure(
printer,
invalidField(fieldNumber, String.format("Invalid timestamp format in %s", fieldName)),
invalidField(
fieldNumber,
invalidCustomPropertyFieldFormat(
fieldNumber, fieldName, "timeInterval", "start:end")),
csvRecord);
}
return timestampMap;
Expand All @@ -506,6 +521,7 @@ private void validateAndUpdateExtension(
int fieldNumber,
String fieldName,
Object fieldValue,
String customPropertyType,
Map<String, Object> extensionMap,
JsonSchema jsonSchema)
throws IOException {
Expand All @@ -518,7 +534,8 @@ private void validateAndUpdateExtension(
if (!validationMessages.isEmpty()) {
importFailure(
printer,
invalidCustomPropertyValue(fieldNumber, fieldName, validationMessages.toString()),
invalidCustomPropertyValue(
fieldNumber, fieldName, customPropertyType, validationMessages.toString()),
csvRecord);
} else {
extensionMap.put(fieldName, fieldValue); // Add to extensionMap if valid
Expand Down Expand Up @@ -753,13 +770,28 @@ public static String invalidExtension(int field, String key, String value) {
+ key
+ ", Value = "
+ value
+ " .Extensions should be of format customPropertyName:customPropertyValue";
+ " . Extensions should be of format customPropertyName:customPropertyValue";
return String.format(FIELD_ERROR_MSG, CsvErrorType.INVALID_FIELD, field + 1, error);
}

public static String invalidCustomPropertyKey(int field, String key) {
String error = String.format("Unknown custom field: %s", key);
return String.format(FIELD_ERROR_MSG, CsvErrorType.INVALID_FIELD, field + 1, error);
}

public static String invalidCustomPropertyValue(
int field, String key, String fieldType, String value) {
String error =
String.format("Invalid value of Key = %s of type %s, Value = %s", key, fieldType, value);
return String.format(FIELD_ERROR_MSG, CsvErrorType.INVALID_FIELD, field + 1, error);
}

public static String invalidCustomPropertyValue(int field, String key, String value) {
public static String invalidCustomPropertyFieldFormat(
int field, String fieldName, String fieldType, String propertyConfig) {
String error =
"Invalid key-value pair in extension string: Key = " + key + ", Value = " + value;
String.format(
"Custom field %s value of type %s is not as per defined format %s",
fieldName, fieldType, propertyConfig);
return String.format(FIELD_ERROR_MSG, CsvErrorType.INVALID_FIELD, field + 1, error);
}

Expand Down
31 changes: 31 additions & 0 deletions openmetadata-service/src/test/java/org/openmetadata/csv/CsvUtilTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,10 +16,14 @@
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.openmetadata.common.utils.CommonUtil.listOf;

import com.fasterxml.jackson.databind.JsonNode;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.fasterxml.jackson.databind.node.ObjectNode;
import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
import org.junit.jupiter.api.Test;
import org.openmetadata.schema.entity.type.CustomProperty;
import org.openmetadata.schema.type.EntityReference;
import org.openmetadata.schema.type.TagLabel;

Expand Down Expand Up @@ -74,6 +78,33 @@ void testAddRecord() {
expectedRecord.add("t1;t2");
List<TagLabel> tags = listOf(new TagLabel().withTagFQN("t1"), new TagLabel().withTagFQN("t2"));
assertEquals(expectedRecord, CsvUtil.addTagLabels(actualRecord, tags));

// Add extension
expectedRecord.add(null);
assertEquals(expectedRecord, CsvUtil.addExtension(actualRecord, null)); // Null extension

ObjectMapper mapper = new ObjectMapper();
ObjectNode jsonNode = mapper.createObjectNode();

// Add new custom property stringCp of type string
CustomProperty stringCp =
new CustomProperty()
.withName("stringCp")
.withDescription("string type custom property")
.withPropertyType(
new EntityReference().withFullyQualifiedName("string").withType("type"));
JsonNode stringCpValue =
mapper.convertValue("String; input; with; semicolon\n And new line", JsonNode.class);
jsonNode.set("stringCp", stringCpValue);

// Add new custom property queryCp of type sqlQuery
JsonNode queryCpValue =
mapper.convertValue("SELECT * FROM table WHERE column = 'value';", JsonNode.class);
jsonNode.set("queryCp", queryCpValue);

expectedRecord.add(
"\"stringCp:String; input; with; semicolon\n And new line\";\"queryCp:SELECT * FROM table WHERE column = 'value';\"");
assertEquals(expectedRecord, CsvUtil.addExtension(actualRecord, jsonNode));
}

public static void assertCsv(String expectedCsv, String actualCsv) {
Expand Down
12 changes: 12 additions & 0 deletions ...metadata-service/src/test/java/org/openmetadata/service/resources/EntityResourceTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -389,6 +389,18 @@ public abstract class EntityResourceTest<T extends EntityInterface, K extends Cr

public static Type INT_TYPE;
public static Type STRING_TYPE;
public static Type EMAIL_TYPE;
public static Type DATECP_TYPE;
public static Type DATETIMECP_TYPE;
public static Type TIMECP_TYPE;
public static Type DURATION_TYPE;
public static Type MARKDOWN_TYPE;
public static Type ENTITY_REFERENCE_TYPE;
public static Type ENTITY_REFERENCE_LIST_TYPE;
public static Type TIME_INTERVAL_TYPE;
public static Type NUMBER_TYPE;
public static Type SQLQUERY_TYPE;
public static Type TIMESTAMP_TYPE;

public static Type ENUM_TYPE;

Expand Down
Loading

0 comments on commit e5176f9

Please sign in to comment.