Skip to content

Commit

Permalink
Merge branch 'main' into feature-import-export-custom-property
Browse files Browse the repository at this point in the history
  • Loading branch information
@sonika-shah
sonika-shah authored Sep 29, 2024
2 parents eaa9e86 + 35bef56 commit bf56784
Show file tree
Hide file tree
Showing 44 changed files with 1,380 additions and 266 deletions.
4 changes: 2 additions & 2 deletions openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/CollectionDAO.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1402,7 +1402,7 @@ List<List<String>> listCountByEntityLink(
+ " SELECT te.type, te.taskStatus, te.id "
+ " FROM thread_entity te "
+ " WHERE MATCH(te.taskAssigneesIds) AGAINST (:userTeamJsonMysql IN BOOLEAN MODE) "
+ ") AS combined "
+ ") AS combined WHERE combined.type is not NULL "
+ "GROUP BY combined.type, combined.taskStatus;",
connectionType = MYSQL)
@ConnectionAwareSqlQuery(
Expand Down Expand Up @@ -1436,7 +1436,7 @@ List<List<String>> listCountByEntityLink(
+ " SELECT te.type, te.taskStatus, te.id "
+ " FROM thread_entity te "
+ " WHERE to_tsvector('simple', taskAssigneesIds) @@ to_tsquery('simple', :userTeamJsonPostgres) "
+ ") AS combined "
+ ") AS combined WHERE combined.type is not NULL "
+ "GROUP BY combined.type, combined.taskStatus;",
connectionType = POSTGRES)
@RegisterRowMapper(OwnerCountFieldMapper.class)
Expand Down
126 changes: 102 additions & 24 deletions openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/EntityRepository.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -67,6 +67,7 @@
import static org.openmetadata.service.util.EntityUtil.tagLabelMatch;

import com.fasterxml.jackson.databind.JsonNode;
import com.fasterxml.jackson.databind.node.ArrayNode;
import com.fasterxml.jackson.databind.node.ObjectNode;
import com.google.common.annotations.VisibleForTesting;
import com.google.common.cache.CacheBuilder;
Expand Down Expand Up @@ -100,6 +101,7 @@
import java.util.function.BiPredicate;
import java.util.function.Function;
import java.util.stream.Collectors;
import java.util.stream.StreamSupport;
import javax.json.JsonPatch;
import javax.validation.constraints.NotNull;
import javax.ws.rs.core.Response.Status;
Expand Down Expand Up @@ -143,6 +145,7 @@
import org.openmetadata.schema.type.api.BulkOperationResult;
import org.openmetadata.schema.type.api.BulkResponse;
import org.openmetadata.schema.type.csv.CsvImportResult;
import org.openmetadata.schema.type.customproperties.EnumWithDescriptionsConfig;
import org.openmetadata.schema.utils.EntityInterfaceUtil;
import org.openmetadata.service.Entity;
import org.openmetadata.service.OpenMetadataApplicationConfig;
Expand Down Expand Up @@ -1426,41 +1429,116 @@ private void validateExtension(T entity) {
}
String customPropertyType = TypeRegistry.getCustomPropertyType(entityType, fieldName);
String propertyConfig = TypeRegistry.getCustomPropertyConfig(entityType, fieldName);
DateTimeFormatter formatter;
try {
if ("date-cp".equals(customPropertyType)) {
DateTimeFormatter inputFormatter =
DateTimeFormatter.ofPattern(Objects.requireNonNull(propertyConfig), Locale.ENGLISH);

// Parse the input string into a TemporalAccessor
TemporalAccessor date = inputFormatter.parse(fieldValue.textValue());

// Create a formatter for the desired output format
DateTimeFormatter outputFormatter =
DateTimeFormatter.ofPattern(propertyConfig, Locale.ENGLISH);
((ObjectNode) jsonNode).put(fieldName, outputFormatter.format(date));
} else if ("dateTime-cp".equals(customPropertyType)) {
formatter = DateTimeFormatter.ofPattern(Objects.requireNonNull(propertyConfig));
LocalDateTime dateTime = LocalDateTime.parse(fieldValue.textValue(), formatter);
((ObjectNode) jsonNode).put(fieldName, dateTime.format(formatter));
} else if ("time-cp".equals(customPropertyType)) {
formatter = DateTimeFormatter.ofPattern(Objects.requireNonNull(propertyConfig));
LocalTime time = LocalTime.parse(fieldValue.textValue(), formatter);
((ObjectNode) jsonNode).put(fieldName, time.format(formatter));
}
try {
validateAndUpdateExtensionBasedOnPropertyType(
entity,
(ObjectNode) jsonNode,
fieldName,
fieldValue,
customPropertyType,
propertyConfig);
} catch (DateTimeParseException e) {
throw new IllegalArgumentException(
CatalogExceptionMessage.dateTimeValidationError(
fieldName, TypeRegistry.getCustomPropertyConfig(entityType, fieldName)));
CatalogExceptionMessage.dateTimeValidationError(fieldName, propertyConfig));
}
Set<ValidationMessage> validationMessages = jsonSchema.validate(fieldValue);

Set<ValidationMessage> validationMessages = jsonSchema.validate(entry.getValue());
if (!validationMessages.isEmpty()) {
throw new IllegalArgumentException(
CatalogExceptionMessage.jsonValidationError(fieldName, validationMessages.toString()));
}
}
}

private void validateAndUpdateExtensionBasedOnPropertyType(
T entity,
ObjectNode jsonNode,
String fieldName,
JsonNode fieldValue,
String customPropertyType,
String propertyConfig) {

switch (customPropertyType) {
case "date-cp", "dateTime-cp", "time-cp" -> {
String formattedValue =
getFormattedDateTimeField(
fieldValue.textValue(), customPropertyType, propertyConfig, fieldName);
jsonNode.put(fieldName, formattedValue);
}
case "enumWithDescriptions" -> handleEnumWithDescriptions(
fieldName, fieldValue, propertyConfig, jsonNode, entity);
default -> {}
}
}

private String getFormattedDateTimeField(
String fieldValue, String customPropertyType, String propertyConfig, String fieldName) {
DateTimeFormatter formatter;

try {
return switch (customPropertyType) {
case "date-cp" -> {
DateTimeFormatter inputFormatter =
DateTimeFormatter.ofPattern(propertyConfig, Locale.ENGLISH);
TemporalAccessor date = inputFormatter.parse(fieldValue);
DateTimeFormatter outputFormatter =
DateTimeFormatter.ofPattern(propertyConfig, Locale.ENGLISH);
yield outputFormatter.format(date);
}
case "dateTime-cp" -> {
formatter = DateTimeFormatter.ofPattern(propertyConfig);
LocalDateTime dateTime = LocalDateTime.parse(fieldValue, formatter);
yield dateTime.format(formatter);
}
case "time-cp" -> {
formatter = DateTimeFormatter.ofPattern(propertyConfig);
LocalTime time = LocalTime.parse(fieldValue, formatter);
yield time.format(formatter);
}
default -> throw new IllegalArgumentException(
"Unsupported customPropertyType: " + customPropertyType);
};
} catch (DateTimeParseException e) {
throw new IllegalArgumentException(
CatalogExceptionMessage.dateTimeValidationError(fieldName, propertyConfig));
}
}

private void handleEnumWithDescriptions(
String fieldName, JsonNode fieldValue, String propertyConfig, ObjectNode jsonNode, T entity) {
JsonNode propertyConfigNode = JsonUtils.readTree(propertyConfig);
EnumWithDescriptionsConfig config =
JsonUtils.treeToValue(propertyConfigNode, EnumWithDescriptionsConfig.class);

if (!config.getMultiSelect() && fieldValue.size() > 1) {
throw new IllegalArgumentException(
"Only one key is allowed for non-multiSelect enumWithDescriptions");
}
// Replace each enumWithDescriptions key in the fieldValue with the corresponding object from
// the propertyConfig
Map<String, JsonNode> keyToObjectMap =
StreamSupport.stream(propertyConfigNode.get("values").spliterator(), false)
.collect(Collectors.toMap(node -> node.get("key").asText(), node -> node));

if (fieldValue.isArray()) {
ArrayNode newArray = JsonUtils.getObjectNode().arrayNode();
fieldValue.forEach(
valueNode -> {
String key = valueNode.isTextual() ? valueNode.asText() : valueNode.get("key").asText();
JsonNode valueObject = keyToObjectMap.get(key);

if (valueObject == null) {
throw new IllegalArgumentException("Key not found in propertyConfig: " + key);
}
newArray.add(valueNode.isTextual() ? valueObject : valueNode);
});

jsonNode.replace(fieldName, newArray);
entity.setExtension(JsonUtils.treeToValue(jsonNode, Object.class));
}
}

public final void storeExtension(EntityInterface entity) {
JsonNode jsonNode = JsonUtils.valueToTree(entity.getExtension());
Iterator<Entry<String, JsonNode>> customFields = jsonNode.fields();
Expand Down
83 changes: 65 additions & 18 deletions openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/TypeRepository.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,7 @@
import java.util.List;
import java.util.Set;
import java.util.UUID;
import java.util.stream.Collectors;
import javax.ws.rs.core.UriInfo;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.lang3.tuple.Triple;
Expand All @@ -40,6 +41,8 @@
import org.openmetadata.schema.type.Include;
import org.openmetadata.schema.type.Relationship;
import org.openmetadata.schema.type.customproperties.EnumConfig;
import org.openmetadata.schema.type.customproperties.EnumWithDescriptionsConfig;
import org.openmetadata.schema.type.customproperties.Value;
import org.openmetadata.service.Entity;
import org.openmetadata.service.TypeRegistry;
import org.openmetadata.service.resources.types.TypeResource;
Expand Down Expand Up @@ -169,24 +172,9 @@ private List<CustomProperty> getCustomProperties(Type type) {

private void validateProperty(CustomProperty customProperty) {
switch (customProperty.getPropertyType().getName()) {
case "enum" -> {
CustomPropertyConfig config = customProperty.getCustomPropertyConfig();
if (config != null) {
EnumConfig enumConfig = JsonUtils.convertValue(config.getConfig(), EnumConfig.class);
if (enumConfig == null
|| (enumConfig.getValues() != null && enumConfig.getValues().isEmpty())) {
throw new IllegalArgumentException(
"Enum Custom Property Type must have EnumConfig populated with values.");
} else if (enumConfig.getValues() != null
&& enumConfig.getValues().stream().distinct().count()
!= enumConfig.getValues().size()) {
throw new IllegalArgumentException(
"Enum Custom Property values cannot have duplicates.");
}
} else {
throw new IllegalArgumentException("Enum Custom Property Type must have EnumConfig.");
}
}
case "enum" -> validateEnumConfig(customProperty.getCustomPropertyConfig());
case "enumWithDescriptions" -> validateEnumWithDescriptionsConfig(
customProperty.getCustomPropertyConfig());
case "date-cp" -> validateDateFormat(
customProperty.getCustomPropertyConfig(), getDateTokens(), "Invalid date format");
case "dateTime-cp" -> validateDateFormat(
Expand Down Expand Up @@ -229,6 +217,44 @@ private Set<Character> getTimeTokens() {
return Set.of('H', 'h', 'm', 's', 'a', 'S');
}

private void validateEnumConfig(CustomPropertyConfig config) {
if (config != null) {
EnumConfig enumConfig = JsonUtils.convertValue(config.getConfig(), EnumConfig.class);
if (enumConfig == null
|| (enumConfig.getValues() != null && enumConfig.getValues().isEmpty())) {
throw new IllegalArgumentException(
"Enum Custom Property Type must have EnumConfig populated with values.");
} else if (enumConfig.getValues() != null
&& enumConfig.getValues().stream().distinct().count() != enumConfig.getValues().size()) {
throw new IllegalArgumentException("Enum Custom Property values cannot have duplicates.");
}
} else {
throw new IllegalArgumentException("Enum Custom Property Type must have EnumConfig.");
}
}

private void validateEnumWithDescriptionsConfig(CustomPropertyConfig config) {
if (config != null) {
EnumWithDescriptionsConfig enumWithDescriptionsConfig =
JsonUtils.convertValue(config.getConfig(), EnumWithDescriptionsConfig.class);
if (enumWithDescriptionsConfig == null
|| (enumWithDescriptionsConfig.getValues() != null
&& enumWithDescriptionsConfig.getValues().isEmpty())) {
throw new IllegalArgumentException(
"EnumWithDescriptions Custom Property Type must have customPropertyConfig populated with values.");
}
JsonUtils.validateJsonSchema(config.getConfig(), EnumWithDescriptionsConfig.class);
if (enumWithDescriptionsConfig.getValues().stream().map(Value::getKey).distinct().count()
!= enumWithDescriptionsConfig.getValues().size()) {
throw new IllegalArgumentException(
"EnumWithDescriptions Custom Property key cannot have duplicates.");
}
} else {
throw new IllegalArgumentException(
"EnumWithDescriptions Custom Property Type must have customPropertyConfig.");
}
}

/** Handles entity updated from PUT and POST operation. */
public class TypeUpdater extends EntityUpdater {
public TypeUpdater(Type original, Type updated, Operation operation) {
Expand Down Expand Up @@ -387,6 +413,27 @@ private void validatePropertyConfigUpdate(
throw new IllegalArgumentException(
"Existing Enum Custom Property values cannot be removed.");
}
} else if (origProperty.getPropertyType().getName().equals("enumWithDescriptions")) {
EnumWithDescriptionsConfig origConfig =
JsonUtils.convertValue(
origProperty.getCustomPropertyConfig().getConfig(),
EnumWithDescriptionsConfig.class);
EnumWithDescriptionsConfig updatedConfig =
JsonUtils.convertValue(
updatedProperty.getCustomPropertyConfig().getConfig(),
EnumWithDescriptionsConfig.class);
HashSet<String> updatedValues =
updatedConfig.getValues().stream()
.map(Value::getKey)
.collect(Collectors.toCollection(HashSet::new));
if (updatedValues.size() != updatedConfig.getValues().size()) {
throw new IllegalArgumentException(
"EnumWithDescriptions Custom Property values cannot have duplicates.");
} else if (!updatedValues.containsAll(
origConfig.getValues().stream().map(Value::getKey).collect(Collectors.toSet()))) {
throw new IllegalArgumentException(
"Existing EnumWithDescriptions Custom Property values cannot be removed.");
}
}
}
}
Expand Down
7 changes: 3 additions & 4 deletions ...ta-service/src/main/java/org/openmetadata/service/resources/apis/APIEndpointResource.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -47,7 +47,6 @@
import org.openmetadata.schema.api.data.CreateAPIEndpoint;
import org.openmetadata.schema.api.data.RestoreEntity;
import org.openmetadata.schema.entity.data.APIEndpoint;
import org.openmetadata.schema.entity.data.Topic;
import org.openmetadata.schema.type.ChangeEvent;
import org.openmetadata.schema.type.EntityHistory;
import org.openmetadata.schema.type.Include;
Expand Down Expand Up @@ -370,17 +369,17 @@ public Response updateAPIEndpoint(
@PUT
@Operation(
operationId = "createOrUpdateAPIEndpoint",
summary = "Update topic",
summary = "Update API Endpoint",
description =
"Create a API Endpoint, it it does not exist or update an existing API Endpoint.",
responses = {
@ApiResponse(
responseCode = "200",
description = "The updated topic ",
description = "The updated api endpoint ",
content =
@Content(
mediaType = "application/json",
schema = @Schema(implementation = Topic.class)))
schema = @Schema(implementation = APIEndpoint.class)))
})
public Response createOrUpdate(
@Context UriInfo uriInfo,
Expand Down
11 changes: 5 additions & 6 deletions ...-service/src/main/java/org/openmetadata/service/resources/databases/DatabaseResource.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -51,7 +51,6 @@
import org.openmetadata.schema.api.data.CreateDatabase;
import org.openmetadata.schema.api.data.RestoreEntity;
import org.openmetadata.schema.entity.data.Database;
import org.openmetadata.schema.entity.data.Table;
import org.openmetadata.schema.type.ChangeEvent;
import org.openmetadata.schema.type.DatabaseProfilerConfig;
import org.openmetadata.schema.type.EntityHistory;
Expand Down Expand Up @@ -566,15 +565,15 @@ public Response restoreDatabase(
@Operation(
operationId = "addDataProfilerConfig",
summary = "Add database profile config",
description = "Add database profile config to the table.",
description = "Add database profile config to the database.",
responses = {
@ApiResponse(
responseCode = "200",
description = "Successfully updated the Database ",
content =
@Content(
mediaType = "application/json",
schema = @Schema(implementation = Table.class)))
schema = @Schema(implementation = Database.class)))
})
public Database addDataProfilerConfig(
@Context UriInfo uriInfo,
Expand All @@ -595,15 +594,15 @@ public Database addDataProfilerConfig(
@Operation(
operationId = "getDataProfilerConfig",
summary = "Get database profile config",
description = "Get database profile config to the table.",
description = "Get database profile config to the database.",
responses = {
@ApiResponse(
responseCode = "200",
description = "Successfully updated the Database ",
content =
@Content(
mediaType = "application/json",
schema = @Schema(implementation = Table.class)))
schema = @Schema(implementation = Database.class)))
})
public Database getDataProfilerConfig(
@Context UriInfo uriInfo,
Expand Down Expand Up @@ -633,7 +632,7 @@ public Database getDataProfilerConfig(
content =
@Content(
mediaType = "application/json",
schema = @Schema(implementation = Table.class)))
schema = @Schema(implementation = Database.class)))
})
public Database deleteDataProfilerConfig(
@Context UriInfo uriInfo,
Expand Down
Loading

0 comments on commit bf56784

Please sign in to comment.