Merge branch 'main' into fix-lineage-pw

openmetadata-service/src/main/java/org/openmetadata/service/TypeRegistry.java

@@ -112,7 +112,8 @@ public static String getCustomPropertyType(String entityType, String propertyNam
  }
  }
  }
- return null;
+ throw EntityNotFoundException.byMessage(
+ CatalogExceptionMessage.entityNotFound(Entity.TYPE, String.valueOf(type)));
  }
 
  public static String getCustomPropertyConfig(String entityType, String propertyName) {
@@ -122,7 +123,13 @@ public static String getCustomPropertyConfig(String entityType, String propertyN
  if (property.getName().equals(propertyName)
  && property.getCustomPropertyConfig() != null
  && property.getCustomPropertyConfig().getConfig() != null) {
- return property.getCustomPropertyConfig().getConfig().toString();
+ Object config = property.getCustomPropertyConfig().getConfig();
+ if (config instanceof String || config instanceof Integer) {
+ return config.toString(); // for simple type config return as string
+ } else {
+ return JsonUtils.pojoToJson(
+ config); // for complex object in config return as JSON string
+ }
  }
  }
  }

openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/CollectionDAO.java

@@ -1402,7 +1402,7 @@ List<List<String>> listCountByEntityLink(
  + " SELECT te.type, te.taskStatus, te.id "
  + " FROM thread_entity te "
  + " WHERE MATCH(te.taskAssigneesIds) AGAINST (:userTeamJsonMysql IN BOOLEAN MODE) "
- + ") AS combined "
+ + ") AS combined WHERE combined.type is not NULL "
  + "GROUP BY combined.type, combined.taskStatus;",
  connectionType = MYSQL)
  @ConnectionAwareSqlQuery(
@@ -1436,7 +1436,7 @@ List<List<String>> listCountByEntityLink(
  + " SELECT te.type, te.taskStatus, te.id "
  + " FROM thread_entity te "
  + " WHERE to_tsvector('simple', taskAssigneesIds) @@ to_tsquery('simple', :userTeamJsonPostgres) "
- + ") AS combined "
+ + ") AS combined WHERE combined.type is not NULL "
  + "GROUP BY combined.type, combined.taskStatus;",
  connectionType = POSTGRES)
  @RegisterRowMapper(OwnerCountFieldMapper.class)

openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/EntityRepository.java

@@ -67,6 +67,7 @@
 import static org.openmetadata.service.util.EntityUtil.tagLabelMatch;
 
 import com.fasterxml.jackson.databind.JsonNode;
+import com.fasterxml.jackson.databind.node.ArrayNode;
 import com.fasterxml.jackson.databind.node.ObjectNode;
 import com.google.common.annotations.VisibleForTesting;
 import com.google.common.cache.CacheBuilder;
@@ -100,6 +101,7 @@
 import java.util.function.BiPredicate;
 import java.util.function.Function;
 import java.util.stream.Collectors;
+import java.util.stream.StreamSupport;
 import javax.json.JsonPatch;
 import javax.validation.constraints.NotNull;
 import javax.ws.rs.core.Response.Status;
@@ -143,6 +145,7 @@
 import org.openmetadata.schema.type.api.BulkOperationResult;
 import org.openmetadata.schema.type.api.BulkResponse;
 import org.openmetadata.schema.type.csv.CsvImportResult;
+import org.openmetadata.schema.type.customproperties.EnumWithDescriptionsConfig;
 import org.openmetadata.schema.utils.EntityInterfaceUtil;
 import org.openmetadata.service.Entity;
 import org.openmetadata.service.OpenMetadataApplicationConfig;
@@ -1426,41 +1429,116 @@ private void validateExtension(T entity) {
  }
  String customPropertyType = TypeRegistry.getCustomPropertyType(entityType, fieldName);
  String propertyConfig = TypeRegistry.getCustomPropertyConfig(entityType, fieldName);
- DateTimeFormatter formatter;
- try {
- if ("date-cp".equals(customPropertyType)) {
- DateTimeFormatter inputFormatter =
- DateTimeFormatter.ofPattern(Objects.requireNonNull(propertyConfig), Locale.ENGLISH);
 
- // Parse the input string into a TemporalAccessor
- TemporalAccessor date = inputFormatter.parse(fieldValue.textValue());
-
- // Create a formatter for the desired output format
- DateTimeFormatter outputFormatter =
- DateTimeFormatter.ofPattern(propertyConfig, Locale.ENGLISH);
- ((ObjectNode) jsonNode).put(fieldName, outputFormatter.format(date));
- } else if ("dateTime-cp".equals(customPropertyType)) {
- formatter = DateTimeFormatter.ofPattern(Objects.requireNonNull(propertyConfig));
- LocalDateTime dateTime = LocalDateTime.parse(fieldValue.textValue(), formatter);
- ((ObjectNode) jsonNode).put(fieldName, dateTime.format(formatter));
- } else if ("time-cp".equals(customPropertyType)) {
- formatter = DateTimeFormatter.ofPattern(Objects.requireNonNull(propertyConfig));
- LocalTime time = LocalTime.parse(fieldValue.textValue(), formatter);
- ((ObjectNode) jsonNode).put(fieldName, time.format(formatter));
- }
+ try {
+ validateAndUpdateExtensionBasedOnPropertyType(
+ entity,
+ (ObjectNode) jsonNode,
+ fieldName,
+ fieldValue,
+ customPropertyType,
+ propertyConfig);
  } catch (DateTimeParseException e) {
  throw new IllegalArgumentException(
- CatalogExceptionMessage.dateTimeValidationError(
- fieldName, TypeRegistry.getCustomPropertyConfig(entityType, fieldName)));
+ CatalogExceptionMessage.dateTimeValidationError(fieldName, propertyConfig));
  }
- Set<ValidationMessage> validationMessages = jsonSchema.validate(fieldValue);
+
+ Set<ValidationMessage> validationMessages = jsonSchema.validate(entry.getValue());
  if (!validationMessages.isEmpty()) {
  throw new IllegalArgumentException(
  CatalogExceptionMessage.jsonValidationError(fieldName, validationMessages.toString()));
  }
  }
  }
 
+ private void validateAndUpdateExtensionBasedOnPropertyType(
+ T entity,
+ ObjectNode jsonNode,
+ String fieldName,
+ JsonNode fieldValue,
+ String customPropertyType,
+ String propertyConfig) {
+
+ switch (customPropertyType) {
+ case "date-cp", "dateTime-cp", "time-cp" -> {
+ String formattedValue =
+ getFormattedDateTimeField(
+ fieldValue.textValue(), customPropertyType, propertyConfig, fieldName);
+ jsonNode.put(fieldName, formattedValue);
+ }
+ case "enumWithDescriptions" -> handleEnumWithDescriptions(
+ fieldName, fieldValue, propertyConfig, jsonNode, entity);
+ default -> {}
+ }
+ }
+
+ private String getFormattedDateTimeField(
+ String fieldValue, String customPropertyType, String propertyConfig, String fieldName) {
+ DateTimeFormatter formatter;
+
+ try {
+ return switch (customPropertyType) {
+ case "date-cp" -> {
+ DateTimeFormatter inputFormatter =
+ DateTimeFormatter.ofPattern(propertyConfig, Locale.ENGLISH);
+ TemporalAccessor date = inputFormatter.parse(fieldValue);
+ DateTimeFormatter outputFormatter =
+ DateTimeFormatter.ofPattern(propertyConfig, Locale.ENGLISH);
+ yield outputFormatter.format(date);
+ }
+ case "dateTime-cp" -> {
+ formatter = DateTimeFormatter.ofPattern(propertyConfig);
+ LocalDateTime dateTime = LocalDateTime.parse(fieldValue, formatter);
+ yield dateTime.format(formatter);
+ }
+ case "time-cp" -> {
+ formatter = DateTimeFormatter.ofPattern(propertyConfig);
+ LocalTime time = LocalTime.parse(fieldValue, formatter);
+ yield time.format(formatter);
+ }
+ default -> throw new IllegalArgumentException(
+ "Unsupported customPropertyType: " + customPropertyType);
+ };
+ } catch (DateTimeParseException e) {
+ throw new IllegalArgumentException(
+ CatalogExceptionMessage.dateTimeValidationError(fieldName, propertyConfig));
+ }
+ }
+
+ private void handleEnumWithDescriptions(
+ String fieldName, JsonNode fieldValue, String propertyConfig, ObjectNode jsonNode, T entity) {
+ JsonNode propertyConfigNode = JsonUtils.readTree(propertyConfig);
+ EnumWithDescriptionsConfig config =
+ JsonUtils.treeToValue(propertyConfigNode, EnumWithDescriptionsConfig.class);
+
+ if (!config.getMultiSelect() && fieldValue.size() > 1) {
+ throw new IllegalArgumentException(
+ "Only one key is allowed for non-multiSelect enumWithDescriptions");
+ }
+ // Replace each enumWithDescriptions key in the fieldValue with the corresponding object from
+ // the propertyConfig
+ Map<String, JsonNode> keyToObjectMap =
+ StreamSupport.stream(propertyConfigNode.get("values").spliterator(), false)
+ .collect(Collectors.toMap(node -> node.get("key").asText(), node -> node));
+
+ if (fieldValue.isArray()) {
+ ArrayNode newArray = JsonUtils.getObjectNode().arrayNode();
+ fieldValue.forEach(
+ valueNode -> {
+ String key = valueNode.isTextual() ? valueNode.asText() : valueNode.get("key").asText();
+ JsonNode valueObject = keyToObjectMap.get(key);
+
+ if (valueObject == null) {
+ throw new IllegalArgumentException("Key not found in propertyConfig: " + key);
+ }
+ newArray.add(valueNode.isTextual() ? valueObject : valueNode);
+ });
+
+ jsonNode.replace(fieldName, newArray);
+ entity.setExtension(JsonUtils.treeToValue(jsonNode, Object.class));
+ }
+ }
+
  public final void storeExtension(EntityInterface entity) {
  JsonNode jsonNode = JsonUtils.valueToTree(entity.getExtension());
  Iterator<Entry<String, JsonNode>> customFields = jsonNode.fields();

openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/TypeRepository.java

@@ -28,6 +28,7 @@
 import java.util.List;
 import java.util.Set;
 import java.util.UUID;
+import java.util.stream.Collectors;
 import javax.ws.rs.core.UriInfo;
 import lombok.extern.slf4j.Slf4j;
 import org.apache.commons.lang3.tuple.Triple;
@@ -40,6 +41,8 @@
 import org.openmetadata.schema.type.Include;
 import org.openmetadata.schema.type.Relationship;
 import org.openmetadata.schema.type.customproperties.EnumConfig;
+import org.openmetadata.schema.type.customproperties.EnumWithDescriptionsConfig;
+import org.openmetadata.schema.type.customproperties.Value;
 import org.openmetadata.service.Entity;
 import org.openmetadata.service.TypeRegistry;
 import org.openmetadata.service.resources.types.TypeResource;
@@ -169,24 +172,9 @@ private List<CustomProperty> getCustomProperties(Type type) {
 
  private void validateProperty(CustomProperty customProperty) {
  switch (customProperty.getPropertyType().getName()) {
- case "enum" -> {
- CustomPropertyConfig config = customProperty.getCustomPropertyConfig();
- if (config != null) {
- EnumConfig enumConfig = JsonUtils.convertValue(config.getConfig(), EnumConfig.class);
- if (enumConfig == null
- || (enumConfig.getValues() != null && enumConfig.getValues().isEmpty())) {
- throw new IllegalArgumentException(
- "Enum Custom Property Type must have EnumConfig populated with values.");
- } else if (enumConfig.getValues() != null
- && enumConfig.getValues().stream().distinct().count()
- != enumConfig.getValues().size()) {
- throw new IllegalArgumentException(
- "Enum Custom Property values cannot have duplicates.");
- }
- } else {
- throw new IllegalArgumentException("Enum Custom Property Type must have EnumConfig.");
- }
- }
+ case "enum" -> validateEnumConfig(customProperty.getCustomPropertyConfig());
+ case "enumWithDescriptions" -> validateEnumWithDescriptionsConfig(
+ customProperty.getCustomPropertyConfig());
  case "date-cp" -> validateDateFormat(
  customProperty.getCustomPropertyConfig(), getDateTokens(), "Invalid date format");
  case "dateTime-cp" -> validateDateFormat(
@@ -229,6 +217,44 @@ private Set<Character> getTimeTokens() {
  return Set.of('H', 'h', 'm', 's', 'a', 'S');
  }
 
+ private void validateEnumConfig(CustomPropertyConfig config) {
+ if (config != null) {
+ EnumConfig enumConfig = JsonUtils.convertValue(config.getConfig(), EnumConfig.class);
+ if (enumConfig == null
+ || (enumConfig.getValues() != null && enumConfig.getValues().isEmpty())) {
+ throw new IllegalArgumentException(
+ "Enum Custom Property Type must have EnumConfig populated with values.");
+ } else if (enumConfig.getValues() != null
+ && enumConfig.getValues().stream().distinct().count() != enumConfig.getValues().size()) {
+ throw new IllegalArgumentException("Enum Custom Property values cannot have duplicates.");
+ }
+ } else {
+ throw new IllegalArgumentException("Enum Custom Property Type must have EnumConfig.");
+ }
+ }
+
+ private void validateEnumWithDescriptionsConfig(CustomPropertyConfig config) {
+ if (config != null) {
+ EnumWithDescriptionsConfig enumWithDescriptionsConfig =
+ JsonUtils.convertValue(config.getConfig(), EnumWithDescriptionsConfig.class);
+ if (enumWithDescriptionsConfig == null
+ || (enumWithDescriptionsConfig.getValues() != null
+ && enumWithDescriptionsConfig.getValues().isEmpty())) {
+ throw new IllegalArgumentException(
+ "EnumWithDescriptions Custom Property Type must have customPropertyConfig populated with values.");
+ }
+ JsonUtils.validateJsonSchema(config.getConfig(), EnumWithDescriptionsConfig.class);
+ if (enumWithDescriptionsConfig.getValues().stream().map(Value::getKey).distinct().count()
+ != enumWithDescriptionsConfig.getValues().size()) {
+ throw new IllegalArgumentException(
+ "EnumWithDescriptions Custom Property key cannot have duplicates.");
+ }
+ } else {
+ throw new IllegalArgumentException(
+ "EnumWithDescriptions Custom Property Type must have customPropertyConfig.");
+ }
+ }
+
  /** Handles entity updated from PUT and POST operation. */
  public class TypeUpdater extends EntityUpdater {
  public TypeUpdater(Type original, Type updated, Operation operation) {
@@ -387,6 +413,27 @@ private void validatePropertyConfigUpdate(
  throw new IllegalArgumentException(
  "Existing Enum Custom Property values cannot be removed.");
  }
+ } else if (origProperty.getPropertyType().getName().equals("enumWithDescriptions")) {
+ EnumWithDescriptionsConfig origConfig =
+ JsonUtils.convertValue(
+ origProperty.getCustomPropertyConfig().getConfig(),
+ EnumWithDescriptionsConfig.class);
+ EnumWithDescriptionsConfig updatedConfig =
+ JsonUtils.convertValue(
+ updatedProperty.getCustomPropertyConfig().getConfig(),
+ EnumWithDescriptionsConfig.class);
+ HashSet<String> updatedValues =
+ updatedConfig.getValues().stream()
+ .map(Value::getKey)
+ .collect(Collectors.toCollection(HashSet::new));
+ if (updatedValues.size() != updatedConfig.getValues().size()) {
+ throw new IllegalArgumentException(
+ "EnumWithDescriptions Custom Property values cannot have duplicates.");
+ } else if (!updatedValues.containsAll(
+ origConfig.getValues().stream().map(Value::getKey).collect(Collectors.toSet()))) {
+ throw new IllegalArgumentException(
+ "Existing EnumWithDescriptions Custom Property values cannot be removed.");
+ }
  }
  }
  }

openmetadata-service/src/main/java/org/openmetadata/service/resources/apis/APIEndpointResource.java

@@ -47,7 +47,6 @@
 import org.openmetadata.schema.api.data.CreateAPIEndpoint;
 import org.openmetadata.schema.api.data.RestoreEntity;
 import org.openmetadata.schema.entity.data.APIEndpoint;
-import org.openmetadata.schema.entity.data.Topic;
 import org.openmetadata.schema.type.ChangeEvent;
 import org.openmetadata.schema.type.EntityHistory;
 import org.openmetadata.schema.type.Include;
@@ -370,17 +369,17 @@ public Response updateAPIEndpoint(
  @PUT
  @Operation(
  operationId = "createOrUpdateAPIEndpoint",
- summary = "Update topic",
+ summary = "Update API Endpoint",
  description =
  "Create a API Endpoint, it it does not exist or update an existing API Endpoint.",
  responses = {
  @ApiResponse(
  responseCode = "200",
- description = "The updated topic ",
+ description = "The updated api endpoint ",
  content =
  @Content(
  mediaType = "application/json",
- schema = @Schema(implementation = Topic.class)))
+ schema = @Schema(implementation = APIEndpoint.class)))
  })
  public Response createOrUpdate(
  @Context UriInfo uriInfo,