diff --git a/bootstrap/sql/migrations/native/1.5.6/mysql/postDataMigrationSQLScript.sql b/bootstrap/sql/migrations/native/1.5.6/mysql/postDataMigrationSQLScript.sql new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/bootstrap/sql/migrations/native/1.5.6/mysql/schemaChanges.sql b/bootstrap/sql/migrations/native/1.5.6/mysql/schemaChanges.sql new file mode 100644 index 000000000000..e6c09213ef7a --- /dev/null +++ b/bootstrap/sql/migrations/native/1.5.6/mysql/schemaChanges.sql @@ -0,0 +1,5 @@ +-- Delete Search Indexing Application +DELETE er FROM entity_relationship er JOIN installed_apps ia ON er.fromId = ia.id OR er.toId = ia.id WHERE ia.name = 'SearchIndexingApplication'; +DELETE er FROM entity_relationship er JOIN apps_marketplace ia ON er.fromId = ia.id OR er.toId = ia.id WHERE ia.name = 'SearchIndexingApplication'; +DELETE FROM installed_apps where name = 'SearchIndexingApplication'; +DELETE FROM apps_marketplace where name = 'SearchIndexingApplication'; \ No newline at end of file diff --git a/bootstrap/sql/migrations/native/1.5.6/postgres/postDataMigrationSQLScript.sql b/bootstrap/sql/migrations/native/1.5.6/postgres/postDataMigrationSQLScript.sql new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/bootstrap/sql/migrations/native/1.5.6/postgres/schemaChanges.sql b/bootstrap/sql/migrations/native/1.5.6/postgres/schemaChanges.sql new file mode 100644 index 000000000000..4a37bbc9d39c --- /dev/null +++ b/bootstrap/sql/migrations/native/1.5.6/postgres/schemaChanges.sql @@ -0,0 +1,5 @@ +-- Delete Search Indexing Application +DELETE FROM entity_relationship er USING installed_apps ia WHERE (er.fromId = ia.id OR er.toId = ia.id) AND ia.name = 'SearchIndexingApplication'; +DELETE FROM entity_relationship er USING apps_marketplace ia WHERE (er.fromId = ia.id OR er.toId = ia.id) AND ia.name = 'SearchIndexingApplication'; +DELETE FROM installed_apps where name = 'SearchIndexingApplication'; +DELETE FROM apps_marketplace where name = 'SearchIndexingApplication'; \ No newline at end of file diff --git a/openmetadata-service/src/main/java/org/openmetadata/csv/CsvUtil.java b/openmetadata-service/src/main/java/org/openmetadata/csv/CsvUtil.java index 9dc7c44dd228..1d6ed073a1a9 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/csv/CsvUtil.java +++ b/openmetadata-service/src/main/java/org/openmetadata/csv/CsvUtil.java @@ -17,14 +17,20 @@ import static org.openmetadata.common.utils.CommonUtil.listOrEmpty; import static org.openmetadata.common.utils.CommonUtil.nullOrEmpty; +import com.fasterxml.jackson.databind.ObjectMapper; import java.io.IOException; +import java.io.StringReader; import java.io.StringWriter; import java.util.ArrayList; import java.util.Arrays; +import java.util.Collections; import java.util.List; +import java.util.Map; +import java.util.regex.Pattern; import java.util.stream.Collectors; import org.apache.commons.csv.CSVFormat; import org.apache.commons.csv.CSVFormat.Builder; +import org.apache.commons.csv.CSVParser; import org.apache.commons.csv.CSVPrinter; import org.apache.commons.csv.CSVRecord; import org.openmetadata.schema.type.EntityReference; @@ -39,6 +45,8 @@ public final class CsvUtil { public static final String ENTITY_TYPE_SEPARATOR = ":"; public static final String LINE_SEPARATOR = "\r\n"; + public static final String INTERNAL_ARRAY_SEPARATOR = "|"; + private CsvUtil() { // Utility class hides the constructor } @@ -94,6 +102,62 @@ public static List fieldToEntities(String field) { return field == null ? null : listOf(field.split(ENTITY_TYPE_SEPARATOR)); } + public static List fieldToInternalArray(String field) { + // Split a fieldValue that contains multiple elements of an array separated by + // INTERNAL_ARRAY_SEPARATOR + if (field == null || field.isBlank()) { + return Collections.emptyList(); + } + return listOf(field.split(Pattern.quote(INTERNAL_ARRAY_SEPARATOR))); + } + + /** + * Parses a field containing key-value pairs separated by semicolons, correctly handling quotes. + * Each key-value pair may also be enclosed in quotes, especially if it contains delimiter like (SEPARATOR , FIELD_SEPARATOR). + * Input Example: + * "key1:value1;key2:value2;\"key3:value;with;semicolon\"" + * Output: [key1:value1, key2:value2, key3:value;with;semicolon] + * + */ + public static List fieldToExtensionStrings(String field) throws IOException { + if (field == null || field.isBlank()) { + return List.of(); + } + + // Replace semicolons within quoted strings with a placeholder + String preprocessedField = + Pattern.compile("\"([^\"]*)\"") // Matches content inside double quotes + .matcher(field) + .replaceAll(mr -> "\"" + mr.group(1).replace(";", "__SEMICOLON__") + "\""); + + preprocessedField = preprocessedField.replace("\n", "\\n").replace("\"", "\\\""); + + CSVFormat format = + CSVFormat.DEFAULT + .withDelimiter(';') + .withQuote('"') + .withRecordSeparator(null) + .withIgnoreSurroundingSpaces(true) + .withIgnoreEmptyLines(true) + .withEscape('\\'); // Use backslash for escaping special characters + + try (CSVParser parser = CSVParser.parse(new StringReader(preprocessedField), format)) { + return parser.getRecords().stream() + .flatMap(CSVRecord::stream) + .map( + value -> + value + .replace("__SEMICOLON__", ";") + .replace("\\n", "\n")) // Restore original semicolons and newlines + .map( + value -> + value.startsWith("\"") && value.endsWith("\"") // Remove outer quotes if present + ? value.substring(1, value.length() - 1) + : value) + .toList(); + } + } + public static String quote(String field) { return String.format("\"%s\"", field); } @@ -205,4 +269,89 @@ private static String quoteCsvField(String str) { } return str; } + + public static List addExtension(List csvRecord, Object extension) { + if (extension == null) { + csvRecord.add(null); + return csvRecord; + } + + ObjectMapper objectMapper = new ObjectMapper(); + Map extensionMap = objectMapper.convertValue(extension, Map.class); + + String extensionString = + extensionMap.entrySet().stream() + .map( + entry -> { + String key = entry.getKey(); + Object value = entry.getValue(); + return CsvUtil.quoteCsvField(key + ENTITY_TYPE_SEPARATOR + formatValue(value)); + }) + .collect(Collectors.joining(FIELD_SEPARATOR)); + + csvRecord.add(extensionString); + return csvRecord; + } + + private static String formatValue(Object value) { + if (value instanceof Map) { + return formatMapValue((Map) value); + } + + if (value instanceof List) { + return formatListValue((List) value); + } + + return value != null ? value.toString() : ""; + } + + private static String formatMapValue(Map valueMap) { + if (isEntityReference(valueMap)) { + return formatEntityReference(valueMap); + } else if (isTimeInterval(valueMap)) { + return formatTimeInterval(valueMap); + } + + return valueMap.toString(); + } + + private static String formatListValue(List list) { + if (list.isEmpty()) { + return ""; + } + + if (list.get(0) instanceof Map && isEnumWithDescriptions((Map) list.get(0))) { + return list.stream() + .map(item -> ((Map) item).get("key").toString()) + .collect(Collectors.joining(INTERNAL_ARRAY_SEPARATOR)); + } else if (list.get(0) instanceof Map) { + return list.stream() + .map(item -> formatMapValue((Map) item)) + .collect(Collectors.joining(INTERNAL_ARRAY_SEPARATOR)); + } else { + return list.stream() + .map(Object::toString) + .collect(Collectors.joining(INTERNAL_ARRAY_SEPARATOR)); + } + } + + private static boolean isEntityReference(Map valueMap) { + return valueMap.containsKey("type") && valueMap.containsKey("fullyQualifiedName"); + } + + private static boolean isTimeInterval(Map valueMap) { + return valueMap.containsKey("start") && valueMap.containsKey("end"); + } + + private static boolean isEnumWithDescriptions(Map valueMap) { + return valueMap.containsKey("key") && valueMap.containsKey("description"); + } + + private static String formatEntityReference(Map valueMap) { + return valueMap.get("type") + ENTITY_TYPE_SEPARATOR + valueMap.get("fullyQualifiedName"); + } + + private static String formatTimeInterval(Map valueMap) { + return valueMap.get("start") + ENTITY_TYPE_SEPARATOR + valueMap.get("end"); + } } diff --git a/openmetadata-service/src/main/java/org/openmetadata/csv/EntityCsv.java b/openmetadata-service/src/main/java/org/openmetadata/csv/EntityCsv.java index 2a651d6b25bd..c169465526e9 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/csv/EntityCsv.java +++ b/openmetadata-service/src/main/java/org/openmetadata/csv/EntityCsv.java @@ -16,21 +16,38 @@ import static org.openmetadata.common.utils.CommonUtil.listOf; import static org.openmetadata.common.utils.CommonUtil.listOrEmpty; import static org.openmetadata.common.utils.CommonUtil.nullOrEmpty; +import static org.openmetadata.csv.CsvUtil.ENTITY_TYPE_SEPARATOR; import static org.openmetadata.csv.CsvUtil.FIELD_SEPARATOR; +import static org.openmetadata.csv.CsvUtil.fieldToEntities; +import static org.openmetadata.csv.CsvUtil.fieldToExtensionStrings; +import static org.openmetadata.csv.CsvUtil.fieldToInternalArray; import static org.openmetadata.csv.CsvUtil.recordToString; +import com.fasterxml.jackson.databind.JsonNode; +import com.networknt.schema.JsonSchema; +import com.networknt.schema.ValidationMessage; import java.io.IOException; import java.io.Reader; import java.io.StringReader; import java.io.StringWriter; +import java.time.LocalDateTime; +import java.time.LocalTime; +import java.time.format.DateTimeFormatter; +import java.time.format.DateTimeParseException; +import java.time.temporal.TemporalAccessor; import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; import java.util.Comparator; import java.util.HashMap; import java.util.List; +import java.util.Locale; import java.util.Map; +import java.util.Optional; +import java.util.Set; import java.util.UUID; +import java.util.stream.Collectors; +import java.util.stream.StreamSupport; import javax.ws.rs.core.Response; import lombok.extern.slf4j.Slf4j; import org.apache.commons.csv.CSVFormat; @@ -51,7 +68,9 @@ import org.openmetadata.schema.type.csv.CsvFile; import org.openmetadata.schema.type.csv.CsvHeader; import org.openmetadata.schema.type.csv.CsvImportResult; +import org.openmetadata.schema.type.customproperties.EnumWithDescriptionsConfig; import org.openmetadata.service.Entity; +import org.openmetadata.service.TypeRegistry; import org.openmetadata.service.jdbi3.EntityRepository; import org.openmetadata.service.util.EntityUtil; import org.openmetadata.service.util.JsonUtils; @@ -309,6 +328,292 @@ protected final List getTagLabels( return tagLabels; } + public Map getExtension(CSVPrinter printer, CSVRecord csvRecord, int fieldNumber) + throws IOException { + String extensionString = csvRecord.get(fieldNumber); + if (nullOrEmpty(extensionString)) { + return null; + } + + Map extensionMap = new HashMap<>(); + + for (String extensions : fieldToExtensionStrings(extensionString)) { + // Split on the first occurrence of ENTITY_TYPE_SEPARATOR to get key-value pair + int separatorIndex = extensions.indexOf(ENTITY_TYPE_SEPARATOR); + + if (separatorIndex == -1) { + importFailure(printer, invalidExtension(fieldNumber, extensions, "null"), csvRecord); + continue; + } + + String key = extensions.substring(0, separatorIndex); + String value = extensions.substring(separatorIndex + 1); + + if (key.isEmpty() || value.isEmpty()) { + importFailure(printer, invalidExtension(fieldNumber, key, value), csvRecord); + } else { + extensionMap.put(key, value); + } + } + + validateExtension(printer, fieldNumber, csvRecord, extensionMap); + return extensionMap; + } + + private void validateExtension( + CSVPrinter printer, int fieldNumber, CSVRecord csvRecord, Map extensionMap) + throws IOException { + for (Map.Entry entry : extensionMap.entrySet()) { + String fieldName = entry.getKey(); + Object fieldValue = entry.getValue(); + + JsonSchema jsonSchema = TypeRegistry.instance().getSchema(entityType, fieldName); + if (jsonSchema == null) { + importFailure(printer, invalidCustomPropertyKey(fieldNumber, fieldName), csvRecord); + return; + } + String customPropertyType = TypeRegistry.getCustomPropertyType(entityType, fieldName); + String propertyConfig = TypeRegistry.getCustomPropertyConfig(entityType, fieldName); + + switch (customPropertyType) { + case "entityReference", "entityReferenceList" -> { + boolean isList = "entityReferenceList".equals(customPropertyType); + fieldValue = + parseEntityReferences(printer, csvRecord, fieldNumber, fieldValue.toString(), isList); + } + case "date-cp", "dateTime-cp", "time-cp" -> fieldValue = + getFormattedDateTimeField( + printer, + csvRecord, + fieldNumber, + fieldName, + fieldValue.toString(), + customPropertyType, + propertyConfig); + case "enum" -> { + List enumKeys = listOrEmpty(fieldToInternalArray(fieldValue.toString())); + fieldValue = enumKeys.isEmpty() ? null : enumKeys; + } + case "timeInterval" -> fieldValue = + handleTimeInterval(printer, csvRecord, fieldNumber, fieldName, fieldValue); + case "number", "integer", "timestamp" -> { + try { + fieldValue = Long.parseLong(fieldValue.toString()); + } catch (NumberFormatException e) { + importFailure( + printer, + invalidCustomPropertyValue( + fieldNumber, fieldName, customPropertyType, fieldValue.toString()), + csvRecord); + fieldValue = null; + } + } + case "enumWithDescriptions" -> { + fieldValue = + parseEnumWithDescriptions( + printer, + csvRecord, + fieldNumber, + fieldName, + fieldValue.toString(), + propertyConfig); + } + default -> {} + } + // Validate the field against the JSON schema + validateAndUpdateExtension( + printer, + csvRecord, + fieldNumber, + fieldName, + fieldValue, + customPropertyType, + extensionMap, + jsonSchema); + } + } + + private Object parseEntityReferences( + CSVPrinter printer, CSVRecord csvRecord, int fieldNumber, String fieldValue, boolean isList) + throws IOException { + List entityReferences = new ArrayList<>(); + + List entityRefStrings = + isList + ? listOrEmpty(fieldToInternalArray(fieldValue)) + : Collections.singletonList(fieldValue); + + for (String entityRefStr : entityRefStrings) { + List entityRefTypeAndValue = listOrEmpty(fieldToEntities(entityRefStr)); + + if (entityRefTypeAndValue.size() == 2) { + EntityReference entityRef = + getEntityReference( + printer, + csvRecord, + fieldNumber, + entityRefTypeAndValue.get(0), + entityRefTypeAndValue.get(1)); + Optional.ofNullable(entityRef).ifPresent(entityReferences::add); + } + } + + return isList ? entityReferences : entityReferences.isEmpty() ? null : entityReferences.get(0); + } + + private Object parseEnumWithDescriptions( + CSVPrinter printer, + CSVRecord csvRecord, + int fieldNumber, + String fieldName, + String fieldValue, + String propertyConfig) + throws IOException { + List enumKeys = listOrEmpty(fieldToInternalArray(fieldValue)); + List enumObjects = new ArrayList<>(); + + JsonNode propertyConfigNode = JsonUtils.readTree(propertyConfig); + if (propertyConfigNode == null) { + importFailure( + printer, + invalidCustomPropertyFieldFormat( + fieldNumber, + fieldName, + "enumWithDescriptions", + "Invalid propertyConfig of enumWithDescriptions: " + fieldValue), + csvRecord); + return null; + } + + Map keyToObjectMap = + StreamSupport.stream(propertyConfigNode.get("values").spliterator(), false) + .collect(Collectors.toMap(node -> node.get("key").asText(), node -> node)); + EnumWithDescriptionsConfig config = + JsonUtils.treeToValue(propertyConfigNode, EnumWithDescriptionsConfig.class); + if (!config.getMultiSelect() && enumKeys.size() > 1) { + importFailure( + printer, + invalidCustomPropertyFieldFormat( + fieldNumber, + fieldName, + "enumWithDescriptions", + "only one key is allowed for non-multiSelect enumWithDescriptions"), + csvRecord); + return null; + } + + for (String key : enumKeys) { + try { + JsonNode valueObject = keyToObjectMap.get(key); + if (valueObject == null) { + importFailure( + printer, + invalidCustomPropertyValue( + fieldNumber, + fieldName, + "enumWithDescriptions", + key + " not found in propertyConfig of " + fieldName), + csvRecord); + return null; + } + enumObjects.add(valueObject); + } catch (Exception e) { + importFailure(printer, e.getMessage(), csvRecord); + } + } + + return enumObjects; + } + + protected String getFormattedDateTimeField( + CSVPrinter printer, + CSVRecord csvRecord, + int fieldNumber, + String fieldName, + String fieldValue, + String fieldType, + String propertyConfig) + throws IOException { + try { + DateTimeFormatter formatter = DateTimeFormatter.ofPattern(propertyConfig, Locale.ENGLISH); + + return switch (fieldType) { + case "date-cp" -> { + TemporalAccessor date = formatter.parse(fieldValue); + yield formatter.format(date); + } + case "dateTime-cp" -> { + LocalDateTime dateTime = LocalDateTime.parse(fieldValue, formatter); + yield dateTime.format(formatter); + } + case "time-cp" -> { + LocalTime time = LocalTime.parse(fieldValue, formatter); + yield time.format(formatter); + } + default -> throw new IllegalStateException("Unexpected value: " + fieldType); + }; + } catch (DateTimeParseException e) { + importFailure( + printer, + invalidCustomPropertyFieldFormat(fieldNumber, fieldName, fieldType, propertyConfig), + csvRecord); + return null; + } + } + + private Map handleTimeInterval( + CSVPrinter printer, CSVRecord csvRecord, int fieldNumber, String fieldName, Object fieldValue) + throws IOException { + List timestampValues = fieldToEntities(fieldValue.toString()); + Map timestampMap = new HashMap<>(); + if (timestampValues.size() == 2) { + try { + timestampMap.put("start", Long.parseLong(timestampValues.get(0))); + timestampMap.put("end", Long.parseLong(timestampValues.get(1))); + } catch (NumberFormatException e) { + importFailure( + printer, + invalidCustomPropertyValue( + fieldNumber, fieldName, "timeInterval", fieldValue.toString()), + csvRecord); + return null; + } + } else { + importFailure( + printer, + invalidCustomPropertyFieldFormat(fieldNumber, fieldName, "timeInterval", "start:end"), + csvRecord); + return null; + } + return timestampMap; + } + + private void validateAndUpdateExtension( + CSVPrinter printer, + CSVRecord csvRecord, + int fieldNumber, + String fieldName, + Object fieldValue, + String customPropertyType, + Map extensionMap, + JsonSchema jsonSchema) + throws IOException { + if (fieldValue != null) { + JsonNode jsonNodeValue = JsonUtils.convertValue(fieldValue, JsonNode.class); + + Set validationMessages = jsonSchema.validate(jsonNodeValue); + if (!validationMessages.isEmpty()) { + importFailure( + printer, + invalidCustomPropertyValue( + fieldNumber, fieldName, customPropertyType, validationMessages.toString()), + csvRecord); + } else { + extensionMap.put(fieldName, fieldValue); + } + } + } + public static String[] getResultHeaders(List csvHeaders) { List importResultsCsvHeader = listOf(IMPORT_STATUS_HEADER, IMPORT_STATUS_DETAILS); importResultsCsvHeader.addAll(CsvUtil.getHeaders(csvHeaders)); @@ -530,6 +835,37 @@ public static String invalidOwner(int field) { return String.format(FIELD_ERROR_MSG, CsvErrorType.INVALID_FIELD, field + 1, error); } + public static String invalidExtension(int field, String key, String value) { + String error = + "Invalid key-value pair in extension string: Key = " + + key + + ", Value = " + + value + + " . Extensions should be of format customPropertyName:customPropertyValue"; + return String.format(FIELD_ERROR_MSG, CsvErrorType.INVALID_FIELD, field + 1, error); + } + + public static String invalidCustomPropertyKey(int field, String key) { + String error = String.format("Unknown custom field: %s", key); + return String.format(FIELD_ERROR_MSG, CsvErrorType.INVALID_FIELD, field + 1, error); + } + + public static String invalidCustomPropertyValue( + int field, String key, String fieldType, String value) { + String error = + String.format("Invalid value of Key = %s of type %s, Value = %s", key, fieldType, value); + return String.format(FIELD_ERROR_MSG, CsvErrorType.INVALID_FIELD, field + 1, error); + } + + public static String invalidCustomPropertyFieldFormat( + int field, String fieldName, String fieldType, String propertyConfig) { + String error = + String.format( + "Custom field %s value of type %s is not as per defined format %s", + fieldName, fieldType, propertyConfig); + return String.format(FIELD_ERROR_MSG, CsvErrorType.INVALID_FIELD, field + 1, error); + } + public static String invalidBoolean(int field, String fieldValue) { String error = String.format("Field %s should be either 'true' of 'false'", fieldValue); return String.format(FIELD_ERROR_MSG, CsvErrorType.INVALID_FIELD, field + 1, error); @@ -577,7 +913,7 @@ private void setFinalStatus() { ApiStatus status = ApiStatus.FAILURE; if (importResult.getNumberOfRowsPassed().equals(importResult.getNumberOfRowsProcessed())) { status = ApiStatus.SUCCESS; - } else if (importResult.getNumberOfRowsPassed() > 1) { + } else if (importResult.getNumberOfRowsPassed() >= 1) { status = ApiStatus.PARTIAL_SUCCESS; } importResult.setStatus(status); diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/TypeRegistry.java b/openmetadata-service/src/main/java/org/openmetadata/service/TypeRegistry.java index 96cd9d143007..a294a79cc51b 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/TypeRegistry.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/TypeRegistry.java @@ -69,10 +69,17 @@ private void addCustomProperty( String customPropertyFQN = getCustomPropertyFQN(entityType, propertyName); CUSTOM_PROPERTIES.put(customPropertyFQN, customProperty); - JsonSchema jsonSchema = - JsonUtils.getJsonSchema(TYPES.get(customProperty.getPropertyType().getName()).getSchema()); - CUSTOM_PROPERTY_SCHEMAS.put(customPropertyFQN, jsonSchema); - LOG.info("Adding custom property {} with JSON schema {}", customPropertyFQN, jsonSchema); + try { + JsonSchema jsonSchema = + JsonUtils.getJsonSchema( + TYPES.get(customProperty.getPropertyType().getName()).getSchema()); + CUSTOM_PROPERTY_SCHEMAS.put(customPropertyFQN, jsonSchema); + LOG.info("Adding custom property {} with JSON schema {}", customPropertyFQN, jsonSchema); + + } catch (Exception e) { + CUSTOM_PROPERTIES.remove(customPropertyFQN); + LOG.info("Failed to add custom property {}: {}", customPropertyFQN, e.getMessage()); + } } public JsonSchema getSchema(String entityType, String propertyName) { diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/apps/bundles/searchIndex/SearchIndexApp.java b/openmetadata-service/src/main/java/org/openmetadata/service/apps/bundles/searchIndex/SearchIndexApp.java index 977f82cd5e0f..adb45d380433 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/apps/bundles/searchIndex/SearchIndexApp.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/apps/bundles/searchIndex/SearchIndexApp.java @@ -163,7 +163,21 @@ public void startApp(JobExecutionContext jobExecutionContext) { } } + private void cleanUpStaleJobsFromRuns() { + try { + collectionDAO + .appExtensionTimeSeriesDao() + .markStaleEntriesStopped(getApp().getId().toString()); + } catch (Exception ex) { + LOG.error("Failed in Marking Stale Entries Stopped."); + } + } + private void initializeJob() { + // Remove any Stale Jobs + cleanUpStaleJobsFromRuns(); + + // Initialize New Job int totalRecords = getTotalRequestToProcess(jobData.getEntities(), collectionDAO); this.jobData.setStats( new Stats() diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/CollectionDAO.java b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/CollectionDAO.java index 576808e57ef7..0b83fcbd04c2 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/CollectionDAO.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/CollectionDAO.java @@ -4252,6 +4252,16 @@ interface AppExtensionTimeSeries { connectionType = POSTGRES) void insert(@Bind("json") String json); + @ConnectionAwareSqlUpdate( + value = + "UPDATE apps_extension_time_series SET json = JSON_SET(json, '$.status', 'stopped') where appId=:appId AND JSON_UNQUOTE(JSON_EXTRACT(json_column_name, '$.status')) = 'running'", + connectionType = MYSQL) + @ConnectionAwareSqlUpdate( + value = + "UPDATE apps_extension_time_series SET json = jsonb_set(json, '{status}', '\"stopped\"') WHERE appId = :appId AND json->>'status' = 'running'", + connectionType = POSTGRES) + void markStaleEntriesStopped(@Bind("appId") String appId); + @ConnectionAwareSqlUpdate( value = "UPDATE apps_extension_time_series set json = :json where appId=:appId and timestamp=:timestamp", diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/GlossaryRepository.java b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/GlossaryRepository.java index 38b8267de757..1bfd75e94635 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/GlossaryRepository.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/GlossaryRepository.java @@ -20,6 +20,7 @@ import static org.openmetadata.csv.CsvUtil.FIELD_SEPARATOR; import static org.openmetadata.csv.CsvUtil.addEntityReference; import static org.openmetadata.csv.CsvUtil.addEntityReferences; +import static org.openmetadata.csv.CsvUtil.addExtension; import static org.openmetadata.csv.CsvUtil.addField; import static org.openmetadata.csv.CsvUtil.addOwners; import static org.openmetadata.csv.CsvUtil.addReviewers; @@ -157,7 +158,7 @@ public String exportToCsv(String name, String user) throws IOException { ListFilter filter = new ListFilter(Include.NON_DELETED).addQueryParam("parent", name); List terms = repository.listAll( - repository.getFields("owners,reviewers,tags,relatedTerms,synonyms"), filter); + repository.getFields("owners,reviewers,tags,relatedTerms,synonyms,extension"), filter); terms.sort(Comparator.comparing(EntityInterface::getFullyQualifiedName)); return new GlossaryCsv(glossary, user).exportCsv(terms); } @@ -200,7 +201,8 @@ protected void createEntity(CSVPrinter printer, List csvRecords) thro printer, csvRecord, List.of(Pair.of(7, TagLabel.TagSource.CLASSIFICATION)))) .withReviewers(getOwners(printer, csvRecord, 8)) .withOwners(getOwners(printer, csvRecord, 9)) - .withStatus(getTermStatus(printer, csvRecord)); + .withStatus(getTermStatus(printer, csvRecord)) + .withExtension(getExtension(printer, csvRecord, 11)); if (processRecord) { createEntity(printer, csvRecord, glossaryTerm); } @@ -219,7 +221,10 @@ private List getTermReferences(CSVPrinter printer, CSVRecord csvR if (termRefList.size() % 2 != 0) { // List should have even numbered terms - termName and endPoint importFailure( - printer, invalidField(6, "Term references should termName;endpoint"), csvRecord); + printer, + invalidField( + 6, "Term References should be given in the format referenceName:endpoint url."), + csvRecord); processRecord = false; return null; } @@ -265,6 +270,7 @@ protected void addRecord(CsvFile csvFile, GlossaryTerm entity) { addReviewers(recordList, entity.getReviewers()); addOwners(recordList, entity.getOwners()); addField(recordList, entity.getStatus().value()); + addExtension(recordList, entity.getExtension()); addRecord(csvFile, recordList); } diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/search/elasticsearch/ElasticSearchClient.java b/openmetadata-service/src/main/java/org/openmetadata/service/search/elasticsearch/ElasticSearchClient.java index c65cab8de183..7acefe878022 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/search/elasticsearch/ElasticSearchClient.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/search/elasticsearch/ElasticSearchClient.java @@ -2228,6 +2228,7 @@ public RestHighLevelClient createElasticSearchClient(ElasticSearchConfiguration requestConfigBuilder .setConnectTimeout(esConfig.getConnectionTimeoutSecs() * 1000) .setSocketTimeout(esConfig.getSocketTimeoutSecs() * 1000)); + restClientBuilder.setCompressionEnabled(true); return new RestHighLevelClientBuilder(restClientBuilder.build()) .setApiCompatibilityMode(true) .build(); diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/search/elasticsearch/ElasticSearchIndexSink.java b/openmetadata-service/src/main/java/org/openmetadata/service/search/elasticsearch/ElasticSearchIndexSink.java index 553788aa1581..dedd9ac74cff 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/search/elasticsearch/ElasticSearchIndexSink.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/search/elasticsearch/ElasticSearchIndexSink.java @@ -28,10 +28,10 @@ public class ElasticSearchIndexSink implements Sink { private final StepStats stats = new StepStats(); private final SearchRepository searchRepository; - private final int maxPayLoadSizeInBytes; + private final long maxPayLoadSizeInBytes; public ElasticSearchIndexSink( - SearchRepository searchRepository, int total, int maxPayLoadSizeInBytes) { + SearchRepository searchRepository, int total, long maxPayLoadSizeInBytes) { this.searchRepository = searchRepository; this.maxPayLoadSizeInBytes = maxPayLoadSizeInBytes; this.stats.withTotalRecords(total).withSuccessRecords(0).withFailedRecords(0); diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/search/opensearch/OpenSearchClient.java b/openmetadata-service/src/main/java/org/openmetadata/service/search/opensearch/OpenSearchClient.java index 8a1c5b93b461..6e1b3551b93c 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/search/opensearch/OpenSearchClient.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/search/opensearch/OpenSearchClient.java @@ -2193,6 +2193,8 @@ public RestHighLevelClient createOpenSearchClient(ElasticSearchConfiguration esC requestConfigBuilder .setConnectTimeout(esConfig.getConnectionTimeoutSecs() * 1000) .setSocketTimeout(esConfig.getSocketTimeoutSecs() * 1000)); + restClientBuilder.setCompressionEnabled(true); + restClientBuilder.setChunkedEnabled(true); return new RestHighLevelClient(restClientBuilder); } catch (Exception e) { LOG.error("Failed to create open search client ", e); diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/search/opensearch/OpenSearchIndexSink.java b/openmetadata-service/src/main/java/org/openmetadata/service/search/opensearch/OpenSearchIndexSink.java index 7dc0766b5987..205b65cf3eac 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/search/opensearch/OpenSearchIndexSink.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/search/opensearch/OpenSearchIndexSink.java @@ -29,9 +29,9 @@ public class OpenSearchIndexSink implements Sink { private final StepStats stats = new StepStats(); private final SearchRepository searchRepository; - private final int maxPayLoadSizeInBytes; + private final long maxPayLoadSizeInBytes; - public OpenSearchIndexSink(SearchRepository repository, int total, int maxPayLoadSizeInBytes) { + public OpenSearchIndexSink(SearchRepository repository, int total, long maxPayLoadSizeInBytes) { this.searchRepository = repository; this.maxPayLoadSizeInBytes = maxPayLoadSizeInBytes; this.stats.withTotalRecords(total).withSuccessRecords(0).withFailedRecords(0); diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/util/OpenMetadataOperations.java b/openmetadata-service/src/main/java/org/openmetadata/service/util/OpenMetadataOperations.java index 754867160730..903f40eaa687 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/util/OpenMetadataOperations.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/util/OpenMetadataOperations.java @@ -270,8 +270,8 @@ public Integer reIndex( int batchSize, @Option( names = {"-p", "--payload-size"}, - defaultValue = "10485760") - int payloadSize, + defaultValue = "104857600l") + long payloadSize, @Option( names = {"--recreate-indexes"}, defaultValue = "true") @@ -295,7 +295,7 @@ public Integer reIndex( } private int executeSearchReindexApp( - String appName, int batchSize, int payloadSize, boolean recreateIndexes) { + String appName, int batchSize, long payloadSize, boolean recreateIndexes) { AppRepository appRepository = (AppRepository) Entity.getEntityRepository(Entity.APPLICATION); App originalSearchIndexApp = appRepository.getByName(null, appName, appRepository.getFields("id")); diff --git a/openmetadata-service/src/main/resources/json/data/app/SearchIndexingApplication.json b/openmetadata-service/src/main/resources/json/data/app/SearchIndexingApplication.json index 285124f81f5a..3169c3634a42 100644 --- a/openmetadata-service/src/main/resources/json/data/app/SearchIndexingApplication.json +++ b/openmetadata-service/src/main/resources/json/data/app/SearchIndexingApplication.json @@ -45,6 +45,7 @@ ], "recreateIndex": false, "batchSize": "100", + "payLoadSize": 104857600, "searchIndexMappingLanguage": "EN" }, "appSchedule": { diff --git a/openmetadata-service/src/main/resources/json/data/appMarketPlaceDefinition/SearchIndexingApplication.json b/openmetadata-service/src/main/resources/json/data/appMarketPlaceDefinition/SearchIndexingApplication.json index c8bb323a9142..8690cd5864d5 100644 --- a/openmetadata-service/src/main/resources/json/data/appMarketPlaceDefinition/SearchIndexingApplication.json +++ b/openmetadata-service/src/main/resources/json/data/appMarketPlaceDefinition/SearchIndexingApplication.json @@ -60,6 +60,7 @@ ], "recreateIndex": false, "batchSize": "100", + "payLoadSize": 104857600, "searchIndexMappingLanguage": "EN" } } diff --git a/openmetadata-service/src/main/resources/json/data/glossary/glossaryCsvDocumentation.json b/openmetadata-service/src/main/resources/json/data/glossary/glossaryCsvDocumentation.json index c8de3da46d39..47750c710b29 100644 --- a/openmetadata-service/src/main/resources/json/data/glossary/glossaryCsvDocumentation.json +++ b/openmetadata-service/src/main/resources/json/data/glossary/glossaryCsvDocumentation.json @@ -86,7 +86,7 @@ ] }, { - "name": "status", + "name": "glossaryStatus", "required": false, "description": "Status of the glossary term. Allowed values `Draft`, `Approved`, or `Deprecated`", "examples": [ @@ -94,6 +94,22 @@ "`Approved`", "`Deprecated`" ] + }, + { + "name": "extension", + "required": false, + "description": "Custom property values added to the glossary term. Each field value (property and its value) is separated by `;` and internal values can be separated by `|`. For `entityReferenceList` type property, pass `type1:fqn1|type2:fqn2`. For single `entityReference` type property, pass `type:fqn`. Similarly, for `enumMultiSelect`, pass values separated by `|`, and for `enumSingleSelect`, pass a single value along with the property name. For `timeInterval` property type, pass the `startTime:endTime` to the property name. If the field value itself contains delimiter values like `,` and `;` or newline they need to be quoted, and the quotation needs to be further escaped. In general, if passing multiple field values separated by `;`, the extension column value needs to be quoted.", + "examples": [ + "`customAttribute1:value1;customAttribute2:value2`", + "`\"dateCp:18-09-2024;dateTimeCp:18-09-2024 01:09:34;durationCp:PT5H30M10S;emailCp:admin@open-metadata.org\"`", + "`entRefListCp:searchIndex:elasticsearch_sample.table_search_index|databaseSchema:Glue.default.information_schema|databaseSchema:sample_data.ecommerce_db.shopify|database:Glue.default|`", + "`\"entRefCp:user:\"\"aaron.singh2\"\"\"`", + "`\"enumMultiSelectCp:val3|val2|val1|val4|val5;enumSingleSelectCp:singleVal1\"`", + "`\"timeCp:10:08:45;timeIntervalCp:1726142300000:17261420000;timeStampCp:1726142400000\"`", + "`\"integerCp:7777;numberCp:123456\"`", + "`\"\"\"queryCp:select col,row from table where id ='30';\"\";stringcp:sample string content\"`", + "`markdownCp:# Sample Markdown Text`" + ] } ] } \ No newline at end of file diff --git a/openmetadata-service/src/test/java/org/openmetadata/csv/CsvUtilTest.java b/openmetadata-service/src/test/java/org/openmetadata/csv/CsvUtilTest.java index e7e063516ce2..c76b9bb79ecb 100644 --- a/openmetadata-service/src/test/java/org/openmetadata/csv/CsvUtilTest.java +++ b/openmetadata-service/src/test/java/org/openmetadata/csv/CsvUtilTest.java @@ -16,10 +16,14 @@ import static org.junit.jupiter.api.Assertions.assertEquals; import static org.openmetadata.common.utils.CommonUtil.listOf; +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.node.ObjectNode; import java.util.ArrayList; import java.util.Collections; import java.util.List; import org.junit.jupiter.api.Test; +import org.openmetadata.schema.entity.type.CustomProperty; import org.openmetadata.schema.type.EntityReference; import org.openmetadata.schema.type.TagLabel; @@ -74,6 +78,33 @@ void testAddRecord() { expectedRecord.add("t1;t2"); List tags = listOf(new TagLabel().withTagFQN("t1"), new TagLabel().withTagFQN("t2")); assertEquals(expectedRecord, CsvUtil.addTagLabels(actualRecord, tags)); + + // Add extension + expectedRecord.add(null); + assertEquals(expectedRecord, CsvUtil.addExtension(actualRecord, null)); // Null extension + + ObjectMapper mapper = new ObjectMapper(); + ObjectNode jsonNode = mapper.createObjectNode(); + + // Add new custom property stringCp of type string + CustomProperty stringCp = + new CustomProperty() + .withName("stringCp") + .withDescription("string type custom property") + .withPropertyType( + new EntityReference().withFullyQualifiedName("string").withType("type")); + JsonNode stringCpValue = + mapper.convertValue("String; input; with; semicolon\n And new line", JsonNode.class); + jsonNode.set("stringCp", stringCpValue); + + // Add new custom property queryCp of type sqlQuery + JsonNode queryCpValue = + mapper.convertValue("SELECT * FROM table WHERE column = 'value';", JsonNode.class); + jsonNode.set("queryCp", queryCpValue); + + expectedRecord.add( + "\"stringCp:String; input; with; semicolon\n And new line\";\"queryCp:SELECT * FROM table WHERE column = 'value';\""); + assertEquals(expectedRecord, CsvUtil.addExtension(actualRecord, jsonNode)); } public static void assertCsv(String expectedCsv, String actualCsv) { diff --git a/openmetadata-service/src/test/java/org/openmetadata/service/resources/EntityResourceTest.java b/openmetadata-service/src/test/java/org/openmetadata/service/resources/EntityResourceTest.java index 1eca6b4777b7..c4bd1aed2528 100644 --- a/openmetadata-service/src/test/java/org/openmetadata/service/resources/EntityResourceTest.java +++ b/openmetadata-service/src/test/java/org/openmetadata/service/resources/EntityResourceTest.java @@ -389,6 +389,18 @@ public abstract class EntityResourceTest true); - assertSummary(result, ApiStatus.FAILURE, 2, 1, 1); + assertSummary(result, ApiStatus.PARTIAL_SUCCESS, 2, 1, 1); String[] expectedRows = { resultsHeader, getFailedRecord(record, "[name must match \"^((?!::).)*$\"]") }; assertRows(result, expectedRows); // Create glossaryTerm with invalid parent - record = "invalidParent,g1,dsp1,dsc1,h1;h2;h3,,term1;http://term1,Tier.Tier1,,,"; + record = "invalidParent,g1,dsp1,dsc1,h1;h2;h3,,term1;http://term1,Tier.Tier1,,,,"; csv = createCsv(GlossaryCsv.HEADERS, listOf(record), null); result = importCsv(glossaryName, csv, false); Awaitility.await().atMost(4, TimeUnit.SECONDS).until(() -> true); - assertSummary(result, ApiStatus.FAILURE, 2, 1, 1); + assertSummary(result, ApiStatus.PARTIAL_SUCCESS, 2, 1, 1); expectedRows = new String[] { resultsHeader, @@ -524,16 +534,158 @@ record = "invalidParent,g1,dsp1,dsc1,h1;h2;h3,,term1;http://term1,Tier.Tier1,,," }; assertRows(result, expectedRows); + // Create glossaryTerm with Invalid references + record = ",g1,dsp1,dsc1,h1;h2;h3,,term1:http://term1,,,,,"; + csv = createCsv(GlossaryCsv.HEADERS, listOf(record), null); + result = importCsv(glossaryName, csv, false); + assertSummary(result, ApiStatus.PARTIAL_SUCCESS, 2, 1, 1); + expectedRows = + new String[] { + resultsHeader, + getFailedRecord( + record, + invalidField( + 6, "Term References should be given in the format referenceName:endpoint url.")) + }; + assertRows(result, expectedRows); + // Create glossaryTerm with invalid tags field - record = ",g1,dsp1,dsc1,h1;h2;h3,,term1;http://term1,Tag.invalidTag,,,"; + record = ",g1,dsp1,dsc1,h1;h2;h3,,term1;http://term1,Tag.invalidTag,,,,"; csv = createCsv(GlossaryCsv.HEADERS, listOf(record), null); result = importCsv(glossaryName, csv, false); - assertSummary(result, ApiStatus.FAILURE, 2, 1, 1); + assertSummary(result, ApiStatus.PARTIAL_SUCCESS, 2, 1, 1); expectedRows = new String[] { resultsHeader, getFailedRecord(record, entityNotFound(7, Entity.TAG, "Tag.invalidTag")) }; assertRows(result, expectedRows); + + // Create glossaryTerm with Invalid extension column format + record = ",g1,dsp1,dsc1,h1;h2;h3,,term1;http://term1,PII.None,,,,glossaryTermDateCp"; + csv = createCsv(GlossaryCsv.HEADERS, listOf(record), null); + result = importCsv(glossaryName, csv, false); + assertSummary(result, ApiStatus.PARTIAL_SUCCESS, 2, 1, 1); + expectedRows = + new String[] { + resultsHeader, getFailedRecord(record, invalidExtension(11, "glossaryTermDateCp", null)) + }; + assertRows(result, expectedRows); + + // Create glossaryTerm with Invalid custom property key + String invalidCustomPropertyKeyRecord = + ",g1,dsp1,dsc1,h1;h2;h3,,term1;http://term1,PII.None,,,,invalidCustomProperty:someValue"; + csv = createCsv(GlossaryCsv.HEADERS, listOf(invalidCustomPropertyKeyRecord), null); + result = importCsv(glossaryName, csv, false); + Awaitility.await().atMost(4, TimeUnit.SECONDS).until(() -> true); + assertSummary(result, ApiStatus.PARTIAL_SUCCESS, 2, 1, 1); + expectedRows = + new String[] { + resultsHeader, + getFailedRecord( + invalidCustomPropertyKeyRecord, invalidCustomPropertyKey(11, "invalidCustomProperty")) + }; + assertRows(result, expectedRows); + + // Create glossaryTerm with Invalid custom property value + CustomProperty glossaryTermIntegerCp = + new CustomProperty() + .withName("glossaryTermIntegerCp") + .withDescription("integer type custom property") + .withPropertyType(INT_TYPE.getEntityReference()); + TypeResourceTest typeResourceTest = new TypeResourceTest(); + Type entityType = + typeResourceTest.getEntityByName( + Entity.GLOSSARY_TERM, "customProperties", ADMIN_AUTH_HEADERS); + entityType = + typeResourceTest.addAndCheckCustomProperty( + entityType.getId(), glossaryTermIntegerCp, OK, ADMIN_AUTH_HEADERS); + String invalidIntValueRecord = + ",g1,dsp1,dsc1,h1;h2;h3,,term1;http://term1,PII.None,,,,glossaryTermIntegerCp:11s22"; + csv = createCsv(GlossaryCsv.HEADERS, listOf(invalidIntValueRecord), null); + result = importCsv(glossaryName, csv, false); + Awaitility.await().atMost(4, TimeUnit.SECONDS).until(() -> true); + assertSummary(result, ApiStatus.PARTIAL_SUCCESS, 2, 1, 1); + expectedRows = + new String[] { + resultsHeader, + getFailedRecord( + invalidIntValueRecord, + invalidCustomPropertyValue( + 11, "glossaryTermIntegerCp", INT_TYPE.getDisplayName(), "11s22")) + }; + assertRows(result, expectedRows); + + // Create glossaryTerm with Invalid custom property value's format + CustomProperty glossaryTermDateCp = + new CustomProperty() + .withName("glossaryTermDateCp") + .withDescription("dd-MM-yyyy format time") + .withPropertyType(DATECP_TYPE.getEntityReference()) + .withCustomPropertyConfig(new CustomPropertyConfig().withConfig("dd-MM-yyyy")); + entityType = + typeResourceTest.getEntityByName( + Entity.GLOSSARY_TERM, "customProperties", ADMIN_AUTH_HEADERS); + entityType = + typeResourceTest.addAndCheckCustomProperty( + entityType.getId(), glossaryTermDateCp, OK, ADMIN_AUTH_HEADERS); + String invalidDateFormatRecord = + ",g1,dsp1,dsc1,h1;h2;h3,,term1;http://term1,PII.None,,,,glossaryTermDateCp:invalid-date-format"; + csv = createCsv(GlossaryCsv.HEADERS, listOf(invalidDateFormatRecord), null); + result = importCsv(glossaryName, csv, false); + Awaitility.await().atMost(4, TimeUnit.SECONDS).until(() -> true); + assertSummary(result, ApiStatus.PARTIAL_SUCCESS, 2, 1, 1); + expectedRows = + new String[] { + resultsHeader, + getFailedRecord( + invalidDateFormatRecord, + invalidCustomPropertyFieldFormat( + 11, "glossaryTermDateCp", DATECP_TYPE.getDisplayName(), "dd-MM-yyyy")) + }; + assertRows(result, expectedRows); + + // Create glossaryTerm with Invalid custom property of type enumWithDescriptions + CustomProperty glossaryTermEnumCp = + new CustomProperty() + .withName("glossaryTermEnumWithDescriptionsCp") + .withDescription("enumWithDescriptions type custom property with multiselect = true") + .withPropertyType(ENUM_WITH_DESCRIPTIONS_TYPE.getEntityReference()) + .withCustomPropertyConfig( + new CustomPropertyConfig() + .withConfig( + Map.of( + "values", + List.of( + Map.of("key", "key1", "description", "description1"), + Map.of("key", "key2", "description", "description2")), + "multiSelect", + true))); + entityType = + typeResourceTest.getEntityByName( + Entity.GLOSSARY_TERM, "customProperties", ADMIN_AUTH_HEADERS); + entityType = + typeResourceTest.addAndCheckCustomProperty( + entityType.getId(), glossaryTermEnumCp, OK, ADMIN_AUTH_HEADERS); + String invalidEnumWithDescriptionRecord = + ",g1,dsp1,dsc1,h1;h2;h3,,term1;http://term1,PII.None,,,,glossaryTermEnumWithDescriptionsCp:key1|key3"; + String invalidEnumWithDescriptionValue = + ",g1,dsp1,dsc1,h1;h2;h3,,term1;http://term1,PII.None,,,,glossaryTermEnumWithDescriptionsCp:key1|key3"; + csv = createCsv(GlossaryCsv.HEADERS, listOf(invalidEnumWithDescriptionValue), null); + result = importCsv(glossaryName, csv, false); + Awaitility.await().atMost(4, TimeUnit.SECONDS).until(() -> true); + assertSummary(result, ApiStatus.PARTIAL_SUCCESS, 2, 1, 1); + expectedRows = + new String[] { + resultsHeader, + getFailedRecord( + invalidEnumWithDescriptionRecord, + invalidCustomPropertyValue( + 11, + "glossaryTermEnumWithDescriptionsCp", + ENUM_WITH_DESCRIPTIONS_TYPE.getDisplayName(), + "key3 not found in propertyConfig of glossaryTermEnumWithDescriptionsCp")) + }; + assertRows(result, expectedRows); } @Test @@ -544,38 +696,174 @@ void testGlossaryImportExport() throws IOException { String team11 = TEAM11.getName(); List reviewerRef = listOf(user1, user2).stream().sorted(Comparator.naturalOrder()).toList(); + // PUT valid custom fields to the entity type + // Create instances of CustomPropertyConfig + CustomPropertyConfig dateTimeConfig = + new CustomPropertyConfig().withConfig("dd-MM-yyyy HH:mm:ss"); + CustomPropertyConfig timeConfig = new CustomPropertyConfig().withConfig("HH:mm:ss"); + CustomPropertyConfig enumConfig = + new CustomPropertyConfig() + .withConfig( + Map.of( + "values", + List.of("val1", "val2", "val3", "val4", "val5", "valwith\"quote\""), + "multiSelect", + true)); + + // PUT valid custom fields to the entity type + TypeResourceTest typeResourceTest = new TypeResourceTest(); + Type entityType = + typeResourceTest.getEntityByName( + Entity.GLOSSARY_TERM, "customProperties", ADMIN_AUTH_HEADERS); + + CustomProperty[] customProperties = { + new CustomProperty() + .withName("glossaryTermEmailCp") + .withDescription("email type custom property") + .withPropertyType(EMAIL_TYPE.getEntityReference()), + new CustomProperty() + .withName("glossaryTermDateCp") + .withDescription("dd-MM-yyyy format time") + .withPropertyType(DATECP_TYPE.getEntityReference()) + .withCustomPropertyConfig(new CustomPropertyConfig().withConfig("dd-MM-yyyy")), + new CustomProperty() + .withName("glossaryTermDateTimeCp") + .withDescription("dd-MM-yyyy HH:mm:ss format dateTime") + .withPropertyType(DATETIMECP_TYPE.getEntityReference()) + .withCustomPropertyConfig(dateTimeConfig), + new CustomProperty() + .withName("glossaryTermTimeCp") + .withDescription("HH:mm:ss format time") + .withPropertyType(TIMECP_TYPE.getEntityReference()) + .withCustomPropertyConfig(timeConfig), + new CustomProperty() + .withName("glossaryTermIntegerCp") + .withDescription("integer type custom property") + .withPropertyType(INT_TYPE.getEntityReference()), + new CustomProperty() + .withName("glossaryTermDurationCp") + .withDescription("duration type custom property") + .withPropertyType(DURATION_TYPE.getEntityReference()), + new CustomProperty() + .withName("glossaryTermMarkdownCp") + .withDescription("markdown type custom property") + .withPropertyType(MARKDOWN_TYPE.getEntityReference()), + new CustomProperty() + .withName("glossaryTermStringCp") + .withDescription("string type custom property") + .withPropertyType(STRING_TYPE.getEntityReference()), + new CustomProperty() + .withName("glossaryTermEntRefCp") + .withDescription("entity Reference type custom property") // value includes fqn of entity + .withPropertyType(ENTITY_REFERENCE_TYPE.getEntityReference()) + .withCustomPropertyConfig(new CustomPropertyConfig().withConfig(List.of("user"))), + new CustomProperty() + .withName("glossaryTermEntRefListCp") + .withDescription( + "entity Reference List type custom property") // value includes list of fqn of + .withPropertyType(ENTITY_REFERENCE_LIST_TYPE.getEntityReference()) + .withCustomPropertyConfig( + new CustomPropertyConfig() + .withConfig( + List.of( + Entity.TABLE, + Entity.STORED_PROCEDURE, + Entity.DATABASE_SCHEMA, + Entity.DATABASE, + Entity.DASHBOARD, + Entity.DASHBOARD_DATA_MODEL, + Entity.PIPELINE, + Entity.TOPIC, + Entity.CONTAINER, + Entity.SEARCH_INDEX, + Entity.MLMODEL, + Entity.GLOSSARY_TERM))), + new CustomProperty() + .withName("glossaryTermTimeIntervalCp") + .withDescription("timeInterval type custom property in format starttime:endtime") + .withPropertyType(TIME_INTERVAL_TYPE.getEntityReference()), + new CustomProperty() + .withName("glossaryTermNumberCp") + .withDescription("numberCp") + .withPropertyType(INT_TYPE.getEntityReference()), + new CustomProperty() + .withName("glossaryTermQueryCp") + .withDescription("queryCp desc") + .withPropertyType(SQLQUERY_TYPE.getEntityReference()), + new CustomProperty() + .withName("glossaryTermTimestampCp") + .withDescription("timestamp type custom property") + .withPropertyType(TIMESTAMP_TYPE.getEntityReference()), + new CustomProperty() + .withName("glossaryTermEnumCpSingle") + .withDescription("enum type custom property with multiselect = false") + .withPropertyType(ENUM_TYPE.getEntityReference()) + .withCustomPropertyConfig( + new CustomPropertyConfig() + .withConfig( + Map.of( + "values", + List.of("single1", "single2", "single3", "single4", "\"single5\""), + "multiSelect", + false))), + new CustomProperty() + .withName("glossaryTermEnumCpMulti") + .withDescription("enum type custom property with multiselect = true") + .withPropertyType(ENUM_TYPE.getEntityReference()) + .withCustomPropertyConfig(enumConfig), + new CustomProperty() + .withName("glossaryTermEnumWithDescriptionsCp") + .withDescription("enumWithDescriptions type custom property with multiselect = true") + .withPropertyType(ENUM_WITH_DESCRIPTIONS_TYPE.getEntityReference()) + .withCustomPropertyConfig( + new CustomPropertyConfig() + .withConfig( + Map.of( + "values", + List.of( + Map.of("key", "key1", "description", "description1"), + Map.of("key", "key2", "description", "description2")), + "multiSelect", + true))) + }; - // CSV Header "parent" "name" "displayName" "description" "synonyms" "relatedTerms" "references" - // "tags", "reviewers", "owners", "status" + for (CustomProperty customProperty : customProperties) { + entityType = + typeResourceTest.addAndCheckCustomProperty( + entityType.getId(), customProperty, OK, ADMIN_AUTH_HEADERS); + } + // CSV Header "parent", "name", "displayName", "description", "synonyms", "relatedTerms", + // "references", + // "tags", "reviewers", "owners", "status", "extension" // Create two records List createRecords = listOf( String.format( - ",g1,dsp1,\"dsc1,1\",h1;h2;h3,,term1;http://term1,PII.None,user:%s,user:%s,%s", - reviewerRef.get(0), user1, "Approved"), + ",g1,dsp1,\"dsc1,1\",h1;h2;h3,,term1;http://term1,PII.None,user:%s,user:%s,%s,\"glossaryTermDateCp:18-09-2024;glossaryTermDateTimeCp:18-09-2024 01:09:34;glossaryTermDurationCp:PT5H30M10S;glossaryTermEmailCp:admin@open-metadata.org;glossaryTermEntRefCp:team:\"\"%s\"\";glossaryTermEntRefListCp:user:\"\"%s\"\"|user:\"\"%s\"\"\"", + reviewerRef.get(0), user1, "Approved", team11, user1, user2), String.format( - ",g2,dsp2,dsc3,h1;h3;h3,,term2;https://term2,PII.NonSensitive,,user:%s,%s", + ",g2,dsp2,dsc3,h1;h3;h3,,term2;https://term2,PII.NonSensitive,,user:%s,%s,\"glossaryTermEnumCpMulti:val3|val2|val1|val4|val5;glossaryTermEnumCpSingle:singleVal1;glossaryTermIntegerCp:7777;glossaryTermMarkdownCp:# Sample Markdown Text;glossaryTermNumberCp:123456;\"\"glossaryTermQueryCp:select col,row from table where id ='30';\"\";glossaryTermStringCp:sample string content;glossaryTermTimeCp:10:08:45;glossaryTermTimeIntervalCp:1726142300000:17261420000;glossaryTermTimestampCp:1726142400000\"", user1, "Approved"), String.format( - "importExportTest.g1,g11,dsp2,dsc11,h1;h3;h3,,,,user:%s,team:%s,%s", + "importExportTest.g1,g11,dsp2,dsc11,h1;h3;h3,,,,user:%s,team:%s,%s,glossaryTermEnumWithDescriptionsCp:key1|key2", reviewerRef.get(0), team11, "Draft")); // Update terms with change in description List updateRecords = listOf( String.format( - ",g1,dsp1,new-dsc1,h1;h2;h3,,term1;http://term1,PII.None,user:%s,user:%s,%s", - reviewerRef.get(0), user1, "Approved"), + ",g1,dsp1,new-dsc1,h1;h2;h3,,term1;http://term1,PII.None,user:%s,user:%s,%s,\"glossaryTermDateCp:18-09-2024;glossaryTermDateTimeCp:18-09-2024 01:09:34;glossaryTermDurationCp:PT5H30M10S;glossaryTermEmailCp:admin@open-metadata.org;glossaryTermEntRefCp:team:\"\"%s\"\";glossaryTermEntRefListCp:user:\"\"%s\"\"|user:\"\"%s\"\"\"", + reviewerRef.get(0), user1, "Approved", team11, user1, user2), String.format( - ",g2,dsp2,new-dsc3,h1;h3;h3,,term2;https://term2,PII.NonSensitive,user:%s,user:%s,%s", + ",g2,dsp2,new-dsc3,h1;h3;h3,,term2;https://term2,PII.NonSensitive,user:%s,user:%s,%s,\"glossaryTermEnumCpMulti:val3|val2|val1|val4|val5;glossaryTermEnumCpSingle:singleVal1;glossaryTermIntegerCp:7777;glossaryTermMarkdownCp:# Sample Markdown Text;glossaryTermNumberCp:123456;\"\"glossaryTermQueryCp:select col,row from table where id ='30';\"\";glossaryTermStringCp:sample string content;glossaryTermTimeCp:10:08:45;glossaryTermTimeIntervalCp:1726142300000:17261420000;glossaryTermTimestampCp:1726142400000\"", user1, user2, "Approved"), String.format( - "importExportTest.g1,g11,dsp2,new-dsc11,h1;h3;h3,,,,user:%s,team:%s,%s", + "importExportTest.g1,g11,dsp2,new-dsc11,h1;h3;h3,,,,user:%s,team:%s,%s,glossaryTermEnumWithDescriptionsCp:key1|key2", reviewerRef.get(0), team11, "Draft")); // Add new row to existing rows List newRecords = - listOf(",g3,dsp0,dsc0,h1;h2;h3,,term0;http://term0,PII.Sensitive,,,Approved"); + listOf(",g3,dsp0,dsc0,h1;h2;h3,,term0;http://term0,PII.Sensitive,,,Approved,"); testImportExport( glossary.getName(), GlossaryCsv.HEADERS, createRecords, updateRecords, newRecords); } diff --git a/openmetadata-service/src/test/java/org/openmetadata/service/resources/metadata/TypeResourceTest.java b/openmetadata-service/src/test/java/org/openmetadata/service/resources/metadata/TypeResourceTest.java index c8d3b3a35d76..18a371eb20e2 100644 --- a/openmetadata-service/src/test/java/org/openmetadata/service/resources/metadata/TypeResourceTest.java +++ b/openmetadata-service/src/test/java/org/openmetadata/service/resources/metadata/TypeResourceTest.java @@ -71,8 +71,20 @@ public TypeResourceTest() { public void setupTypes() throws HttpResponseException { INT_TYPE = getEntityByName("integer", "", ADMIN_AUTH_HEADERS); STRING_TYPE = getEntityByName("string", "", ADMIN_AUTH_HEADERS); + EMAIL_TYPE = getEntityByName("email", "", ADMIN_AUTH_HEADERS); ENUM_TYPE = getEntityByName("enum", "", ADMIN_AUTH_HEADERS); ENUM_WITH_DESCRIPTIONS_TYPE = getEntityByName("enumWithDescriptions", "", ADMIN_AUTH_HEADERS); + DATECP_TYPE = getEntityByName("date-cp", "", ADMIN_AUTH_HEADERS); + DATETIMECP_TYPE = getEntityByName("dateTime-cp", "", ADMIN_AUTH_HEADERS); + TIMECP_TYPE = getEntityByName("time-cp", "", ADMIN_AUTH_HEADERS); + DURATION_TYPE = getEntityByName("duration", "", ADMIN_AUTH_HEADERS); + MARKDOWN_TYPE = getEntityByName("markdown", "", ADMIN_AUTH_HEADERS); + ENTITY_REFERENCE_TYPE = getEntityByName("entityReference", "", ADMIN_AUTH_HEADERS); + ENTITY_REFERENCE_LIST_TYPE = getEntityByName("entityReferenceList", "", ADMIN_AUTH_HEADERS); + TIME_INTERVAL_TYPE = getEntityByName("timeInterval", "", ADMIN_AUTH_HEADERS); + NUMBER_TYPE = getEntityByName("number", "", ADMIN_AUTH_HEADERS); + SQLQUERY_TYPE = getEntityByName("sqlQuery", "", ADMIN_AUTH_HEADERS); + TIMESTAMP_TYPE = getEntityByName("timestamp", "", ADMIN_AUTH_HEADERS); } @Override diff --git a/openmetadata-service/src/test/java/org/openmetadata/service/resources/teams/TeamResourceTest.java b/openmetadata-service/src/test/java/org/openmetadata/service/resources/teams/TeamResourceTest.java index 7748f9d9a516..e766f1db028f 100644 --- a/openmetadata-service/src/test/java/org/openmetadata/service/resources/teams/TeamResourceTest.java +++ b/openmetadata-service/src/test/java/org/openmetadata/service/resources/teams/TeamResourceTest.java @@ -803,7 +803,7 @@ void testImportInvalidCsv() throws IOException { String record = getRecord(1, GROUP, team.getName(), "", false, "", "invalidPolicy"); String csv = createCsv(TeamCsv.HEADERS, listOf(record), null); CsvImportResult result = importCsv(team.getName(), csv, false); - assertSummary(result, ApiStatus.FAILURE, 2, 1, 1); + assertSummary(result, ApiStatus.PARTIAL_SUCCESS, 2, 1, 1); String[] expectedRows = { resultsHeader, getFailedRecord(record, EntityCsv.entityNotFound(8, Entity.POLICY, "invalidPolicy")) @@ -814,7 +814,7 @@ void testImportInvalidCsv() throws IOException { record = getRecord(1, GROUP, team.getName(), "", false, "invalidRole", ""); csv = createCsv(TeamCsv.HEADERS, listOf(record), null); result = importCsv(team.getName(), csv, false); - assertSummary(result, ApiStatus.FAILURE, 2, 1, 1); + assertSummary(result, ApiStatus.PARTIAL_SUCCESS, 2, 1, 1); expectedRows = new String[] { resultsHeader, @@ -826,7 +826,7 @@ record = getRecord(1, GROUP, team.getName(), "", false, "invalidRole", ""); record = getRecord(1, GROUP, team.getName(), "user:invalidOwner", false, "", ""); csv = createCsv(TeamCsv.HEADERS, listOf(record), null); result = importCsv(team.getName(), csv, false); - assertSummary(result, ApiStatus.FAILURE, 2, 1, 1); + assertSummary(result, ApiStatus.PARTIAL_SUCCESS, 2, 1, 1); expectedRows = new String[] { resultsHeader, @@ -838,7 +838,7 @@ record = getRecord(1, GROUP, team.getName(), "user:invalidOwner", false, "", "") record = getRecord(1, GROUP, "invalidParent", "", false, "", ""); csv = createCsv(TeamCsv.HEADERS, listOf(record), null); result = importCsv(team.getName(), csv, false); - assertSummary(result, ApiStatus.FAILURE, 2, 1, 1); + assertSummary(result, ApiStatus.PARTIAL_SUCCESS, 2, 1, 1); expectedRows = new String[] { resultsHeader, getFailedRecord(record, EntityCsv.entityNotFound(4, TEAM, "invalidParent")) @@ -849,7 +849,7 @@ resultsHeader, getFailedRecord(record, EntityCsv.entityNotFound(4, TEAM, "invali record = getRecord(1, GROUP, TEAM21.getName(), "", false, "", ""); csv = createCsv(TeamCsv.HEADERS, listOf(record), null); result = importCsv(team.getName(), csv, false); - assertSummary(result, ApiStatus.FAILURE, 2, 1, 1); + assertSummary(result, ApiStatus.PARTIAL_SUCCESS, 2, 1, 1); expectedRows = new String[] { resultsHeader, diff --git a/openmetadata-service/src/test/java/org/openmetadata/service/resources/teams/UserResourceTest.java b/openmetadata-service/src/test/java/org/openmetadata/service/resources/teams/UserResourceTest.java index 225518833fd7..79f7195a6611 100644 --- a/openmetadata-service/src/test/java/org/openmetadata/service/resources/teams/UserResourceTest.java +++ b/openmetadata-service/src/test/java/org/openmetadata/service/resources/teams/UserResourceTest.java @@ -1122,7 +1122,7 @@ void testImportInvalidCsv() throws IOException { String record = "invalid::User,,,user@domain.com,,,team-invalidCsv,"; String csv = createCsv(UserCsv.HEADERS, listOf(record), null); CsvImportResult result = importCsv(team.getName(), csv, false); - assertSummary(result, ApiStatus.FAILURE, 2, 1, 1); + assertSummary(result, ApiStatus.PARTIAL_SUCCESS, 2, 1, 1); String[] expectedRows = { resultsHeader, getFailedRecord(record, "[name must match \"^((?!::).)*$\"]") }; @@ -1134,7 +1134,7 @@ resultsHeader, getFailedRecord(record, "[name must match \"^((?!::).)*$\"]") record = "user,,,user@domain.com,,,invalidTeam,"; csv = createCsv(UserCsv.HEADERS, listOf(record), null); result = importCsv(team.getName(), csv, false); - assertSummary(result, ApiStatus.FAILURE, 2, 1, 1); + assertSummary(result, ApiStatus.PARTIAL_SUCCESS, 2, 1, 1); expectedRows = new String[] { resultsHeader, @@ -1146,7 +1146,7 @@ record = "user,,,user@domain.com,,,invalidTeam,"; record = "user,,,user@domain.com,,,team-invalidCsv,invalidRole"; csv = createCsv(UserCsv.HEADERS, listOf(record), null); result = importCsv(team.getName(), csv, false); - assertSummary(result, ApiStatus.FAILURE, 2, 1, 1); + assertSummary(result, ApiStatus.PARTIAL_SUCCESS, 2, 1, 1); expectedRows = new String[] { resultsHeader, diff --git a/openmetadata-spec/src/main/resources/json/schema/entity/applications/configuration/internal/searchIndexingAppConfig.json b/openmetadata-spec/src/main/resources/json/schema/entity/applications/configuration/internal/searchIndexingAppConfig.json index e432d259f5e4..f05af337987c 100644 --- a/openmetadata-spec/src/main/resources/json/schema/entity/applications/configuration/internal/searchIndexingAppConfig.json +++ b/openmetadata-spec/src/main/resources/json/schema/entity/applications/configuration/internal/searchIndexingAppConfig.json @@ -38,6 +38,12 @@ "type": "integer", "default": 100 }, + "payLoadSize": { + "description": "Maximum number of events sent in a batch (Default 100).", + "type": "integer", + "existingJavaType": "java.lang.Long", + "default": 104857600 + }, "searchIndexMappingLanguage": { "description": "Recreate Indexes with updated Language", "$ref": "../../../../configuration/elasticSearchConfiguration.json#/definitions/searchIndexMappingLanguage" diff --git a/openmetadata-spec/src/main/resources/json/schema/system/eventPublisherJob.json b/openmetadata-spec/src/main/resources/json/schema/system/eventPublisherJob.json index e6f6b9922975..4a4f8b123f76 100644 --- a/openmetadata-spec/src/main/resources/json/schema/system/eventPublisherJob.json +++ b/openmetadata-spec/src/main/resources/json/schema/system/eventPublisherJob.json @@ -102,9 +102,10 @@ "type": "integer" }, "payLoadSize": { - "description": "Payload size in bytes depending on config", + "description": "Payload size in bytes depending on config.", "type": "integer", - "default": 10485760 + "existingJavaType": "java.lang.Long", + "default": 104857600 }, "searchIndexMappingLanguage": { "description": "Recreate Indexes with updated Language", diff --git a/openmetadata-ui/src/main/resources/ui/package.json b/openmetadata-ui/src/main/resources/ui/package.json index 8c8ef24b6bbc..3c9f2acba118 100644 --- a/openmetadata-ui/src/main/resources/ui/package.json +++ b/openmetadata-ui/src/main/resources/ui/package.json @@ -52,6 +52,7 @@ "@fontsource/poppins": "^5.0.0", "@fontsource/source-code-pro": "^5.0.0", "@github/g-emoji-element": "^1.1.5", + "@inovua/reactdatagrid-community": "^5.10.2", "@okta/okta-auth-js": "^7.5.0", "@okta/okta-react": "^6.4.3", "@rjsf/core": "5.15.0", diff --git a/openmetadata-ui/src/main/resources/ui/playwright/constant/glossaryImportExport.ts b/openmetadata-ui/src/main/resources/ui/playwright/constant/glossaryImportExport.ts new file mode 100644 index 000000000000..54d6db070312 --- /dev/null +++ b/openmetadata-ui/src/main/resources/ui/playwright/constant/glossaryImportExport.ts @@ -0,0 +1,21 @@ +export const CUSTOM_PROPERTIES_TYPES = { + STRING: 'String', + MARKDOWN: 'Markdown', + SQL_QUERY: 'Sql Query', + ENUM_WITH_DESCRIPTIONS: 'Enum With Descriptions', +}; + +export const FIELD_VALUES_CUSTOM_PROPERTIES = { + STRING: 'This is "testing" string;', + MARKDOWN: `## Overview +This project is designed to **simplify** and *automate* daily tasks. It aims to: +- Increase productivity +- Reduce manual effort +- Provide real-time data insights + +## Features +1. **Task Management**: Organize tasks efficiently with custom tags. +2. **Real-Time Analytics**: Get up-to-date insights on task progress. +3. **Automation**: Automate repetitive workflows using custom scripts.`, + SQL_QUERY: 'SELECT * FROM table_name WHERE id="20";', +}; diff --git a/openmetadata-ui/src/main/resources/ui/playwright/e2e/Features/ActivityFeed.spec.ts b/openmetadata-ui/src/main/resources/ui/playwright/e2e/Features/ActivityFeed.spec.ts index 6aeeac2e0318..77a6edd21c83 100644 --- a/openmetadata-ui/src/main/resources/ui/playwright/e2e/Features/ActivityFeed.spec.ts +++ b/openmetadata-ui/src/main/resources/ui/playwright/e2e/Features/ActivityFeed.spec.ts @@ -10,7 +10,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -import { test as base, expect, Page } from '@playwright/test'; +import { expect, Page, test as base } from '@playwright/test'; import { PolicyClass, PolicyRulesType, @@ -43,8 +43,8 @@ import { checkTaskCount, createDescriptionTask, createTagTask, - TASK_OPEN_FETCH_LINK, TaskDetails, + TASK_OPEN_FETCH_LINK, } from '../../utils/task'; import { performUserLogin } from '../../utils/user'; diff --git a/openmetadata-ui/src/main/resources/ui/playwright/e2e/Pages/GlossaryImportExport.spec.ts b/openmetadata-ui/src/main/resources/ui/playwright/e2e/Pages/GlossaryImportExport.spec.ts index 0e782585822f..c28eeba14f22 100644 --- a/openmetadata-ui/src/main/resources/ui/playwright/e2e/Pages/GlossaryImportExport.spec.ts +++ b/openmetadata-ui/src/main/resources/ui/playwright/e2e/Pages/GlossaryImportExport.spec.ts @@ -11,29 +11,58 @@ * limitations under the License. */ import { expect, test } from '@playwright/test'; +import { CUSTOM_PROPERTIES_ENTITIES } from '../../constant/customProperty'; +import { CUSTOM_PROPERTIES_TYPES } from '../../constant/glossaryImportExport'; +import { GlobalSettingOptions } from '../../constant/settings'; import { SidebarItem } from '../../constant/sidebar'; import { Glossary } from '../../support/glossary/Glossary'; import { GlossaryTerm } from '../../support/glossary/GlossaryTerm'; -import { createNewPage, redirectToHomePage } from '../../utils/common'; +import { UserClass } from '../../support/user/UserClass'; +import { + createNewPage, + redirectToHomePage, + toastNotification, + uuid, +} from '../../utils/common'; +import { + addCustomPropertiesForEntity, + deleteCreatedProperty, +} from '../../utils/customProperty'; import { selectActiveGlossary } from '../../utils/glossary'; -import { sidebarClick } from '../../utils/sidebar'; +import { + createGlossaryTermRowDetails, + fillGlossaryRowDetails, + validateImportStatus, +} from '../../utils/importUtils'; +import { settingClick, sidebarClick } from '../../utils/sidebar'; // use the admin user to login test.use({ storageState: 'playwright/.auth/admin.json', }); -const glossary = new Glossary(); -const glossaryTerm1 = new GlossaryTerm(glossary); +const user1 = new UserClass(); +const user2 = new UserClass(); +const glossary1 = new Glossary(); +const glossary2 = new Glossary(); +const glossaryTerm1 = new GlossaryTerm(glossary1); +const glossaryTerm2 = new GlossaryTerm(glossary2); +const propertiesList = Object.values(CUSTOM_PROPERTIES_TYPES); -test.describe('Bulk Import Export', () => { - test.slow(); +const propertyListName: Record = {}; + +test.describe('Glossary Bulk Import Export', () => { + test.slow(true); test.beforeAll('setup pre-test', async ({ browser }) => { const { apiContext, afterAction } = await createNewPage(browser); - await glossary.create(apiContext); + await user1.create(apiContext); + await user2.create(apiContext); + await glossary1.create(apiContext); + await glossary2.create(apiContext); await glossaryTerm1.create(apiContext); + await glossaryTerm2.create(apiContext); await afterAction(); }); @@ -41,7 +70,10 @@ test.describe('Bulk Import Export', () => { test.afterAll('Cleanup', async ({ browser }) => { const { apiContext, afterAction } = await createNewPage(browser); - await glossary.delete(apiContext); + await user1.delete(apiContext); + await user2.delete(apiContext); + await glossary1.delete(apiContext); + await glossary2.delete(apiContext); await afterAction(); }); @@ -50,16 +82,34 @@ test.describe('Bulk Import Export', () => { await redirectToHomePage(page); }); - test('Import and Export Glossary', async ({ page }) => { - await test.step('Export data', async () => { + test('Glossary Bulk Import Export', async ({ page }) => { + await test.step('create custom properties for extension edit', async () => { + for (const property of propertiesList) { + const entity = CUSTOM_PROPERTIES_ENTITIES.entity_glossaryTerm; + const propertyName = `pwcustomproperty${entity.name}test${uuid()}`; + propertyListName[property] = propertyName; + + await settingClick(page, GlobalSettingOptions.GLOSSARY_TERM, true); + + await addCustomPropertiesForEntity({ + page, + propertyName, + customPropertyData: entity, + customType: property, + enumWithDescriptionConfig: entity.enumWithDescriptionConfig, + }); + } + }); + + await test.step('should export data glossary term details', async () => { await sidebarClick(page, SidebarItem.GLOSSARY); - await selectActiveGlossary(page, glossary.data.displayName); + await selectActiveGlossary(page, glossary1.data.displayName); const downloadPromise = page.waitForEvent('download'); await page.click('[data-testid="manage-button"]'); await page.click('[data-testid="export-button-description"]'); - await page.fill('#fileName', glossary.data.displayName); + await page.fill('#fileName', glossary1.data.displayName); await page.click('#submit-button'); const download = await downloadPromise; @@ -67,39 +117,97 @@ test.describe('Bulk Import Export', () => { await download.saveAs('downloads/' + download.suggestedFilename()); }); - await test.step('Import data', async () => { - await selectActiveGlossary(page, glossary.data.displayName); - await page.click('[data-testid="manage-button"]'); - await page.click('[data-testid="import-button-description"]'); - const fileInput = await page.$('[type="file"]'); - await fileInput?.setInputFiles([ - 'downloads/' + glossary.data.displayName + '.csv', - ]); - - // Adding manual wait for the file to load - await page.waitForTimeout(500); - - await expect( - page.getByText('Number of rows: 2 | Passed: 2') - ).toBeVisible(); - - await expect(page.getByTestId('import-result-table')).toBeVisible(); - - await expect(page.getByTestId('preview-cancel-button')).toBeVisible(); - - const glossaryImport = page.waitForResponse( - '/api/v1/glossaries/name/*/import?dryRun=false' - ); - await page.getByTestId('import-button').click(); - await glossaryImport; - - const glossaryResponse = page.waitForResponse('/api/v1/glossaryTerms?**'); - await page.getByTestId('preview-button').click(); - await glossaryResponse; - - await expect(page.getByTestId('entity-header-display-name')).toHaveText( - glossary.responseData.displayName - ); + await test.step( + 'should import and edit with one additional glossaryTerm', + async () => { + await sidebarClick(page, SidebarItem.GLOSSARY); + await selectActiveGlossary(page, glossary1.data.displayName); + await page.click('[data-testid="manage-button"]'); + await page.click('[data-testid="import-button-description"]'); + const fileInput = await page.$('[type="file"]'); + await fileInput?.setInputFiles([ + 'downloads/' + glossary1.data.displayName + '.csv', + ]); + + // Adding manual wait for the file to load + await page.waitForTimeout(500); + + // Adding some assertion to make sure that CSV loaded correctly + await expect( + page.locator('.InovuaReactDataGrid__header-layout') + ).toBeVisible(); + await expect(page.getByTestId('add-row-btn')).toBeVisible(); + await expect(page.getByRole('button', { name: 'Next' })).toBeVisible(); + await expect( + page.getByRole('button', { name: 'Previous' }) + ).toBeVisible(); + + await page.click('[data-testid="add-row-btn"]'); + + // click on last row first cell + await page.click( + '.InovuaReactDataGrid__row--last > .InovuaReactDataGrid__row-cell-wrap > .InovuaReactDataGrid__cell--first' + ); + // Click on first cell and edit + await fillGlossaryRowDetails( + { + ...createGlossaryTermRowDetails(), + owners: [user1.responseData?.['displayName']], + reviewers: [user2.responseData?.['displayName']], + relatedTerm: { + parent: glossary2.data.name, + name: glossaryTerm2.data.name, + }, + }, + page, + propertyListName + ); + + await page.getByRole('button', { name: 'Next' }).click(); + const loader = page.locator( + '.inovua-react-toolkit-load-mask__background-layer' + ); + + await loader.waitFor({ state: 'hidden' }); + + await validateImportStatus(page, { + passed: '3', + processed: '3', + failed: '0', + }); + + await page.waitForSelector('.InovuaReactDataGrid__header-layout', { + state: 'visible', + }); + + const rowStatus = ['Entity updated', 'Entity created']; + + await expect(page.locator('[data-props-id="details"]')).toHaveText( + rowStatus + ); + + await page.getByRole('button', { name: 'Update' }).click(); + await page + .locator('.inovua-react-toolkit-load-mask__background-layer') + .waitFor({ state: 'detached' }); + + await toastNotification( + page, + `Glossaryterm ${glossary1.responseData.fullyQualifiedName} details updated successfully` + ); + } + ); + + await test.step('delete custom properties', async () => { + for (const propertyName of Object.values(propertyListName)) { + await settingClick(page, GlobalSettingOptions.GLOSSARY_TERM, true); + + await page.waitForLoadState('networkidle'); + + await page.getByTestId('loader').waitFor({ state: 'detached' }); + + await deleteCreatedProperty(page, propertyName); + } }); }); }); diff --git a/openmetadata-ui/src/main/resources/ui/playwright/utils/importUtils.ts b/openmetadata-ui/src/main/resources/ui/playwright/utils/importUtils.ts new file mode 100644 index 000000000000..1003eda7f7bf --- /dev/null +++ b/openmetadata-ui/src/main/resources/ui/playwright/utils/importUtils.ts @@ -0,0 +1,340 @@ +/* + * Copyright 2024 Collate. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * http://www.apache.org/licenses/LICENSE-2.0 + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import { expect, Page } from '@playwright/test'; +import { CUSTOM_PROPERTIES_ENTITIES } from '../constant/customProperty'; +import { + CUSTOM_PROPERTIES_TYPES, + FIELD_VALUES_CUSTOM_PROPERTIES, +} from '../constant/glossaryImportExport'; +import { descriptionBox, uuid } from './common'; + +export const createGlossaryTermRowDetails = () => { + return { + name: `playwright,glossaryTerm ${uuid()}`, + displayName: 'Playwright,Glossary Term', + description: `Playwright GlossaryTerm description. + Neque porro quisquam est qui dolorem ipsum quia dolor sit amet, consectetur, adipisci velit... + There is no one who loves pain itself, who seeks after it and wants to have it, simply because it is pain..`, + tag: 'PII.Sensitive', + synonyms: 'playwright,glossaryTerm,testing', + references: 'data;http:sandbox.com', + }; +}; + +export const fillTextInputDetails = async (page: Page, text: string) => { + await page.locator('.InovuaReactDataGrid__cell--cell-active').press('Enter'); + + await page.locator('.ant-layout-content').getByRole('textbox').fill(text); + await page + .locator('.ant-layout-content') + .getByRole('textbox') + .press('Enter', { delay: 100 }); +}; + +export const fillDescriptionDetails = async ( + page: Page, + description: string +) => { + await page.locator('.InovuaReactDataGrid__cell--cell-active').press('Enter'); + await page.click( + '.toastui-editor-md-container > .toastui-editor > .ProseMirror' + ); + + await page.fill( + '.toastui-editor-md-container > .toastui-editor > .ProseMirror', + description + ); + + await page.click('[data-testid="save"]'); + await page.click('.InovuaReactDataGrid__cell--cell-active'); +}; + +export const fillOwnerDetails = async (page: Page, owners: string[]) => { + await page + .locator('.InovuaReactDataGrid__cell--cell-active') + .press('Enter', { delay: 100 }); + + const userListResponse = page.waitForResponse( + '/api/v1/users?limit=*&isBot=false*' + ); + await page.getByRole('tab', { name: 'Users' }).click(); + await userListResponse; + await page.waitForSelector('[data-testid="loader"]', { state: 'detached' }); + + await page.click('[data-testid="owner-select-users-search-bar"]'); + + for (const owner of owners) { + await page.locator('[data-testid="owner-select-users-search-bar"]').clear(); + await page.keyboard.type(owner); + await page.waitForResponse( + `/api/v1/search/query?q=*${owner}*%20AND%20isBot:false&from=0&size=25&index=user_search_index` + ); + + await page.getByRole('listitem', { name: owner }).click(); + } + + await page.getByTestId('selectable-list-update-btn').click(); + + await page.click('.InovuaReactDataGrid__cell--cell-active'); +}; + +export const fillTagDetails = async (page: Page, tag: string) => { + await page + .locator('.InovuaReactDataGrid__cell--cell-active') + .press('Enter', { delay: 100 }); + + await page.click('[data-testid="tag-selector"]'); + await page.locator('[data-testid="tag-selector"] input').fill(tag); + await page.click(`[data-testid="tag-${tag}"]`); + await page.click('[data-testid="inline-save-btn"]'); + await page.click('.InovuaReactDataGrid__cell--cell-active'); +}; + +export const fillGlossaryTermDetails = async ( + page: Page, + glossary: { parent: string; name: string } +) => { + await page + .locator('.InovuaReactDataGrid__cell--cell-active') + .press('Enter', { delay: 100 }); + + await page.waitForSelector('[data-testid="loader"]', { state: 'detached' }); + + await page.click('[data-testid="tag-selector"]'); + await page.locator('[data-testid="tag-selector"] input').fill(glossary.name); + await page.getByTestId(`tag-"${glossary.parent}"."${glossary.name}"`).click(); + await page.click('[data-testid="saveAssociatedTag"]'); + await page.click('.InovuaReactDataGrid__cell--cell-active'); +}; + +export const fillDomainDetails = async ( + page: Page, + domains: { name: string; displayName: string } +) => { + await page + .locator('.InovuaReactDataGrid__cell--cell-active') + .press('Enter', { delay: 100 }); + + await page.click('[data-testid="selectable-list"] [data-testid="searchbar"]'); + await page + .locator('[data-testid="selectable-list"] [data-testid="searchbar"]') + .fill(domains.name); + + await page.click(`.ant-popover [title="${domains.displayName}"]`); + await page.waitForTimeout(100); +}; + +const editGlossaryCustomProperty = async ( + page: Page, + propertyName: string, + type: string +) => { + await page + .locator( + `[data-testid=${propertyName}] [data-testid='edit-icon-right-panel']` + ) + .click(); + + if (type === CUSTOM_PROPERTIES_TYPES.STRING) { + await page + .getByTestId('value-input') + .fill(FIELD_VALUES_CUSTOM_PROPERTIES.STRING); + await page.getByTestId('inline-save-btn').click(); + } + + if (type === CUSTOM_PROPERTIES_TYPES.MARKDOWN) { + await page.waitForSelector(descriptionBox, { state: 'visible' }); + + await page + .locator(descriptionBox) + .fill(FIELD_VALUES_CUSTOM_PROPERTIES.MARKDOWN); + + await page.getByTestId('markdown-editor').getByTestId('save').click(); + + await page.waitForSelector(descriptionBox, { + state: 'detached', + }); + } + + if (type === CUSTOM_PROPERTIES_TYPES.SQL_QUERY) { + await page + .getByTestId('code-mirror-container') + .getByRole('textbox') + .fill(FIELD_VALUES_CUSTOM_PROPERTIES.SQL_QUERY); + + await page.getByTestId('inline-save-btn').click(); + } + + if (type === CUSTOM_PROPERTIES_TYPES.ENUM_WITH_DESCRIPTIONS) { + await page.getByTestId('enum-with-description-select').click(); + + await page.waitForSelector('.ant-select-dropdown', { + state: 'visible', + }); + + // await page + // .getByRole('option', { + // name: CUSTOM_PROPERTIES_ENTITIES.entity_glossaryTerm + // .enumWithDescriptionConfig.values[0].key, + // }) + // .click(); + + await page + .locator('span') + .filter({ + hasText: + CUSTOM_PROPERTIES_ENTITIES.entity_glossaryTerm + .enumWithDescriptionConfig.values[0].key, + }) + .click(); + + await page.getByTestId('inline-save-btn').click(); + } +}; + +export const fillCustomPropertyDetails = async ( + page: Page, + propertyListName: Record +) => { + await page + .locator('.InovuaReactDataGrid__cell--cell-active') + .press('Enter', { delay: 100 }); + + // Wait for the loader to disappear + await page.waitForSelector('.ant-skeleton-content', { state: 'hidden' }); + + for (const propertyName of Object.values(CUSTOM_PROPERTIES_TYPES)) { + await editGlossaryCustomProperty( + page, + propertyListName[propertyName], + propertyName + ); + } + + await page.getByTestId('save').click(); + + await expect(page.locator('.ant-modal-wrap')).not.toBeVisible(); + + await page.click('.InovuaReactDataGrid__cell--cell-active'); +}; + +export const fillGlossaryRowDetails = async ( + row: { + name: string; + displayName: string; + description: string; + synonyms: string; + relatedTerm: { + name: string; + parent: string; + }; + references: string; + tag: string; + reviewers: string[]; + owners: string[]; + }, + page: Page, + propertyListName: Record +) => { + await page + .locator('.InovuaReactDataGrid__cell--cell-active') + .press('ArrowRight'); + + await fillTextInputDetails(page, row.name); + + await page + .locator('.InovuaReactDataGrid__cell--cell-active') + .press('ArrowRight'); + + await fillTextInputDetails(page, row.displayName); + + // Navigate to next cell and make cell editable + await page + .locator('.InovuaReactDataGrid__cell--cell-active') + .press('ArrowRight', { delay: 100 }); + + await fillDescriptionDetails(page, row.description); + + await page + .locator('.InovuaReactDataGrid__cell--cell-active') + .press('ArrowRight', { delay: 100 }); + + await fillTextInputDetails(page, row.synonyms); + + await page + .locator('.InovuaReactDataGrid__cell--cell-active') + .press('ArrowRight', { delay: 100 }); + + await fillGlossaryTermDetails(page, row.relatedTerm); + + await page + .locator('.InovuaReactDataGrid__cell--cell-active') + .press('ArrowRight', { delay: 100 }); + + await fillTextInputDetails(page, row.references); + + await page + .locator('.InovuaReactDataGrid__cell--cell-active') + .press('ArrowRight', { delay: 100 }); + + await fillTagDetails(page, row.tag); + + await page + .locator('.InovuaReactDataGrid__cell--cell-active') + .press('ArrowRight', { delay: 100 }); + + await fillOwnerDetails(page, row.reviewers); + + await page + .locator('.InovuaReactDataGrid__cell--cell-active') + .press('ArrowRight', { delay: 100 }); + + await fillOwnerDetails(page, row.owners); + + await page + .locator('.InovuaReactDataGrid__cell--cell-active') + .press('ArrowRight', { delay: 100 }); + + await fillCustomPropertyDetails(page, propertyListName); +}; + +export const validateImportStatus = async ( + page: Page, + status: { passed: string; failed: string; processed: string } +) => { + await page.waitForSelector('[data-testid="processed-row"]'); + const processedRow = await page.$eval( + '[data-testid="processed-row"]', + (el) => el.textContent + ); + + expect(processedRow).toBe(status.processed); + + const passedRow = await page.$eval( + '[data-testid="passed-row"]', + (el) => el.textContent + ); + + expect(passedRow).toBe(status.passed); + + const failedRow = await page.$eval( + '[data-testid="failed-row"]', + (el) => el.textContent + ); + + expect(failedRow).toBe(status.failed); + + await page.waitForSelector('.InovuaReactDataGrid__header-layout', { + state: 'visible', + }); +}; diff --git a/openmetadata-ui/src/main/resources/ui/src/components/APIEndpoint/APIEndpointDetails/APIEndpointDetails.tsx b/openmetadata-ui/src/main/resources/ui/src/components/APIEndpoint/APIEndpointDetails/APIEndpointDetails.tsx index b3a65ac6d1e8..b0c3036d2aae 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/APIEndpoint/APIEndpointDetails/APIEndpointDetails.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/components/APIEndpoint/APIEndpointDetails/APIEndpointDetails.tsx @@ -19,6 +19,7 @@ import { useTranslation } from 'react-i18next'; import { useHistory, useParams } from 'react-router-dom'; import { getEntityDetailsPath } from '../../../constants/constants'; import { FEED_COUNT_INITIAL_DATA } from '../../../constants/entity.constants'; +import { COMMON_RESIZABLE_PANEL_CONFIG } from '../../../constants/ResizablePanel.constants'; import LineageProvider from '../../../context/LineageProvider/LineageProvider'; import { EntityTabs, EntityType } from '../../../enums/entity.enum'; import { Tag } from '../../../generated/entity/classification/tag'; @@ -325,8 +326,7 @@ const APIEndpointDetails: React.FC = ({ /> ), - minWidth: 800, - flex: 0.87, + ...COMMON_RESIZABLE_PANEL_CONFIG.LEFT_PANEL, }} secondPanel={{ children: ( @@ -350,8 +350,7 @@ const APIEndpointDetails: React.FC = ({ /> ), - minWidth: 320, - flex: 0.13, + ...COMMON_RESIZABLE_PANEL_CONFIG.RIGHT_PANEL, className: 'entity-resizable-right-panel-container entity-resizable-panel-container', }} diff --git a/openmetadata-ui/src/main/resources/ui/src/components/BulkImport/BulkEntityImport.component.tsx b/openmetadata-ui/src/main/resources/ui/src/components/BulkImport/BulkEntityImport.component.tsx new file mode 100644 index 000000000000..d1067f96632b --- /dev/null +++ b/openmetadata-ui/src/main/resources/ui/src/components/BulkImport/BulkEntityImport.component.tsx @@ -0,0 +1,392 @@ +/* + * Copyright 2024 Collate. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * http://www.apache.org/licenses/LICENSE-2.0 + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import ReactDataGrid from '@inovua/reactdatagrid-community'; +import '@inovua/reactdatagrid-community/index.css'; +import { + TypeColumn, + TypeComputedProps, +} from '@inovua/reactdatagrid-community/types'; +import { Button, Card, Col, Row, Space, Typography } from 'antd'; +import { AxiosError } from 'axios'; +import React, { MutableRefObject, useCallback, useMemo, useState } from 'react'; +import { useTranslation } from 'react-i18next'; +import { usePapaParse } from 'react-papaparse'; + +import { capitalize } from 'lodash'; +import { + ENTITY_IMPORT_STEPS, + VALIDATION_STEP, +} from '../../constants/BulkImport.constant'; +import { CSVImportResult } from '../../generated/type/csvImportResult'; +import { + getCSVStringFromColumnsAndDataSource, + getEntityColumnsAndDataSourceFromCSV, +} from '../../utils/CSV/CSV.utils'; +import csvUtilsClassBase from '../../utils/CSV/CSVUtilsClassBase'; +import { showErrorToast, showSuccessToast } from '../../utils/ToastUtils'; +import { ImportStatus } from '../common/EntityImport/ImportStatus/ImportStatus.component'; +import Stepper from '../Settings/Services/Ingestion/IngestionStepper/IngestionStepper.component'; +import { UploadFile } from '../UploadFile/UploadFile'; +import './bulk-entity-import.style.less'; +import { BulkImportProps } from './BulkEntityImport.interface'; + +let inEdit = false; + +const BulkEntityImport = ({ + entityType, + fqn, + onValidateCsvString, + onSuccess, +}: BulkImportProps) => { + const [activeStep, setActiveStep] = useState( + VALIDATION_STEP.UPLOAD + ); + const { t } = useTranslation(); + const [isValidating, setIsValidating] = useState(false); + const [validationData, setValidationData] = useState(); + const [columns, setColumns] = useState([]); + const [dataSource, setDataSource] = useState[]>([]); + const { readString } = usePapaParse(); + const [validateCSVData, setValidateCSVData] = + useState<{ columns: TypeColumn[]; dataSource: Record[] }>(); + const [gridRef, setGridRef] = useState< + MutableRefObject + >({ current: null }); + + const filterColumns = useMemo( + () => + columns?.filter( + (col) => + !csvUtilsClassBase.hideImportsColumnList().includes(col.name ?? '') + ), + [columns] + ); + + const focusToGrid = useCallback(() => { + setGridRef((ref) => { + ref.current?.focus(); + + return ref; + }); + }, [setGridRef]); + + const onCSVReadComplete = useCallback( + (results: { data: string[][] }) => { + // results.data is returning data with unknown type + const { columns, dataSource } = getEntityColumnsAndDataSourceFromCSV( + results.data as string[][], + entityType + ); + setDataSource(dataSource); + setColumns(columns); + + setActiveStep(VALIDATION_STEP.EDIT_VALIDATE); + setTimeout(focusToGrid, 500); + }, + [entityType, setDataSource, setColumns, setActiveStep, focusToGrid] + ); + + const handleLoadData = useCallback( + async (e: ProgressEvent) => { + try { + const result = e.target?.result as string; + + const validationResponse = await onValidateCsvString(result, true); + + if (['failure', 'aborted'].includes(validationResponse?.status ?? '')) { + setValidationData(validationResponse); + + setActiveStep(VALIDATION_STEP.UPLOAD); + + return; + } + + if (result) { + readString(result, { + worker: true, + skipEmptyLines: true, + complete: onCSVReadComplete, + }); + } + } catch (error) { + showErrorToast(error as AxiosError); + } + }, + [onCSVReadComplete] + ); + + const onEditComplete = useCallback( + ({ value, columnId, rowId }) => { + const data = [...dataSource]; + data[rowId][columnId] = value; + + setDataSource(data); + }, + [dataSource] + ); + + const handleBack = () => { + if (activeStep === VALIDATION_STEP.UPDATE) { + setActiveStep(VALIDATION_STEP.EDIT_VALIDATE); + } else { + setActiveStep(VALIDATION_STEP.UPLOAD); + } + }; + + const handleValidate = async () => { + setIsValidating(true); + setValidateCSVData(undefined); + try { + // Call the validate API + const csvData = getCSVStringFromColumnsAndDataSource(columns, dataSource); + + const response = await onValidateCsvString( + csvData, + activeStep === VALIDATION_STEP.EDIT_VALIDATE + ); + + if (activeStep === VALIDATION_STEP.UPDATE) { + if (response?.status === 'failure') { + setValidationData(response); + readString(response?.importResultsCsv ?? '', { + worker: true, + skipEmptyLines: true, + complete: (results) => { + // results.data is returning data with unknown type + setValidateCSVData( + getEntityColumnsAndDataSourceFromCSV( + results.data as string[][], + entityType + ) + ); + }, + }); + setActiveStep(VALIDATION_STEP.UPDATE); + } else { + showSuccessToast( + t('message.entity-details-updated', { + entityType: capitalize(entityType), + fqn, + }) + ); + onSuccess(); + } + } else if (activeStep === VALIDATION_STEP.EDIT_VALIDATE) { + setValidationData(response); + setActiveStep(VALIDATION_STEP.UPDATE); + readString(response?.importResultsCsv ?? '', { + worker: true, + skipEmptyLines: true, + complete: (results) => { + // results.data is returning data with unknown type + setValidateCSVData( + getEntityColumnsAndDataSourceFromCSV( + results.data as string[][], + entityType + ) + ); + }, + }); + } + } catch (error) { + showErrorToast(error as AxiosError); + } finally { + setIsValidating(false); + } + }; + + const onEditStart = () => { + inEdit = true; + }; + + const onEditStop = () => { + requestAnimationFrame(() => { + inEdit = false; + gridRef.current?.focus(); + }); + }; + + const onKeyDown = (event: KeyboardEvent) => { + if (inEdit) { + if (event.key === 'Escape') { + const [rowIndex, colIndex] = gridRef.current?.computedActiveCell ?? [ + 0, 0, + ]; + const column = gridRef.current?.getColumnBy(colIndex); + + gridRef.current?.cancelEdit?.({ + rowIndex, + columnId: column?.name ?? '', + }); + } + + return; + } + const grid = gridRef.current; + if (!grid) { + return; + } + let [rowIndex, colIndex] = grid.computedActiveCell ?? [0, 0]; + + if (event.key === ' ' || event.key === 'Enter') { + const column = grid.getColumnBy(colIndex); + grid.startEdit?.({ columnId: column.name ?? '', rowIndex }); + event.preventDefault(); + + return; + } + if (event.key !== 'Tab') { + return; + } + event.preventDefault(); + event.stopPropagation(); + + const direction = event.shiftKey ? -1 : 1; + + const columns = grid.visibleColumns; + const rowCount = grid.count; + + colIndex += direction; + if (colIndex === -1) { + colIndex = columns.length - 1; + rowIndex -= 1; + } + if (colIndex === columns.length) { + rowIndex += 1; + colIndex = 0; + } + if (rowIndex < 0 || rowIndex === rowCount) { + return; + } + + grid?.setActiveCell([rowIndex, colIndex]); + }; + + const handleAddRow = useCallback(() => { + setDataSource((data) => { + setTimeout(() => { + gridRef.current?.scrollToId(data.length + ''); + gridRef.current?.focus(); + }, 1); + + return [...data, { id: data.length + '' }]; + }); + }, [gridRef]); + + const handleRetryCsvUpload = () => { + setValidationData(undefined); + + setActiveStep(VALIDATION_STEP.UPLOAD); + }; + + return ( + + + + + + {activeStep === 0 && ( + <> + {validationData?.abortReason ? ( + + + + {t('label.aborted')}{' '} + {validationData.abortReason} + + + + + + + ) : ( + + )} + + )} + {activeStep === 1 && ( + + )} + {activeStep === 2 && validationData && ( + + + + + + {validateCSVData && ( + + )} + + + )} + + {activeStep > 0 && ( + + {activeStep === 1 && ( + + )} +
+ {activeStep > 0 && ( + + )} + {activeStep < 3 && ( + + )} +
+ + )} + + ); +}; + +export default BulkEntityImport; diff --git a/openmetadata-ui/src/main/resources/ui/src/components/BulkImport/BulkEntityImport.interface.ts b/openmetadata-ui/src/main/resources/ui/src/components/BulkImport/BulkEntityImport.interface.ts new file mode 100644 index 000000000000..a65a5ff11009 --- /dev/null +++ b/openmetadata-ui/src/main/resources/ui/src/components/BulkImport/BulkEntityImport.interface.ts @@ -0,0 +1,24 @@ +/* + * Copyright 2024 Collate. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * http://www.apache.org/licenses/LICENSE-2.0 + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import { EntityType } from '../../enums/entity.enum'; +import { CSVImportResult } from '../../generated/type/csvImportResult'; + +export interface BulkImportProps { + entityType: EntityType; + fqn: string; + onValidateCsvString: ( + data: string, + dryRun?: boolean + ) => Promise; + onSuccess: () => void; +} diff --git a/openmetadata-ui/src/main/resources/ui/src/components/BulkImport/bulk-entity-import.style.less b/openmetadata-ui/src/main/resources/ui/src/components/BulkImport/bulk-entity-import.style.less new file mode 100644 index 000000000000..a015f108f4fb --- /dev/null +++ b/openmetadata-ui/src/main/resources/ui/src/components/BulkImport/bulk-entity-import.style.less @@ -0,0 +1,18 @@ +/* + * Copyright 2024 Collate. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * http://www.apache.org/licenses/LICENSE-2.0 + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +.reserve-right-sidebar { + .import-footer { + // Right side padding 20 + 64 width of sidebar + padding-right: 84px; + } +} diff --git a/openmetadata-ui/src/main/resources/ui/src/components/Dashboard/DashboardDetails/DashboardDetails.component.tsx b/openmetadata-ui/src/main/resources/ui/src/components/Dashboard/DashboardDetails/DashboardDetails.component.tsx index 7410e12f75fa..43403585cfdd 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/Dashboard/DashboardDetails/DashboardDetails.component.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/components/Dashboard/DashboardDetails/DashboardDetails.component.tsx @@ -28,6 +28,7 @@ import { getEntityDetailsPath, } from '../../../constants/constants'; import { FEED_COUNT_INITIAL_DATA } from '../../../constants/entity.constants'; +import { COMMON_RESIZABLE_PANEL_CONFIG } from '../../../constants/ResizablePanel.constants'; import LineageProvider from '../../../context/LineageProvider/LineageProvider'; import { usePermissionProvider } from '../../../context/PermissionProvider/PermissionProvider'; import { ResourceEntity } from '../../../context/PermissionProvider/PermissionProvider.interface'; @@ -652,8 +653,7 @@ const DashboardDetails = ({ )} ), - minWidth: 800, - flex: 0.87, + ...COMMON_RESIZABLE_PANEL_CONFIG.LEFT_PANEL, }} secondPanel={{ children: ( @@ -677,8 +677,7 @@ const DashboardDetails = ({ /> ), - minWidth: 320, - flex: 0.13, + ...COMMON_RESIZABLE_PANEL_CONFIG.RIGHT_PANEL, className: 'entity-resizable-right-panel-container entity-resizable-panel-container', }} diff --git a/openmetadata-ui/src/main/resources/ui/src/components/Dashboard/DataModel/DataModels/DataModelDetails.component.tsx b/openmetadata-ui/src/main/resources/ui/src/components/Dashboard/DataModel/DataModels/DataModelDetails.component.tsx index 1d9a808c9b8a..e844a0d696c1 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/Dashboard/DataModel/DataModels/DataModelDetails.component.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/components/Dashboard/DataModel/DataModels/DataModelDetails.component.tsx @@ -23,6 +23,7 @@ import { getVersionPath, } from '../../../../constants/constants'; import { FEED_COUNT_INITIAL_DATA } from '../../../../constants/entity.constants'; +import { COMMON_RESIZABLE_PANEL_CONFIG } from '../../../../constants/ResizablePanel.constants'; import LineageProvider from '../../../../context/LineageProvider/LineageProvider'; import { CSMode } from '../../../../enums/codemirror.enum'; import { EntityTabs, EntityType } from '../../../../enums/entity.enum'; @@ -259,8 +260,7 @@ const DataModelDetails = ({ /> ), - minWidth: 800, - flex: 0.87, + ...COMMON_RESIZABLE_PANEL_CONFIG.LEFT_PANEL, }} secondPanel={{ children: ( @@ -286,8 +286,7 @@ const DataModelDetails = ({ /> ), - minWidth: 320, - flex: 0.13, + ...COMMON_RESIZABLE_PANEL_CONFIG.RIGHT_PANEL, className: 'entity-resizable-right-panel-container entity-resizable-panel-container', }} diff --git a/openmetadata-ui/src/main/resources/ui/src/components/Domain/DomainTabs/DataProductsTab/DataProductsTab.component.tsx b/openmetadata-ui/src/main/resources/ui/src/components/Domain/DomainTabs/DataProductsTab/DataProductsTab.component.tsx index 22a50405b386..9a807da1c622 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/Domain/DomainTabs/DataProductsTab/DataProductsTab.component.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/components/Domain/DomainTabs/DataProductsTab/DataProductsTab.component.tsx @@ -22,6 +22,7 @@ import React, { } from 'react'; import { useTranslation } from 'react-i18next'; import { PAGE_SIZE_LARGE } from '../../../../constants/constants'; +import { COMMON_RESIZABLE_PANEL_CONFIG } from '../../../../constants/ResizablePanel.constants'; import { ERROR_PLACEHOLDER_TYPE } from '../../../../enums/common.enum'; import { EntityType } from '../../../../enums/entity.enum'; import { SearchIndex } from '../../../../enums/search.enum'; @@ -142,8 +143,7 @@ const DataProductsTab = forwardRef( ))} ), - minWidth: 800, - flex: 0.87, + ...COMMON_RESIZABLE_PANEL_CONFIG.LEFT_PANEL, }} pageTitle={t('label.domain')} secondPanel={{ @@ -158,8 +158,7 @@ const DataProductsTab = forwardRef( handleClosePanel={() => setSelectedCard(undefined)} /> ), - minWidth: 320, - flex: 0.13, + ...COMMON_RESIZABLE_PANEL_CONFIG.RIGHT_PANEL, className: 'entity-summary-resizable-right-panel-container domain-resizable-panel-container', }} diff --git a/openmetadata-ui/src/main/resources/ui/src/components/Domain/DomainTabs/DocumentationTab/DocumentationTab.component.tsx b/openmetadata-ui/src/main/resources/ui/src/components/Domain/DomainTabs/DocumentationTab/DocumentationTab.component.tsx index dedea541f7cc..2378134a3d78 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/Domain/DomainTabs/DocumentationTab/DocumentationTab.component.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/components/Domain/DomainTabs/DocumentationTab/DocumentationTab.component.tsx @@ -22,6 +22,7 @@ import { UserTeamSelectableList } from '../../../../components/common/UserTeamSe import DomainTypeSelectForm from '../../../../components/Domain/DomainTypeSelectForm/DomainTypeSelectForm.component'; import { DE_ACTIVE_COLOR } from '../../../../constants/constants'; import { EntityField } from '../../../../constants/Feeds.constants'; +import { COMMON_RESIZABLE_PANEL_CONFIG } from '../../../../constants/ResizablePanel.constants'; import { usePermissionProvider } from '../../../../context/PermissionProvider/PermissionProvider'; import { ResourceEntity } from '../../../../context/PermissionProvider/PermissionProvider.interface'; import { EntityType, TabSpecificField } from '../../../../enums/entity.enum'; @@ -191,8 +192,7 @@ const DocumentationTab = ({ /> ), - minWidth: 800, - flex: 0.75, + ...COMMON_RESIZABLE_PANEL_CONFIG.LEFT_PANEL, }} secondPanel={{ children: ( @@ -374,8 +374,7 @@ const DocumentationTab = ({ )} ), - minWidth: 320, - flex: 0.25, + ...COMMON_RESIZABLE_PANEL_CONFIG.RIGHT_PANEL, className: 'entity-resizable-right-panel-container domain-resizable-panel-container', }} diff --git a/openmetadata-ui/src/main/resources/ui/src/components/Glossary/GlossaryDetails/GlossaryDetails.component.tsx b/openmetadata-ui/src/main/resources/ui/src/components/Glossary/GlossaryDetails/GlossaryDetails.component.tsx index 6f3ede3e481f..0ffba4c955a2 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/Glossary/GlossaryDetails/GlossaryDetails.component.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/components/Glossary/GlossaryDetails/GlossaryDetails.component.tsx @@ -20,6 +20,7 @@ import { useHistory, useParams } from 'react-router-dom'; import { getGlossaryTermDetailsPath } from '../../../constants/constants'; import { FEED_COUNT_INITIAL_DATA } from '../../../constants/entity.constants'; import { EntityField } from '../../../constants/Feeds.constants'; +import { COMMON_RESIZABLE_PANEL_CONFIG } from '../../../constants/ResizablePanel.constants'; import { EntityType } from '../../../enums/entity.enum'; import { Glossary } from '../../../generated/entity/data/glossary'; import { ChangeDescription } from '../../../generated/entity/type'; @@ -179,8 +180,7 @@ const GlossaryDetails = ({ ), - minWidth: 800, - flex: 0.75, + ...COMMON_RESIZABLE_PANEL_CONFIG.LEFT_PANEL, }} secondPanel={{ children: ( @@ -195,8 +195,7 @@ const GlossaryDetails = ({ onUpdate={handleGlossaryUpdate} /> ), - minWidth: 320, - flex: 0.25, + ...COMMON_RESIZABLE_PANEL_CONFIG.RIGHT_PANEL, className: 'entity-resizable-right-panel-container glossary-resizable-panel-container', }} diff --git a/openmetadata-ui/src/main/resources/ui/src/components/Glossary/GlossaryDetailsRightPanel/GlossaryDetailsRightPanel.component.tsx b/openmetadata-ui/src/main/resources/ui/src/components/Glossary/GlossaryDetailsRightPanel/GlossaryDetailsRightPanel.component.tsx index 9f23582e98ec..c753cc350695 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/Glossary/GlossaryDetailsRightPanel/GlossaryDetailsRightPanel.component.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/components/Glossary/GlossaryDetailsRightPanel/GlossaryDetailsRightPanel.component.tsx @@ -288,7 +288,7 @@ const GlossaryDetailsRightPanel = ({ )} - + {!isGlossary && selectedData && ( ), - minWidth: 800, - flex: 0.75, + ...COMMON_RESIZABLE_PANEL_CONFIG.LEFT_PANEL, }} secondPanel={{ children: ( @@ -205,8 +205,7 @@ const GlossaryOverviewTab = ({ onUpdate={onUpdate} /> ), - minWidth: 320, - flex: 0.25, + ...COMMON_RESIZABLE_PANEL_CONFIG.RIGHT_PANEL, className: 'entity-resizable-right-panel-container', }} /> diff --git a/openmetadata-ui/src/main/resources/ui/src/components/Glossary/ImportGlossary/ImportGlossary.test.tsx b/openmetadata-ui/src/main/resources/ui/src/components/Glossary/ImportGlossary/ImportGlossary.test.tsx index ba285d3a4042..41e22ea9fc98 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/Glossary/ImportGlossary/ImportGlossary.test.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/components/Glossary/ImportGlossary/ImportGlossary.test.tsx @@ -13,6 +13,7 @@ import { act, fireEvent, render, screen } from '@testing-library/react'; import React from 'react'; import { CSVImportResult } from '../../../generated/type/csvImportResult'; +import { importGlossaryInCSVFormat } from '../../../rest/glossaryAPI'; import ImportGlossary from './ImportGlossary'; const mockPush = jest.fn(); @@ -36,11 +37,18 @@ jest.mock('../../common/Loader/Loader', () => jest.fn().mockReturnValue(
Loader
) ); -jest.mock('../ImportResult/GlossaryImportResult.component', () => ({ - GlossaryImportResult: jest - .fn() - .mockReturnValue(
GlossaryImportResult
), -})); +jest.mock('../../BulkImport/BulkEntityImport.component', () => + jest.fn().mockImplementation(({ onSuccess, onValidateCsvString }) => ( +
+ + +

BulkEntityImport

+
+ )) +); jest.mock('../../../rest/glossaryAPI', () => ({ importGlossaryInCSVFormat: jest @@ -80,17 +88,34 @@ describe('Import Glossary', () => { expect(await screen.findByTestId('breadcrumb')).toBeInTheDocument(); expect(await screen.findByTestId('title')).toBeInTheDocument(); - expect(await screen.findByTestId('entity-import')).toBeInTheDocument(); + expect(screen.getByText('BulkEntityImport')).toBeInTheDocument(); + expect(screen.getByText('SuccessButton')).toBeInTheDocument(); + expect(screen.getByText('ValidateCsvButton')).toBeInTheDocument(); }); - it('GlossaryImportResult should visible', async () => { + it('should redirect the page when onSuccess get triggered', async () => { render(); - const importBtn = await screen.findByTestId('import'); + const successButton = screen.getByText('SuccessButton'); await act(async () => { - fireEvent.click(importBtn); + fireEvent.click(successButton); }); - expect(await screen.findByText('GlossaryImportResult')).toBeInTheDocument(); + expect(mockPush).toHaveBeenCalled(); + }); + + it('should call the importGlossaryInCSVFormat api when validate props is trigger', async () => { + render(); + + const successButton = screen.getByText('ValidateCsvButton'); + await act(async () => { + fireEvent.click(successButton); + }); + + expect(importGlossaryInCSVFormat).toHaveBeenCalledWith( + 'Glossary1', + 'markdown: This is test', + true + ); }); }); diff --git a/openmetadata-ui/src/main/resources/ui/src/components/Glossary/ImportGlossary/ImportGlossary.tsx b/openmetadata-ui/src/main/resources/ui/src/components/Glossary/ImportGlossary/ImportGlossary.tsx index 39628da2edf6..2ae67b4ee5af 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/Glossary/ImportGlossary/ImportGlossary.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/components/Glossary/ImportGlossary/ImportGlossary.tsx @@ -12,17 +12,16 @@ */ import { Col, Row, Typography } from 'antd'; import { AxiosError } from 'axios'; -import React, { FC, useMemo, useState } from 'react'; +import React, { FC, useMemo } from 'react'; import { useTranslation } from 'react-i18next'; import { useHistory } from 'react-router-dom'; -import { CSVImportResult } from '../../../generated/type/csvImportResult'; +import { EntityType } from '../../../enums/entity.enum'; import { importGlossaryInCSVFormat } from '../../../rest/glossaryAPI'; import { getGlossaryPath } from '../../../utils/RouterUtils'; import { showErrorToast } from '../../../utils/ToastUtils'; -import { EntityImport } from '../../common/EntityImport/EntityImport.component'; +import BulkEntityImport from '../../BulkImport/BulkEntityImport.component'; import TitleBreadcrumb from '../../common/TitleBreadcrumb/TitleBreadcrumb.component'; import { TitleBreadcrumbProps } from '../../common/TitleBreadcrumb/TitleBreadcrumb.interface'; -import { GlossaryImportResult } from '../ImportResult/GlossaryImportResult.component'; import './import-glossary.less'; interface Props { @@ -32,7 +31,6 @@ interface Props { const ImportGlossary: FC = ({ glossaryName }) => { const { t } = useTranslation(); const history = useHistory(); - const [csvImportResult, setCsvImportResult] = useState(); const breadcrumbList: TitleBreadcrumbProps['titleLinks'] = useMemo( () => [ @@ -53,10 +51,13 @@ const ImportGlossary: FC = ({ glossaryName }) => { history.push(getGlossaryPath(glossaryName)); }; - const handleImportCsv = async (name: string, data: string, dryRun = true) => { + const handleImportCsv = async (data: string, dryRun = true) => { try { - const response = await importGlossaryInCSVFormat(name, data, dryRun); - setCsvImportResult(response); + const response = await importGlossaryInCSVFormat( + glossaryName, + data, + dryRun + ); return response; } catch (error) { @@ -67,7 +68,7 @@ const ImportGlossary: FC = ({ glossaryName }) => { }; return ( - + @@ -79,17 +80,12 @@ const ImportGlossary: FC = ({ glossaryName }) => { - - {csvImportResult ? ( - - ) : ( - <> - )} - + ); diff --git a/openmetadata-ui/src/main/resources/ui/src/components/Glossary/ImportResult/GlossaryImportResult.component.tsx b/openmetadata-ui/src/main/resources/ui/src/components/Glossary/ImportResult/GlossaryImportResult.component.tsx deleted file mode 100644 index 1399f883a3b5..000000000000 --- a/openmetadata-ui/src/main/resources/ui/src/components/Glossary/ImportResult/GlossaryImportResult.component.tsx +++ /dev/null @@ -1,234 +0,0 @@ -/* - * Copyright 2023 Collate. - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * http://www.apache.org/licenses/LICENSE-2.0 - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -import { Space, Typography } from 'antd'; -import { ColumnsType } from 'antd/lib/table'; -import { isEmpty } from 'lodash'; -import React, { FC, useEffect, useMemo, useState } from 'react'; -import { useTranslation } from 'react-i18next'; -import { usePapaParse } from 'react-papaparse'; -import { ReactComponent as FailBadgeIcon } from '../../../assets/svg/fail-badge.svg'; -import { ReactComponent as SuccessBadgeIcon } from '../../../assets/svg/success-badge.svg'; -import Table from '../../../components/common/Table/Table'; -import { GLOSSARY_IMPORT_STATUS } from '../../../constants/Glossary.constant'; -import { - CSVImportResult, - Status, -} from '../../../generated/type/csvImportResult'; -import { parseCSV } from '../../../utils/EntityImport/EntityImportUtils'; -import { GlossaryCSVRecord } from '../ImportGlossary/ImportGlossary.interface'; - -interface Props { - csvImportResult: CSVImportResult; -} - -export const GlossaryImportResult: FC = ({ csvImportResult }) => { - const { readString } = usePapaParse(); - const { t } = useTranslation(); - const [parsedRecords, setParsedRecords] = useState([]); - const [loading, setIsLoading] = useState(false); - - const columns: ColumnsType = useMemo( - () => [ - { - title: t('label.status'), - dataIndex: 'status', - key: 'status', - fixed: true, - render: ( - status: GlossaryCSVRecord['status'], - record: GlossaryCSVRecord - ) => { - return ( - - {status === Status.Failure ? ( - <> - - {record.details} - - ) : ( - - )} - - ); - }, - }, - { - title: t('label.parent'), - dataIndex: 'parent', - key: 'parent', - fixed: true, - render: (parent: GlossaryCSVRecord['parent']) => { - return ( - - {isEmpty(parent) ? '--' : parent} - - ); - }, - }, - { - title: t('label.name'), - dataIndex: 'name*', - key: 'name', - fixed: true, - render: (name: GlossaryCSVRecord['name*']) => { - return ( - - {name} - - ); - }, - }, - { - title: t('label.display-name'), - dataIndex: 'displayName', - key: 'displayName', - render: (displayName: GlossaryCSVRecord['displayName']) => { - return ( - - {isEmpty(displayName) ? '--' : displayName} - - ); - }, - }, - { - title: t('label.description'), - dataIndex: 'description', - key: 'description', - width: 300, - render: (description: GlossaryCSVRecord['description']) => { - return ( - - {isEmpty(description) ? '--' : description} - - ); - }, - }, - { - title: t('label.synonym-lowercase-plural'), - dataIndex: 'synonyms', - key: 'synonyms', - render: (synonyms: GlossaryCSVRecord['synonyms']) => { - const value = synonyms?.split(';').join(', '); - - return ( - - {isEmpty(synonyms) ? '--' : value} - - ); - }, - }, - { - title: t('label.related-term-plural'), - dataIndex: 'relatedTerms', - key: 'relatedTerms', - render: (relatedTerms: GlossaryCSVRecord['relatedTerms']) => { - const value = relatedTerms?.split(';').join(', '); - - return ( - - {isEmpty(relatedTerms) ? '--' : value} - - ); - }, - }, - { - title: t('label.reference-plural'), - dataIndex: 'references', - key: 'relatedTerms', - render: (references: GlossaryCSVRecord['references']) => { - return ( - - {isEmpty(references) ? '--' : references} - - ); - }, - }, - { - title: t('label.tag-plural'), - dataIndex: 'tags', - key: 'tags', - render: (tags: GlossaryCSVRecord['tags']) => { - const value = tags?.split(';').join(', '); - - return ( - - {isEmpty(tags) ? '--' : value} - - ); - }, - }, - ], - [] - ); - - const parseCsvFile = () => { - if (csvImportResult.importResultsCsv) { - readString(csvImportResult.importResultsCsv, { - worker: true, - complete: (results) => { - // results.data is returning data with unknown type - setParsedRecords( - parseCSV(results.data as string[][]).map( - (value) => ({ - ...value, - key: value['name*'], - status: GLOSSARY_IMPORT_STATUS.includes(value['details'] ?? '') - ? Status.Success - : Status.Failure, - }) - ) - ); - setIsLoading(false); - }, - }); - } - }; - - useEffect(() => { - setIsLoading(true); - parseCsvFile(); - }, [csvImportResult.importResultsCsv]); - - return ( - - ); -}; diff --git a/openmetadata-ui/src/main/resources/ui/src/components/Glossary/ImportResult/GlossaryImportResult.test.tsx b/openmetadata-ui/src/main/resources/ui/src/components/Glossary/ImportResult/GlossaryImportResult.test.tsx deleted file mode 100644 index f505ab4faea2..000000000000 --- a/openmetadata-ui/src/main/resources/ui/src/components/Glossary/ImportResult/GlossaryImportResult.test.tsx +++ /dev/null @@ -1,141 +0,0 @@ -/* - * Copyright 2023 Collate. - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * http://www.apache.org/licenses/LICENSE-2.0 - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -import { - findByTestId, - findByText, - getAllByRole, - render, - screen, -} from '@testing-library/react'; -import React from 'react'; -import { CSVImportResult } from '../../../generated/type/csvImportResult'; -import { GlossaryImportResult } from './GlossaryImportResult.component'; - -const mockCsvImportResult = { - dryRun: true, - status: 'success', - numberOfRowsProcessed: 3, - numberOfRowsPassed: 3, - numberOfRowsFailed: 0, - importResultsCsv: `status,details,parent,name*,displayName,description,synonyms,relatedTerms,references,tags\r -success,Entity created,,Glossary2 Term,Glossary2 Term displayName,Description for Glossary2 Term,,,,\r -success,Entity created,,Glossary2 term2,Glossary2 term2 displayname,"Description, data.","ter1,term2",,,\r -failure,#INVALID_FIELD: Field 6 error - Entity First Name not found,test,Glossary3 term3,Glossary3 term3 displayname,"Description2, data.","ter3,term4",,,\r`, -}; - -describe('Import Results component should work properly', () => { - it('Should render the results', async () => { - render( - - ); - - expect( - await screen.findByTestId('import-result-table') - ).toBeInTheDocument(); - }); - - it('Should render the parsed result', async () => { - const { container } = render( - - ); - - const tableRows = getAllByRole(container, 'row'); - - expect(tableRows).toHaveLength(4); - - const firstRow = tableRows[1]; - - const rowStatus = await findByTestId(firstRow, 'success-badge'); - const rowName = await findByText(firstRow, 'Glossary2 Term'); - const rowDisplayName = await findByText( - firstRow, - 'Glossary2 Term displayName' - ); - const rowDescription = await findByText( - firstRow, - 'Description for Glossary2 Term' - ); - - expect(rowStatus).toBeInTheDocument(); - expect(rowName).toBeInTheDocument(); - expect(rowDisplayName).toBeInTheDocument(); - expect(rowDescription).toBeInTheDocument(); - }); - - it('Should render the parsed result properly with special character', async () => { - const { container } = render( - - ); - - const tableRows = getAllByRole(container, 'row'); - - expect(tableRows).toHaveLength(4); - - const secondRow = tableRows[2]; - - const rowStatus = await findByTestId(secondRow, 'success-badge'); - const rowDisplayName = await findByText( - secondRow, - 'Glossary2 term2 displayname' - ); - const rowName = await findByText(secondRow, 'Glossary2 term2'); - const rowDescription = await findByText(secondRow, 'Description, data.'); - const synonym = await findByText(secondRow, 'ter1,term2'); - - expect(rowStatus).toBeInTheDocument(); - expect(rowName).toBeInTheDocument(); - expect(rowDisplayName).toBeInTheDocument(); - expect(rowDescription).toBeInTheDocument(); - expect(synonym).toBeInTheDocument(); - }); - - it('Should render the parsed result even if its failed row', async () => { - const { container } = render( - - ); - - const tableRows = getAllByRole(container, 'row'); - - expect(tableRows).toHaveLength(4); - - const thirdRow = tableRows[3]; - - const rowStatus = await findByTestId(thirdRow, 'failure-badge'); - const errorMsg = await findByText( - thirdRow, - '#INVALID_FIELD: Field 6 error - Entity First Name not found' - ); - const rowDisplayName = await findByText( - thirdRow, - 'Glossary3 term3 displayname' - ); - const rowName = await findByText(thirdRow, 'Glossary3 term3'); - const rowDescription = await findByText(thirdRow, 'Description2, data.'); - const synonym = await findByText(thirdRow, 'ter3,term4'); - - expect(rowStatus).toBeInTheDocument(); - expect(errorMsg).toBeInTheDocument(); - expect(rowName).toBeInTheDocument(); - expect(rowDisplayName).toBeInTheDocument(); - expect(rowDescription).toBeInTheDocument(); - expect(synonym).toBeInTheDocument(); - }); -}); diff --git a/openmetadata-ui/src/main/resources/ui/src/components/Metric/MetricDetails/MetricDetails.tsx b/openmetadata-ui/src/main/resources/ui/src/components/Metric/MetricDetails/MetricDetails.tsx index a8bb7ef12d69..dd4fbc326dfb 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/Metric/MetricDetails/MetricDetails.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/components/Metric/MetricDetails/MetricDetails.tsx @@ -19,6 +19,7 @@ import { useTranslation } from 'react-i18next'; import { useHistory, useParams } from 'react-router-dom'; import { getEntityDetailsPath, ROUTES } from '../../../constants/constants'; import { FEED_COUNT_INITIAL_DATA } from '../../../constants/entity.constants'; +import { COMMON_RESIZABLE_PANEL_CONFIG } from '../../../constants/ResizablePanel.constants'; import LineageProvider from '../../../context/LineageProvider/LineageProvider'; import { EntityTabs, EntityType } from '../../../enums/entity.enum'; import { Tag } from '../../../generated/entity/classification/tag'; @@ -312,8 +313,7 @@ const MetricDetails: React.FC = ({ /> ), - minWidth: 800, - flex: 0.87, + ...COMMON_RESIZABLE_PANEL_CONFIG.LEFT_PANEL, }} secondPanel={{ children: ( @@ -346,8 +346,7 @@ const MetricDetails: React.FC = ({ /> ), - minWidth: 320, - flex: 0.13, + ...COMMON_RESIZABLE_PANEL_CONFIG.RIGHT_PANEL, className: 'entity-resizable-right-panel-container entity-resizable-panel-container', }} diff --git a/openmetadata-ui/src/main/resources/ui/src/components/MlModel/MlModelDetail/MlModelDetail.component.tsx b/openmetadata-ui/src/main/resources/ui/src/components/MlModel/MlModelDetail/MlModelDetail.component.tsx index 590684e9d731..77fc45d9b2f6 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/MlModel/MlModelDetail/MlModelDetail.component.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/components/MlModel/MlModelDetail/MlModelDetail.component.tsx @@ -21,6 +21,7 @@ import { useTranslation } from 'react-i18next'; import { useHistory, useParams } from 'react-router-dom'; import { getEntityDetailsPath } from '../../../constants/constants'; import { FEED_COUNT_INITIAL_DATA } from '../../../constants/entity.constants'; +import { COMMON_RESIZABLE_PANEL_CONFIG } from '../../../constants/ResizablePanel.constants'; import LineageProvider from '../../../context/LineageProvider/LineageProvider'; import { usePermissionProvider } from '../../../context/PermissionProvider/PermissionProvider'; import { ResourceEntity } from '../../../context/PermissionProvider/PermissionProvider.interface'; @@ -426,8 +427,7 @@ const MlModelDetail: FC = ({ /> ), - minWidth: 800, - flex: 0.87, + ...COMMON_RESIZABLE_PANEL_CONFIG.LEFT_PANEL, }} secondPanel={{ children: ( @@ -451,8 +451,7 @@ const MlModelDetail: FC = ({ /> ), - minWidth: 320, - flex: 0.13, + ...COMMON_RESIZABLE_PANEL_CONFIG.RIGHT_PANEL, className: 'entity-resizable-right-panel-container entity-resizable-panel-container', }} diff --git a/openmetadata-ui/src/main/resources/ui/src/components/Modals/ModalWithCustomProperty/ModalWithCustomPropertyEditor.component.tsx b/openmetadata-ui/src/main/resources/ui/src/components/Modals/ModalWithCustomProperty/ModalWithCustomPropertyEditor.component.tsx new file mode 100644 index 000000000000..e98d9324e1c8 --- /dev/null +++ b/openmetadata-ui/src/main/resources/ui/src/components/Modals/ModalWithCustomProperty/ModalWithCustomPropertyEditor.component.tsx @@ -0,0 +1,176 @@ +/* + * Copyright 2024 Collate. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * http://www.apache.org/licenses/LICENSE-2.0 + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import { Button, Modal, Typography } from 'antd'; +import React, { useCallback, useEffect, useMemo, useState } from 'react'; +import { useTranslation } from 'react-i18next'; + +import { AxiosError } from 'axios'; +import { isObject } from 'lodash'; +import { EntityType } from '../../../enums/entity.enum'; +import { GlossaryTerm } from '../../../generated/entity/data/glossaryTerm'; +import { EnumConfig, Type, ValueClass } from '../../../generated/entity/type'; +import { getTypeByFQN } from '../../../rest/metadataTypeAPI'; +import { + convertCustomPropertyStringToEntityExtension, + convertEntityExtensionToCustomPropertyString, +} from '../../../utils/CSV/CSV.utils'; +import { showErrorToast } from '../../../utils/ToastUtils'; +import { CustomPropertyTable } from '../../common/CustomPropertyTable/CustomPropertyTable'; +import Loader from '../../common/Loader/Loader'; +import { + ExtensionDataProps, + ModalWithCustomPropertyEditorProps, +} from './ModalWithMarkdownEditor.interface'; + +export const ModalWithCustomPropertyEditor = ({ + header, + entityType, + value, + onSave, + onCancel, + visible, +}: ModalWithCustomPropertyEditorProps) => { + const { t } = useTranslation(); + const [isLoading, setIsLoading] = useState(false); + const [isSaveLoading, setIsSaveLoading] = useState(false); + const [customPropertyValue, setCustomPropertyValue] = + useState(); + const [customPropertyTypes, setCustomPropertyTypes] = useState(); + + const enumWithDescriptionsKeyPairValues = useMemo(() => { + const valuesWithEnumKey: Record = {}; + + customPropertyTypes?.customProperties?.forEach((property) => { + if (property.propertyType.name === 'enumWithDescriptions') { + valuesWithEnumKey[property.name] = ( + property.customPropertyConfig?.config as EnumConfig + ).values as ValueClass[]; + } + }); + + return valuesWithEnumKey; + }, [customPropertyTypes]); + + const fetchTypeDetail = async () => { + setIsLoading(true); + try { + const response = await getTypeByFQN(entityType); + setCustomPropertyTypes(response); + setCustomPropertyValue( + convertCustomPropertyStringToEntityExtension(value ?? '', response) + ); + } catch (err) { + showErrorToast(err as AxiosError); + } finally { + setIsLoading(false); + } + }; + + const handleSaveData = async () => { + setIsSaveLoading(true); + await onSave( + convertEntityExtensionToCustomPropertyString( + customPropertyValue, + customPropertyTypes + ) + ); + setIsSaveLoading(false); + }; + + // EnumWithDescriptions values are change only contain keys, + // so we need to modify the extension data to include descriptions for them to display in the table + const modifyExtensionData = useCallback( + (extension: ExtensionDataProps) => { + const modifiedExtension = Object.entries(extension).reduce( + (acc, [key, value]) => { + if (enumWithDescriptionsKeyPairValues[key]) { + return { + ...acc, + [key]: (value as string[] | ValueClass[]).map((item) => { + if (isObject(item)) { + return item; + } + + return { + key: item, + description: enumWithDescriptionsKeyPairValues[key].find( + (val) => val.key === item + )?.description, + }; + }), + }; + } + + return { ...acc, [key]: value }; + }, + {} + ); + + return modifiedExtension; + }, + [enumWithDescriptionsKeyPairValues] + ); + + const onExtensionUpdate = async (data: GlossaryTerm) => { + setCustomPropertyValue(modifyExtensionData(data.extension)); + }; + + useEffect(() => { + fetchTypeDetail(); + }, []); + + return ( + + {t('label.cancel')} + , + , + ]} + maskClosable={false} + open={visible} + title={{header}} + width={650} + onCancel={onCancel}> + {isLoading ? ( + + ) : ( + + )} + + ); +}; diff --git a/openmetadata-ui/src/main/resources/ui/src/components/Modals/ModalWithCustomProperty/ModalWithMarkdownEditor.interface.ts b/openmetadata-ui/src/main/resources/ui/src/components/Modals/ModalWithCustomProperty/ModalWithMarkdownEditor.interface.ts new file mode 100644 index 000000000000..5fa33f3b60b4 --- /dev/null +++ b/openmetadata-ui/src/main/resources/ui/src/components/Modals/ModalWithCustomProperty/ModalWithMarkdownEditor.interface.ts @@ -0,0 +1,35 @@ +/* + * Copyright 2024 Collate. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * http://www.apache.org/licenses/LICENSE-2.0 + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import { EntityType } from '../../../enums/entity.enum'; +import { EntityReference, ValueClass } from '../../../generated/entity/type'; + +export type ExtensionDataTypes = + | string + | string[] + | EntityReference + | EntityReference[] + | ValueClass[] + | { start: string; end: string }; + +export interface ExtensionDataProps { + [key: string]: ExtensionDataTypes; +} + +export type ModalWithCustomPropertyEditorProps = { + entityType: EntityType; + header: string; + value?: string; + onSave: (extension?: string) => Promise; + onCancel?: () => void; + visible: boolean; +}; diff --git a/openmetadata-ui/src/main/resources/ui/src/components/Pipeline/PipelineDetails/PipelineDetails.component.tsx b/openmetadata-ui/src/main/resources/ui/src/components/Pipeline/PipelineDetails/PipelineDetails.component.tsx index ba430f9804b9..7ea1244a54d9 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/Pipeline/PipelineDetails/PipelineDetails.component.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/components/Pipeline/PipelineDetails/PipelineDetails.component.tsx @@ -30,6 +30,7 @@ import { } from '../../../constants/constants'; import { FEED_COUNT_INITIAL_DATA } from '../../../constants/entity.constants'; import { PIPELINE_TASK_TABS } from '../../../constants/pipeline.constants'; +import { COMMON_RESIZABLE_PANEL_CONFIG } from '../../../constants/ResizablePanel.constants'; import LineageProvider from '../../../context/LineageProvider/LineageProvider'; import { usePermissionProvider } from '../../../context/PermissionProvider/PermissionProvider'; import { ResourceEntity } from '../../../context/PermissionProvider/PermissionProvider.interface'; @@ -648,8 +649,7 @@ const PipelineDetails = ({ ), - minWidth: 800, - flex: 0.87, + ...COMMON_RESIZABLE_PANEL_CONFIG.LEFT_PANEL, }} secondPanel={{ children: ( @@ -673,8 +673,7 @@ const PipelineDetails = ({ /> ), - minWidth: 320, - flex: 0.13, + ...COMMON_RESIZABLE_PANEL_CONFIG.RIGHT_PANEL, className: 'entity-resizable-right-panel-container entity-resizable-panel-container', }} diff --git a/openmetadata-ui/src/main/resources/ui/src/components/Topic/TopicDetails/TopicDetails.component.tsx b/openmetadata-ui/src/main/resources/ui/src/components/Topic/TopicDetails/TopicDetails.component.tsx index cca6405f5121..809a5c9397cb 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/Topic/TopicDetails/TopicDetails.component.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/components/Topic/TopicDetails/TopicDetails.component.tsx @@ -20,6 +20,7 @@ import { useTranslation } from 'react-i18next'; import { useHistory, useParams } from 'react-router-dom'; import { getEntityDetailsPath } from '../../../constants/constants'; import { FEED_COUNT_INITIAL_DATA } from '../../../constants/entity.constants'; +import { COMMON_RESIZABLE_PANEL_CONFIG } from '../../../constants/ResizablePanel.constants'; import LineageProvider from '../../../context/LineageProvider/LineageProvider'; import { ERROR_PLACEHOLDER_TYPE } from '../../../enums/common.enum'; import { EntityTabs, EntityType } from '../../../enums/entity.enum'; @@ -337,8 +338,7 @@ const TopicDetails: React.FC = ({ /> ), - minWidth: 800, - flex: 0.87, + ...COMMON_RESIZABLE_PANEL_CONFIG.LEFT_PANEL, }} secondPanel={{ children: ( @@ -362,8 +362,7 @@ const TopicDetails: React.FC = ({ /> ), - minWidth: 320, - flex: 0.13, + ...COMMON_RESIZABLE_PANEL_CONFIG.RIGHT_PANEL, className: 'entity-resizable-right-panel-container entity-resizable-panel-container', }} diff --git a/openmetadata-ui/src/main/resources/ui/src/components/UploadFile/UploadFile.interface.ts b/openmetadata-ui/src/main/resources/ui/src/components/UploadFile/UploadFile.interface.ts new file mode 100644 index 000000000000..811a36b951cc --- /dev/null +++ b/openmetadata-ui/src/main/resources/ui/src/components/UploadFile/UploadFile.interface.ts @@ -0,0 +1,24 @@ +/* + * Copyright 2024 Collate. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * http://www.apache.org/licenses/LICENSE-2.0 + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import { RcFile } from 'antd/lib/upload'; + +declare type BeforeUploadValueType = void | boolean | string | Blob | File; + +export interface UploadFileProps { + fileType: string; + beforeUpload?: ( + file: RcFile, + FileList: RcFile[] + ) => BeforeUploadValueType | Promise; + onCSVUploaded: (event: ProgressEvent) => void; +} diff --git a/openmetadata-ui/src/main/resources/ui/src/components/UploadFile/UploadFile.tsx b/openmetadata-ui/src/main/resources/ui/src/components/UploadFile/UploadFile.tsx new file mode 100644 index 000000000000..f594d2b0e08d --- /dev/null +++ b/openmetadata-ui/src/main/resources/ui/src/components/UploadFile/UploadFile.tsx @@ -0,0 +1,79 @@ +/* + * Copyright 2024 Collate. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * http://www.apache.org/licenses/LICENSE-2.0 + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import { Space, Typography, UploadProps } from 'antd'; +import Dragger from 'antd/lib/upload/Dragger'; +import { AxiosError } from 'axios'; +import { t } from 'i18next'; +import React, { FC, useCallback, useState } from 'react'; +import { ReactComponent as ImportIcon } from '../../assets/svg/ic-drag-drop.svg'; +import { Transi18next } from '../../utils/CommonUtils'; +import { showErrorToast } from '../../utils/ToastUtils'; +import Loader from '../common/Loader/Loader'; +import { UploadFileProps } from './UploadFile.interface'; + +export const UploadFile: FC = ({ + fileType, + beforeUpload, + onCSVUploaded, +}) => { + const [uploading, setUploading] = useState(false); + + const handleUpload: UploadProps['customRequest'] = useCallback( + (options) => { + setUploading(true); + try { + const reader = new FileReader(); + reader.onload = onCSVUploaded; + reader.onerror = () => { + throw t('server.unexpected-error'); + }; + reader.readAsText(options.file as Blob); + } catch (error) { + showErrorToast(error as AxiosError); + } finally { + setUploading(false); + } + }, + [onCSVUploaded] + ); + + return uploading ? ( + + ) : ( + + + + + } + values={{ + text: t('label.browse'), + }} + /> + + + + ); +}; diff --git a/openmetadata-ui/src/main/resources/ui/src/components/common/CustomPropertyTable/CustomPropertyTable.tsx b/openmetadata-ui/src/main/resources/ui/src/components/common/CustomPropertyTable/CustomPropertyTable.tsx index 4bf4cb047006..b82da885bb1b 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/common/CustomPropertyTable/CustomPropertyTable.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/components/common/CustomPropertyTable/CustomPropertyTable.tsx @@ -13,6 +13,7 @@ import { Col, Divider, Row, Skeleton, Typography } from 'antd'; import { AxiosError } from 'axios'; +import classNames from 'classnames'; import { isEmpty, isUndefined } from 'lodash'; import React, { Fragment, @@ -43,6 +44,7 @@ import { } from '../../../utils/EntityVersionUtils'; import { showErrorToast } from '../../../utils/ToastUtils'; import ErrorPlaceHolder from '../ErrorWithPlaceholder/ErrorPlaceHolder'; +import './custom-property-table.less'; import { CustomPropertyProps, ExtentionEntities, @@ -252,25 +254,32 @@ export const CustomPropertyTable = ({ {isRenderedInRightPanel ? ( - <> +
{dataSource.map((record, index) => ( - +
+ +
{index !== dataSource.length - 1 && ( - + )} ))} - +
) : ( {dataSource.map((record) => ( diff --git a/openmetadata-ui/src/main/resources/ui/src/components/common/CustomPropertyTable/PropertyValue.tsx b/openmetadata-ui/src/main/resources/ui/src/components/common/CustomPropertyTable/PropertyValue.tsx index 72add4ef09e4..d8838d060dac 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/common/CustomPropertyTable/PropertyValue.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/components/common/CustomPropertyTable/PropertyValue.tsx @@ -56,7 +56,10 @@ import { ICON_DIMENSION, VALIDATION_MESSAGES, } from '../../../constants/constants'; -import { ENUM_WITH_DESCRIPTION } from '../../../constants/CustomProperty.constants'; +import { + ENUM_WITH_DESCRIPTION, + INLINE_PROPERTY_TYPES, +} from '../../../constants/CustomProperty.constants'; import { TIMESTAMP_UNIX_IN_MILLISECONDS_REGEX } from '../../../constants/regex.constants'; import { CSMode } from '../../../enums/codemirror.enum'; import { SearchIndex } from '../../../enums/search.enum'; @@ -91,18 +94,23 @@ export const PropertyValue: FC = ({ property, isRenderedInRightPanel = false, }) => { - const { propertyName, propertyType, value } = useMemo(() => { - const propertyName = property.name; - const propertyType = property.propertyType; + const { propertyName, propertyType, value, isInlineProperty } = + useMemo(() => { + const propertyName = property.name; + const propertyType = property.propertyType; + const isInlineProperty = INLINE_PROPERTY_TYPES.includes( + propertyType.name ?? '' + ); - const value = extension?.[propertyName]; + const value = extension?.[propertyName]; - return { - propertyName, - propertyType, - value, - }; - }, [property, extension]); + return { + propertyName, + propertyType, + value, + isInlineProperty, + }; + }, [property, extension]); const [showInput, setShowInput] = useState(false); const [isLoading, setIsLoading] = useState(false); @@ -617,6 +625,7 @@ export const PropertyValue: FC = ({ return ( = ({ isTeam={item.type === 'team'} name={item.name ?? ''} type="circle" - width="24" + width="20" /> ) : ( searchClassBase.getEntityIcon(item.type) @@ -865,7 +874,7 @@ export const PropertyValue: FC = ({ } type="text"> {getEntityName(item)} @@ -900,14 +909,14 @@ export const PropertyValue: FC = ({ icon={
+ style={{ width: '20px', display: 'flex' }}> {['user', 'team'].includes(item.type) ? ( ) : ( searchClassBase.getEntityIcon(item.type) @@ -916,7 +925,7 @@ export const PropertyValue: FC = ({ } type="text"> {getEntityName(item)} @@ -964,7 +973,7 @@ export const PropertyValue: FC = ({ default: return ( {value} @@ -1001,8 +1010,45 @@ export const PropertyValue: FC = ({ setIsOverflowing(isOverflowing); }, [property, extension, contentRef, value]); + const customPropertyInlineElement = ( +
+
+
+ + {getEntityName(property)} + +
+ +
+ {showInput ? getPropertyInput() : getValueElement()} + {hasEditPermissions && !showInput && ( + + + + )} +
+
+ +
+ ); + const customPropertyElement = ( - +
@@ -1030,50 +1076,51 @@ export const PropertyValue: FC = ({ - - +
{showInput ? getPropertyInput() : getValueElement()} - +
{isOverflowing && !showInput && ( - - - + )} - + ); if (isRenderedInRightPanel) { return ( -
- {customPropertyElement} +
+ {isInlineProperty ? customPropertyInlineElement : customPropertyElement}
); } return ( {customPropertyElement} diff --git a/openmetadata-ui/src/main/resources/ui/src/components/common/CustomPropertyTable/custom-property-table.less b/openmetadata-ui/src/main/resources/ui/src/components/common/CustomPropertyTable/custom-property-table.less new file mode 100644 index 000000000000..bbf4e91c436b --- /dev/null +++ b/openmetadata-ui/src/main/resources/ui/src/components/common/CustomPropertyTable/custom-property-table.less @@ -0,0 +1,18 @@ +.custom-property-right-panel-container { + background: #f8f8f8; + padding: 14px; + border-radius: 10px; + + .custom-property-right-panel-card { + background: #fff; + padding: 14px; + } + .custom-property-right-panel-card.top-border-radius { + border-top-left-radius: 10px; + border-top-right-radius: 10px; + } + .custom-property-right-panel-card.bottom-border-radius { + border-bottom-left-radius: 10px; + border-bottom-right-radius: 10px; + } +} diff --git a/openmetadata-ui/src/main/resources/ui/src/components/common/CustomPropertyTable/property-value.less b/openmetadata-ui/src/main/resources/ui/src/components/common/CustomPropertyTable/property-value.less index 3f505fd3b3d8..6dcbfa1f9cb4 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/common/CustomPropertyTable/property-value.less +++ b/openmetadata-ui/src/main/resources/ui/src/components/common/CustomPropertyTable/property-value.less @@ -49,3 +49,18 @@ padding: 4px; background: @enum-tag-bg-color; } + +.sql-query-custom-property { + .ant-space-item:first-child { + width: 100%; + } +} +.custom-property-card { + .ant-card-body { + overflow-x: scroll; + } +} + +.custom-property-card-right-panel { + overflow-x: scroll; +} diff --git a/openmetadata-ui/src/main/resources/ui/src/components/common/EntityImport/EntityImport.component.tsx b/openmetadata-ui/src/main/resources/ui/src/components/common/EntityImport/EntityImport.component.tsx index b8c5ef60fcc9..3c89da64af8a 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/common/EntityImport/EntityImport.component.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/components/common/EntityImport/EntityImport.component.tsx @@ -10,31 +10,20 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -import { - Affix, - Button, - Card, - Col, - Row, - Space, - Typography, - UploadProps, -} from 'antd'; -import Dragger from 'antd/lib/upload/Dragger'; +import { Affix, Button, Card, Col, Row, Space, Typography } from 'antd'; import { AxiosError } from 'axios'; import { isUndefined } from 'lodash'; import React, { useMemo, useState } from 'react'; import { useTranslation } from 'react-i18next'; -import { ReactComponent as ImportIcon } from '../../../assets/svg/ic-drag-drop.svg'; import { ReactComponent as SuccessBadgeIcon } from '../../../assets/svg/success-badge.svg'; import { STEPS_FOR_IMPORT_ENTITY } from '../../../constants/entity.constants'; import { CSVImportResult, Status, } from '../../../generated/type/csvImportResult'; -import { Transi18next } from '../../../utils/CommonUtils'; import { showErrorToast } from '../../../utils/ToastUtils'; import Stepper from '../../Settings/Services/Ingestion/IngestionStepper/IngestionStepper.component'; +import { UploadFile } from '../../UploadFile/UploadFile'; import Loader from '../Loader/Loader'; import './entity-import.style.less'; import { EntityImportProps } from './EntityImport.interface'; @@ -54,7 +43,6 @@ export const EntityImport = ({ const [csvFileResult, setCsvFileResult] = useState(''); const [csvImportResult, setCsvImportResult] = useState(); const [activeStep, setActiveStep] = useState(1); - const [uploading, setUploading] = useState(false); const { isFailure, isAborted } = useMemo(() => { const status = csvImportResult?.status; @@ -71,7 +59,6 @@ export const EntityImport = ({ try { const result = e.target?.result as string; if (result) { - setUploading(true); const response = await onImport(entityName, result); if (response) { @@ -82,27 +69,6 @@ export const EntityImport = ({ } } catch (error) { setCsvImportResult(undefined); - } finally { - setUploading(false); - } - }; - - const handleUpload: UploadProps['customRequest'] = (options) => { - setIsLoading(true); - try { - const reader = new FileReader(); - - reader.readAsText(options.file as Blob); - - reader.addEventListener('load', handleLoadData); - - reader.addEventListener('error', () => { - throw t('server.unexpected-error'); - }); - } catch (error) { - showErrorToast(error as AxiosError); - } finally { - setIsLoading(false); } }; @@ -136,39 +102,14 @@ export const EntityImport = ({ <> {activeStep === 1 && (
- {uploading ? ( - - ) : ( - { - setFileName(file.name); - }} - className="file-dragger-wrapper p-lg bg-white" - customRequest={handleUpload} - data-testid="upload-file-widget" - multiple={false} - showUploadList={false}> - - - - - } - values={{ - text: t('label.browse'), - }} - /> - - - - )} + { + setFileName(file.name); + }} + fileType=".csv" + onCSVUploaded={handleLoadData} + /> +