From 1ec2981e867368d452094fe65da131caaec269f9 Mon Sep 17 00:00:00 2001 From: Chirag Madlani <12962843+chirag-madlani@users.noreply.github.com> Date: Mon, 16 Sep 2024 18:09:49 +0530 Subject: [PATCH 01/39] fix import issue --- .../src/main/resources/ui/src/utils/ServiceUtilClassBase.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/openmetadata-ui/src/main/resources/ui/src/utils/ServiceUtilClassBase.ts b/openmetadata-ui/src/main/resources/ui/src/utils/ServiceUtilClassBase.ts index 02eadcd8bf12..061666e80370 100644 --- a/openmetadata-ui/src/main/resources/ui/src/utils/ServiceUtilClassBase.ts +++ b/openmetadata-ui/src/main/resources/ui/src/utils/ServiceUtilClassBase.ts @@ -11,7 +11,7 @@ * limitations under the License. */ -import { capitalize, toLower } from 'lodash'; +import { capitalize, get, toLower } from 'lodash'; import MetricIcon from '../assets/svg/metric.svg'; import { AIRBYTE, From 8708738ec66c995af26413e1cdbe35651caf99fc Mon Sep 17 00:00:00 2001 From: sonikashah Date: Thu, 19 Sep 2024 15:13:40 +0530 Subject: [PATCH 02/39] Feat : Allow Custom Property Update in Bulk Upload --- .../java/org/openmetadata/csv/CsvUtil.java | 162 ++++++++++++ .../java/org/openmetadata/csv/EntityCsv.java | 233 ++++++++++++++++++ .../openmetadata/service/TypeRegistry.java | 11 +- .../service/jdbi3/DatabaseRepository.java | 7 +- .../jdbi3/DatabaseSchemaRepository.java | 8 +- .../jdbi3/DatabaseServiceRepository.java | 7 +- .../service/jdbi3/GlossaryRepository.java | 7 +- .../service/jdbi3/TableRepository.java | 30 ++- .../database/databaseCsvDocumentation.json | 16 ++ .../databaseSchemaCsvDocumentation.json | 16 ++ .../databaseServiceCsvDocumentation.json | 16 ++ .../glossary/glossaryCsvDocumentation.json | 16 ++ .../data/table/tableCsvDocumentation.json | 16 ++ .../databases/DatabaseResourceTest.java | 8 +- .../databases/DatabaseSchemaResourceTest.java | 8 +- .../databases/TableResourceTest.java | 16 +- .../glossary/GlossaryResourceTest.java | 20 +- 17 files changed, 548 insertions(+), 49 deletions(-) diff --git a/openmetadata-service/src/main/java/org/openmetadata/csv/CsvUtil.java b/openmetadata-service/src/main/java/org/openmetadata/csv/CsvUtil.java index 9dc7c44dd228..10870c99e965 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/csv/CsvUtil.java +++ b/openmetadata-service/src/main/java/org/openmetadata/csv/CsvUtil.java @@ -17,11 +17,15 @@ import static org.openmetadata.common.utils.CommonUtil.listOrEmpty; import static org.openmetadata.common.utils.CommonUtil.nullOrEmpty; +import com.fasterxml.jackson.databind.ObjectMapper; import java.io.IOException; import java.io.StringWriter; import java.util.ArrayList; import java.util.Arrays; +import java.util.Collections; import java.util.List; +import java.util.Map; +import java.util.regex.Pattern; import java.util.stream.Collectors; import org.apache.commons.csv.CSVFormat; import org.apache.commons.csv.CSVFormat.Builder; @@ -39,6 +43,8 @@ public final class CsvUtil { public static final String ENTITY_TYPE_SEPARATOR = ":"; public static final String LINE_SEPARATOR = "\r\n"; + public static final String INTERNAL_ARRAY_SEPARATOR = "|"; + private CsvUtil() { // Utility class hides the constructor } @@ -94,6 +100,80 @@ public static List fieldToEntities(String field) { return field == null ? null : listOf(field.split(ENTITY_TYPE_SEPARATOR)); } + public static List fieldToInternalArray(String field) { + // Split a fieldValue that contains multiple elements of an array separated by + // INTERNAL_ARRAY_SEPARATOR + if (field == null || field.isBlank()) { + return Collections.emptyList(); + } + return listOf(field.split(Pattern.quote(INTERNAL_ARRAY_SEPARATOR))); + } + + /** + * Parses a field containing key-value pairs separated by semicolons, correctly handling quotes. + * Each key-value pair may also be enclosed in quotes, especially if it contains delimiter like (SEPARATOR , FIELD_SEPARATOR). + * + * Input Example: + * "key1:value1;key2:value2;\"key3:value;with;semicolon\"" + * Output: ["key1:value1", "key2:value2", "key3:value;with;semicolon"] + * + * @param field The input string with key-value pairs. + * @return A list of key-value pairs, handling quotes and semicolons correctly. + */ + public static List fieldToExtensionStrings(String field) throws IOException { + if (field == null || field.isBlank()) { + return List.of(); // Return empty list if input is null or blank + } + + List result = new ArrayList<>(); + StringBuilder currentField = new StringBuilder(); + boolean inQuotes = false; // Track whether we are inside quotes + + // Iterate through each character in the field + for (int i = 0; i < field.length(); i++) { + char c = field.charAt(i); + + if (c == '"') { + if (inQuotes && i + 1 < field.length() && field.charAt(i + 1) == '"') { + currentField.append('"'); // Handle escaped quote ("" -> ") + i++; // Skip the next character + } else { + inQuotes = !inQuotes; // Toggle quote state + currentField.append(c); // Keep the quote as part of the field + } + } else if (c == FIELD_SEPARATOR.charAt(0) && !inQuotes) { + addFieldToResult(result, currentField); // Add the field when semicolon is outside quotes + currentField.setLength(0); // Reset buffer for next field + } else { + currentField.append(c); // Continue building the field + } + } + + // Add the last field + addFieldToResult(result, currentField); + + return result; + } + + /** + * Adds the processed field to the result list, removing surrounding quotes if present. + * + * @param result List to hold parsed fields. + * @param currentField The current field being processed. + */ + private static void addFieldToResult(List result, StringBuilder currentField) { + String fieldStr = currentField.toString(); + + // Remove surrounding quotes if field contains special characters and is quoted + if ((fieldStr.contains(SEPARATOR) || fieldStr.contains(FIELD_SEPARATOR)) + && fieldStr.startsWith("\"") + && fieldStr.endsWith("\"")) { + fieldStr = fieldStr.substring(1, fieldStr.length() - 1); + } + + result.add(fieldStr); + } + public static String quote(String field) { return String.format("\"%s\"", field); } @@ -205,4 +285,86 @@ private static String quoteCsvField(String str) { } return str; } + + public static void addExtension(List csvRecord, Object extension) { + if (extension == null) { + csvRecord.add(null); + return; + } + + ObjectMapper objectMapper = new ObjectMapper(); + Map extensionMap = objectMapper.convertValue(extension, Map.class); + + String extensionString = + extensionMap.entrySet().stream() + .map( + entry -> { + String key = entry.getKey(); + Object value = entry.getValue(); + return CsvUtil.quoteCsvField(key + ENTITY_TYPE_SEPARATOR + formatValue(value)); + }) + .collect(Collectors.joining(FIELD_SEPARATOR)); + + csvRecord.add(extensionString); + } + + private static String formatValue(Object value) { + // Handle Map (e.g., entity reference or date interval) + if (value instanceof Map) { + return formatMapValue((Map) value); + } + + // Handle List (e.g., Entity Reference List or multi-select Enum List) + if (value instanceof List) { + return formatListValue((List) value); + } + + // Fallback for simple types + return value != null ? value.toString() : ""; + } + + private static String formatMapValue(Map valueMap) { + if (isEntityReference(valueMap)) { + return formatEntityReference(valueMap); + } else if (isTimeInterval(valueMap)) { + return formatTimeInterval(valueMap); + } + + // If no specific format, return the raw map string + return valueMap.toString(); + } + + private static String formatListValue(List list) { + if (list.isEmpty()) { + return ""; + } + + if (list.get(0) instanceof Map) { + // Handle a list of entity references or maps + return list.stream() + .map(item -> formatMapValue((Map) item)) + .collect(Collectors.joining(INTERNAL_ARRAY_SEPARATOR)); + } else { + // Handle a simple list of strings or numbers + return list.stream() + .map(Object::toString) + .collect(Collectors.joining(INTERNAL_ARRAY_SEPARATOR)); + } + } + + private static boolean isEntityReference(Map valueMap) { + return valueMap.containsKey("type") && valueMap.containsKey("fullyQualifiedName"); + } + + private static boolean isTimeInterval(Map valueMap) { + return valueMap.containsKey("start") && valueMap.containsKey("end"); + } + + private static String formatEntityReference(Map valueMap) { + return valueMap.get("type") + ENTITY_TYPE_SEPARATOR + valueMap.get("fullyQualifiedName"); + } + + private static String formatTimeInterval(Map valueMap) { + return valueMap.get("start") + ENTITY_TYPE_SEPARATOR + valueMap.get("end"); + } } diff --git a/openmetadata-service/src/main/java/org/openmetadata/csv/EntityCsv.java b/openmetadata-service/src/main/java/org/openmetadata/csv/EntityCsv.java index 2a651d6b25bd..ea683498e929 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/csv/EntityCsv.java +++ b/openmetadata-service/src/main/java/org/openmetadata/csv/EntityCsv.java @@ -16,20 +16,35 @@ import static org.openmetadata.common.utils.CommonUtil.listOf; import static org.openmetadata.common.utils.CommonUtil.listOrEmpty; import static org.openmetadata.common.utils.CommonUtil.nullOrEmpty; +import static org.openmetadata.csv.CsvUtil.ENTITY_TYPE_SEPARATOR; import static org.openmetadata.csv.CsvUtil.FIELD_SEPARATOR; +import static org.openmetadata.csv.CsvUtil.fieldToEntities; +import static org.openmetadata.csv.CsvUtil.fieldToExtensionStrings; +import static org.openmetadata.csv.CsvUtil.fieldToInternalArray; import static org.openmetadata.csv.CsvUtil.recordToString; +import com.fasterxml.jackson.databind.JsonNode; +import com.networknt.schema.JsonSchema; +import com.networknt.schema.ValidationMessage; import java.io.IOException; import java.io.Reader; import java.io.StringReader; import java.io.StringWriter; +import java.time.LocalDateTime; +import java.time.LocalTime; +import java.time.format.DateTimeFormatter; +import java.time.format.DateTimeParseException; +import java.time.temporal.TemporalAccessor; import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; import java.util.Comparator; import java.util.HashMap; import java.util.List; +import java.util.Locale; import java.util.Map; +import java.util.Optional; +import java.util.Set; import java.util.UUID; import javax.ws.rs.core.Response; import lombok.extern.slf4j.Slf4j; @@ -52,6 +67,7 @@ import org.openmetadata.schema.type.csv.CsvHeader; import org.openmetadata.schema.type.csv.CsvImportResult; import org.openmetadata.service.Entity; +import org.openmetadata.service.TypeRegistry; import org.openmetadata.service.jdbi3.EntityRepository; import org.openmetadata.service.util.EntityUtil; import org.openmetadata.service.util.JsonUtils; @@ -309,6 +325,207 @@ protected final List getTagLabels( return tagLabels; } + public Map getExtension(CSVPrinter printer, CSVRecord csvRecord, int fieldNumber) + throws IOException { + String extensionString = csvRecord.get(fieldNumber); + if (nullOrEmpty(extensionString)) { + return null; + } + + // Parse the extension string into a map of key-value pairs + Map extensionMap = new HashMap<>(); + + for (String extensions : fieldToExtensionStrings(extensionString)) { + // Split on the first occurrence of ENTITY_TYPE_SEPARATOR to get key-value pair + int separatorIndex = extensions.indexOf(ENTITY_TYPE_SEPARATOR); + + if (separatorIndex == -1) { + // No separator found, invalid entry + importFailure(printer, invalidExtension(fieldNumber, extensions, "null"), csvRecord); + continue; + } + + String key = extensions.substring(0, separatorIndex); // Get the key part + String value = extensions.substring(separatorIndex + 1); // Get the value part + + // Validate that the key and value are present + if (key.isEmpty() || value.isEmpty()) { + importFailure(printer, invalidExtension(fieldNumber, key, value), csvRecord); + } else { + extensionMap.put(key, value); // Add to the map + } + } + + validateExtension(printer, fieldNumber, csvRecord, extensionMap); + return extensionMap; + } + + private void validateExtension( + CSVPrinter printer, int fieldNumber, CSVRecord csvRecord, Map extensionMap) + throws IOException { + for (Map.Entry entry : extensionMap.entrySet()) { + String fieldName = entry.getKey(); + Object fieldValue = entry.getValue(); + + // Fetch the JSON schema and property type for the given field name + JsonSchema jsonSchema = TypeRegistry.instance().getSchema(entityType, fieldName); + if (jsonSchema == null) { + importFailure(printer, "Unknown custom field: " + fieldName, csvRecord); + return; + } + String customPropertyType = TypeRegistry.getCustomPropertyType(entityType, fieldName); + String propertyConfig = TypeRegistry.getCustomPropertyConfig(entityType, fieldName); + + // Validate field based on the custom property type + + switch (customPropertyType) { + case "entityReference", "entityReferenceList" -> { + boolean isList = "entityReferenceList".equals(customPropertyType); + fieldValue = + parseEntityReferences(fieldValue.toString(), printer, csvRecord, fieldNumber, isList); + } + case "date-cp", "dateTime-cp", "time-cp" -> fieldValue = + getFormattedDateTimeField( + customPropertyType, + fieldValue.toString(), + propertyConfig, + fieldName, + printer, + csvRecord); + case "enum", "enumList" -> { + List enumKeys = listOrEmpty(fieldToInternalArray(fieldValue.toString())); + fieldValue = enumKeys.isEmpty() ? null : enumKeys; + } + case "timeInterval" -> fieldValue = + handleTimeInterval( + printer, csvRecord, fieldNumber, fieldName, fieldValue, extensionMap, jsonSchema); + case "number", "integer", "timestamp" -> fieldValue = Long.parseLong(fieldValue.toString()); + default -> {} + } + // Validate the field against the JSON schema + validateAndUpdateExtension( + printer, csvRecord, fieldNumber, fieldName, fieldValue, extensionMap, jsonSchema); + } + } + + private Object parseEntityReferences( + String fieldValue, CSVPrinter printer, CSVRecord csvRecord, int fieldNumber, boolean isList) + throws IOException { + List entityReferences = new ArrayList<>(); + + // Split the field into individual references or handle as single entity + List entityRefStrings = + isList + ? listOrEmpty(fieldToInternalArray(fieldValue)) // Split by 'INTERNAL_ARRAY_SEPARATOR' + : Collections.singletonList(fieldValue); // Single entity reference + + // Process each entity reference string + for (String entityRefStr : entityRefStrings) { + List entityRefTypeAndValue = listOrEmpty(fieldToEntities(entityRefStr)); + + if (entityRefTypeAndValue.size() == 2) { + EntityReference entityRef = + getEntityReference( + printer, + csvRecord, + fieldNumber, + entityRefTypeAndValue.get(0), + entityRefTypeAndValue.get(1)); + Optional.ofNullable(entityRef).ifPresent(entityReferences::add); + } + } + + return isList ? entityReferences : entityReferences.isEmpty() ? null : entityReferences.get(0); + } + + protected String getFormattedDateTimeField( + String fieldType, + String fieldValue, + String propertyConfig, + String fieldName, + CSVPrinter printer, + CSVRecord csvRecord) + throws IOException { + try { + DateTimeFormatter formatter = DateTimeFormatter.ofPattern(propertyConfig, Locale.ENGLISH); + + return switch (fieldType) { + case "date-cp" -> { + TemporalAccessor date = formatter.parse(fieldValue); + yield formatter.format(date); + // Parse and format as date + } + case "dateTime-cp" -> { + LocalDateTime dateTime = LocalDateTime.parse(fieldValue, formatter); + yield dateTime.format(formatter); + // Parse and format as LocalDateTime + } + case "time-cp" -> { + LocalTime time = LocalTime.parse(fieldValue, formatter); + yield time.format(formatter); + // Parse and format as LocalTime + } + default -> throw new IllegalStateException("Unexpected value: " + fieldType); + }; + } catch (DateTimeParseException e) { + importFailure( + printer, + String.format( + "Custom field %s value is not as per defined format %s", fieldName, propertyConfig), + csvRecord); + return null; + } + } + + private Map handleTimeInterval( + CSVPrinter printer, + CSVRecord csvRecord, + int fieldNumber, + String fieldName, + Object fieldValue, + Map extensionMap, + JsonSchema jsonSchema) + throws IOException { + List timestampValues = fieldToEntities(fieldValue.toString()); + Map timestampMap = new HashMap<>(); + if (timestampValues.size() == 2) { + timestampMap.put("start", Long.parseLong(timestampValues.get(0))); + timestampMap.put("end", Long.parseLong(timestampValues.get(1))); + } else { + importFailure( + printer, + invalidField(fieldNumber, String.format("Invalid timestamp format in %s", fieldName)), + csvRecord); + } + return timestampMap; + } + + private void validateAndUpdateExtension( + CSVPrinter printer, + CSVRecord csvRecord, + int fieldNumber, + String fieldName, + Object fieldValue, + Map extensionMap, + JsonSchema jsonSchema) + throws IOException { + // Convert the field value into a JsonNode + if (fieldValue != null) { + JsonNode jsonNodeValue = JsonUtils.convertValue(fieldValue, JsonNode.class); + + // Validate the field value using the JSON schema + Set validationMessages = jsonSchema.validate(jsonNodeValue); + if (!validationMessages.isEmpty()) { + importFailure( + printer, + invalidCustomPropertyValue(fieldNumber, fieldName, validationMessages.toString()), + csvRecord); + } else { + extensionMap.put(fieldName, fieldValue); // Add to extensionMap if valid + } + } + } + public static String[] getResultHeaders(List csvHeaders) { List importResultsCsvHeader = listOf(IMPORT_STATUS_HEADER, IMPORT_STATUS_DETAILS); importResultsCsvHeader.addAll(CsvUtil.getHeaders(csvHeaders)); @@ -530,6 +747,22 @@ public static String invalidOwner(int field) { return String.format(FIELD_ERROR_MSG, CsvErrorType.INVALID_FIELD, field + 1, error); } + public static String invalidExtension(int field, String key, String value) { + String error = + "Invalid key-value pair in extension string: Key = " + + key + + ", Value = " + + value + + " .Extensions should be of format customPropertyName:customPropertyValue"; + return String.format(FIELD_ERROR_MSG, CsvErrorType.INVALID_FIELD, field + 1, error); + } + + public static String invalidCustomPropertyValue(int field, String key, String value) { + String error = + "Invalid key-value pair in extension string: Key = " + key + ", Value = " + value; + return String.format(FIELD_ERROR_MSG, CsvErrorType.INVALID_FIELD, field + 1, error); + } + public static String invalidBoolean(int field, String fieldValue) { String error = String.format("Field %s should be either 'true' of 'false'", fieldValue); return String.format(FIELD_ERROR_MSG, CsvErrorType.INVALID_FIELD, field + 1, error); diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/TypeRegistry.java b/openmetadata-service/src/main/java/org/openmetadata/service/TypeRegistry.java index ff01df62db3f..96cd9d143007 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/TypeRegistry.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/TypeRegistry.java @@ -112,7 +112,8 @@ public static String getCustomPropertyType(String entityType, String propertyNam } } } - return null; + throw EntityNotFoundException.byMessage( + CatalogExceptionMessage.entityNotFound(Entity.TYPE, String.valueOf(type))); } public static String getCustomPropertyConfig(String entityType, String propertyName) { @@ -122,7 +123,13 @@ public static String getCustomPropertyConfig(String entityType, String propertyN if (property.getName().equals(propertyName) && property.getCustomPropertyConfig() != null && property.getCustomPropertyConfig().getConfig() != null) { - return property.getCustomPropertyConfig().getConfig().toString(); + Object config = property.getCustomPropertyConfig().getConfig(); + if (config instanceof String || config instanceof Integer) { + return config.toString(); // for simple type config return as string + } else { + return JsonUtils.pojoToJson( + config); // for complex object in config return as JSON string + } } } } diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/DatabaseRepository.java b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/DatabaseRepository.java index 71a8adb2e0e6..71ac680b8461 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/DatabaseRepository.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/DatabaseRepository.java @@ -13,6 +13,7 @@ package org.openmetadata.service.jdbi3; +import static org.openmetadata.csv.CsvUtil.addExtension; import static org.openmetadata.csv.CsvUtil.addField; import static org.openmetadata.csv.CsvUtil.addGlossaryTerms; import static org.openmetadata.csv.CsvUtil.addOwners; @@ -112,7 +113,7 @@ public String exportToCsv(String name, String user) throws IOException { (DatabaseSchemaRepository) Entity.getEntityRepository(DATABASE_SCHEMA); ListFilter filter = new ListFilter(Include.NON_DELETED).addQueryParam("database", name); List schemas = - repository.listAll(repository.getFields("owners,tags,domain"), filter); + repository.listAll(repository.getFields("owners,tags,domain,extension"), filter); schemas.sort(Comparator.comparing(EntityInterface::getFullyQualifiedName)); return new DatabaseCsv(database, user).exportCsv(schemas); } @@ -269,7 +270,8 @@ protected void createEntity(CSVPrinter printer, List csvRecords) thro .withTags(tagLabels) .withRetentionPeriod(csvRecord.get(7)) .withSourceUrl(csvRecord.get(8)) - .withDomain(getEntityReference(printer, csvRecord, 9, Entity.DOMAIN)); + .withDomain(getEntityReference(printer, csvRecord, 9, Entity.DOMAIN)) + .withExtension(getExtension(printer, csvRecord, 10)); if (processRecord) { createEntity(printer, csvRecord, schema); } @@ -293,6 +295,7 @@ protected void addRecord(CsvFile csvFile, DatabaseSchema entity) { ? "" : entity.getDomain().getFullyQualifiedName(); addField(recordList, domain); + addExtension(recordList, entity.getExtension()); addRecord(csvFile, recordList); } } diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/DatabaseSchemaRepository.java b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/DatabaseSchemaRepository.java index 8f477f20aaea..664327fbc3b1 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/DatabaseSchemaRepository.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/DatabaseSchemaRepository.java @@ -14,6 +14,7 @@ package org.openmetadata.service.jdbi3; import static org.openmetadata.common.utils.CommonUtil.nullOrEmpty; +import static org.openmetadata.csv.CsvUtil.addExtension; import static org.openmetadata.csv.CsvUtil.addField; import static org.openmetadata.csv.CsvUtil.addGlossaryTerms; import static org.openmetadata.csv.CsvUtil.addOwners; @@ -184,7 +185,8 @@ public String exportToCsv(String name, String user) throws IOException { DatabaseSchema schema = getByName(null, name, Fields.EMPTY_FIELDS); // Validate database schema TableRepository repository = (TableRepository) Entity.getEntityRepository(TABLE); ListFilter filter = new ListFilter(Include.NON_DELETED).addQueryParam("databaseSchema", name); - List tables = repository.listAll(repository.getFields("owners,tags,domain"), filter); + List
tables = + repository.listAll(repository.getFields("owners,tags,domain,extension"), filter); tables.sort(Comparator.comparing(EntityInterface::getFullyQualifiedName)); return new DatabaseSchemaCsv(schema, user).exportCsv(tables); } @@ -301,7 +303,8 @@ protected void createEntity(CSVPrinter printer, List csvRecords) thro .withRetentionPeriod(csvRecord.get(7)) .withSourceUrl(csvRecord.get(8)) .withColumns(nullOrEmpty(table.getColumns()) ? new ArrayList<>() : table.getColumns()) - .withDomain(getEntityReference(printer, csvRecord, 9, Entity.DOMAIN)); + .withDomain(getEntityReference(printer, csvRecord, 9, Entity.DOMAIN)) + .withExtension(getExtension(printer, csvRecord, 10)); if (processRecord) { createEntity(printer, csvRecord, table); @@ -326,6 +329,7 @@ protected void addRecord(CsvFile csvFile, Table entity) { ? "" : entity.getDomain().getFullyQualifiedName(); addField(recordList, domain); + addExtension(recordList, entity.getExtension()); addRecord(csvFile, recordList); } } diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/DatabaseServiceRepository.java b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/DatabaseServiceRepository.java index 1d44a7dbd3b0..de7f4da1abdb 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/DatabaseServiceRepository.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/DatabaseServiceRepository.java @@ -13,6 +13,7 @@ package org.openmetadata.service.jdbi3; +import static org.openmetadata.csv.CsvUtil.addExtension; import static org.openmetadata.csv.CsvUtil.addField; import static org.openmetadata.csv.CsvUtil.addGlossaryTerms; import static org.openmetadata.csv.CsvUtil.addOwners; @@ -68,7 +69,7 @@ public String exportToCsv(String name, String user) throws IOException { DatabaseRepository repository = (DatabaseRepository) Entity.getEntityRepository(DATABASE); ListFilter filter = new ListFilter(Include.NON_DELETED).addQueryParam("service", name); List databases = - repository.listAll(repository.getFields("owners,tags,domain"), filter); + repository.listAll(repository.getFields("owners,tags,domain,extension"), filter); databases.sort(Comparator.comparing(EntityInterface::getFullyQualifiedName)); return new DatabaseServiceCsv(databaseService, user).exportCsv(databases); } @@ -122,7 +123,8 @@ protected void createEntity(CSVPrinter printer, List csvRecords) thro .withDescription(csvRecord.get(2)) .withOwners(getOwners(printer, csvRecord, 3)) .withTags(tagLabels) - .withDomain(getEntityReference(printer, csvRecord, 7, Entity.DOMAIN)); + .withDomain(getEntityReference(printer, csvRecord, 7, Entity.DOMAIN)) + .withExtension(getExtension(printer, csvRecord, 8)); if (processRecord) { createEntity(printer, csvRecord, database); @@ -145,6 +147,7 @@ protected void addRecord(CsvFile csvFile, Database entity) { ? "" : entity.getDomain().getFullyQualifiedName(); addField(recordList, domain); + addExtension(recordList, entity.getExtension()); addRecord(csvFile, recordList); } } diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/GlossaryRepository.java b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/GlossaryRepository.java index 38b8267de757..2c9dc64b0468 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/GlossaryRepository.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/GlossaryRepository.java @@ -20,6 +20,7 @@ import static org.openmetadata.csv.CsvUtil.FIELD_SEPARATOR; import static org.openmetadata.csv.CsvUtil.addEntityReference; import static org.openmetadata.csv.CsvUtil.addEntityReferences; +import static org.openmetadata.csv.CsvUtil.addExtension; import static org.openmetadata.csv.CsvUtil.addField; import static org.openmetadata.csv.CsvUtil.addOwners; import static org.openmetadata.csv.CsvUtil.addReviewers; @@ -157,7 +158,7 @@ public String exportToCsv(String name, String user) throws IOException { ListFilter filter = new ListFilter(Include.NON_DELETED).addQueryParam("parent", name); List terms = repository.listAll( - repository.getFields("owners,reviewers,tags,relatedTerms,synonyms"), filter); + repository.getFields("owners,reviewers,tags,relatedTerms,synonyms,extension"), filter); terms.sort(Comparator.comparing(EntityInterface::getFullyQualifiedName)); return new GlossaryCsv(glossary, user).exportCsv(terms); } @@ -200,7 +201,8 @@ protected void createEntity(CSVPrinter printer, List csvRecords) thro printer, csvRecord, List.of(Pair.of(7, TagLabel.TagSource.CLASSIFICATION)))) .withReviewers(getOwners(printer, csvRecord, 8)) .withOwners(getOwners(printer, csvRecord, 9)) - .withStatus(getTermStatus(printer, csvRecord)); + .withStatus(getTermStatus(printer, csvRecord)) + .withExtension(getExtension(printer, csvRecord, 11)); if (processRecord) { createEntity(printer, csvRecord, glossaryTerm); } @@ -265,6 +267,7 @@ protected void addRecord(CsvFile csvFile, GlossaryTerm entity) { addReviewers(recordList, entity.getReviewers()); addOwners(recordList, entity.getOwners()); addField(recordList, entity.getStatus().value()); + addExtension(recordList, entity.getExtension()); addRecord(csvFile, recordList); } diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/TableRepository.java b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/TableRepository.java index c15c18ad2c5c..51c2b90a7e89 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/TableRepository.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/TableRepository.java @@ -17,6 +17,7 @@ import static org.openmetadata.common.utils.CommonUtil.listOf; import static org.openmetadata.common.utils.CommonUtil.listOrEmpty; import static org.openmetadata.common.utils.CommonUtil.nullOrEmpty; +import static org.openmetadata.csv.CsvUtil.addExtension; import static org.openmetadata.csv.CsvUtil.addField; import static org.openmetadata.csv.CsvUtil.addGlossaryTerms; import static org.openmetadata.csv.CsvUtil.addOwners; @@ -785,7 +786,8 @@ public Table applySuggestion(EntityInterface entity, String columnFQN, Suggestio @Override public String exportToCsv(String name, String user) throws IOException { // Validate table - Table table = getByName(null, name, new Fields(allowedFields, "owners,domain,tags,columns")); + Table table = + getByName(null, name, new Fields(allowedFields, "owners,domain,tags,columns,extension")); return new TableCsv(table, user).exportCsv(listOf(table)); } @@ -1196,7 +1198,8 @@ protected void createEntity(CSVPrinter printer, List csvRecords) thro .withTags(tagLabels != null && tagLabels.isEmpty() ? null : tagLabels) .withRetentionPeriod(csvRecord.get(7)) .withSourceUrl(csvRecord.get(8)) - .withDomain(getEntityReference(printer, csvRecord, 9, Entity.DOMAIN)); + .withDomain(getEntityReference(printer, csvRecord, 9, Entity.DOMAIN)) + .withExtension(getExtension(printer, csvRecord, 10)); ImportResult importResult = updateColumn(printer, csvRecord); if (importResult.result().equals(IMPORT_FAILED)) { importFailure(printer, importResult.details(), csvRecord); @@ -1234,7 +1237,7 @@ public void updateColumns( } public ImportResult updateColumn(CSVPrinter printer, CSVRecord csvRecord) throws IOException { - String columnFqn = csvRecord.get(10); + String columnFqn = csvRecord.get(11); Column column = findColumn(table.getColumns(), columnFqn); boolean columnExists = column != null; if (column == null) { @@ -1245,22 +1248,22 @@ public ImportResult updateColumn(CSVPrinter printer, CSVRecord csvRecord) throws .withFullyQualifiedName( table.getFullyQualifiedName() + Entity.SEPARATOR + columnFqn); } - column.withDisplayName(csvRecord.get(11)); - column.withDescription(csvRecord.get(12)); - column.withDataTypeDisplay(csvRecord.get(13)); + column.withDisplayName(csvRecord.get(12)); + column.withDescription(csvRecord.get(13)); + column.withDataTypeDisplay(csvRecord.get(14)); column.withDataType( - nullOrEmpty(csvRecord.get(14)) ? null : ColumnDataType.fromValue(csvRecord.get(14))); - column.withArrayDataType( nullOrEmpty(csvRecord.get(15)) ? null : ColumnDataType.fromValue(csvRecord.get(15))); + column.withArrayDataType( + nullOrEmpty(csvRecord.get(16)) ? null : ColumnDataType.fromValue(csvRecord.get(16))); column.withDataLength( - nullOrEmpty(csvRecord.get(16)) ? null : Integer.parseInt(csvRecord.get(16))); + nullOrEmpty(csvRecord.get(17)) ? null : Integer.parseInt(csvRecord.get(17))); List tagLabels = getTagLabels( printer, csvRecord, List.of( - Pair.of(17, TagLabel.TagSource.CLASSIFICATION), - Pair.of(18, TagLabel.TagSource.GLOSSARY))); + Pair.of(18, TagLabel.TagSource.CLASSIFICATION), + Pair.of(19, TagLabel.TagSource.GLOSSARY))); column.withTags(nullOrEmpty(tagLabels) ? null : tagLabels); column.withOrdinalPosition(nullOrEmpty(table.getColumns()) ? 0 : table.getColumns().size()); @@ -1322,6 +1325,7 @@ protected void addRecord(CsvFile csvFile, Table entity) { ? "" : entity.getDomain().getFullyQualifiedName(); addField(recordList, domain); + addExtension(recordList, entity.getExtension()); if (!nullOrEmpty(table.getColumns())) { addRecord(csvFile, recordList, table.getColumns().get(0), false); @@ -1330,7 +1334,7 @@ protected void addRecord(CsvFile csvFile, Table entity) { } } else { // Create a dummy Entry for the Column - for (int i = 0; i < 9; i++) { + for (int i = 0; i < 10; i++) { addField(recordList, (String) null); // Add empty fields for table information } addRecord(csvFile, recordList); @@ -1340,7 +1344,7 @@ protected void addRecord(CsvFile csvFile, Table entity) { private void addRecord( CsvFile csvFile, List recordList, Column column, boolean emptyTableDetails) { if (emptyTableDetails) { - for (int i = 0; i < 10; i++) { + for (int i = 0; i < 11; i++) { addField(recordList, (String) null); // Add empty fields for table information } } diff --git a/openmetadata-service/src/main/resources/json/data/database/databaseCsvDocumentation.json b/openmetadata-service/src/main/resources/json/data/database/databaseCsvDocumentation.json index 6357f94217f9..cf56f1d6e0d7 100644 --- a/openmetadata-service/src/main/resources/json/data/database/databaseCsvDocumentation.json +++ b/openmetadata-service/src/main/resources/json/data/database/databaseCsvDocumentation.json @@ -84,6 +84,22 @@ "examples": [ "Marketing", "Sales" ] + }, + { + "name": "extension", + "required": false, + "description": "Custom property values added to the database schema. Each field value (property and its value) is separated by `;` and internal values can be separated by `|`. For `entityReferenceList` type property, pass `type1:fqn1|type2:fqn2`. For single `entityReference` type property, pass `type:fqn`. Similarly, for `enumMultiSelect`, pass values separated by `|`, and for `enumSingleSelect`, pass a single value along with the property name. For `timeInterval` property type, pass the `startTime:endTime` to the property name. If the field value itself contains delimiter values like `,` and `;` or newline they need to be quoted, and the quotation needs to be further escaped. In general, if passing multiple field values separated by `;`, the extension column value needs to be quoted.", + "examples": [ + "`customAttribute1:value1;customAttribute2:value2`", + "`\"dateCp:18-09-2024;dateTimeCp:18-09-2024 01:09:34;durationCp:PT5H30M10S;emailCp:admin@open-metadata.org\"`", + "`entRefListCp:searchIndex:elasticsearch_sample.table_search_index|databaseSchema:Glue.default.information_schema|databaseSchema:sample_data.ecommerce_db.shopify|database:Glue.default|`", + "`\"entRefCp:user:\"\"aaron.singh2\"\"\"`", + "`\"enumMultiSelectCp:val3|val2|val1|val4|val5;enumSingleSelectCp:singleVal1\"`", + "`\"timeCp:10:08:45;timeIntervalCp:1726142300000:17261420000;timeStampCp:1726142400000\"`", + "`\"integerCp:7777;numberCp:123456\"`", + "`\"\"\"queryCp:select col,row from table where id ='30';\"\"\";stringcp:sample string content\"`", + "`markdownCp:# Sample Markdown Text`" + ] } ] } \ No newline at end of file diff --git a/openmetadata-service/src/main/resources/json/data/databaseSchema/databaseSchemaCsvDocumentation.json b/openmetadata-service/src/main/resources/json/data/databaseSchema/databaseSchemaCsvDocumentation.json index 55d0fd342eaf..15fabdcf6336 100644 --- a/openmetadata-service/src/main/resources/json/data/databaseSchema/databaseSchemaCsvDocumentation.json +++ b/openmetadata-service/src/main/resources/json/data/databaseSchema/databaseSchemaCsvDocumentation.json @@ -84,6 +84,22 @@ "examples": [ "Marketing", "Sales" ] + }, + { + "name": "extension", + "required": false, + "description": "Custom property values added to the database. Each field value (property and its value) is separated by `;` and internal values can be separated by `|`. For `entityReferenceList` type property, pass `type1:fqn1|type2:fqn2`. For single `entityReference` type property, pass `type:fqn`. Similarly, for `enumMultiSelect`, pass values separated by `|`, and for `enumSingleSelect`, pass a single value along with the property name. For `timeInterval` property type, pass the `startTime:endTime` to the property name. If the field value itself contains delimiter values like `,` and `;` or newline they need to be quoted, and the quotation needs to be further escaped. In general, if passing multiple field values separated by `;`, the extension column value needs to be quoted.", + "examples": [ + "`customAttribute1:value1;customAttribute2:value2`", + "`\"dateCp:18-09-2024;dateTimeCp:18-09-2024 01:09:34;durationCp:PT5H30M10S;emailCp:admin@open-metadata.org\"`", + "`entRefListCp:searchIndex:elasticsearch_sample.table_search_index|databaseSchema:Glue.default.information_schema|databaseSchema:sample_data.ecommerce_db.shopify|database:Glue.default|`", + "`\"entRefCp:user:\"\"aaron.singh2\"\"\"`", + "`\"enumMultiSelectCp:val3|val2|val1|val4|val5;enumSingleSelectCp:singleVal1\"`", + "`\"timeCp:10:08:45;timeIntervalCp:1726142300000:17261420000;timeStampCp:1726142400000\"`", + "`\"integerCp:7777;numberCp:123456\"`", + "`\"\"\"queryCp:select col,row from table where id ='30';\"\"\";stringcp:sample string content\"`", + "`markdownCp:# Sample Markdown Text`" + ] } ] } \ No newline at end of file diff --git a/openmetadata-service/src/main/resources/json/data/databaseService/databaseServiceCsvDocumentation.json b/openmetadata-service/src/main/resources/json/data/databaseService/databaseServiceCsvDocumentation.json index 6c4346b69256..67eb3a214432 100644 --- a/openmetadata-service/src/main/resources/json/data/databaseService/databaseServiceCsvDocumentation.json +++ b/openmetadata-service/src/main/resources/json/data/databaseService/databaseServiceCsvDocumentation.json @@ -68,6 +68,22 @@ "examples": [ "Marketing", "Sales" ] + }, + { + "name": "extension", + "required": false, + "description": "Custom property values added to the database schema. Each field value (property and its value) is separated by `;` and internal values can be separated by `|`. For `entityReferenceList` type property, pass `type1:fqn1|type2:fqn2`. For single `entityReference` type property, pass `type:fqn`. Similarly, for `enumMultiSelect`, pass values separated by `|`, and for `enumSingleSelect`, pass a single value along with the property name. For `timeInterval` property type, pass the `startTime:endTime` to the property name. If the field value itself contains delimiter values like `,` and `;` or newline they need to be quoted, and the quotation needs to be further escaped. In general, if passing multiple field values separated by `;`, the extension column value needs to be quoted.", + "examples": [ + "`customAttribute1:value1;customAttribute2:value2`", + "`\"dateCp:18-09-2024;dateTimeCp:18-09-2024 01:09:34;durationCp:PT5H30M10S;emailCp:admin@open-metadata.org\"`", + "`entRefListCp:searchIndex:elasticsearch_sample.table_search_index|databaseSchema:Glue.default.information_schema|databaseSchema:sample_data.ecommerce_db.shopify|database:Glue.default|`", + "`\"entRefCp:user:\"\"aaron.singh2\"\"\"`", + "`\"enumMultiSelectCp:val3|val2|val1|val4|val5;enumSingleSelectCp:singleVal1\"`", + "`\"timeCp:10:08:45;timeIntervalCp:1726142300000:17261420000;timeStampCp:1726142400000\"`", + "`\"integerCp:7777;numberCp:123456\"`", + "`\"\"\"queryCp:select col,row from table where id ='30';\"\"\";stringcp:sample string content\"`", + "`markdownCp:# Sample Markdown Text`" + ] } ] } \ No newline at end of file diff --git a/openmetadata-service/src/main/resources/json/data/glossary/glossaryCsvDocumentation.json b/openmetadata-service/src/main/resources/json/data/glossary/glossaryCsvDocumentation.json index c8de3da46d39..9d77ac22da1b 100644 --- a/openmetadata-service/src/main/resources/json/data/glossary/glossaryCsvDocumentation.json +++ b/openmetadata-service/src/main/resources/json/data/glossary/glossaryCsvDocumentation.json @@ -94,6 +94,22 @@ "`Approved`", "`Deprecated`" ] + }, + { + "name": "extension", + "required": false, + "description": "Custom property values added to the glossary term. Each field value (property and its value) is separated by `;` and internal values can be separated by `|`. For `entityReferenceList` type property, pass `type1:fqn1|type2:fqn2`. For single `entityReference` type property, pass `type:fqn`. Similarly, for `enumMultiSelect`, pass values separated by `|`, and for `enumSingleSelect`, pass a single value along with the property name. For `timeInterval` property type, pass the `startTime:endTime` to the property name. If the field value itself contains delimiter values like `,` and `;` or newline they need to be quoted, and the quotation needs to be further escaped. In general, if passing multiple field values separated by `;`, the extension column value needs to be quoted.", + "examples": [ + "`customAttribute1:value1;customAttribute2:value2`", + "`\"dateCp:18-09-2024;dateTimeCp:18-09-2024 01:09:34;durationCp:PT5H30M10S;emailCp:admin@open-metadata.org\"`", + "`entRefListCp:searchIndex:elasticsearch_sample.table_search_index|databaseSchema:Glue.default.information_schema|databaseSchema:sample_data.ecommerce_db.shopify|database:Glue.default|`", + "`\"entRefCp:user:\"\"aaron.singh2\"\"\"`", + "`\"enumMultiSelectCp:val3|val2|val1|val4|val5;enumSingleSelectCp:singleVal1\"`", + "`\"timeCp:10:08:45;timeIntervalCp:1726142300000:17261420000;timeStampCp:1726142400000\"`", + "`\"integerCp:7777;numberCp:123456\"`", + "`\"\"\"queryCp:select col,row from table where id ='30';\"\"\";stringcp:sample string content\"`", + "`markdownCp:# Sample Markdown Text`" + ] } ] } \ No newline at end of file diff --git a/openmetadata-service/src/main/resources/json/data/table/tableCsvDocumentation.json b/openmetadata-service/src/main/resources/json/data/table/tableCsvDocumentation.json index c528965ab0f7..5a1b36d36e00 100644 --- a/openmetadata-service/src/main/resources/json/data/table/tableCsvDocumentation.json +++ b/openmetadata-service/src/main/resources/json/data/table/tableCsvDocumentation.json @@ -85,6 +85,22 @@ "Marketing", "Sales" ] }, + { + "name": "extension", + "required": false, + "description": "Custom property values added to the table. Each field value (property and its value) is separated by `;` and internal values can be separated by `|`. For `entityReferenceList` type property, pass `type1:fqn1|type2:fqn2`. For single `entityReference` type property, pass `type:fqn`. Similarly, for `enumMultiSelect`, pass values separated by `|`, and for `enumSingleSelect`, pass a single value along with the property name. For `timeInterval` property type, pass the `startTime:endTime` to the property name. If the field value itself contains delimiter values like `,` and `;` or newline they need to be quoted, and the quotation needs to be further escaped. In general, if passing multiple field values separated by `;`, the extension column value needs to be quoted.", + "examples": [ + "`customAttribute1:value1;customAttribute2:value2`", + "`\"dateCp:18-09-2024;dateTimeCp:18-09-2024 01:09:34;durationCp:PT5H30M10S;emailCp:admin@open-metadata.org\"`", + "`entRefListCp:searchIndex:elasticsearch_sample.table_search_index|databaseSchema:Glue.default.information_schema|databaseSchema:sample_data.ecommerce_db.shopify|database:Glue.default|`", + "`\"entRefCp:user:\"\"aaron.singh2\"\"\"`", + "`\"enumMultiSelectCp:val3|val2|val1|val4|val5;enumSingleSelectCp:singleVal1\"`", + "`\"timeCp:10:08:45;timeIntervalCp:1726142300000:17261420000;timeStampCp:1726142400000\"`", + "`\"integerCp:7777;numberCp:123456\"`", + "`\"\"\"queryCp:select col,row from table where id ='30';\"\"\";stringcp:sample string content\"`", + "`markdownCp:# Sample Markdown Text`" + ] + }, { "name": "column.fullyQualifiedName", "required": true, diff --git a/openmetadata-service/src/test/java/org/openmetadata/service/resources/databases/DatabaseResourceTest.java b/openmetadata-service/src/test/java/org/openmetadata/service/resources/databases/DatabaseResourceTest.java index 599cdd79a11a..ea7d996e3774 100644 --- a/openmetadata-service/src/test/java/org/openmetadata/service/resources/databases/DatabaseResourceTest.java +++ b/openmetadata-service/src/test/java/org/openmetadata/service/resources/databases/DatabaseResourceTest.java @@ -121,7 +121,7 @@ void testImportInvalidCsv() { // Headers: name, displayName, description, owner, tags, retentionPeriod, sourceUrl, domain // Update databaseSchema with invalid tags field String resultsHeader = recordToString(EntityCsv.getResultHeaders(DatabaseCsv.HEADERS)); - String record = "s1,dsp1,dsc1,,Tag.invalidTag,,,,,"; + String record = "s1,dsp1,dsc1,,Tag.invalidTag,,,,,,"; String csv = createCsv(DatabaseCsv.HEADERS, listOf(record), null); CsvImportResult result = importCsv(databaseName, csv, false); assertSummary(result, ApiStatus.FAILURE, 2, 1, 1); @@ -132,7 +132,7 @@ resultsHeader, getFailedRecord(record, entityNotFound(4, "tag", "Tag.invalidTag" assertRows(result, expectedRows); // invalid tag it will give error. - record = "non-existing,dsp1,dsc1,,Tag.invalidTag,,,,,"; + record = "non-existing,dsp1,dsc1,,Tag.invalidTag,,,,,,"; csv = createCsv(DatabaseSchemaCsv.HEADERS, listOf(record), null); result = importCsv(databaseName, csv, false); assertSummary(result, ApiStatus.FAILURE, 2, 1, 1); @@ -144,7 +144,7 @@ resultsHeader, getFailedRecord(record, entityNotFound(4, "tag", "Tag.invalidTag" // databaseSchema will be created if it does not exist String schemaFqn = FullyQualifiedName.add(database.getFullyQualifiedName(), "non-existing"); - record = "non-existing,dsp1,dsc1,,,,,,,"; + record = "non-existing,dsp1,dsc1,,,,,,,,"; csv = createCsv(DatabaseSchemaCsv.HEADERS, listOf(record), null); result = importCsv(databaseName, csv, false); assertSummary(result, ApiStatus.SUCCESS, 2, 2, 0); @@ -168,7 +168,7 @@ void testImportExport() throws IOException { // Update terms with change in description String record = String.format( - "s1,dsp1,new-dsc1,user:%s,,,Tier.Tier1,P23DT23H,http://test.com,%s", + "s1,dsp1,new-dsc1,user:%s,,,Tier.Tier1,P23DT23H,http://test.com,%s,", user1, escapeCsv(DOMAIN.getFullyQualifiedName())); // Update created entity with changes diff --git a/openmetadata-service/src/test/java/org/openmetadata/service/resources/databases/DatabaseSchemaResourceTest.java b/openmetadata-service/src/test/java/org/openmetadata/service/resources/databases/DatabaseSchemaResourceTest.java index d5ce6a919a6f..46ba6a5cda92 100644 --- a/openmetadata-service/src/test/java/org/openmetadata/service/resources/databases/DatabaseSchemaResourceTest.java +++ b/openmetadata-service/src/test/java/org/openmetadata/service/resources/databases/DatabaseSchemaResourceTest.java @@ -120,7 +120,7 @@ void testImportInvalidCsv() { // Headers: name, displayName, description, owner, tags, retentionPeriod, sourceUrl, domain // Create table with invalid tags field String resultsHeader = recordToString(EntityCsv.getResultHeaders(DatabaseSchemaCsv.HEADERS)); - String record = "s1,dsp1,dsc1,,Tag.invalidTag,,,,,"; + String record = "s1,dsp1,dsc1,,Tag.invalidTag,,,,,,"; String csv = createCsv(DatabaseSchemaCsv.HEADERS, listOf(record), null); CsvImportResult result = importCsv(schemaName, csv, false); assertSummary(result, ApiStatus.FAILURE, 2, 1, 1); @@ -131,7 +131,7 @@ resultsHeader, getFailedRecord(record, entityNotFound(4, "tag", "Tag.invalidTag" assertRows(result, expectedRows); // Tag will cause failure - record = "non-existing,dsp1,dsc1,,Tag.invalidTag,,,,,"; + record = "non-existing,dsp1,dsc1,,Tag.invalidTag,,,,,,"; csv = createCsv(DatabaseSchemaCsv.HEADERS, listOf(record), null); result = importCsv(schemaName, csv, false); assertSummary(result, ApiStatus.FAILURE, 2, 1, 1); @@ -142,7 +142,7 @@ resultsHeader, getFailedRecord(record, entityNotFound(4, "tag", "Tag.invalidTag" assertRows(result, expectedRows); // non-existing table will cause - record = "non-existing,dsp1,dsc1,,,,,,,"; + record = "non-existing,dsp1,dsc1,,,,,,,,"; String tableFqn = FullyQualifiedName.add(schema.getFullyQualifiedName(), "non-existing"); csv = createCsv(DatabaseSchemaCsv.HEADERS, listOf(record), null); result = importCsv(schemaName, csv, false); @@ -167,7 +167,7 @@ void testImportExport() throws IOException { List updateRecords = listOf( String.format( - "s1,dsp1,new-dsc1,user:%s,,,Tier.Tier1,P23DT23H,http://test.com,%s", + "s1,dsp1,new-dsc1,user:%s,,,Tier.Tier1,P23DT23H,http://test.com,%s,", user1, escapeCsv(DOMAIN.getFullyQualifiedName()))); // Update created entity with changes diff --git a/openmetadata-service/src/test/java/org/openmetadata/service/resources/databases/TableResourceTest.java b/openmetadata-service/src/test/java/org/openmetadata/service/resources/databases/TableResourceTest.java index a1ea2989e051..4a3999d55746 100644 --- a/openmetadata-service/src/test/java/org/openmetadata/service/resources/databases/TableResourceTest.java +++ b/openmetadata-service/src/test/java/org/openmetadata/service/resources/databases/TableResourceTest.java @@ -2726,7 +2726,7 @@ void testImportInvalidCsv() { // Headers: name, displayName, description, owner, tags, retentionPeriod, sourceUrl, domain // Create table with invalid tags field String resultsHeader = recordToString(EntityCsv.getResultHeaders(TableCsv.HEADERS)); - String record = "s1,dsp1,dsc1,,Tag.invalidTag,,,,,,c1,c1,c1,,INT,,,,"; + String record = "s1,dsp1,dsc1,,Tag.invalidTag,,,,,,,c1,c1,c1,,INT,,,,"; String csv = createCsv(TableCsv.HEADERS, listOf(record), null); CsvImportResult result = importCsv(tableName, csv, false); assertSummary(result, ApiStatus.FAILURE, 2, 1, 1); @@ -2738,19 +2738,19 @@ void testImportInvalidCsv() { assertRows(result, expectedRows); // Add an invalid column tag - record = "s1,dsp1,dsc1,,,,,,,,c1,,,,INT,,,Tag.invalidTag,"; + record = "s1,dsp1,dsc1,,,,,,,,,c1,,,,INT,,,Tag.invalidTag,"; csv = createCsv(TableCsv.HEADERS, listOf(record), null); result = importCsv(tableName, csv, false); assertSummary(result, ApiStatus.FAILURE, 2, 1, 1); expectedRows = new String[] { resultsHeader, - getFailedRecord(record, EntityCsv.entityNotFound(17, "tag", "Tag.invalidTag")) + getFailedRecord(record, EntityCsv.entityNotFound(18, "tag", "Tag.invalidTag")) }; assertRows(result, expectedRows); // Update a non-existing column, this should create a new column with name "nonExistingColumn" - record = "s1,dsp1,dsc1,,,,,,,,nonExistingColumn,,,,INT,,,,"; + record = "s1,dsp1,dsc1,,,,,,,,,nonExistingColumn,,,,INT,,,,"; csv = createCsv(TableCsv.HEADERS, listOf(record), null); result = importCsv(tableName, csv, false); assertSummary(result, ApiStatus.SUCCESS, 2, 2, 0); @@ -2776,12 +2776,12 @@ void testImportExport() throws IOException { List updateRecords = listOf( String.format( - "s1,dsp1,new-dsc1,user:%s,,,Tier.Tier1,P23DT23H,http://test.com,%s,c1," + "s1,dsp1,new-dsc1,user:%s,,,Tier.Tier1,P23DT23H,http://test.com,%s,,c1," + "dsp1-new,desc1,type,STRUCT,,,PII.Sensitive,", user1, escapeCsv(DOMAIN.getFullyQualifiedName())), - ",,,,,,,,,,c1.c11,dsp11-new,desc11,type1,INT,,,PII.Sensitive,", - ",,,,,,,,,,c2,,,type1,INT,,,,", - ",,,,,,,,,,c3,,,type1,INT,,,,"); + ",,,,,,,,,,,c1.c11,dsp11-new,desc11,type1,INT,,,PII.Sensitive,", + ",,,,,,,,,,,c2,,,type1,INT,,,,", + ",,,,,,,,,,,c3,,,type1,INT,,,,"); // Update created entity with changes importCsvAndValidate(table.getFullyQualifiedName(), TableCsv.HEADERS, null, updateRecords); diff --git a/openmetadata-service/src/test/java/org/openmetadata/service/resources/glossary/GlossaryResourceTest.java b/openmetadata-service/src/test/java/org/openmetadata/service/resources/glossary/GlossaryResourceTest.java index d3f032f40111..997a2d38b5c0 100644 --- a/openmetadata-service/src/test/java/org/openmetadata/service/resources/glossary/GlossaryResourceTest.java +++ b/openmetadata-service/src/test/java/org/openmetadata/service/resources/glossary/GlossaryResourceTest.java @@ -501,7 +501,7 @@ void testImportInvalidCsv() { // Create glossaryTerm with invalid name (due to ::) String resultsHeader = recordToString(EntityCsv.getResultHeaders(GlossaryCsv.HEADERS)); - String record = ",g::1,dsp1,dsc1,,,,,,,"; + String record = ",g::1,dsp1,dsc1,,,,,,,,"; String csv = createCsv(GlossaryCsv.HEADERS, listOf(record), null); CsvImportResult result = importCsv(glossaryName, csv, false); Awaitility.await().atMost(4, TimeUnit.SECONDS).until(() -> true); @@ -512,7 +512,7 @@ resultsHeader, getFailedRecord(record, "[name must match \"^((?!::).)*$\"]") assertRows(result, expectedRows); // Create glossaryTerm with invalid parent - record = "invalidParent,g1,dsp1,dsc1,h1;h2;h3,,term1;http://term1,Tier.Tier1,,,"; + record = "invalidParent,g1,dsp1,dsc1,h1;h2;h3,,term1;http://term1,Tier.Tier1,,,,"; csv = createCsv(GlossaryCsv.HEADERS, listOf(record), null); result = importCsv(glossaryName, csv, false); Awaitility.await().atMost(4, TimeUnit.SECONDS).until(() -> true); @@ -525,7 +525,7 @@ record = "invalidParent,g1,dsp1,dsc1,h1;h2;h3,,term1;http://term1,Tier.Tier1,,," assertRows(result, expectedRows); // Create glossaryTerm with invalid tags field - record = ",g1,dsp1,dsc1,h1;h2;h3,,term1;http://term1,Tag.invalidTag,,,"; + record = ",g1,dsp1,dsc1,h1;h2;h3,,term1;http://term1,Tag.invalidTag,,,,"; csv = createCsv(GlossaryCsv.HEADERS, listOf(record), null); result = importCsv(glossaryName, csv, false); assertSummary(result, ApiStatus.FAILURE, 2, 1, 1); @@ -551,31 +551,31 @@ void testGlossaryImportExport() throws IOException { List createRecords = listOf( String.format( - ",g1,dsp1,\"dsc1,1\",h1;h2;h3,,term1;http://term1,PII.None,user:%s,user:%s,%s", + ",g1,dsp1,\"dsc1,1\",h1;h2;h3,,term1;http://term1,PII.None,user:%s,user:%s,%s,", reviewerRef.get(0), user1, "Approved"), String.format( - ",g2,dsp2,dsc3,h1;h3;h3,,term2;https://term2,PII.NonSensitive,,user:%s,%s", + ",g2,dsp2,dsc3,h1;h3;h3,,term2;https://term2,PII.NonSensitive,,user:%s,%s,", user1, "Approved"), String.format( - "importExportTest.g1,g11,dsp2,dsc11,h1;h3;h3,,,,user:%s,team:%s,%s", + "importExportTest.g1,g11,dsp2,dsc11,h1;h3;h3,,,,user:%s,team:%s,%s,", reviewerRef.get(0), team11, "Draft")); // Update terms with change in description List updateRecords = listOf( String.format( - ",g1,dsp1,new-dsc1,h1;h2;h3,,term1;http://term1,PII.None,user:%s,user:%s,%s", + ",g1,dsp1,new-dsc1,h1;h2;h3,,term1;http://term1,PII.None,user:%s,user:%s,%s,", reviewerRef.get(0), user1, "Approved"), String.format( - ",g2,dsp2,new-dsc3,h1;h3;h3,,term2;https://term2,PII.NonSensitive,user:%s,user:%s,%s", + ",g2,dsp2,new-dsc3,h1;h3;h3,,term2;https://term2,PII.NonSensitive,user:%s,user:%s,%s,", user1, user2, "Approved"), String.format( - "importExportTest.g1,g11,dsp2,new-dsc11,h1;h3;h3,,,,user:%s,team:%s,%s", + "importExportTest.g1,g11,dsp2,new-dsc11,h1;h3;h3,,,,user:%s,team:%s,%s,", reviewerRef.get(0), team11, "Draft")); // Add new row to existing rows List newRecords = - listOf(",g3,dsp0,dsc0,h1;h2;h3,,term0;http://term0,PII.Sensitive,,,Approved"); + listOf(",g3,dsp0,dsc0,h1;h2;h3,,term0;http://term0,PII.Sensitive,,,Approved,"); testImportExport( glossary.getName(), GlossaryCsv.HEADERS, createRecords, updateRecords, newRecords); } From 58fc964e8e36e61d28e82681f3c470cb884e8a77 Mon Sep 17 00:00:00 2001 From: sonikashah Date: Thu, 19 Sep 2024 19:52:47 +0530 Subject: [PATCH 03/39] Feat : Allow Custom Property Update in Bulk Upload --- .../java/org/openmetadata/csv/CsvUtil.java | 27 ++++++++++++------- 1 file changed, 17 insertions(+), 10 deletions(-) diff --git a/openmetadata-service/src/main/java/org/openmetadata/csv/CsvUtil.java b/openmetadata-service/src/main/java/org/openmetadata/csv/CsvUtil.java index 10870c99e965..88b945b31e76 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/csv/CsvUtil.java +++ b/openmetadata-service/src/main/java/org/openmetadata/csv/CsvUtil.java @@ -120,14 +120,15 @@ public static List fieldToInternalArray(String field) { * @param field The input string with key-value pairs. * @return A list of key-value pairs, handling quotes and semicolons correctly. */ - public static List fieldToExtensionStrings(String field) throws IOException { + public static List fieldToExtensionStrings(String field) { if (field == null || field.isBlank()) { - return List.of(); // Return empty list if input is null or blank + return List.of(); } List result = new ArrayList<>(); StringBuilder currentField = new StringBuilder(); - boolean inQuotes = false; // Track whether we are inside quotes + // Track whether we are inside quotes + boolean inQuotes = false; // Iterate through each character in the field for (int i = 0; i < field.length(); i++) { @@ -135,17 +136,23 @@ public static List fieldToExtensionStrings(String field) throws IOExcept if (c == '"') { if (inQuotes && i + 1 < field.length() && field.charAt(i + 1) == '"') { - currentField.append('"'); // Handle escaped quote ("" -> ") - i++; // Skip the next character + // Handle escaped quote ("" -> ") + currentField.append('"'); + i++; } else { - inQuotes = !inQuotes; // Toggle quote state - currentField.append(c); // Keep the quote as part of the field + // Toggle quote state + inQuotes = !inQuotes; + // Keep the quote as part of the field + currentField.append(c); } } else if (c == FIELD_SEPARATOR.charAt(0) && !inQuotes) { - addFieldToResult(result, currentField); // Add the field when semicolon is outside quotes - currentField.setLength(0); // Reset buffer for next field + // Add the field when semicolon is outside quotes + addFieldToResult(result, currentField); + // Reset buffer for next field + currentField.setLength(0); } else { - currentField.append(c); // Continue building the field + // Continue building the field + currentField.append(c); } } From 04ea4bc8d6c8c8730f7060f296573d263e474117 Mon Sep 17 00:00:00 2001 From: Ashish Gupta Date: Tue, 24 Sep 2024 23:34:40 +0530 Subject: [PATCH 04/39] supported editable imports in glossary page --- .../src/main/resources/ui/package.json | 1 + .../BulkImport/BulkEntityImport.component.tsx | 380 ++++++++++++++++++ .../BulkImport/BulkEntityImport.interface.ts | 25 ++ .../BulkImport/bulk-entity-import.style.less | 18 + .../ImportGlossary/ImportGlossary.tsx | 37 +- .../GlossaryImportResult.component.tsx | 234 ----------- .../GlossaryImportResult.test.tsx | 141 ------- .../UploadFile/UploadFile.interface.ts | 24 ++ .../src/components/UploadFile/UploadFile.tsx | 79 ++++ .../EntityImport/EntityImport.component.tsx | 79 +--- .../ui/src/constants/BulkImport.constant.ts | 35 ++ .../ui/src/locale/languages/en-us.json | 3 + .../resources/ui/src/utils/CSV/CSV.utils.tsx | 129 ++++++ .../ui/src/utils/CSV/CSVUtilsClassBase.tsx | 236 +++++++++++ .../main/resources/ui/webpack.config.dev.js | 4 + .../main/resources/ui/webpack.config.prod.js | 4 + .../src/main/resources/ui/yarn.lock | 67 ++- 17 files changed, 1031 insertions(+), 465 deletions(-) create mode 100644 openmetadata-ui/src/main/resources/ui/src/components/BulkImport/BulkEntityImport.component.tsx create mode 100644 openmetadata-ui/src/main/resources/ui/src/components/BulkImport/BulkEntityImport.interface.ts create mode 100644 openmetadata-ui/src/main/resources/ui/src/components/BulkImport/bulk-entity-import.style.less delete mode 100644 openmetadata-ui/src/main/resources/ui/src/components/Glossary/ImportResult/GlossaryImportResult.component.tsx delete mode 100644 openmetadata-ui/src/main/resources/ui/src/components/Glossary/ImportResult/GlossaryImportResult.test.tsx create mode 100644 openmetadata-ui/src/main/resources/ui/src/components/UploadFile/UploadFile.interface.ts create mode 100644 openmetadata-ui/src/main/resources/ui/src/components/UploadFile/UploadFile.tsx create mode 100644 openmetadata-ui/src/main/resources/ui/src/constants/BulkImport.constant.ts create mode 100644 openmetadata-ui/src/main/resources/ui/src/utils/CSV/CSV.utils.tsx create mode 100644 openmetadata-ui/src/main/resources/ui/src/utils/CSV/CSVUtilsClassBase.tsx diff --git a/openmetadata-ui/src/main/resources/ui/package.json b/openmetadata-ui/src/main/resources/ui/package.json index 1d6d1cf54d9b..785e29ffd57e 100644 --- a/openmetadata-ui/src/main/resources/ui/package.json +++ b/openmetadata-ui/src/main/resources/ui/package.json @@ -52,6 +52,7 @@ "@fontsource/poppins": "^5.0.0", "@fontsource/source-code-pro": "^5.0.0", "@github/g-emoji-element": "^1.1.5", + "@inovua/reactdatagrid-community": "^5.10.2", "@okta/okta-auth-js": "^7.5.0", "@okta/okta-react": "^6.4.3", "@rjsf/core": "5.15.0", diff --git a/openmetadata-ui/src/main/resources/ui/src/components/BulkImport/BulkEntityImport.component.tsx b/openmetadata-ui/src/main/resources/ui/src/components/BulkImport/BulkEntityImport.component.tsx new file mode 100644 index 000000000000..bc199c9cabd9 --- /dev/null +++ b/openmetadata-ui/src/main/resources/ui/src/components/BulkImport/BulkEntityImport.component.tsx @@ -0,0 +1,380 @@ +/* + * Copyright 2024 Collate. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * http://www.apache.org/licenses/LICENSE-2.0 + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import ReactDataGrid from '@inovua/reactdatagrid-community'; +import '@inovua/reactdatagrid-community/index.css'; +import { + TypeColumn, + TypeComputedProps, +} from '@inovua/reactdatagrid-community/types'; +import { Button, Card, Col, Row, Space, Typography } from 'antd'; +import { AxiosError } from 'axios'; +import React, { MutableRefObject, useCallback, useState } from 'react'; +import { useTranslation } from 'react-i18next'; +import { usePapaParse } from 'react-papaparse'; + +import { + ENTITY_IMPORT_STEPS, + VALIDATION_STEP, +} from '../../constants/BulkImport.constant'; +import { CSVImportResult } from '../../generated/type/csvImportResult'; +import { + getCSVStringFromColumnsAndDataSource, + getEntityColumnsAndDataSourceFromCSV, +} from '../../utils/CSV/CSV.utils'; +import { showErrorToast, showSuccessToast } from '../../utils/ToastUtils'; +import { ImportStatus } from '../common/EntityImport/ImportStatus/ImportStatus.component'; +import Stepper from '../Settings/Services/Ingestion/IngestionStepper/IngestionStepper.component'; +import { UploadFile } from '../UploadFile/UploadFile'; +import './bulk-entity-import.style.less'; +import { BulkImportProps } from './BulkEntityImport.interface'; + +let inEdit = false; + +const BulkEntityImport = ({ + entityType, + fqn, + onValidateCsvString, + onSuccess, + hideAddButton, +}: BulkImportProps) => { + const [activeStep, setActiveStep] = useState( + VALIDATION_STEP.UPLOAD + ); + const { t } = useTranslation(); + const [isValidating, setIsValidating] = useState(false); + const [validationData, setValidationData] = useState(); + const [columns, setColumns] = useState([]); + const [dataSource, setDataSource] = useState[]>([]); + const { readString } = usePapaParse(); + const [validateCSVData, setValidateCSVData] = + useState<{ columns: TypeColumn[]; dataSource: Record[] }>(); + const [gridRef, setGridRef] = useState< + MutableRefObject + >({ current: null }); + + const focusToGrid = useCallback(() => { + setGridRef((ref) => { + ref.current?.focus(); + + return ref; + }); + }, [setGridRef]); + + const onCSVReadComplete = useCallback( + (results: { data: string[][] }) => { + // results.data is returning data with unknown type + const { columns, dataSource } = getEntityColumnsAndDataSourceFromCSV( + results.data as string[][] + ); + setDataSource(dataSource); + setColumns(columns); + + setActiveStep(VALIDATION_STEP.EDIT_VALIDATE); + setTimeout(focusToGrid, 500); + }, + [setDataSource, setColumns, setActiveStep, focusToGrid] + ); + + const validateCsvString = useCallback( + async (csvData: string) => await onValidateCsvString(csvData, true), + [] + ); + + const handleLoadData = useCallback( + async (e: ProgressEvent) => { + try { + const result = e.target?.result as string; + + const validationResponse = await validateCsvString(result); + + if (['failure', 'aborted'].includes(validationResponse?.status ?? '')) { + setValidationData(validationResponse); + + setActiveStep(VALIDATION_STEP.UPLOAD); + + return; + } + + if (result) { + readString(result, { + worker: true, + skipEmptyLines: true, + complete: onCSVReadComplete, + }); + } + } catch (error) { + showErrorToast(error as AxiosError); + } + }, + [onCSVReadComplete] + ); + + const onEditComplete = useCallback( + ({ value, columnId, rowId }) => { + const data = [...dataSource]; + data[rowId][columnId] = value; + + setDataSource(data); + }, + [dataSource] + ); + + const handleBack = () => { + if (activeStep === VALIDATION_STEP.UPDATE) { + setActiveStep(VALIDATION_STEP.EDIT_VALIDATE); + } else { + setActiveStep(VALIDATION_STEP.UPLOAD); + } + }; + + const handleValidate = async () => { + setIsValidating(true); + setValidateCSVData(undefined); + try { + // Call the validate API + const csvData = getCSVStringFromColumnsAndDataSource(columns, dataSource); + + const response = await onValidateCsvString( + csvData, + activeStep === VALIDATION_STEP.EDIT_VALIDATE + ); + + if (activeStep === VALIDATION_STEP.UPDATE) { + if (response?.status === 'failure') { + setValidationData(response); + readString(response?.importResultsCsv ?? '', { + worker: true, + skipEmptyLines: true, + complete: (results) => { + // results.data is returning data with unknown type + setValidateCSVData( + getEntityColumnsAndDataSourceFromCSV(results.data as string[][]) + ); + }, + }); + setActiveStep(VALIDATION_STEP.UPDATE); + } else { + showSuccessToast( + t('message.entity-details-updated', { + entityType, + fqn, + }) + ); + onSuccess(); + } + } else if (activeStep === VALIDATION_STEP.EDIT_VALIDATE) { + setValidationData(response); + setActiveStep(VALIDATION_STEP.UPDATE); + readString(response?.importResultsCsv ?? '', { + worker: true, + skipEmptyLines: true, + complete: (results) => { + // results.data is returning data with unknown type + setValidateCSVData( + getEntityColumnsAndDataSourceFromCSV(results.data as string[][]) + ); + }, + }); + } + } catch (error) { + showErrorToast(error as AxiosError); + } finally { + setIsValidating(false); + } + }; + + const onEditStart = () => { + inEdit = true; + }; + + const onEditStop = () => { + requestAnimationFrame(() => { + inEdit = false; + gridRef.current?.focus(); + }); + }; + + const onKeyDown = (event: KeyboardEvent) => { + if (inEdit) { + if (event.key === 'Escape') { + const [rowIndex, colIndex] = gridRef.current?.computedActiveCell ?? [ + 0, 0, + ]; + const column = gridRef.current?.getColumnBy(colIndex); + + gridRef.current?.cancelEdit?.({ + rowIndex, + columnId: column?.name ?? '', + }); + } + + return; + } + const grid = gridRef.current; + if (!grid) { + return; + } + let [rowIndex, colIndex] = grid.computedActiveCell ?? [0, 0]; + + if (event.key === ' ' || event.key === 'Enter') { + const column = grid.getColumnBy(colIndex); + grid.startEdit?.({ columnId: column.name ?? '', rowIndex }); + event.preventDefault(); + + return; + } + if (event.key !== 'Tab') { + return; + } + event.preventDefault(); + event.stopPropagation(); + + const direction = event.shiftKey ? -1 : 1; + + const columns = grid.visibleColumns; + const rowCount = grid.count; + + colIndex += direction; + if (colIndex === -1) { + colIndex = columns.length - 1; + rowIndex -= 1; + } + if (colIndex === columns.length) { + rowIndex += 1; + colIndex = 0; + } + if (rowIndex < 0 || rowIndex === rowCount) { + return; + } + + grid?.setActiveCell([rowIndex, colIndex]); + }; + + const handleAddRow = useCallback(() => { + setDataSource((data) => { + setTimeout(() => { + gridRef.current?.scrollToId(data.length + ''); + gridRef.current?.focus(); + }, 1); + + return [...data, { id: data.length + '' }]; + }); + }, [gridRef]); + + const handleRetryCsvUpload = () => { + setValidationData(undefined); + + setActiveStep(VALIDATION_STEP.UPLOAD); + }; + + return ( + + + + + + {activeStep === 0 && ( + <> + {validationData?.abortReason ? ( + + + + {t('label.aborted')}{' '} + {validationData.abortReason} + + + + + + + ) : ( + + )} + + )} + {activeStep === 1 && ( + + )} + {activeStep === 2 && validationData && ( + + + + + + {validateCSVData && ( + + )} + + + )} + + {activeStep > 0 && ( + + {activeStep === 1 && !hideAddButton && ( + + )} +
+ {activeStep > 0 && ( + + )} + {activeStep < 3 && ( + + )} +
+ + )} + + ); +}; + +export default BulkEntityImport; diff --git a/openmetadata-ui/src/main/resources/ui/src/components/BulkImport/BulkEntityImport.interface.ts b/openmetadata-ui/src/main/resources/ui/src/components/BulkImport/BulkEntityImport.interface.ts new file mode 100644 index 000000000000..0669ad9f9b33 --- /dev/null +++ b/openmetadata-ui/src/main/resources/ui/src/components/BulkImport/BulkEntityImport.interface.ts @@ -0,0 +1,25 @@ +/* + * Copyright 2024 Collate. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * http://www.apache.org/licenses/LICENSE-2.0 + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import { EntityType } from '../../enums/entity.enum'; +import { CSVImportResult } from '../../generated/type/csvImportResult'; + +export interface BulkImportProps { + entityType: EntityType; + fqn: string; + hideAddButton?: boolean; + onValidateCsvString: ( + data: string, + dryRun?: boolean + ) => Promise; + onSuccess: () => void; +} diff --git a/openmetadata-ui/src/main/resources/ui/src/components/BulkImport/bulk-entity-import.style.less b/openmetadata-ui/src/main/resources/ui/src/components/BulkImport/bulk-entity-import.style.less new file mode 100644 index 000000000000..a015f108f4fb --- /dev/null +++ b/openmetadata-ui/src/main/resources/ui/src/components/BulkImport/bulk-entity-import.style.less @@ -0,0 +1,18 @@ +/* + * Copyright 2024 Collate. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * http://www.apache.org/licenses/LICENSE-2.0 + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +.reserve-right-sidebar { + .import-footer { + // Right side padding 20 + 64 width of sidebar + padding-right: 84px; + } +} diff --git a/openmetadata-ui/src/main/resources/ui/src/components/Glossary/ImportGlossary/ImportGlossary.tsx b/openmetadata-ui/src/main/resources/ui/src/components/Glossary/ImportGlossary/ImportGlossary.tsx index 39628da2edf6..ec28e9d70a52 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/Glossary/ImportGlossary/ImportGlossary.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/components/Glossary/ImportGlossary/ImportGlossary.tsx @@ -12,17 +12,16 @@ */ import { Col, Row, Typography } from 'antd'; import { AxiosError } from 'axios'; -import React, { FC, useMemo, useState } from 'react'; +import React, { FC, useMemo } from 'react'; import { useTranslation } from 'react-i18next'; import { useHistory } from 'react-router-dom'; -import { CSVImportResult } from '../../../generated/type/csvImportResult'; +import { EntityType } from '../../../enums/entity.enum'; import { importGlossaryInCSVFormat } from '../../../rest/glossaryAPI'; import { getGlossaryPath } from '../../../utils/RouterUtils'; import { showErrorToast } from '../../../utils/ToastUtils'; -import { EntityImport } from '../../common/EntityImport/EntityImport.component'; +import BulkEntityImport from '../../BulkImport/BulkEntityImport.component'; import TitleBreadcrumb from '../../common/TitleBreadcrumb/TitleBreadcrumb.component'; import { TitleBreadcrumbProps } from '../../common/TitleBreadcrumb/TitleBreadcrumb.interface'; -import { GlossaryImportResult } from '../ImportResult/GlossaryImportResult.component'; import './import-glossary.less'; interface Props { @@ -32,7 +31,6 @@ interface Props { const ImportGlossary: FC = ({ glossaryName }) => { const { t } = useTranslation(); const history = useHistory(); - const [csvImportResult, setCsvImportResult] = useState(); const breadcrumbList: TitleBreadcrumbProps['titleLinks'] = useMemo( () => [ @@ -53,10 +51,13 @@ const ImportGlossary: FC = ({ glossaryName }) => { history.push(getGlossaryPath(glossaryName)); }; - const handleImportCsv = async (name: string, data: string, dryRun = true) => { + const handleImportCsv = async (data: string, dryRun = true) => { try { - const response = await importGlossaryInCSVFormat(name, data, dryRun); - setCsvImportResult(response); + const response = await importGlossaryInCSVFormat( + glossaryName, + data, + dryRun + ); return response; } catch (error) { @@ -67,7 +68,7 @@ const ImportGlossary: FC = ({ glossaryName }) => { }; return ( - + @@ -79,17 +80,13 @@ const ImportGlossary: FC = ({ glossaryName }) => { - - {csvImportResult ? ( - - ) : ( - <> - )} - + ); diff --git a/openmetadata-ui/src/main/resources/ui/src/components/Glossary/ImportResult/GlossaryImportResult.component.tsx b/openmetadata-ui/src/main/resources/ui/src/components/Glossary/ImportResult/GlossaryImportResult.component.tsx deleted file mode 100644 index 1399f883a3b5..000000000000 --- a/openmetadata-ui/src/main/resources/ui/src/components/Glossary/ImportResult/GlossaryImportResult.component.tsx +++ /dev/null @@ -1,234 +0,0 @@ -/* - * Copyright 2023 Collate. - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * http://www.apache.org/licenses/LICENSE-2.0 - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -import { Space, Typography } from 'antd'; -import { ColumnsType } from 'antd/lib/table'; -import { isEmpty } from 'lodash'; -import React, { FC, useEffect, useMemo, useState } from 'react'; -import { useTranslation } from 'react-i18next'; -import { usePapaParse } from 'react-papaparse'; -import { ReactComponent as FailBadgeIcon } from '../../../assets/svg/fail-badge.svg'; -import { ReactComponent as SuccessBadgeIcon } from '../../../assets/svg/success-badge.svg'; -import Table from '../../../components/common/Table/Table'; -import { GLOSSARY_IMPORT_STATUS } from '../../../constants/Glossary.constant'; -import { - CSVImportResult, - Status, -} from '../../../generated/type/csvImportResult'; -import { parseCSV } from '../../../utils/EntityImport/EntityImportUtils'; -import { GlossaryCSVRecord } from '../ImportGlossary/ImportGlossary.interface'; - -interface Props { - csvImportResult: CSVImportResult; -} - -export const GlossaryImportResult: FC = ({ csvImportResult }) => { - const { readString } = usePapaParse(); - const { t } = useTranslation(); - const [parsedRecords, setParsedRecords] = useState([]); - const [loading, setIsLoading] = useState(false); - - const columns: ColumnsType = useMemo( - () => [ - { - title: t('label.status'), - dataIndex: 'status', - key: 'status', - fixed: true, - render: ( - status: GlossaryCSVRecord['status'], - record: GlossaryCSVRecord - ) => { - return ( - - {status === Status.Failure ? ( - <> - - {record.details} - - ) : ( - - )} - - ); - }, - }, - { - title: t('label.parent'), - dataIndex: 'parent', - key: 'parent', - fixed: true, - render: (parent: GlossaryCSVRecord['parent']) => { - return ( - - {isEmpty(parent) ? '--' : parent} - - ); - }, - }, - { - title: t('label.name'), - dataIndex: 'name*', - key: 'name', - fixed: true, - render: (name: GlossaryCSVRecord['name*']) => { - return ( - - {name} - - ); - }, - }, - { - title: t('label.display-name'), - dataIndex: 'displayName', - key: 'displayName', - render: (displayName: GlossaryCSVRecord['displayName']) => { - return ( - - {isEmpty(displayName) ? '--' : displayName} - - ); - }, - }, - { - title: t('label.description'), - dataIndex: 'description', - key: 'description', - width: 300, - render: (description: GlossaryCSVRecord['description']) => { - return ( - - {isEmpty(description) ? '--' : description} - - ); - }, - }, - { - title: t('label.synonym-lowercase-plural'), - dataIndex: 'synonyms', - key: 'synonyms', - render: (synonyms: GlossaryCSVRecord['synonyms']) => { - const value = synonyms?.split(';').join(', '); - - return ( - - {isEmpty(synonyms) ? '--' : value} - - ); - }, - }, - { - title: t('label.related-term-plural'), - dataIndex: 'relatedTerms', - key: 'relatedTerms', - render: (relatedTerms: GlossaryCSVRecord['relatedTerms']) => { - const value = relatedTerms?.split(';').join(', '); - - return ( - - {isEmpty(relatedTerms) ? '--' : value} - - ); - }, - }, - { - title: t('label.reference-plural'), - dataIndex: 'references', - key: 'relatedTerms', - render: (references: GlossaryCSVRecord['references']) => { - return ( - - {isEmpty(references) ? '--' : references} - - ); - }, - }, - { - title: t('label.tag-plural'), - dataIndex: 'tags', - key: 'tags', - render: (tags: GlossaryCSVRecord['tags']) => { - const value = tags?.split(';').join(', '); - - return ( - - {isEmpty(tags) ? '--' : value} - - ); - }, - }, - ], - [] - ); - - const parseCsvFile = () => { - if (csvImportResult.importResultsCsv) { - readString(csvImportResult.importResultsCsv, { - worker: true, - complete: (results) => { - // results.data is returning data with unknown type - setParsedRecords( - parseCSV(results.data as string[][]).map( - (value) => ({ - ...value, - key: value['name*'], - status: GLOSSARY_IMPORT_STATUS.includes(value['details'] ?? '') - ? Status.Success - : Status.Failure, - }) - ) - ); - setIsLoading(false); - }, - }); - } - }; - - useEffect(() => { - setIsLoading(true); - parseCsvFile(); - }, [csvImportResult.importResultsCsv]); - - return ( -
- ); -}; diff --git a/openmetadata-ui/src/main/resources/ui/src/components/Glossary/ImportResult/GlossaryImportResult.test.tsx b/openmetadata-ui/src/main/resources/ui/src/components/Glossary/ImportResult/GlossaryImportResult.test.tsx deleted file mode 100644 index f505ab4faea2..000000000000 --- a/openmetadata-ui/src/main/resources/ui/src/components/Glossary/ImportResult/GlossaryImportResult.test.tsx +++ /dev/null @@ -1,141 +0,0 @@ -/* - * Copyright 2023 Collate. - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * http://www.apache.org/licenses/LICENSE-2.0 - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -import { - findByTestId, - findByText, - getAllByRole, - render, - screen, -} from '@testing-library/react'; -import React from 'react'; -import { CSVImportResult } from '../../../generated/type/csvImportResult'; -import { GlossaryImportResult } from './GlossaryImportResult.component'; - -const mockCsvImportResult = { - dryRun: true, - status: 'success', - numberOfRowsProcessed: 3, - numberOfRowsPassed: 3, - numberOfRowsFailed: 0, - importResultsCsv: `status,details,parent,name*,displayName,description,synonyms,relatedTerms,references,tags\r -success,Entity created,,Glossary2 Term,Glossary2 Term displayName,Description for Glossary2 Term,,,,\r -success,Entity created,,Glossary2 term2,Glossary2 term2 displayname,"Description, data.","ter1,term2",,,\r -failure,#INVALID_FIELD: Field 6 error - Entity First Name not found,test,Glossary3 term3,Glossary3 term3 displayname,"Description2, data.","ter3,term4",,,\r`, -}; - -describe('Import Results component should work properly', () => { - it('Should render the results', async () => { - render( - - ); - - expect( - await screen.findByTestId('import-result-table') - ).toBeInTheDocument(); - }); - - it('Should render the parsed result', async () => { - const { container } = render( - - ); - - const tableRows = getAllByRole(container, 'row'); - - expect(tableRows).toHaveLength(4); - - const firstRow = tableRows[1]; - - const rowStatus = await findByTestId(firstRow, 'success-badge'); - const rowName = await findByText(firstRow, 'Glossary2 Term'); - const rowDisplayName = await findByText( - firstRow, - 'Glossary2 Term displayName' - ); - const rowDescription = await findByText( - firstRow, - 'Description for Glossary2 Term' - ); - - expect(rowStatus).toBeInTheDocument(); - expect(rowName).toBeInTheDocument(); - expect(rowDisplayName).toBeInTheDocument(); - expect(rowDescription).toBeInTheDocument(); - }); - - it('Should render the parsed result properly with special character', async () => { - const { container } = render( - - ); - - const tableRows = getAllByRole(container, 'row'); - - expect(tableRows).toHaveLength(4); - - const secondRow = tableRows[2]; - - const rowStatus = await findByTestId(secondRow, 'success-badge'); - const rowDisplayName = await findByText( - secondRow, - 'Glossary2 term2 displayname' - ); - const rowName = await findByText(secondRow, 'Glossary2 term2'); - const rowDescription = await findByText(secondRow, 'Description, data.'); - const synonym = await findByText(secondRow, 'ter1,term2'); - - expect(rowStatus).toBeInTheDocument(); - expect(rowName).toBeInTheDocument(); - expect(rowDisplayName).toBeInTheDocument(); - expect(rowDescription).toBeInTheDocument(); - expect(synonym).toBeInTheDocument(); - }); - - it('Should render the parsed result even if its failed row', async () => { - const { container } = render( - - ); - - const tableRows = getAllByRole(container, 'row'); - - expect(tableRows).toHaveLength(4); - - const thirdRow = tableRows[3]; - - const rowStatus = await findByTestId(thirdRow, 'failure-badge'); - const errorMsg = await findByText( - thirdRow, - '#INVALID_FIELD: Field 6 error - Entity First Name not found' - ); - const rowDisplayName = await findByText( - thirdRow, - 'Glossary3 term3 displayname' - ); - const rowName = await findByText(thirdRow, 'Glossary3 term3'); - const rowDescription = await findByText(thirdRow, 'Description2, data.'); - const synonym = await findByText(thirdRow, 'ter3,term4'); - - expect(rowStatus).toBeInTheDocument(); - expect(errorMsg).toBeInTheDocument(); - expect(rowName).toBeInTheDocument(); - expect(rowDisplayName).toBeInTheDocument(); - expect(rowDescription).toBeInTheDocument(); - expect(synonym).toBeInTheDocument(); - }); -}); diff --git a/openmetadata-ui/src/main/resources/ui/src/components/UploadFile/UploadFile.interface.ts b/openmetadata-ui/src/main/resources/ui/src/components/UploadFile/UploadFile.interface.ts new file mode 100644 index 000000000000..811a36b951cc --- /dev/null +++ b/openmetadata-ui/src/main/resources/ui/src/components/UploadFile/UploadFile.interface.ts @@ -0,0 +1,24 @@ +/* + * Copyright 2024 Collate. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * http://www.apache.org/licenses/LICENSE-2.0 + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import { RcFile } from 'antd/lib/upload'; + +declare type BeforeUploadValueType = void | boolean | string | Blob | File; + +export interface UploadFileProps { + fileType: string; + beforeUpload?: ( + file: RcFile, + FileList: RcFile[] + ) => BeforeUploadValueType | Promise; + onCSVUploaded: (event: ProgressEvent) => void; +} diff --git a/openmetadata-ui/src/main/resources/ui/src/components/UploadFile/UploadFile.tsx b/openmetadata-ui/src/main/resources/ui/src/components/UploadFile/UploadFile.tsx new file mode 100644 index 000000000000..f594d2b0e08d --- /dev/null +++ b/openmetadata-ui/src/main/resources/ui/src/components/UploadFile/UploadFile.tsx @@ -0,0 +1,79 @@ +/* + * Copyright 2024 Collate. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * http://www.apache.org/licenses/LICENSE-2.0 + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import { Space, Typography, UploadProps } from 'antd'; +import Dragger from 'antd/lib/upload/Dragger'; +import { AxiosError } from 'axios'; +import { t } from 'i18next'; +import React, { FC, useCallback, useState } from 'react'; +import { ReactComponent as ImportIcon } from '../../assets/svg/ic-drag-drop.svg'; +import { Transi18next } from '../../utils/CommonUtils'; +import { showErrorToast } from '../../utils/ToastUtils'; +import Loader from '../common/Loader/Loader'; +import { UploadFileProps } from './UploadFile.interface'; + +export const UploadFile: FC = ({ + fileType, + beforeUpload, + onCSVUploaded, +}) => { + const [uploading, setUploading] = useState(false); + + const handleUpload: UploadProps['customRequest'] = useCallback( + (options) => { + setUploading(true); + try { + const reader = new FileReader(); + reader.onload = onCSVUploaded; + reader.onerror = () => { + throw t('server.unexpected-error'); + }; + reader.readAsText(options.file as Blob); + } catch (error) { + showErrorToast(error as AxiosError); + } finally { + setUploading(false); + } + }, + [onCSVUploaded] + ); + + return uploading ? ( + + ) : ( + + + + + } + values={{ + text: t('label.browse'), + }} + /> + + + + ); +}; diff --git a/openmetadata-ui/src/main/resources/ui/src/components/common/EntityImport/EntityImport.component.tsx b/openmetadata-ui/src/main/resources/ui/src/components/common/EntityImport/EntityImport.component.tsx index b8c5ef60fcc9..3c89da64af8a 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/common/EntityImport/EntityImport.component.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/components/common/EntityImport/EntityImport.component.tsx @@ -10,31 +10,20 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -import { - Affix, - Button, - Card, - Col, - Row, - Space, - Typography, - UploadProps, -} from 'antd'; -import Dragger from 'antd/lib/upload/Dragger'; +import { Affix, Button, Card, Col, Row, Space, Typography } from 'antd'; import { AxiosError } from 'axios'; import { isUndefined } from 'lodash'; import React, { useMemo, useState } from 'react'; import { useTranslation } from 'react-i18next'; -import { ReactComponent as ImportIcon } from '../../../assets/svg/ic-drag-drop.svg'; import { ReactComponent as SuccessBadgeIcon } from '../../../assets/svg/success-badge.svg'; import { STEPS_FOR_IMPORT_ENTITY } from '../../../constants/entity.constants'; import { CSVImportResult, Status, } from '../../../generated/type/csvImportResult'; -import { Transi18next } from '../../../utils/CommonUtils'; import { showErrorToast } from '../../../utils/ToastUtils'; import Stepper from '../../Settings/Services/Ingestion/IngestionStepper/IngestionStepper.component'; +import { UploadFile } from '../../UploadFile/UploadFile'; import Loader from '../Loader/Loader'; import './entity-import.style.less'; import { EntityImportProps } from './EntityImport.interface'; @@ -54,7 +43,6 @@ export const EntityImport = ({ const [csvFileResult, setCsvFileResult] = useState(''); const [csvImportResult, setCsvImportResult] = useState(); const [activeStep, setActiveStep] = useState(1); - const [uploading, setUploading] = useState(false); const { isFailure, isAborted } = useMemo(() => { const status = csvImportResult?.status; @@ -71,7 +59,6 @@ export const EntityImport = ({ try { const result = e.target?.result as string; if (result) { - setUploading(true); const response = await onImport(entityName, result); if (response) { @@ -82,27 +69,6 @@ export const EntityImport = ({ } } catch (error) { setCsvImportResult(undefined); - } finally { - setUploading(false); - } - }; - - const handleUpload: UploadProps['customRequest'] = (options) => { - setIsLoading(true); - try { - const reader = new FileReader(); - - reader.readAsText(options.file as Blob); - - reader.addEventListener('load', handleLoadData); - - reader.addEventListener('error', () => { - throw t('server.unexpected-error'); - }); - } catch (error) { - showErrorToast(error as AxiosError); - } finally { - setIsLoading(false); } }; @@ -136,39 +102,14 @@ export const EntityImport = ({ <> {activeStep === 1 && ( - {uploading ? ( - - ) : ( - { - setFileName(file.name); - }} - className="file-dragger-wrapper p-lg bg-white" - customRequest={handleUpload} - data-testid="upload-file-widget" - multiple={false} - showUploadList={false}> - - - - - } - values={{ - text: t('label.browse'), - }} - /> - - - - )} + { + setFileName(file.name); + }} + fileType=".csv" + onCSVUploaded={handleLoadData} + /> + - {activeStep === 1 && !hideAddButton && ( + {activeStep === 1 && ( diff --git a/openmetadata-ui/src/main/resources/ui/src/components/BulkImport/BulkEntityImport.interface.ts b/openmetadata-ui/src/main/resources/ui/src/components/BulkImport/BulkEntityImport.interface.ts index 0669ad9f9b33..a65a5ff11009 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/BulkImport/BulkEntityImport.interface.ts +++ b/openmetadata-ui/src/main/resources/ui/src/components/BulkImport/BulkEntityImport.interface.ts @@ -16,7 +16,6 @@ import { CSVImportResult } from '../../generated/type/csvImportResult'; export interface BulkImportProps { entityType: EntityType; fqn: string; - hideAddButton?: boolean; onValidateCsvString: ( data: string, dryRun?: boolean diff --git a/openmetadata-ui/src/main/resources/ui/src/components/Glossary/ImportGlossary/ImportGlossary.tsx b/openmetadata-ui/src/main/resources/ui/src/components/Glossary/ImportGlossary/ImportGlossary.tsx index ec28e9d70a52..ab6cd178d70f 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/Glossary/ImportGlossary/ImportGlossary.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/components/Glossary/ImportGlossary/ImportGlossary.tsx @@ -81,7 +81,6 @@ const ImportGlossary: FC = ({ glossaryName }) => { { + const { t } = useTranslation(); + const [isLoading, setIsLoading] = useState(false); + const [isSaveLoading, setIsSaveLoading] = useState(false); + const [customPropertyValue, setCustomPropertyValue] = + useState({}); + const [customPropertyTypes, setCustomPropertyTypes] = useState(); + + const fetchTypeDetail = async () => { + setIsLoading(true); + try { + const response = await getTypeByFQN(entityType); + setCustomPropertyTypes(response); + setCustomPropertyValue( + convertCustomPropertyStringToEntityExtension(value, response) + ); + } catch (err) { + showErrorToast(err as AxiosError); + } finally { + setIsLoading(false); + } + }; + + const handleSaveData = async () => { + setIsSaveLoading(true); + await onSave( + convertEntityExtensionToCustomPropertyString( + customPropertyValue, + customPropertyTypes + ) + ); + setIsSaveLoading(false); + }; + + const onExtensionUpdate = async (data: GlossaryTerm) => { + setCustomPropertyValue(data.extension); + }; + + useEffect(() => { + fetchTypeDetail(); + }, []); + + return ( + + {t('label.cancel')} + , + , + ]} + maskClosable={false} + open={visible} + width="90%" + onCancel={onCancel}> + {isLoading ? ( + + ) : ( + + )} + + ); +}; diff --git a/openmetadata-ui/src/main/resources/ui/src/components/Modals/ModalWithCustomProperty/ModalWithMarkdownEditor.interface.ts b/openmetadata-ui/src/main/resources/ui/src/components/Modals/ModalWithCustomProperty/ModalWithMarkdownEditor.interface.ts new file mode 100644 index 000000000000..b090010a09e8 --- /dev/null +++ b/openmetadata-ui/src/main/resources/ui/src/components/Modals/ModalWithCustomProperty/ModalWithMarkdownEditor.interface.ts @@ -0,0 +1,34 @@ +/* + * Copyright 2024 Collate. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * http://www.apache.org/licenses/LICENSE-2.0 + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import { EntityType } from '../../../enums/entity.enum'; +import { EntityReference } from '../../../generated/entity/type'; + +export type ExtensionDataTypes = + | string + | string[] + | EntityReference + | EntityReference[] + | { start: string; end: string }; + +export interface ExtensionDataProps { + [key: string]: ExtensionDataTypes; +} + +export type ModalWithCustomPropertyEditorProps = { + entityType: EntityType; + header: string; + value: string; + onSave: (extension: string) => Promise; + onCancel?: () => void; + visible: boolean; +}; diff --git a/openmetadata-ui/src/main/resources/ui/src/constants/regex.constants.ts b/openmetadata-ui/src/main/resources/ui/src/constants/regex.constants.ts index fe8adf13a08a..1b0d26ef9b5a 100644 --- a/openmetadata-ui/src/main/resources/ui/src/constants/regex.constants.ts +++ b/openmetadata-ui/src/main/resources/ui/src/constants/regex.constants.ts @@ -54,3 +54,9 @@ export const TASK_SANITIZE_VALUE_REGEX = /^"|"$/g; export const TIMESTAMP_UNIX_IN_MILLISECONDS_REGEX = /^\d{13}$/; export const ALL_ASTERISKS_REGEX = /^\*+$/; + +// Split the input into pairs using `;` and handle quoted strings properly +export const SEMICOLON_SPLITTER = /;(?=(?:(?:[^"]*"){2})*[^"]*$)/; + +// Use regex to check if the string starts and ends with escape characters +export const VALIDATE_ESCAPE_START_END_REGEX = /^(\\+|"+)([\s\S]*?)(\\+|"+)$/; diff --git a/openmetadata-ui/src/main/resources/ui/src/utils/CSV/CSV.utils.tsx b/openmetadata-ui/src/main/resources/ui/src/utils/CSV/CSV.utils.tsx index 9793e7e36a78..6a346379fa76 100644 --- a/openmetadata-ui/src/main/resources/ui/src/utils/CSV/CSV.utils.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/utils/CSV/CSV.utils.tsx @@ -11,11 +11,22 @@ * limitations under the License. */ import { TypeColumn } from '@inovua/reactdatagrid-community/types'; -import { compact, get, isEmpty, startCase } from 'lodash'; +import { compact, get, isEmpty, isUndefined, startCase } from 'lodash'; import React from 'react'; import { ReactComponent as SuccessBadgeIcon } from '../..//assets/svg/success-badge.svg'; import { ReactComponent as FailBadgeIcon } from '../../assets/svg/fail-badge.svg'; +import { + ExtensionDataProps, + ExtensionDataTypes, +} from '../../components/Modals/ModalWithCustomProperty/ModalWithMarkdownEditor.interface'; +import { SEMICOLON_SPLITTER } from '../../constants/regex.constants'; +import { + CustomProperty, + EntityReference, + Type, +} from '../../generated/entity/type'; import { Status } from '../../generated/type/csvImportResult'; +import { removeOuterEscapes } from '../CommonUtils'; import csvUtilsClassBase from './CSVUtilsClassBase'; export interface EditorProps { @@ -110,7 +121,9 @@ export const getCSVStringFromColumnsAndDataSource = ( .map((col) => { const value = get(row, col.name ?? '', ''); const colName = col.name ?? ''; - if ( + if (colName === 'extension') { + return `"${value.replaceAll(new RegExp('"', 'g'), '""')}"`; + } else if ( value.includes(',') || value.includes('\n') || colName.includes('tags') || @@ -127,3 +140,169 @@ export const getCSVStringFromColumnsAndDataSource = ( return [header, ...compact(rows)].join('\n'); }; + +const convertCustomPropertyStringToValueExtensionBasedOnType = ( + value: string, + customProperty: CustomProperty +) => { + switch (customProperty.propertyType.name) { + case 'entityReference': { + const entity = value.split(':'); + + return { + type: entity[0], + fullyQualifiedName: entity[1], + name: entity[1], + } as EntityReference; + } + + case 'entityReferenceList': { + const values = value.split('|'); + + return values.map((entity) => { + const [key, itemValue] = entity.split(':'); + + return { + type: key, + fullyQualifiedName: itemValue, + name: itemValue, + } as EntityReference; + }); + } + case 'enum': { + if (value.includes('|')) { + return value.split('|'); + } else { + return [value]; + } + } + + case 'timeInterval': { + const [start, end] = value.split(':'); + + return { + start, + end, + }; + } + default: + return value.replace(/^["']|["']$/g, '').trim(); + } +}; + +const convertCustomPropertyValueExtensionToStringBasedOnType = ( + value: ExtensionDataTypes, + customProperty: CustomProperty +) => { + switch (customProperty.propertyType.name) { + case 'entityReference': { + const entity = value as EntityReference; + + return `${entity.type}:${entity.fullyQualifiedName ?? ''}`; + } + + case 'entityReferenceList': { + let stringList = ''; + const values = value as unknown as EntityReference[]; + values.forEach((item, index) => { + stringList += `${item.type}:${item.fullyQualifiedName ?? ''}${ + index + 1 === values.length ? '' : '|' + }`; + }); + + return stringList; + } + case 'enum': + return (value as unknown as string[]).map((item) => item).join('|'); + + case 'timeInterval': { + const interval = value as { start: string; end: string }; + + return `${interval.start}:${interval.end}`; + } + + default: + return value; + } +}; + +export const convertCustomPropertyStringToEntityExtension = ( + value: string, + customPropertyType: Type +) => { + if (isUndefined(customPropertyType)) { + return {}; + } + + const keyAndValueTypes: Record = {}; + + const result: ExtensionDataProps = {}; + + customPropertyType.customProperties?.forEach( + (cp) => (keyAndValueTypes[cp.name] = cp) + ); + + // Split the input into pairs using `;` and handle quoted strings properly + const pairs = value.split(SEMICOLON_SPLITTER); + + pairs.forEach((pair) => { + const cleanedText = removeOuterEscapes(pair); + + const [key, ...valueParts] = cleanedText.split(':'); + const value = valueParts.join(':').trim(); // Join back in case of multiple `:` + + // Clean up quotes if they are around the value + if (key && value) { + result[key.trim()] = + convertCustomPropertyStringToValueExtensionBasedOnType( + value, + keyAndValueTypes[key] + ); + } + }); + + return result; +}; + +export const convertEntityExtensionToCustomPropertyString = ( + value: ExtensionDataProps, + customPropertyType?: Type +) => { + if (isUndefined(customPropertyType)) { + return ''; + } + + const keyAndValueTypes: Record = {}; + customPropertyType.customProperties?.forEach( + (cp) => (keyAndValueTypes[cp.name] = cp) + ); + + let convertedString = ''; + + const objectArray = Object.entries(value); + + objectArray.forEach(([key, value], index) => { + const isLastElement = objectArray.length - 1 === index; + if (keyAndValueTypes[key]) { + const stringValue = + convertCustomPropertyValueExtensionToStringBasedOnType( + value, + keyAndValueTypes[key] + ); + + if ( + ['markdown', 'sqlQuery'].includes( + keyAndValueTypes[key].propertyType.name ?? '' + ) + ) { + convertedString += `"${`${key}:${stringValue}`}"${ + isLastElement ? '' : ';' + }`; + } else { + convertedString += `${key}:${stringValue}${isLastElement ? '' : ';'}`; + } + } + }); + + return convertedString; +}; diff --git a/openmetadata-ui/src/main/resources/ui/src/utils/CSV/CSVUtilsClassBase.tsx b/openmetadata-ui/src/main/resources/ui/src/utils/CSV/CSVUtilsClassBase.tsx index 53c142c516b9..163fe52fd7d5 100644 --- a/openmetadata-ui/src/main/resources/ui/src/utils/CSV/CSVUtilsClassBase.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/utils/CSV/CSVUtilsClassBase.tsx @@ -20,6 +20,7 @@ import DomainSelectableList from '../../components/common/DomainSelectableList/D import InlineEdit from '../../components/common/InlineEdit/InlineEdit.component'; import TierCard from '../../components/common/TierCard/TierCard'; import { UserTeamSelectableList } from '../../components/common/UserTeamSelectableList/UserTeamSelectableList.component'; +import { ModalWithCustomPropertyEditor } from '../../components/Modals/ModalWithCustomProperty/ModalWithCustomPropertyEditor.component'; import { ModalWithMarkdownEditor } from '../../components/Modals/ModalWithMarkdownEditor/ModalWithMarkdownEditor'; import { EntityType } from '../../enums/entity.enum'; import { Tag } from '../../generated/entity/classification/tag'; @@ -224,6 +225,27 @@ class CSVUtilsClassBase { ); }; + case 'extension': + return ({ value, ...props }: EditorProps) => { + const handleSave = async (extension: string) => { + props.onChange(extension); + + setTimeout(() => { + props.onComplete(extension); + }, 1); + }; + + return ( + + ); + }; default: return undefined; } diff --git a/openmetadata-ui/src/main/resources/ui/src/utils/CommonUtils.tsx b/openmetadata-ui/src/main/resources/ui/src/utils/CommonUtils.tsx index 96ed498366ef..264eba154b4c 100644 --- a/openmetadata-ui/src/main/resources/ui/src/utils/CommonUtils.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/utils/CommonUtils.tsx @@ -54,7 +54,10 @@ import { LOCALSTORAGE_RECENTLY_VIEWED, } from '../constants/constants'; import { FEED_COUNT_INITIAL_DATA } from '../constants/entity.constants'; -import { UrlEntityCharRegEx } from '../constants/regex.constants'; +import { + UrlEntityCharRegEx, + VALIDATE_ESCAPE_START_END_REGEX, +} from '../constants/regex.constants'; import { SIZE } from '../enums/common.enum'; import { EntityType, FqnPart } from '../enums/entity.enum'; import { PipelineType } from '../generated/entity/services/ingestionPipelines/ingestionPipeline'; @@ -925,3 +928,11 @@ export const isDeleted = (deleted: unknown): boolean => { ? false : true; }; + +export const removeOuterEscapes = (input: string) => { + // Use regex to check if the string starts and ends with escape characters + const match = input.match(VALIDATE_ESCAPE_START_END_REGEX); + + // Return the middle part without the outer escape characters or the original input if no match + return match ? match[2] : input; +}; From 6ec8eb95d33ef1f84abdcad3e5752bf568789f8e Mon Sep 17 00:00:00 2001 From: Ashish Gupta Date: Thu, 26 Sep 2024 21:33:53 +0530 Subject: [PATCH 10/39] fix error in custom property edit modal on new line empty custom property --- .../ModalWithCustomPropertyEditor.component.tsx | 2 +- .../ModalWithMarkdownEditor.interface.ts | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/openmetadata-ui/src/main/resources/ui/src/components/Modals/ModalWithCustomProperty/ModalWithCustomPropertyEditor.component.tsx b/openmetadata-ui/src/main/resources/ui/src/components/Modals/ModalWithCustomProperty/ModalWithCustomPropertyEditor.component.tsx index fd3f0f09beb2..b8eae91cbe01 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/Modals/ModalWithCustomProperty/ModalWithCustomPropertyEditor.component.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/components/Modals/ModalWithCustomProperty/ModalWithCustomPropertyEditor.component.tsx @@ -51,7 +51,7 @@ export const ModalWithCustomPropertyEditor = ({ const response = await getTypeByFQN(entityType); setCustomPropertyTypes(response); setCustomPropertyValue( - convertCustomPropertyStringToEntityExtension(value, response) + convertCustomPropertyStringToEntityExtension(value ?? '', response) ); } catch (err) { showErrorToast(err as AxiosError); diff --git a/openmetadata-ui/src/main/resources/ui/src/components/Modals/ModalWithCustomProperty/ModalWithMarkdownEditor.interface.ts b/openmetadata-ui/src/main/resources/ui/src/components/Modals/ModalWithCustomProperty/ModalWithMarkdownEditor.interface.ts index b090010a09e8..91e0ce67f40b 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/Modals/ModalWithCustomProperty/ModalWithMarkdownEditor.interface.ts +++ b/openmetadata-ui/src/main/resources/ui/src/components/Modals/ModalWithCustomProperty/ModalWithMarkdownEditor.interface.ts @@ -27,7 +27,7 @@ export interface ExtensionDataProps { export type ModalWithCustomPropertyEditorProps = { entityType: EntityType; header: string; - value: string; + value?: string; onSave: (extension: string) => Promise; onCancel?: () => void; visible: boolean; From b4a8f51c496c95760e493420fef633bae07f9280 Mon Sep 17 00:00:00 2001 From: Ashish Gupta Date: Fri, 27 Sep 2024 00:38:14 +0530 Subject: [PATCH 11/39] added entity type from root to support other bulk import entity as well --- .../BulkImport/BulkEntityImport.component.tsx | 18 +++++++++++++----- .../Glossary/ImportGlossary/ImportGlossary.tsx | 2 +- ...ModalWithCustomPropertyEditor.component.tsx | 2 +- .../resources/ui/src/utils/CSV/CSV.utils.tsx | 18 +++++++++++++----- .../ui/src/utils/CSV/CSVUtilsClassBase.tsx | 5 +++-- 5 files changed, 31 insertions(+), 14 deletions(-) diff --git a/openmetadata-ui/src/main/resources/ui/src/components/BulkImport/BulkEntityImport.component.tsx b/openmetadata-ui/src/main/resources/ui/src/components/BulkImport/BulkEntityImport.component.tsx index 0687450ea261..4a7989d3b42f 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/BulkImport/BulkEntityImport.component.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/components/BulkImport/BulkEntityImport.component.tsx @@ -22,6 +22,7 @@ import React, { MutableRefObject, useCallback, useState } from 'react'; import { useTranslation } from 'react-i18next'; import { usePapaParse } from 'react-papaparse'; +import { capitalize } from 'lodash'; import { ENTITY_IMPORT_STEPS, VALIDATION_STEP, @@ -73,7 +74,8 @@ const BulkEntityImport = ({ (results: { data: string[][] }) => { // results.data is returning data with unknown type const { columns, dataSource } = getEntityColumnsAndDataSourceFromCSV( - results.data as string[][] + results.data as string[][], + entityType ); setDataSource(dataSource); setColumns(columns); @@ -81,7 +83,7 @@ const BulkEntityImport = ({ setActiveStep(VALIDATION_STEP.EDIT_VALIDATE); setTimeout(focusToGrid, 500); }, - [setDataSource, setColumns, setActiveStep, focusToGrid] + [entityType, setDataSource, setColumns, setActiveStep, focusToGrid] ); const validateCsvString = useCallback( @@ -157,7 +159,10 @@ const BulkEntityImport = ({ complete: (results) => { // results.data is returning data with unknown type setValidateCSVData( - getEntityColumnsAndDataSourceFromCSV(results.data as string[][]) + getEntityColumnsAndDataSourceFromCSV( + results.data as string[][], + entityType + ) ); }, }); @@ -165,7 +170,7 @@ const BulkEntityImport = ({ } else { showSuccessToast( t('message.entity-details-updated', { - entityType, + entityType: capitalize(entityType), fqn, }) ); @@ -180,7 +185,10 @@ const BulkEntityImport = ({ complete: (results) => { // results.data is returning data with unknown type setValidateCSVData( - getEntityColumnsAndDataSourceFromCSV(results.data as string[][]) + getEntityColumnsAndDataSourceFromCSV( + results.data as string[][], + entityType + ) ); }, }); diff --git a/openmetadata-ui/src/main/resources/ui/src/components/Glossary/ImportGlossary/ImportGlossary.tsx b/openmetadata-ui/src/main/resources/ui/src/components/Glossary/ImportGlossary/ImportGlossary.tsx index ab6cd178d70f..2ae67b4ee5af 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/Glossary/ImportGlossary/ImportGlossary.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/components/Glossary/ImportGlossary/ImportGlossary.tsx @@ -81,7 +81,7 @@ const ImportGlossary: FC = ({ glossaryName }) => { { +export const getColumnConfig = ( + column: string, + entityType: EntityType +): TypeColumn => { const colType = column.split('.').pop() ?? ''; return { @@ -75,16 +79,20 @@ export const getColumnConfig = (column: string): TypeColumn => { name: column, defaultFlex: 1, sortable: false, - renderEditor: csvUtilsClassBase.getEditor(colType), + renderEditor: csvUtilsClassBase.getEditor(colType, entityType), minWidth: COLUMNS_WIDTH[colType] ?? 180, render: column === 'status' ? statusRenderer : undefined, } as TypeColumn; }; -export const getEntityColumnsAndDataSourceFromCSV = (csv: string[][]) => { +export const getEntityColumnsAndDataSourceFromCSV = ( + csv: string[][], + entityType: EntityType +) => { const [cols, ...rows] = csv; - const columns = cols?.map(getColumnConfig) ?? []; + const columns = + cols?.map((column) => getColumnConfig(column, entityType)) ?? []; const dataSource = rows.map((row, idx) => { @@ -145,7 +153,7 @@ const convertCustomPropertyStringToValueExtensionBasedOnType = ( value: string, customProperty: CustomProperty ) => { - switch (customProperty.propertyType.name) { + switch (customProperty?.propertyType.name) { case 'entityReference': { const entity = value.split(':'); diff --git a/openmetadata-ui/src/main/resources/ui/src/utils/CSV/CSVUtilsClassBase.tsx b/openmetadata-ui/src/main/resources/ui/src/utils/CSV/CSVUtilsClassBase.tsx index 163fe52fd7d5..448e076c66c5 100644 --- a/openmetadata-ui/src/main/resources/ui/src/utils/CSV/CSVUtilsClassBase.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/utils/CSV/CSVUtilsClassBase.tsx @@ -32,7 +32,8 @@ import { EditorProps } from './CSV.utils'; class CSVUtilsClassBase { public getEditor( - column: string + column: string, + entityType: EntityType ): ((props: EditorProps) => ReactNode) | undefined { switch (column) { case 'owner': @@ -238,7 +239,7 @@ class CSVUtilsClassBase { return ( Date: Fri, 27 Sep 2024 00:56:45 +0530 Subject: [PATCH 12/39] fix the quote removing due to the regex in the string type --- .../src/main/resources/ui/src/utils/CSV/CSV.utils.tsx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/openmetadata-ui/src/main/resources/ui/src/utils/CSV/CSV.utils.tsx b/openmetadata-ui/src/main/resources/ui/src/utils/CSV/CSV.utils.tsx index b3b00b88113c..42f658c3652c 100644 --- a/openmetadata-ui/src/main/resources/ui/src/utils/CSV/CSV.utils.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/utils/CSV/CSV.utils.tsx @@ -194,7 +194,7 @@ const convertCustomPropertyStringToValueExtensionBasedOnType = ( }; } default: - return value.replace(/^["']|["']$/g, '').trim(); + return value; } }; From e5176f957141e7f7f21f7a44d2cfd92bb0f44c4d Mon Sep 17 00:00:00 2001 From: sonikashah Date: Fri, 27 Sep 2024 10:18:09 +0530 Subject: [PATCH 13/39] Add backend tests , and error msg improvements --- .../java/org/openmetadata/csv/CsvUtil.java | 5 +- .../java/org/openmetadata/csv/EntityCsv.java | 94 ++++--- .../org/openmetadata/csv/CsvUtilTest.java | 31 +++ .../service/resources/EntityResourceTest.java | 12 + .../glossary/GlossaryResourceTest.java | 231 +++++++++++++++++- .../resources/metadata/TypeResourceTest.java | 12 + 6 files changed, 344 insertions(+), 41 deletions(-) diff --git a/openmetadata-service/src/main/java/org/openmetadata/csv/CsvUtil.java b/openmetadata-service/src/main/java/org/openmetadata/csv/CsvUtil.java index ad9a40b98095..09ca7beaba92 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/csv/CsvUtil.java +++ b/openmetadata-service/src/main/java/org/openmetadata/csv/CsvUtil.java @@ -273,10 +273,10 @@ private static String quoteCsvField(String str) { return str; } - public static void addExtension(List csvRecord, Object extension) { + public static List addExtension(List csvRecord, Object extension) { if (extension == null) { csvRecord.add(null); - return; + return csvRecord; } ObjectMapper objectMapper = new ObjectMapper(); @@ -293,6 +293,7 @@ public static void addExtension(List csvRecord, Object extension) { .collect(Collectors.joining(FIELD_SEPARATOR)); csvRecord.add(extensionString); + return csvRecord; } private static String formatValue(Object value) { diff --git a/openmetadata-service/src/main/java/org/openmetadata/csv/EntityCsv.java b/openmetadata-service/src/main/java/org/openmetadata/csv/EntityCsv.java index dde5f6738578..50fc7807e45d 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/csv/EntityCsv.java +++ b/openmetadata-service/src/main/java/org/openmetadata/csv/EntityCsv.java @@ -370,7 +370,7 @@ private void validateExtension( // Fetch the JSON schema and property type for the given field name JsonSchema jsonSchema = TypeRegistry.instance().getSchema(entityType, fieldName); if (jsonSchema == null) { - importFailure(printer, "Unknown custom field: " + fieldName, csvRecord); + importFailure(printer, invalidCustomPropertyKey(fieldNumber, fieldName), csvRecord); return; } String customPropertyType = TypeRegistry.getCustomPropertyType(entityType, fieldName); @@ -382,34 +382,52 @@ private void validateExtension( case "entityReference", "entityReferenceList" -> { boolean isList = "entityReferenceList".equals(customPropertyType); fieldValue = - parseEntityReferences(fieldValue.toString(), printer, csvRecord, fieldNumber, isList); + parseEntityReferences(printer, csvRecord, fieldNumber, fieldValue.toString(), isList); } case "date-cp", "dateTime-cp", "time-cp" -> fieldValue = getFormattedDateTimeField( - customPropertyType, - fieldValue.toString(), - propertyConfig, - fieldName, printer, - csvRecord); + csvRecord, + fieldNumber, + fieldName, + fieldValue.toString(), + customPropertyType, + propertyConfig); case "enum", "enumList" -> { List enumKeys = listOrEmpty(fieldToInternalArray(fieldValue.toString())); fieldValue = enumKeys.isEmpty() ? null : enumKeys; } case "timeInterval" -> fieldValue = - handleTimeInterval( - printer, csvRecord, fieldNumber, fieldName, fieldValue, extensionMap, jsonSchema); - case "number", "integer", "timestamp" -> fieldValue = Long.parseLong(fieldValue.toString()); + handleTimeInterval(printer, csvRecord, fieldNumber, fieldName, fieldValue); + case "number", "integer", "timestamp" -> { + try { + fieldValue = Long.parseLong(fieldValue.toString()); + } catch (NumberFormatException e) { + importFailure( + printer, + invalidCustomPropertyValue( + fieldNumber, fieldName, customPropertyType, fieldValue.toString()), + csvRecord); + fieldValue = null; + } + } default -> {} } // Validate the field against the JSON schema validateAndUpdateExtension( - printer, csvRecord, fieldNumber, fieldName, fieldValue, extensionMap, jsonSchema); + printer, + csvRecord, + fieldNumber, + fieldName, + fieldValue, + customPropertyType, + extensionMap, + jsonSchema); } } private Object parseEntityReferences( - String fieldValue, CSVPrinter printer, CSVRecord csvRecord, int fieldNumber, boolean isList) + CSVPrinter printer, CSVRecord csvRecord, int fieldNumber, String fieldValue, boolean isList) throws IOException { List entityReferences = new ArrayList<>(); @@ -439,12 +457,13 @@ private Object parseEntityReferences( } protected String getFormattedDateTimeField( - String fieldType, - String fieldValue, - String propertyConfig, - String fieldName, CSVPrinter printer, - CSVRecord csvRecord) + CSVRecord csvRecord, + int fieldNumber, + String fieldName, + String fieldValue, + String fieldType, + String propertyConfig) throws IOException { try { DateTimeFormatter formatter = DateTimeFormatter.ofPattern(propertyConfig, Locale.ENGLISH); @@ -470,21 +489,14 @@ protected String getFormattedDateTimeField( } catch (DateTimeParseException e) { importFailure( printer, - String.format( - "Custom field %s value is not as per defined format %s", fieldName, propertyConfig), + invalidCustomPropertyFieldFormat(fieldNumber, fieldName, fieldType, propertyConfig), csvRecord); return null; } } private Map handleTimeInterval( - CSVPrinter printer, - CSVRecord csvRecord, - int fieldNumber, - String fieldName, - Object fieldValue, - Map extensionMap, - JsonSchema jsonSchema) + CSVPrinter printer, CSVRecord csvRecord, int fieldNumber, String fieldName, Object fieldValue) throws IOException { List timestampValues = fieldToEntities(fieldValue.toString()); Map timestampMap = new HashMap<>(); @@ -494,7 +506,10 @@ private Map handleTimeInterval( } else { importFailure( printer, - invalidField(fieldNumber, String.format("Invalid timestamp format in %s", fieldName)), + invalidField( + fieldNumber, + invalidCustomPropertyFieldFormat( + fieldNumber, fieldName, "timeInterval", "start:end")), csvRecord); } return timestampMap; @@ -506,6 +521,7 @@ private void validateAndUpdateExtension( int fieldNumber, String fieldName, Object fieldValue, + String customPropertyType, Map extensionMap, JsonSchema jsonSchema) throws IOException { @@ -518,7 +534,8 @@ private void validateAndUpdateExtension( if (!validationMessages.isEmpty()) { importFailure( printer, - invalidCustomPropertyValue(fieldNumber, fieldName, validationMessages.toString()), + invalidCustomPropertyValue( + fieldNumber, fieldName, customPropertyType, validationMessages.toString()), csvRecord); } else { extensionMap.put(fieldName, fieldValue); // Add to extensionMap if valid @@ -753,13 +770,28 @@ public static String invalidExtension(int field, String key, String value) { + key + ", Value = " + value - + " .Extensions should be of format customPropertyName:customPropertyValue"; + + " . Extensions should be of format customPropertyName:customPropertyValue"; + return String.format(FIELD_ERROR_MSG, CsvErrorType.INVALID_FIELD, field + 1, error); + } + + public static String invalidCustomPropertyKey(int field, String key) { + String error = String.format("Unknown custom field: %s", key); + return String.format(FIELD_ERROR_MSG, CsvErrorType.INVALID_FIELD, field + 1, error); + } + + public static String invalidCustomPropertyValue( + int field, String key, String fieldType, String value) { + String error = + String.format("Invalid value of Key = %s of type %s, Value = %s", key, fieldType, value); return String.format(FIELD_ERROR_MSG, CsvErrorType.INVALID_FIELD, field + 1, error); } - public static String invalidCustomPropertyValue(int field, String key, String value) { + public static String invalidCustomPropertyFieldFormat( + int field, String fieldName, String fieldType, String propertyConfig) { String error = - "Invalid key-value pair in extension string: Key = " + key + ", Value = " + value; + String.format( + "Custom field %s value of type %s is not as per defined format %s", + fieldName, fieldType, propertyConfig); return String.format(FIELD_ERROR_MSG, CsvErrorType.INVALID_FIELD, field + 1, error); } diff --git a/openmetadata-service/src/test/java/org/openmetadata/csv/CsvUtilTest.java b/openmetadata-service/src/test/java/org/openmetadata/csv/CsvUtilTest.java index e7e063516ce2..c76b9bb79ecb 100644 --- a/openmetadata-service/src/test/java/org/openmetadata/csv/CsvUtilTest.java +++ b/openmetadata-service/src/test/java/org/openmetadata/csv/CsvUtilTest.java @@ -16,10 +16,14 @@ import static org.junit.jupiter.api.Assertions.assertEquals; import static org.openmetadata.common.utils.CommonUtil.listOf; +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.node.ObjectNode; import java.util.ArrayList; import java.util.Collections; import java.util.List; import org.junit.jupiter.api.Test; +import org.openmetadata.schema.entity.type.CustomProperty; import org.openmetadata.schema.type.EntityReference; import org.openmetadata.schema.type.TagLabel; @@ -74,6 +78,33 @@ void testAddRecord() { expectedRecord.add("t1;t2"); List tags = listOf(new TagLabel().withTagFQN("t1"), new TagLabel().withTagFQN("t2")); assertEquals(expectedRecord, CsvUtil.addTagLabels(actualRecord, tags)); + + // Add extension + expectedRecord.add(null); + assertEquals(expectedRecord, CsvUtil.addExtension(actualRecord, null)); // Null extension + + ObjectMapper mapper = new ObjectMapper(); + ObjectNode jsonNode = mapper.createObjectNode(); + + // Add new custom property stringCp of type string + CustomProperty stringCp = + new CustomProperty() + .withName("stringCp") + .withDescription("string type custom property") + .withPropertyType( + new EntityReference().withFullyQualifiedName("string").withType("type")); + JsonNode stringCpValue = + mapper.convertValue("String; input; with; semicolon\n And new line", JsonNode.class); + jsonNode.set("stringCp", stringCpValue); + + // Add new custom property queryCp of type sqlQuery + JsonNode queryCpValue = + mapper.convertValue("SELECT * FROM table WHERE column = 'value';", JsonNode.class); + jsonNode.set("queryCp", queryCpValue); + + expectedRecord.add( + "\"stringCp:String; input; with; semicolon\n And new line\";\"queryCp:SELECT * FROM table WHERE column = 'value';\""); + assertEquals(expectedRecord, CsvUtil.addExtension(actualRecord, jsonNode)); } public static void assertCsv(String expectedCsv, String actualCsv) { diff --git a/openmetadata-service/src/test/java/org/openmetadata/service/resources/EntityResourceTest.java b/openmetadata-service/src/test/java/org/openmetadata/service/resources/EntityResourceTest.java index dce74648f4cb..80ec754ae7bf 100644 --- a/openmetadata-service/src/test/java/org/openmetadata/service/resources/EntityResourceTest.java +++ b/openmetadata-service/src/test/java/org/openmetadata/service/resources/EntityResourceTest.java @@ -389,6 +389,18 @@ public abstract class EntityResourceTest true); + assertSummary(result, ApiStatus.PARTIAL_SUCCESS, 2, 1, 1); + expectedRows = + new String[] { + resultsHeader, + getFailedRecord( + invalidCustomPropertyKeyRecord, invalidCustomPropertyKey(11, "invalidCustomProperty")) + }; + assertRows(result, expectedRows); + + // Create glossaryTerm with Invalid custom property value + CustomProperty glossaryTermIntegerCp = + new CustomProperty() + .withName("glossaryTermIntegerCp") + .withDescription("integer type custom property") + .withPropertyType(INT_TYPE.getEntityReference()); + TypeResourceTest typeResourceTest = new TypeResourceTest(); + Type entityType = + typeResourceTest.getEntityByName( + Entity.GLOSSARY_TERM, "customProperties", ADMIN_AUTH_HEADERS); + entityType = + typeResourceTest.addAndCheckCustomProperty( + entityType.getId(), glossaryTermIntegerCp, OK, ADMIN_AUTH_HEADERS); + String invalidIntValueRecord = + ",g1,dsp1,dsc1,h1;h2;h3,,term1;http://term1,PII.None,,,,glossaryTermIntegerCp:11s22"; + csv = createCsv(GlossaryCsv.HEADERS, listOf(invalidIntValueRecord), null); + result = importCsv(glossaryName, csv, false); + Awaitility.await().atMost(4, TimeUnit.SECONDS).until(() -> true); + assertSummary(result, ApiStatus.PARTIAL_SUCCESS, 2, 1, 1); + expectedRows = + new String[] { + resultsHeader, + getFailedRecord( + invalidIntValueRecord, + invalidCustomPropertyValue( + 11, "glossaryTermIntegerCp", INT_TYPE.getDisplayName(), "11s22")) + }; + assertRows(result, expectedRows); + + // Create glossaryTerm with Invalid custom property value's format + CustomProperty glossaryTermDateCp = + new CustomProperty() + .withName("glossaryTermDateCp") + .withDescription("dd-MM-yyyy format time") + .withPropertyType(DATECP_TYPE.getEntityReference()) + .withCustomPropertyConfig(new CustomPropertyConfig().withConfig("dd-MM-yyyy")); + entityType = + typeResourceTest.getEntityByName( + Entity.GLOSSARY_TERM, "customProperties", ADMIN_AUTH_HEADERS); + entityType = + typeResourceTest.addAndCheckCustomProperty( + entityType.getId(), glossaryTermDateCp, OK, ADMIN_AUTH_HEADERS); + String invalidDateFormatRecord = + ",g1,dsp1,dsc1,h1;h2;h3,,term1;http://term1,PII.None,,,,glossaryTermDateCp:invalid-date-format"; + csv = createCsv(GlossaryCsv.HEADERS, listOf(invalidDateFormatRecord), null); + result = importCsv(glossaryName, csv, false); + Awaitility.await().atMost(4, TimeUnit.SECONDS).until(() -> true); + assertSummary(result, ApiStatus.PARTIAL_SUCCESS, 2, 1, 1); + expectedRows = + new String[] { + resultsHeader, + getFailedRecord( + invalidDateFormatRecord, + invalidCustomPropertyFieldFormat( + 11, "glossaryTermDateCp", DATECP_TYPE.getDisplayName(), "dd-MM-yyyy")) + }; + assertRows(result, expectedRows); } @Test @@ -544,17 +637,139 @@ void testGlossaryImportExport() throws IOException { String team11 = TEAM11.getName(); List reviewerRef = listOf(user1, user2).stream().sorted(Comparator.naturalOrder()).toList(); + // PUT valid custom fields to the entity type + // Create instances of CustomPropertyConfig + CustomPropertyConfig dateTimeConfig = + new CustomPropertyConfig().withConfig("dd-MM-yyyy HH:mm:ss"); + CustomPropertyConfig timeConfig = new CustomPropertyConfig().withConfig("HH:mm:ss"); + CustomPropertyConfig enumConfig = + new CustomPropertyConfig() + .withConfig( + Map.of( + "values", + List.of("val1", "val2", "val3", "val4", "val5", "valwith\"quote\""), + "multiSelect", + true)); + + // PUT valid custom fields to the entity type + TypeResourceTest typeResourceTest = new TypeResourceTest(); + Type entityType = + typeResourceTest.getEntityByName( + Entity.GLOSSARY_TERM, "customProperties", ADMIN_AUTH_HEADERS); + + CustomProperty[] customProperties = { + new CustomProperty() + .withName("glossaryTermEmailCp") + .withDescription("email type custom property") + .withPropertyType(EMAIL_TYPE.getEntityReference()), + new CustomProperty() + .withName("glossaryTermDateCp") + .withDescription("dd-MM-yyyy format time") + .withPropertyType(DATECP_TYPE.getEntityReference()) + .withCustomPropertyConfig(new CustomPropertyConfig().withConfig("dd-MM-yyyy")), + new CustomProperty() + .withName("glossaryTermDateTimeCp") + .withDescription("dd-MM-yyyy HH:mm:ss format dateTime") + .withPropertyType(DATETIMECP_TYPE.getEntityReference()) + .withCustomPropertyConfig(dateTimeConfig), + new CustomProperty() + .withName("glossaryTermTimeCp") + .withDescription("HH:mm:ss format time") + .withPropertyType(TIMECP_TYPE.getEntityReference()) + .withCustomPropertyConfig(timeConfig), + new CustomProperty() + .withName("glossaryTermIntegerCp") + .withDescription("integer type custom property") + .withPropertyType(INT_TYPE.getEntityReference()), + new CustomProperty() + .withName("glossaryTermDurationCp") + .withDescription("duration type custom property") + .withPropertyType(DURATION_TYPE.getEntityReference()), + new CustomProperty() + .withName("glossaryTermMarkdownCp") + .withDescription("markdown type custom property") + .withPropertyType(MARKDOWN_TYPE.getEntityReference()), + new CustomProperty() + .withName("glossaryTermStringCp") + .withDescription("string type custom property") + .withPropertyType(STRING_TYPE.getEntityReference()), + new CustomProperty() + .withName("glossaryTermEntRefCp") + .withDescription("entity Reference type custom property") // value includes fqn of entity + .withPropertyType(ENTITY_REFERENCE_TYPE.getEntityReference()) + .withCustomPropertyConfig(new CustomPropertyConfig().withConfig(List.of("user"))), + new CustomProperty() + .withName("glossaryTermEntRefListCp") + .withDescription( + "entity Reference List type custom property") // value includes list of fqn of + .withPropertyType(ENTITY_REFERENCE_LIST_TYPE.getEntityReference()) + .withCustomPropertyConfig( + new CustomPropertyConfig() + .withConfig( + List.of( + Entity.TABLE, + Entity.STORED_PROCEDURE, + Entity.DATABASE_SCHEMA, + Entity.DATABASE, + Entity.DASHBOARD, + Entity.DASHBOARD_DATA_MODEL, + Entity.PIPELINE, + Entity.TOPIC, + Entity.CONTAINER, + Entity.SEARCH_INDEX, + Entity.MLMODEL, + Entity.GLOSSARY_TERM))), + new CustomProperty() + .withName("glossaryTermTimeIntervalCp") + .withDescription("timeInterval type custom property in format starttime:endtime") + .withPropertyType(TIME_INTERVAL_TYPE.getEntityReference()), + new CustomProperty() + .withName("glossaryTermNumberCp") + .withDescription("numberCp") + .withPropertyType(INT_TYPE.getEntityReference()), + new CustomProperty() + .withName("glossaryTermQueryCp") + .withDescription("queryCp desc") + .withPropertyType(SQLQUERY_TYPE.getEntityReference()), + new CustomProperty() + .withName("glossaryTermTimestampCp") + .withDescription("timestamp type custom property") + .withPropertyType(TIMESTAMP_TYPE.getEntityReference()), + new CustomProperty() + .withName("glossaryTermEnumCpSingle") + .withDescription("enum type custom property with multiselect = false") + .withPropertyType(ENUM_TYPE.getEntityReference()) + .withCustomPropertyConfig( + new CustomPropertyConfig() + .withConfig( + Map.of( + "values", + List.of("single1", "single2", "single3", "single4", "\"single5\""), + "multiSelect", + false))), + new CustomProperty() + .withName("glossaryTermEnumCpMulti") + .withDescription("enum type custom property with multiselect = true") + .withPropertyType(ENUM_TYPE.getEntityReference()) + .withCustomPropertyConfig(enumConfig) + }; - // CSV Header "parent" "name" "displayName" "description" "synonyms" "relatedTerms" "references" - // "tags", "reviewers", "owners", "status" + for (CustomProperty customProperty : customProperties) { + entityType = + typeResourceTest.addAndCheckCustomProperty( + entityType.getId(), customProperty, OK, ADMIN_AUTH_HEADERS); + } + // CSV Header "parent", "name", "displayName", "description", "synonyms", "relatedTerms", + // "references", + // "tags", "reviewers", "owners", "status", "extension" // Create two records List createRecords = listOf( String.format( - ",g1,dsp1,\"dsc1,1\",h1;h2;h3,,term1;http://term1,PII.None,user:%s,user:%s,%s,", - reviewerRef.get(0), user1, "Approved"), + ",g1,dsp1,\"dsc1,1\",h1;h2;h3,,term1;http://term1,PII.None,user:%s,user:%s,%s,\"glossaryTermDateCp:18-09-2024;glossaryTermDateTimeCp:18-09-2024 01:09:34;glossaryTermDurationCp:PT5H30M10S;glossaryTermEmailCp:admin@open-metadata.org;glossaryTermEntRefCp:team:\"\"%s\"\";glossaryTermEntRefListCp:user:\"\"%s\"\"|user:\"\"%s\"\"\"", + reviewerRef.get(0), user1, "Approved", team11, user1, user2), String.format( - ",g2,dsp2,dsc3,h1;h3;h3,,term2;https://term2,PII.NonSensitive,,user:%s,%s,", + ",g2,dsp2,dsc3,h1;h3;h3,,term2;https://term2,PII.NonSensitive,,user:%s,%s,\"glossaryTermEnumCpMulti:val3|val2|val1|val4|val5;glossaryTermEnumCpSingle:singleVal1;glossaryTermIntegerCp:7777;glossaryTermMarkdownCp:# Sample Markdown Text;glossaryTermNumberCp:123456;\"\"glossaryTermQueryCp:select col,row from table where id ='30';\"\";glossaryTermStringCp:sample string content;glossaryTermTimeCp:10:08:45;glossaryTermTimeIntervalCp:1726142300000:17261420000;glossaryTermTimestampCp:1726142400000\"", user1, "Approved"), String.format( "importExportTest.g1,g11,dsp2,dsc11,h1;h3;h3,,,,user:%s,team:%s,%s,", @@ -564,10 +779,10 @@ void testGlossaryImportExport() throws IOException { List updateRecords = listOf( String.format( - ",g1,dsp1,new-dsc1,h1;h2;h3,,term1;http://term1,PII.None,user:%s,user:%s,%s,", - reviewerRef.get(0), user1, "Approved"), + ",g1,dsp1,new-dsc1,h1;h2;h3,,term1;http://term1,PII.None,user:%s,user:%s,%s,\"glossaryTermDateCp:18-09-2024;glossaryTermDateTimeCp:18-09-2024 01:09:34;glossaryTermDurationCp:PT5H30M10S;glossaryTermEmailCp:admin@open-metadata.org;glossaryTermEntRefCp:team:\"\"%s\"\";glossaryTermEntRefListCp:user:\"\"%s\"\"|user:\"\"%s\"\"\"", + reviewerRef.get(0), user1, "Approved", team11, user1, user2), String.format( - ",g2,dsp2,new-dsc3,h1;h3;h3,,term2;https://term2,PII.NonSensitive,user:%s,user:%s,%s,", + ",g2,dsp2,new-dsc3,h1;h3;h3,,term2;https://term2,PII.NonSensitive,user:%s,user:%s,%s,\"glossaryTermEnumCpMulti:val3|val2|val1|val4|val5;glossaryTermEnumCpSingle:singleVal1;glossaryTermIntegerCp:7777;glossaryTermMarkdownCp:# Sample Markdown Text;glossaryTermNumberCp:123456;\"\"glossaryTermQueryCp:select col,row from table where id ='30';\"\";glossaryTermStringCp:sample string content;glossaryTermTimeCp:10:08:45;glossaryTermTimeIntervalCp:1726142300000:17261420000;glossaryTermTimestampCp:1726142400000\"", user1, user2, "Approved"), String.format( "importExportTest.g1,g11,dsp2,new-dsc11,h1;h3;h3,,,,user:%s,team:%s,%s,", diff --git a/openmetadata-service/src/test/java/org/openmetadata/service/resources/metadata/TypeResourceTest.java b/openmetadata-service/src/test/java/org/openmetadata/service/resources/metadata/TypeResourceTest.java index 92f050ca44c5..4e64ebf07c03 100644 --- a/openmetadata-service/src/test/java/org/openmetadata/service/resources/metadata/TypeResourceTest.java +++ b/openmetadata-service/src/test/java/org/openmetadata/service/resources/metadata/TypeResourceTest.java @@ -69,7 +69,19 @@ public TypeResourceTest() { public void setupTypes() throws HttpResponseException { INT_TYPE = getEntityByName("integer", "", ADMIN_AUTH_HEADERS); STRING_TYPE = getEntityByName("string", "", ADMIN_AUTH_HEADERS); + EMAIL_TYPE = getEntityByName("email", "", ADMIN_AUTH_HEADERS); ENUM_TYPE = getEntityByName("enum", "", ADMIN_AUTH_HEADERS); + DATECP_TYPE = getEntityByName("date-cp", "", ADMIN_AUTH_HEADERS); + DATETIMECP_TYPE = getEntityByName("dateTime-cp", "", ADMIN_AUTH_HEADERS); + TIMECP_TYPE = getEntityByName("time-cp", "", ADMIN_AUTH_HEADERS); + DURATION_TYPE = getEntityByName("duration", "", ADMIN_AUTH_HEADERS); + MARKDOWN_TYPE = getEntityByName("markdown", "", ADMIN_AUTH_HEADERS); + ENTITY_REFERENCE_TYPE = getEntityByName("entityReference", "", ADMIN_AUTH_HEADERS); + ENTITY_REFERENCE_LIST_TYPE = getEntityByName("entityReferenceList", "", ADMIN_AUTH_HEADERS); + TIME_INTERVAL_TYPE = getEntityByName("timeInterval", "", ADMIN_AUTH_HEADERS); + NUMBER_TYPE = getEntityByName("number", "", ADMIN_AUTH_HEADERS); + SQLQUERY_TYPE = getEntityByName("sqlQuery", "", ADMIN_AUTH_HEADERS); + TIMESTAMP_TYPE = getEntityByName("timestamp", "", ADMIN_AUTH_HEADERS); } @Override From 47519907cda58ca16a69fb5b6c69d2fde0af582f Mon Sep 17 00:00:00 2001 From: mohitdeuex Date: Fri, 27 Sep 2024 11:05:59 +0530 Subject: [PATCH 14/39] GlossaryStatus header change --- .../resources/json/data/glossary/glossaryCsvDocumentation.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/openmetadata-service/src/main/resources/json/data/glossary/glossaryCsvDocumentation.json b/openmetadata-service/src/main/resources/json/data/glossary/glossaryCsvDocumentation.json index 46330823ba32..47750c710b29 100644 --- a/openmetadata-service/src/main/resources/json/data/glossary/glossaryCsvDocumentation.json +++ b/openmetadata-service/src/main/resources/json/data/glossary/glossaryCsvDocumentation.json @@ -86,7 +86,7 @@ ] }, { - "name": "status", + "name": "glossaryStatus", "required": false, "description": "Status of the glossary term. Allowed values `Draft`, `Approved`, or `Deprecated`", "examples": [ From 7d8cf84e4102677dd42da2173123424560d07aef Mon Sep 17 00:00:00 2001 From: Ashish Gupta Date: Fri, 27 Sep 2024 15:01:12 +0530 Subject: [PATCH 15/39] fix unit test and dry run in case of synonyms having quotes in it --- .../ImportGlossary/ImportGlossary.test.tsx | 45 ++++++++++++++----- .../resources/ui/src/utils/CSV/CSV.utils.tsx | 10 +++-- .../ui/src/utils/CSV/CSVUtilsClassBase.tsx | 4 ++ .../src/main/resources/ui/yarn.lock | 15 +++---- 4 files changed, 52 insertions(+), 22 deletions(-) diff --git a/openmetadata-ui/src/main/resources/ui/src/components/Glossary/ImportGlossary/ImportGlossary.test.tsx b/openmetadata-ui/src/main/resources/ui/src/components/Glossary/ImportGlossary/ImportGlossary.test.tsx index ba285d3a4042..41e22ea9fc98 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/Glossary/ImportGlossary/ImportGlossary.test.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/components/Glossary/ImportGlossary/ImportGlossary.test.tsx @@ -13,6 +13,7 @@ import { act, fireEvent, render, screen } from '@testing-library/react'; import React from 'react'; import { CSVImportResult } from '../../../generated/type/csvImportResult'; +import { importGlossaryInCSVFormat } from '../../../rest/glossaryAPI'; import ImportGlossary from './ImportGlossary'; const mockPush = jest.fn(); @@ -36,11 +37,18 @@ jest.mock('../../common/Loader/Loader', () => jest.fn().mockReturnValue(
Loader
) ); -jest.mock('../ImportResult/GlossaryImportResult.component', () => ({ - GlossaryImportResult: jest - .fn() - .mockReturnValue(
GlossaryImportResult
), -})); +jest.mock('../../BulkImport/BulkEntityImport.component', () => + jest.fn().mockImplementation(({ onSuccess, onValidateCsvString }) => ( +
+ + +

BulkEntityImport

+
+ )) +); jest.mock('../../../rest/glossaryAPI', () => ({ importGlossaryInCSVFormat: jest @@ -80,17 +88,34 @@ describe('Import Glossary', () => { expect(await screen.findByTestId('breadcrumb')).toBeInTheDocument(); expect(await screen.findByTestId('title')).toBeInTheDocument(); - expect(await screen.findByTestId('entity-import')).toBeInTheDocument(); + expect(screen.getByText('BulkEntityImport')).toBeInTheDocument(); + expect(screen.getByText('SuccessButton')).toBeInTheDocument(); + expect(screen.getByText('ValidateCsvButton')).toBeInTheDocument(); }); - it('GlossaryImportResult should visible', async () => { + it('should redirect the page when onSuccess get triggered', async () => { render(); - const importBtn = await screen.findByTestId('import'); + const successButton = screen.getByText('SuccessButton'); await act(async () => { - fireEvent.click(importBtn); + fireEvent.click(successButton); }); - expect(await screen.findByText('GlossaryImportResult')).toBeInTheDocument(); + expect(mockPush).toHaveBeenCalled(); + }); + + it('should call the importGlossaryInCSVFormat api when validate props is trigger', async () => { + render(); + + const successButton = screen.getByText('ValidateCsvButton'); + await act(async () => { + fireEvent.click(successButton); + }); + + expect(importGlossaryInCSVFormat).toHaveBeenCalledWith( + 'Glossary1', + 'markdown: This is test', + true + ); }); }); diff --git a/openmetadata-ui/src/main/resources/ui/src/utils/CSV/CSV.utils.tsx b/openmetadata-ui/src/main/resources/ui/src/utils/CSV/CSV.utils.tsx index 42f658c3652c..eadaf59b6a50 100644 --- a/openmetadata-ui/src/main/resources/ui/src/utils/CSV/CSV.utils.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/utils/CSV/CSV.utils.tsx @@ -129,7 +129,11 @@ export const getCSVStringFromColumnsAndDataSource = ( .map((col) => { const value = get(row, col.name ?? '', ''); const colName = col.name ?? ''; - if (colName === 'extension') { + if ( + csvUtilsClassBase + .columnsWithMultipleValuesEscapeNeeded() + .includes(colName) + ) { return `"${value.replaceAll(new RegExp('"', 'g'), '""')}"`; } else if ( value.includes(',') || @@ -160,7 +164,7 @@ const convertCustomPropertyStringToValueExtensionBasedOnType = ( return { type: entity[0], fullyQualifiedName: entity[1], - name: entity[1], + name: removeOuterEscapes(entity[1]), } as EntityReference; } @@ -173,7 +177,7 @@ const convertCustomPropertyStringToValueExtensionBasedOnType = ( return { type: key, fullyQualifiedName: itemValue, - name: itemValue, + name: removeOuterEscapes(itemValue), } as EntityReference; }); } diff --git a/openmetadata-ui/src/main/resources/ui/src/utils/CSV/CSVUtilsClassBase.tsx b/openmetadata-ui/src/main/resources/ui/src/utils/CSV/CSVUtilsClassBase.tsx index 448e076c66c5..40d360f28b66 100644 --- a/openmetadata-ui/src/main/resources/ui/src/utils/CSV/CSVUtilsClassBase.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/utils/CSV/CSVUtilsClassBase.tsx @@ -31,6 +31,10 @@ import Fqn from '../Fqn'; import { EditorProps } from './CSV.utils'; class CSVUtilsClassBase { + public columnsWithMultipleValuesEscapeNeeded() { + return ['extension', 'synonyms']; + } + public getEditor( column: string, entityType: EntityType diff --git a/openmetadata-ui/src/main/resources/ui/yarn.lock b/openmetadata-ui/src/main/resources/ui/yarn.lock index 16936a3ba497..bbfa46056f3c 100644 --- a/openmetadata-ui/src/main/resources/ui/yarn.lock +++ b/openmetadata-ui/src/main/resources/ui/yarn.lock @@ -7751,11 +7751,6 @@ etag@~1.8.1: resolved "https://registry.yarnpkg.com/etag/-/etag-1.8.1.tgz#41ae2eeb65efa62268aebfea83ac7d79299b0887" integrity sha1-Qa4u62XvpiJorr/qg6x9eSmbCIc= -eventemitter3@^2.0.3: - version "2.0.3" - resolved "https://registry.yarnpkg.com/eventemitter3/-/eventemitter3-2.0.3.tgz#b5e1079b59fb5e1ba2771c0a993be060a58c99ba" - integrity sha512-jLN68Dx5kyFHaePoXWPsCGW5qdyZQtLYHkxkg02/Mz6g0kYpDx4FyP6XfArhQdlOC4b8Mv+EMxPo/8La7Tzghg== - eventemitter3@^4.0.0, eventemitter3@^4.0.1, eventemitter3@^4.0.4, eventemitter3@^4.0.7: version "4.0.7" resolved "https://registry.yarnpkg.com/eventemitter3/-/eventemitter3-4.0.7.tgz#2de9b68f6528d5644ef5c59526a1b4a07306169f" @@ -10241,6 +10236,7 @@ lodash._getnative@^3.0.0: version "3.9.1" resolved "https://registry.yarnpkg.com/lodash._getnative/-/lodash._getnative-3.9.1.tgz#570bc7dede46d61cdcde687d65d3eecbaa3aaff5" integrity sha512-RrL9VxMEPyDMHOd9uFbvMe8X55X16/cGM5IgOKgRElQZutpX89iS6vwl64duTV1/16w5JY7tuFNXqoekmh1EmA== + lodash.clonedeep@^4.5.0: version "4.5.0" resolved "https://registry.yarnpkg.com/lodash.clonedeep/-/lodash.clonedeep-4.5.0.tgz#e23f3f9c4f8fbdde872529c1071857a086e5ccef" @@ -10261,6 +10257,11 @@ lodash.isarray@^3.0.0: resolved "https://registry.yarnpkg.com/lodash.isarray/-/lodash.isarray-3.0.4.tgz#79e4eb88c36a8122af86f844aa9bcd851b5fbb55" integrity sha512-JwObCrNJuT0Nnbuecmqr5DgtuBppuCvGD9lxjFpAzwnVtdGoDQ1zig+5W8k5/6Gcn0gZ3936HDAlGd28i7sOGQ== +lodash.isequal@^4.5.0: + version "4.5.0" + resolved "https://registry.yarnpkg.com/lodash.isequal/-/lodash.isequal-4.5.0.tgz#415c4478f2bcc30120c22ce10ed3226f7d3e18e0" + integrity sha512-pDo3lu8Jhfjqls6GkMgpahsF9kCyayhgykjyLMNFTKWrpVdAQtYyB4muAMWozBB4ig/dtWAmsMxLEI8wuz+DYQ== + lodash.keys@^3.1.2: version "3.1.2" resolved "https://registry.yarnpkg.com/lodash.keys/-/lodash.keys-3.1.2.tgz#4dbc0472b156be50a0b286855d1bd0b0c656098a" @@ -10269,10 +10270,6 @@ lodash.keys@^3.1.2: lodash._getnative "^3.0.0" lodash.isarguments "^3.0.0" lodash.isarray "^3.0.0" -lodash.isequal@^4.5.0: - version "4.5.0" - resolved "https://registry.yarnpkg.com/lodash.isequal/-/lodash.isequal-4.5.0.tgz#415c4478f2bcc30120c22ce10ed3226f7d3e18e0" - integrity sha512-pDo3lu8Jhfjqls6GkMgpahsF9kCyayhgykjyLMNFTKWrpVdAQtYyB4muAMWozBB4ig/dtWAmsMxLEI8wuz+DYQ== lodash.merge@^4.6.2: version "4.6.2" From dde2c0b68d8e924211204bcb92bcc3e2061321d8 Mon Sep 17 00:00:00 2001 From: sonikashah Date: Fri, 27 Sep 2024 16:47:46 +0530 Subject: [PATCH 16/39] Remove extension column in CSVs for all entities except glossaryTerm --- .../service/jdbi3/DatabaseRepository.java | 7 ++--- .../jdbi3/DatabaseSchemaRepository.java | 8 ++--- .../jdbi3/DatabaseServiceRepository.java | 7 ++--- .../service/jdbi3/TableRepository.java | 30 ++++++++----------- .../database/databaseCsvDocumentation.json | 16 ---------- .../databaseSchemaCsvDocumentation.json | 16 ---------- .../databaseServiceCsvDocumentation.json | 16 ---------- .../data/table/tableCsvDocumentation.json | 16 ---------- .../databases/DatabaseResourceTest.java | 8 ++--- .../databases/DatabaseSchemaResourceTest.java | 8 ++--- .../databases/TableResourceTest.java | 16 +++++----- 11 files changed, 35 insertions(+), 113 deletions(-) diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/DatabaseRepository.java b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/DatabaseRepository.java index 242473b32879..8ab1f826591a 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/DatabaseRepository.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/DatabaseRepository.java @@ -13,7 +13,6 @@ package org.openmetadata.service.jdbi3; -import static org.openmetadata.csv.CsvUtil.addExtension; import static org.openmetadata.csv.CsvUtil.addField; import static org.openmetadata.csv.CsvUtil.addGlossaryTerms; import static org.openmetadata.csv.CsvUtil.addOwners; @@ -126,7 +125,7 @@ public String exportToCsv(String name, String user) throws IOException { (DatabaseSchemaRepository) Entity.getEntityRepository(DATABASE_SCHEMA); ListFilter filter = new ListFilter(Include.NON_DELETED).addQueryParam("database", name); List schemas = - repository.listAll(repository.getFields("owners,tags,domain,extension"), filter); + repository.listAll(repository.getFields("owners,tags,domain"), filter); schemas.sort(Comparator.comparing(EntityInterface::getFullyQualifiedName)); return new DatabaseCsv(database, user).exportCsv(schemas); } @@ -283,8 +282,7 @@ protected void createEntity(CSVPrinter printer, List csvRecords) thro .withTags(tagLabels) .withRetentionPeriod(csvRecord.get(7)) .withSourceUrl(csvRecord.get(8)) - .withDomain(getEntityReference(printer, csvRecord, 9, Entity.DOMAIN)) - .withExtension(getExtension(printer, csvRecord, 10)); + .withDomain(getEntityReference(printer, csvRecord, 9, Entity.DOMAIN)); if (processRecord) { createEntity(printer, csvRecord, schema); } @@ -308,7 +306,6 @@ protected void addRecord(CsvFile csvFile, DatabaseSchema entity) { ? "" : entity.getDomain().getFullyQualifiedName(); addField(recordList, domain); - addExtension(recordList, entity.getExtension()); addRecord(csvFile, recordList); } } diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/DatabaseSchemaRepository.java b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/DatabaseSchemaRepository.java index 5b810bc094cc..252681ca3d35 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/DatabaseSchemaRepository.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/DatabaseSchemaRepository.java @@ -14,7 +14,6 @@ package org.openmetadata.service.jdbi3; import static org.openmetadata.common.utils.CommonUtil.nullOrEmpty; -import static org.openmetadata.csv.CsvUtil.addExtension; import static org.openmetadata.csv.CsvUtil.addField; import static org.openmetadata.csv.CsvUtil.addGlossaryTerms; import static org.openmetadata.csv.CsvUtil.addOwners; @@ -198,8 +197,7 @@ public String exportToCsv(String name, String user) throws IOException { DatabaseSchema schema = getByName(null, name, Fields.EMPTY_FIELDS); // Validate database schema TableRepository repository = (TableRepository) Entity.getEntityRepository(TABLE); ListFilter filter = new ListFilter(Include.NON_DELETED).addQueryParam("databaseSchema", name); - List
tables = - repository.listAll(repository.getFields("owners,tags,domain,extension"), filter); + List
tables = repository.listAll(repository.getFields("owners,tags,domain"), filter); tables.sort(Comparator.comparing(EntityInterface::getFullyQualifiedName)); return new DatabaseSchemaCsv(schema, user).exportCsv(tables); } @@ -316,8 +314,7 @@ protected void createEntity(CSVPrinter printer, List csvRecords) thro .withRetentionPeriod(csvRecord.get(7)) .withSourceUrl(csvRecord.get(8)) .withColumns(nullOrEmpty(table.getColumns()) ? new ArrayList<>() : table.getColumns()) - .withDomain(getEntityReference(printer, csvRecord, 9, Entity.DOMAIN)) - .withExtension(getExtension(printer, csvRecord, 10)); + .withDomain(getEntityReference(printer, csvRecord, 9, Entity.DOMAIN)); if (processRecord) { createEntity(printer, csvRecord, table); @@ -342,7 +339,6 @@ protected void addRecord(CsvFile csvFile, Table entity) { ? "" : entity.getDomain().getFullyQualifiedName(); addField(recordList, domain); - addExtension(recordList, entity.getExtension()); addRecord(csvFile, recordList); } } diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/DatabaseServiceRepository.java b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/DatabaseServiceRepository.java index de7f4da1abdb..1d44a7dbd3b0 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/DatabaseServiceRepository.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/DatabaseServiceRepository.java @@ -13,7 +13,6 @@ package org.openmetadata.service.jdbi3; -import static org.openmetadata.csv.CsvUtil.addExtension; import static org.openmetadata.csv.CsvUtil.addField; import static org.openmetadata.csv.CsvUtil.addGlossaryTerms; import static org.openmetadata.csv.CsvUtil.addOwners; @@ -69,7 +68,7 @@ public String exportToCsv(String name, String user) throws IOException { DatabaseRepository repository = (DatabaseRepository) Entity.getEntityRepository(DATABASE); ListFilter filter = new ListFilter(Include.NON_DELETED).addQueryParam("service", name); List databases = - repository.listAll(repository.getFields("owners,tags,domain,extension"), filter); + repository.listAll(repository.getFields("owners,tags,domain"), filter); databases.sort(Comparator.comparing(EntityInterface::getFullyQualifiedName)); return new DatabaseServiceCsv(databaseService, user).exportCsv(databases); } @@ -123,8 +122,7 @@ protected void createEntity(CSVPrinter printer, List csvRecords) thro .withDescription(csvRecord.get(2)) .withOwners(getOwners(printer, csvRecord, 3)) .withTags(tagLabels) - .withDomain(getEntityReference(printer, csvRecord, 7, Entity.DOMAIN)) - .withExtension(getExtension(printer, csvRecord, 8)); + .withDomain(getEntityReference(printer, csvRecord, 7, Entity.DOMAIN)); if (processRecord) { createEntity(printer, csvRecord, database); @@ -147,7 +145,6 @@ protected void addRecord(CsvFile csvFile, Database entity) { ? "" : entity.getDomain().getFullyQualifiedName(); addField(recordList, domain); - addExtension(recordList, entity.getExtension()); addRecord(csvFile, recordList); } } diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/TableRepository.java b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/TableRepository.java index 51c2b90a7e89..c15c18ad2c5c 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/TableRepository.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/TableRepository.java @@ -17,7 +17,6 @@ import static org.openmetadata.common.utils.CommonUtil.listOf; import static org.openmetadata.common.utils.CommonUtil.listOrEmpty; import static org.openmetadata.common.utils.CommonUtil.nullOrEmpty; -import static org.openmetadata.csv.CsvUtil.addExtension; import static org.openmetadata.csv.CsvUtil.addField; import static org.openmetadata.csv.CsvUtil.addGlossaryTerms; import static org.openmetadata.csv.CsvUtil.addOwners; @@ -786,8 +785,7 @@ public Table applySuggestion(EntityInterface entity, String columnFQN, Suggestio @Override public String exportToCsv(String name, String user) throws IOException { // Validate table - Table table = - getByName(null, name, new Fields(allowedFields, "owners,domain,tags,columns,extension")); + Table table = getByName(null, name, new Fields(allowedFields, "owners,domain,tags,columns")); return new TableCsv(table, user).exportCsv(listOf(table)); } @@ -1198,8 +1196,7 @@ protected void createEntity(CSVPrinter printer, List csvRecords) thro .withTags(tagLabels != null && tagLabels.isEmpty() ? null : tagLabels) .withRetentionPeriod(csvRecord.get(7)) .withSourceUrl(csvRecord.get(8)) - .withDomain(getEntityReference(printer, csvRecord, 9, Entity.DOMAIN)) - .withExtension(getExtension(printer, csvRecord, 10)); + .withDomain(getEntityReference(printer, csvRecord, 9, Entity.DOMAIN)); ImportResult importResult = updateColumn(printer, csvRecord); if (importResult.result().equals(IMPORT_FAILED)) { importFailure(printer, importResult.details(), csvRecord); @@ -1237,7 +1234,7 @@ public void updateColumns( } public ImportResult updateColumn(CSVPrinter printer, CSVRecord csvRecord) throws IOException { - String columnFqn = csvRecord.get(11); + String columnFqn = csvRecord.get(10); Column column = findColumn(table.getColumns(), columnFqn); boolean columnExists = column != null; if (column == null) { @@ -1248,22 +1245,22 @@ public ImportResult updateColumn(CSVPrinter printer, CSVRecord csvRecord) throws .withFullyQualifiedName( table.getFullyQualifiedName() + Entity.SEPARATOR + columnFqn); } - column.withDisplayName(csvRecord.get(12)); - column.withDescription(csvRecord.get(13)); - column.withDataTypeDisplay(csvRecord.get(14)); + column.withDisplayName(csvRecord.get(11)); + column.withDescription(csvRecord.get(12)); + column.withDataTypeDisplay(csvRecord.get(13)); column.withDataType( - nullOrEmpty(csvRecord.get(15)) ? null : ColumnDataType.fromValue(csvRecord.get(15))); + nullOrEmpty(csvRecord.get(14)) ? null : ColumnDataType.fromValue(csvRecord.get(14))); column.withArrayDataType( - nullOrEmpty(csvRecord.get(16)) ? null : ColumnDataType.fromValue(csvRecord.get(16))); + nullOrEmpty(csvRecord.get(15)) ? null : ColumnDataType.fromValue(csvRecord.get(15))); column.withDataLength( - nullOrEmpty(csvRecord.get(17)) ? null : Integer.parseInt(csvRecord.get(17))); + nullOrEmpty(csvRecord.get(16)) ? null : Integer.parseInt(csvRecord.get(16))); List tagLabels = getTagLabels( printer, csvRecord, List.of( - Pair.of(18, TagLabel.TagSource.CLASSIFICATION), - Pair.of(19, TagLabel.TagSource.GLOSSARY))); + Pair.of(17, TagLabel.TagSource.CLASSIFICATION), + Pair.of(18, TagLabel.TagSource.GLOSSARY))); column.withTags(nullOrEmpty(tagLabels) ? null : tagLabels); column.withOrdinalPosition(nullOrEmpty(table.getColumns()) ? 0 : table.getColumns().size()); @@ -1325,7 +1322,6 @@ protected void addRecord(CsvFile csvFile, Table entity) { ? "" : entity.getDomain().getFullyQualifiedName(); addField(recordList, domain); - addExtension(recordList, entity.getExtension()); if (!nullOrEmpty(table.getColumns())) { addRecord(csvFile, recordList, table.getColumns().get(0), false); @@ -1334,7 +1330,7 @@ protected void addRecord(CsvFile csvFile, Table entity) { } } else { // Create a dummy Entry for the Column - for (int i = 0; i < 10; i++) { + for (int i = 0; i < 9; i++) { addField(recordList, (String) null); // Add empty fields for table information } addRecord(csvFile, recordList); @@ -1344,7 +1340,7 @@ protected void addRecord(CsvFile csvFile, Table entity) { private void addRecord( CsvFile csvFile, List recordList, Column column, boolean emptyTableDetails) { if (emptyTableDetails) { - for (int i = 0; i < 11; i++) { + for (int i = 0; i < 10; i++) { addField(recordList, (String) null); // Add empty fields for table information } } diff --git a/openmetadata-service/src/main/resources/json/data/database/databaseCsvDocumentation.json b/openmetadata-service/src/main/resources/json/data/database/databaseCsvDocumentation.json index 8c526b8e3926..6357f94217f9 100644 --- a/openmetadata-service/src/main/resources/json/data/database/databaseCsvDocumentation.json +++ b/openmetadata-service/src/main/resources/json/data/database/databaseCsvDocumentation.json @@ -84,22 +84,6 @@ "examples": [ "Marketing", "Sales" ] - }, - { - "name": "extension", - "required": false, - "description": "Custom property values added to the database schema. Each field value (property and its value) is separated by `;` and internal values can be separated by `|`. For `entityReferenceList` type property, pass `type1:fqn1|type2:fqn2`. For single `entityReference` type property, pass `type:fqn`. Similarly, for `enumMultiSelect`, pass values separated by `|`, and for `enumSingleSelect`, pass a single value along with the property name. For `timeInterval` property type, pass the `startTime:endTime` to the property name. If the field value itself contains delimiter values like `,` and `;` or newline they need to be quoted, and the quotation needs to be further escaped. In general, if passing multiple field values separated by `;`, the extension column value needs to be quoted.", - "examples": [ - "`customAttribute1:value1;customAttribute2:value2`", - "`\"dateCp:18-09-2024;dateTimeCp:18-09-2024 01:09:34;durationCp:PT5H30M10S;emailCp:admin@open-metadata.org\"`", - "`entRefListCp:searchIndex:elasticsearch_sample.table_search_index|databaseSchema:Glue.default.information_schema|databaseSchema:sample_data.ecommerce_db.shopify|database:Glue.default|`", - "`\"entRefCp:user:\"\"aaron.singh2\"\"\"`", - "`\"enumMultiSelectCp:val3|val2|val1|val4|val5;enumSingleSelectCp:singleVal1\"`", - "`\"timeCp:10:08:45;timeIntervalCp:1726142300000:17261420000;timeStampCp:1726142400000\"`", - "`\"integerCp:7777;numberCp:123456\"`", - "`\"\"\"queryCp:select col,row from table where id ='30';\"\";stringcp:sample string content\"`", - "`markdownCp:# Sample Markdown Text`" - ] } ] } \ No newline at end of file diff --git a/openmetadata-service/src/main/resources/json/data/databaseSchema/databaseSchemaCsvDocumentation.json b/openmetadata-service/src/main/resources/json/data/databaseSchema/databaseSchemaCsvDocumentation.json index 441d0ffde830..55d0fd342eaf 100644 --- a/openmetadata-service/src/main/resources/json/data/databaseSchema/databaseSchemaCsvDocumentation.json +++ b/openmetadata-service/src/main/resources/json/data/databaseSchema/databaseSchemaCsvDocumentation.json @@ -84,22 +84,6 @@ "examples": [ "Marketing", "Sales" ] - }, - { - "name": "extension", - "required": false, - "description": "Custom property values added to the database. Each field value (property and its value) is separated by `;` and internal values can be separated by `|`. For `entityReferenceList` type property, pass `type1:fqn1|type2:fqn2`. For single `entityReference` type property, pass `type:fqn`. Similarly, for `enumMultiSelect`, pass values separated by `|`, and for `enumSingleSelect`, pass a single value along with the property name. For `timeInterval` property type, pass the `startTime:endTime` to the property name. If the field value itself contains delimiter values like `,` and `;` or newline they need to be quoted, and the quotation needs to be further escaped. In general, if passing multiple field values separated by `;`, the extension column value needs to be quoted.", - "examples": [ - "`customAttribute1:value1;customAttribute2:value2`", - "`\"dateCp:18-09-2024;dateTimeCp:18-09-2024 01:09:34;durationCp:PT5H30M10S;emailCp:admin@open-metadata.org\"`", - "`entRefListCp:searchIndex:elasticsearch_sample.table_search_index|databaseSchema:Glue.default.information_schema|databaseSchema:sample_data.ecommerce_db.shopify|database:Glue.default|`", - "`\"entRefCp:user:\"\"aaron.singh2\"\"\"`", - "`\"enumMultiSelectCp:val3|val2|val1|val4|val5;enumSingleSelectCp:singleVal1\"`", - "`\"timeCp:10:08:45;timeIntervalCp:1726142300000:17261420000;timeStampCp:1726142400000\"`", - "`\"integerCp:7777;numberCp:123456\"`", - "`\"\"\"queryCp:select col,row from table where id ='30';\"\";stringcp:sample string content\"`", - "`markdownCp:# Sample Markdown Text`" - ] } ] } \ No newline at end of file diff --git a/openmetadata-service/src/main/resources/json/data/databaseService/databaseServiceCsvDocumentation.json b/openmetadata-service/src/main/resources/json/data/databaseService/databaseServiceCsvDocumentation.json index 427e5388ab29..6c4346b69256 100644 --- a/openmetadata-service/src/main/resources/json/data/databaseService/databaseServiceCsvDocumentation.json +++ b/openmetadata-service/src/main/resources/json/data/databaseService/databaseServiceCsvDocumentation.json @@ -68,22 +68,6 @@ "examples": [ "Marketing", "Sales" ] - }, - { - "name": "extension", - "required": false, - "description": "Custom property values added to the database schema. Each field value (property and its value) is separated by `;` and internal values can be separated by `|`. For `entityReferenceList` type property, pass `type1:fqn1|type2:fqn2`. For single `entityReference` type property, pass `type:fqn`. Similarly, for `enumMultiSelect`, pass values separated by `|`, and for `enumSingleSelect`, pass a single value along with the property name. For `timeInterval` property type, pass the `startTime:endTime` to the property name. If the field value itself contains delimiter values like `,` and `;` or newline they need to be quoted, and the quotation needs to be further escaped. In general, if passing multiple field values separated by `;`, the extension column value needs to be quoted.", - "examples": [ - "`customAttribute1:value1;customAttribute2:value2`", - "`\"dateCp:18-09-2024;dateTimeCp:18-09-2024 01:09:34;durationCp:PT5H30M10S;emailCp:admin@open-metadata.org\"`", - "`entRefListCp:searchIndex:elasticsearch_sample.table_search_index|databaseSchema:Glue.default.information_schema|databaseSchema:sample_data.ecommerce_db.shopify|database:Glue.default|`", - "`\"entRefCp:user:\"\"aaron.singh2\"\"\"`", - "`\"enumMultiSelectCp:val3|val2|val1|val4|val5;enumSingleSelectCp:singleVal1\"`", - "`\"timeCp:10:08:45;timeIntervalCp:1726142300000:17261420000;timeStampCp:1726142400000\"`", - "`\"integerCp:7777;numberCp:123456\"`", - "`\"\"\"queryCp:select col,row from table where id ='30';\"\";stringcp:sample string content\"`", - "`markdownCp:# Sample Markdown Text`" - ] } ] } \ No newline at end of file diff --git a/openmetadata-service/src/main/resources/json/data/table/tableCsvDocumentation.json b/openmetadata-service/src/main/resources/json/data/table/tableCsvDocumentation.json index 3edc15022c12..c528965ab0f7 100644 --- a/openmetadata-service/src/main/resources/json/data/table/tableCsvDocumentation.json +++ b/openmetadata-service/src/main/resources/json/data/table/tableCsvDocumentation.json @@ -85,22 +85,6 @@ "Marketing", "Sales" ] }, - { - "name": "extension", - "required": false, - "description": "Custom property values added to the table. Each field value (property and its value) is separated by `;` and internal values can be separated by `|`. For `entityReferenceList` type property, pass `type1:fqn1|type2:fqn2`. For single `entityReference` type property, pass `type:fqn`. Similarly, for `enumMultiSelect`, pass values separated by `|`, and for `enumSingleSelect`, pass a single value along with the property name. For `timeInterval` property type, pass the `startTime:endTime` to the property name. If the field value itself contains delimiter values like `,` and `;` or newline they need to be quoted, and the quotation needs to be further escaped. In general, if passing multiple field values separated by `;`, the extension column value needs to be quoted.", - "examples": [ - "`customAttribute1:value1;customAttribute2:value2`", - "`\"dateCp:18-09-2024;dateTimeCp:18-09-2024 01:09:34;durationCp:PT5H30M10S;emailCp:admin@open-metadata.org\"`", - "`entRefListCp:searchIndex:elasticsearch_sample.table_search_index|databaseSchema:Glue.default.information_schema|databaseSchema:sample_data.ecommerce_db.shopify|database:Glue.default|`", - "`\"entRefCp:user:\"\"aaron.singh2\"\"\"`", - "`\"enumMultiSelectCp:val3|val2|val1|val4|val5;enumSingleSelectCp:singleVal1\"`", - "`\"timeCp:10:08:45;timeIntervalCp:1726142300000:17261420000;timeStampCp:1726142400000\"`", - "`\"integerCp:7777;numberCp:123456\"`", - "`\"\"\"queryCp:select col,row from table where id ='30';\"\";stringcp:sample string content\"`", - "`markdownCp:# Sample Markdown Text`" - ] - }, { "name": "column.fullyQualifiedName", "required": true, diff --git a/openmetadata-service/src/test/java/org/openmetadata/service/resources/databases/DatabaseResourceTest.java b/openmetadata-service/src/test/java/org/openmetadata/service/resources/databases/DatabaseResourceTest.java index a7374a3ae22d..796479b2ade6 100644 --- a/openmetadata-service/src/test/java/org/openmetadata/service/resources/databases/DatabaseResourceTest.java +++ b/openmetadata-service/src/test/java/org/openmetadata/service/resources/databases/DatabaseResourceTest.java @@ -121,7 +121,7 @@ void testImportInvalidCsv() { // Headers: name, displayName, description, owner, tags, retentionPeriod, sourceUrl, domain // Update databaseSchema with invalid tags field String resultsHeader = recordToString(EntityCsv.getResultHeaders(DatabaseCsv.HEADERS)); - String record = "s1,dsp1,dsc1,,Tag.invalidTag,,,,,,"; + String record = "s1,dsp1,dsc1,,Tag.invalidTag,,,,,"; String csv = createCsv(DatabaseCsv.HEADERS, listOf(record), null); CsvImportResult result = importCsv(databaseName, csv, false); assertSummary(result, ApiStatus.PARTIAL_SUCCESS, 2, 1, 1); @@ -132,7 +132,7 @@ resultsHeader, getFailedRecord(record, entityNotFound(4, "tag", "Tag.invalidTag" assertRows(result, expectedRows); // invalid tag it will give error. - record = "non-existing,dsp1,dsc1,,Tag.invalidTag,,,,,,"; + record = "non-existing,dsp1,dsc1,,Tag.invalidTag,,,,,"; csv = createCsv(DatabaseSchemaCsv.HEADERS, listOf(record), null); result = importCsv(databaseName, csv, false); assertSummary(result, ApiStatus.PARTIAL_SUCCESS, 2, 1, 1); @@ -144,7 +144,7 @@ resultsHeader, getFailedRecord(record, entityNotFound(4, "tag", "Tag.invalidTag" // databaseSchema will be created if it does not exist String schemaFqn = FullyQualifiedName.add(database.getFullyQualifiedName(), "non-existing"); - record = "non-existing,dsp1,dsc1,,,,,,,,"; + record = "non-existing,dsp1,dsc1,,,,,,,"; csv = createCsv(DatabaseSchemaCsv.HEADERS, listOf(record), null); result = importCsv(databaseName, csv, false); assertSummary(result, ApiStatus.SUCCESS, 2, 2, 0); @@ -168,7 +168,7 @@ void testImportExport() throws IOException { // Update terms with change in description String record = String.format( - "s1,dsp1,new-dsc1,user:%s,,,Tier.Tier1,P23DT23H,http://test.com,%s,", + "s1,dsp1,new-dsc1,user:%s,,,Tier.Tier1,P23DT23H,http://test.com,%s", user1, escapeCsv(DOMAIN.getFullyQualifiedName())); // Update created entity with changes diff --git a/openmetadata-service/src/test/java/org/openmetadata/service/resources/databases/DatabaseSchemaResourceTest.java b/openmetadata-service/src/test/java/org/openmetadata/service/resources/databases/DatabaseSchemaResourceTest.java index b06071dd57b8..a4c0a693621c 100644 --- a/openmetadata-service/src/test/java/org/openmetadata/service/resources/databases/DatabaseSchemaResourceTest.java +++ b/openmetadata-service/src/test/java/org/openmetadata/service/resources/databases/DatabaseSchemaResourceTest.java @@ -120,7 +120,7 @@ void testImportInvalidCsv() { // Headers: name, displayName, description, owner, tags, retentionPeriod, sourceUrl, domain // Create table with invalid tags field String resultsHeader = recordToString(EntityCsv.getResultHeaders(DatabaseSchemaCsv.HEADERS)); - String record = "s1,dsp1,dsc1,,Tag.invalidTag,,,,,,"; + String record = "s1,dsp1,dsc1,,Tag.invalidTag,,,,,"; String csv = createCsv(DatabaseSchemaCsv.HEADERS, listOf(record), null); CsvImportResult result = importCsv(schemaName, csv, false); assertSummary(result, ApiStatus.PARTIAL_SUCCESS, 2, 1, 1); @@ -131,7 +131,7 @@ resultsHeader, getFailedRecord(record, entityNotFound(4, "tag", "Tag.invalidTag" assertRows(result, expectedRows); // Tag will cause failure - record = "non-existing,dsp1,dsc1,,Tag.invalidTag,,,,,,"; + record = "non-existing,dsp1,dsc1,,Tag.invalidTag,,,,,"; csv = createCsv(DatabaseSchemaCsv.HEADERS, listOf(record), null); result = importCsv(schemaName, csv, false); assertSummary(result, ApiStatus.PARTIAL_SUCCESS, 2, 1, 1); @@ -142,7 +142,7 @@ resultsHeader, getFailedRecord(record, entityNotFound(4, "tag", "Tag.invalidTag" assertRows(result, expectedRows); // non-existing table will cause - record = "non-existing,dsp1,dsc1,,,,,,,,"; + record = "non-existing,dsp1,dsc1,,,,,,,"; String tableFqn = FullyQualifiedName.add(schema.getFullyQualifiedName(), "non-existing"); csv = createCsv(DatabaseSchemaCsv.HEADERS, listOf(record), null); result = importCsv(schemaName, csv, false); @@ -167,7 +167,7 @@ void testImportExport() throws IOException { List updateRecords = listOf( String.format( - "s1,dsp1,new-dsc1,user:%s,,,Tier.Tier1,P23DT23H,http://test.com,%s,", + "s1,dsp1,new-dsc1,user:%s,,,Tier.Tier1,P23DT23H,http://test.com,%s", user1, escapeCsv(DOMAIN.getFullyQualifiedName()))); // Update created entity with changes diff --git a/openmetadata-service/src/test/java/org/openmetadata/service/resources/databases/TableResourceTest.java b/openmetadata-service/src/test/java/org/openmetadata/service/resources/databases/TableResourceTest.java index 5bdf2f6a28ef..254ebdcec927 100644 --- a/openmetadata-service/src/test/java/org/openmetadata/service/resources/databases/TableResourceTest.java +++ b/openmetadata-service/src/test/java/org/openmetadata/service/resources/databases/TableResourceTest.java @@ -2726,7 +2726,7 @@ void testImportInvalidCsv() { // Headers: name, displayName, description, owner, tags, retentionPeriod, sourceUrl, domain // Create table with invalid tags field String resultsHeader = recordToString(EntityCsv.getResultHeaders(TableCsv.HEADERS)); - String record = "s1,dsp1,dsc1,,Tag.invalidTag,,,,,,,c1,c1,c1,,INT,,,,"; + String record = "s1,dsp1,dsc1,,Tag.invalidTag,,,,,,c1,c1,c1,,INT,,,,"; String csv = createCsv(TableCsv.HEADERS, listOf(record), null); CsvImportResult result = importCsv(tableName, csv, false); assertSummary(result, ApiStatus.PARTIAL_SUCCESS, 2, 1, 1); @@ -2738,19 +2738,19 @@ void testImportInvalidCsv() { assertRows(result, expectedRows); // Add an invalid column tag - record = "s1,dsp1,dsc1,,,,,,,,,c1,,,,INT,,,Tag.invalidTag,"; + record = "s1,dsp1,dsc1,,,,,,,,c1,,,,INT,,,Tag.invalidTag,"; csv = createCsv(TableCsv.HEADERS, listOf(record), null); result = importCsv(tableName, csv, false); assertSummary(result, ApiStatus.PARTIAL_SUCCESS, 2, 1, 1); expectedRows = new String[] { resultsHeader, - getFailedRecord(record, EntityCsv.entityNotFound(18, "tag", "Tag.invalidTag")) + getFailedRecord(record, EntityCsv.entityNotFound(17, "tag", "Tag.invalidTag")) }; assertRows(result, expectedRows); // Update a non-existing column, this should create a new column with name "nonExistingColumn" - record = "s1,dsp1,dsc1,,,,,,,,,nonExistingColumn,,,,INT,,,,"; + record = "s1,dsp1,dsc1,,,,,,,,nonExistingColumn,,,,INT,,,,"; csv = createCsv(TableCsv.HEADERS, listOf(record), null); result = importCsv(tableName, csv, false); assertSummary(result, ApiStatus.SUCCESS, 2, 2, 0); @@ -2776,12 +2776,12 @@ void testImportExport() throws IOException { List updateRecords = listOf( String.format( - "s1,dsp1,new-dsc1,user:%s,,,Tier.Tier1,P23DT23H,http://test.com,%s,,c1," + "s1,dsp1,new-dsc1,user:%s,,,Tier.Tier1,P23DT23H,http://test.com,%s,c1," + "dsp1-new,desc1,type,STRUCT,,,PII.Sensitive,", user1, escapeCsv(DOMAIN.getFullyQualifiedName())), - ",,,,,,,,,,,c1.c11,dsp11-new,desc11,type1,INT,,,PII.Sensitive,", - ",,,,,,,,,,,c2,,,type1,INT,,,,", - ",,,,,,,,,,,c3,,,type1,INT,,,,"); + ",,,,,,,,,,c1.c11,dsp11-new,desc11,type1,INT,,,PII.Sensitive,", + ",,,,,,,,,,c2,,,type1,INT,,,,", + ",,,,,,,,,,c3,,,type1,INT,,,,"); // Update created entity with changes importCsvAndValidate(table.getFullyQualifiedName(), TableCsv.HEADERS, null, updateRecords); From 7a4500bc60d7c360ba0aa257d0eae8870f3dcca7 Mon Sep 17 00:00:00 2001 From: Ashish Gupta Date: Fri, 27 Sep 2024 17:34:01 +0530 Subject: [PATCH 17/39] added editor for reviewers --- .../ui/src/utils/CSV/CSVUtilsClassBase.tsx | 49 +++++++++++++++++++ 1 file changed, 49 insertions(+) diff --git a/openmetadata-ui/src/main/resources/ui/src/utils/CSV/CSVUtilsClassBase.tsx b/openmetadata-ui/src/main/resources/ui/src/utils/CSV/CSVUtilsClassBase.tsx index 40d360f28b66..e2c9711c4519 100644 --- a/openmetadata-ui/src/main/resources/ui/src/utils/CSV/CSVUtilsClassBase.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/utils/CSV/CSVUtilsClassBase.tsx @@ -13,6 +13,7 @@ import { Form } from 'antd'; import { DefaultOptionType } from 'antd/lib/select'; +import { t } from 'i18next'; import { toString } from 'lodash'; import React, { ReactNode } from 'react'; import TreeAsyncSelectList from '../../components/common/AsyncSelectList/TreeAsyncSelectList'; @@ -230,6 +231,54 @@ class CSVUtilsClassBase { ); }; + case 'reviewers': + return ({ value, ...props }: EditorProps) => { + const reviewers = value?.split(';') ?? []; + const reviewersEntityRef = reviewers.map((reviewer) => { + const [type, user] = reviewer.split(':'); + + return { + type, + name: user, + id: user, + } as EntityReference; + }); + + const handleChange = (reviewers?: EntityReference[]) => { + if (!reviewers || reviewers.length === 0) { + props.onChange(); + + setTimeout(() => { + props.onComplete(); + }, 1); + + return; + } + const reviewerText = reviewers + .map((reviewer) => `${reviewer.type}:${reviewer.name}`) + .join(';'); + props.onChange(reviewerText); + + setTimeout(() => { + props.onComplete(reviewerText); + }, 1); + }; + + return ( + + ); + }; case 'extension': return ({ value, ...props }: EditorProps) => { const handleSave = async (extension: string) => { From ce474051b0a62c0c64e261afe9123f806d22814f Mon Sep 17 00:00:00 2001 From: Ashish Gupta Date: Fri, 27 Sep 2024 18:07:48 +0530 Subject: [PATCH 18/39] unit test around csv utils --- .../ui/src/utils/CSV/CSV.utils.test.tsx | 109 ++++++++++++++ .../src/utils/CSV/CSVUtilsClassBase.test.tsx | 133 ++++++++++++++++++ 2 files changed, 242 insertions(+) create mode 100644 openmetadata-ui/src/main/resources/ui/src/utils/CSV/CSV.utils.test.tsx create mode 100644 openmetadata-ui/src/main/resources/ui/src/utils/CSV/CSVUtilsClassBase.test.tsx diff --git a/openmetadata-ui/src/main/resources/ui/src/utils/CSV/CSV.utils.test.tsx b/openmetadata-ui/src/main/resources/ui/src/utils/CSV/CSV.utils.test.tsx new file mode 100644 index 000000000000..0c014da2b2f2 --- /dev/null +++ b/openmetadata-ui/src/main/resources/ui/src/utils/CSV/CSV.utils.test.tsx @@ -0,0 +1,109 @@ +/* + * Copyright 2024 Collate. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * http://www.apache.org/licenses/LICENSE-2.0 + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import { EntityType } from '../../enums/entity.enum'; +import { + getColumnConfig, + getCSVStringFromColumnsAndDataSource, + getEntityColumnsAndDataSourceFromCSV, +} from './CSV.utils'; + +describe('CSVUtils', () => { + describe('getColumnConfig', () => { + it('should return the column configuration object', () => { + const column = 'description'; + const columnConfig = getColumnConfig(column, EntityType.GLOSSARY); + + expect(columnConfig).toBeDefined(); + expect(columnConfig.name).toBe(column); + }); + }); + + describe('getEntityColumnsAndDataSourceFromCSV', () => { + it('should return the columns and data source from the CSV', () => { + const csv = [ + ['col1', 'col2'], + ['value1', 'value2'], + ]; + const { columns, dataSource } = getEntityColumnsAndDataSourceFromCSV( + csv, + EntityType.GLOSSARY + ); + + expect(columns).toHaveLength(2); + expect(dataSource).toHaveLength(1); + }); + }); + + describe('getCSVStringFromColumnsAndDataSource', () => { + it('should return the CSV string from the columns and data source for non-quoted columns', () => { + const columns = [{ name: 'col1' }, { name: 'col2' }]; + const dataSource = [{ col1: 'value1', col2: 'value2' }]; + const csvString = getCSVStringFromColumnsAndDataSource( + columns, + dataSource + ); + + expect(csvString).toBe('col1,col2\nvalue1,value2'); + }); + + it('should return the CSV string from the columns and data source with quoted columns', () => { + const columns = [ + { name: 'tags' }, + { name: 'glossaryTerms' }, + { name: 'description' }, + { name: 'domain' }, + ]; + const dataSource = [ + { + tags: 'value1', + glossaryTerms: 'value2', + description: 'something new', + domain: 'domain1', + }, + ]; + const csvString = getCSVStringFromColumnsAndDataSource( + columns, + dataSource + ); + + expect(csvString).toBe( + 'tags,glossaryTerms,description,domain\n"value1","value2",something new,"domain1"' + ); + }); + + it('should return quoted value if data contains comma', () => { + const columns = [ + { name: 'tags' }, + { name: 'glossaryTerms' }, + { name: 'description' }, + { name: 'domain' }, + ]; + const dataSource = [ + { + tags: 'value,1', + glossaryTerms: 'value_2', + description: 'something#new', + domain: 'domain,1', + }, + ]; + const csvString = getCSVStringFromColumnsAndDataSource( + columns, + dataSource + ); + + expect(csvString).toBe( + `tags,glossaryTerms,description,domain\n"value,1","value_2",something#new,"domain,1"` + ); + }); + }); +}); diff --git a/openmetadata-ui/src/main/resources/ui/src/utils/CSV/CSVUtilsClassBase.test.tsx b/openmetadata-ui/src/main/resources/ui/src/utils/CSV/CSVUtilsClassBase.test.tsx new file mode 100644 index 000000000000..4c8719f75fd6 --- /dev/null +++ b/openmetadata-ui/src/main/resources/ui/src/utils/CSV/CSVUtilsClassBase.test.tsx @@ -0,0 +1,133 @@ +/* + * Copyright 2024 Collate. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * http://www.apache.org/licenses/LICENSE-2.0 + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import React from 'react'; +import { EntityType } from '../../enums/entity.enum'; +import csvUtilsClassBase from './CSVUtilsClassBase'; + +jest.mock( + '../../components/common/AsyncSelectList/TreeAsyncSelectList', + () => ({ + __esModule: true, + default: jest.fn(), + }) +); + +jest.mock( + '../../components/common/DomainSelectableList/DomainSelectableList.component', + () => ({ + __esModule: true, + default: jest.fn(), + }) +); + +jest.mock('../../components/common/InlineEdit/InlineEdit.component', () => ({ + __esModule: true, + default: jest.fn(), +})); + +jest.mock('../../components/common/TierCard/TierCard', () => ({ + __esModule: true, + default: jest.fn(), +})); + +jest.mock( + '../../components/common/UserTeamSelectableList/UserTeamSelectableList.component', + () => ({ + __esModule: true, + UserTeamSelectableList: jest + .fn() + .mockReturnValue(

UserTeamSelectableList

), + }) +); + +jest.mock( + '../../components/Modals/ModalWithMarkdownEditor/ModalWithMarkdownEditor', + () => ({ + __esModule: true, + ModalWithMarkdownEditor: jest.fn(), + }) +); +jest.mock( + '../../components/Modals/ModalWithCustomProperty/ModalWithCustomPropertyEditor.component', + () => ({ + __esModule: true, + ModalWithCustomPropertyEditor: jest.fn(), + }) +); + +describe('CSV utils ClassBase', () => { + describe('getEditor', () => { + it('should return the editor component for the specified column', () => { + const column = 'owner'; + const editor = csvUtilsClassBase.getEditor(column, EntityType.GLOSSARY); + + expect(editor).toBeDefined(); + }); + + it('should return undefined for unknown columns', () => { + const column = 'unknown'; + const editor = csvUtilsClassBase.getEditor(column, EntityType.GLOSSARY); + + expect(editor).toBeUndefined(); + }); + + it('should return the editor component for the "description" column', () => { + const column = 'description'; + const editor = csvUtilsClassBase.getEditor(column, EntityType.GLOSSARY); + + expect(editor).toBeDefined(); + }); + + it('should return the editor component for the "tags" column', () => { + const column = 'tags'; + const editor = csvUtilsClassBase.getEditor(column, EntityType.GLOSSARY); + + expect(editor).toBeDefined(); + }); + + it('should return the editor component for the "glossaryTerms" column', () => { + const column = 'glossaryTerms'; + const editor = csvUtilsClassBase.getEditor(column, EntityType.GLOSSARY); + + expect(editor).toBeDefined(); + }); + + it('should return the editor component for the "tiers" column', () => { + const column = 'tiers'; + const editor = csvUtilsClassBase.getEditor(column, EntityType.GLOSSARY); + + expect(editor).toBeDefined(); + }); + + it('should return the editor component for the "extension" column', () => { + const column = 'extension'; + const editor = csvUtilsClassBase.getEditor(column, EntityType.GLOSSARY); + + expect(editor).toBeDefined(); + }); + + it('should return the editor component for the "reviewers" column', () => { + const column = 'reviewers'; + const editor = csvUtilsClassBase.getEditor(column, EntityType.GLOSSARY); + + expect(editor).toBeDefined(); + }); + + it('should return the editor component for the "domain" column', () => { + const column = 'domain'; + const editor = csvUtilsClassBase.getEditor(column, EntityType.GLOSSARY); + + expect(editor).toBeDefined(); + }); + }); +}); From 998efe8b559158982986e2cf9144cb4a963f5bcb Mon Sep 17 00:00:00 2001 From: Ashish Gupta Date: Fri, 27 Sep 2024 19:10:29 +0530 Subject: [PATCH 19/39] added escape for string too, in case of semicolon comes --- .../components/BulkImport/BulkEntityImport.component.tsx | 7 +------ .../src/main/resources/ui/src/utils/CSV/CSV.utils.tsx | 2 +- 2 files changed, 2 insertions(+), 7 deletions(-) diff --git a/openmetadata-ui/src/main/resources/ui/src/components/BulkImport/BulkEntityImport.component.tsx b/openmetadata-ui/src/main/resources/ui/src/components/BulkImport/BulkEntityImport.component.tsx index 4a7989d3b42f..072b046c9d62 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/BulkImport/BulkEntityImport.component.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/components/BulkImport/BulkEntityImport.component.tsx @@ -86,17 +86,12 @@ const BulkEntityImport = ({ [entityType, setDataSource, setColumns, setActiveStep, focusToGrid] ); - const validateCsvString = useCallback( - async (csvData: string) => await onValidateCsvString(csvData, true), - [] - ); - const handleLoadData = useCallback( async (e: ProgressEvent) => { try { const result = e.target?.result as string; - const validationResponse = await validateCsvString(result); + const validationResponse = await onValidateCsvString(result, true); if (['failure', 'aborted'].includes(validationResponse?.status ?? '')) { setValidationData(validationResponse); diff --git a/openmetadata-ui/src/main/resources/ui/src/utils/CSV/CSV.utils.tsx b/openmetadata-ui/src/main/resources/ui/src/utils/CSV/CSV.utils.tsx index eadaf59b6a50..78b16fb0e082 100644 --- a/openmetadata-ui/src/main/resources/ui/src/utils/CSV/CSV.utils.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/utils/CSV/CSV.utils.tsx @@ -303,7 +303,7 @@ export const convertEntityExtensionToCustomPropertyString = ( ); if ( - ['markdown', 'sqlQuery'].includes( + ['markdown', 'sqlQuery', 'string'].includes( keyAndValueTypes[key].propertyType.name ?? '' ) ) { From 784f2b54fd9d864f251736fb6ac9500aedc996ce Mon Sep 17 00:00:00 2001 From: Ashish Gupta Date: Sat, 28 Sep 2024 02:13:28 +0530 Subject: [PATCH 20/39] added playwright test without extension and supported relatedTerm as editable --- .../e2e/Pages/GlossaryImportExport.spec.ts | 156 ++++++++---- .../ui/playwright/utils/importUtils.ts | 234 ++++++++++++++++++ .../BulkImport/BulkEntityImport.component.tsx | 14 +- ...odalWithCustomPropertyEditor.component.tsx | 2 +- .../ModalWithMarkdownEditor.interface.ts | 2 +- .../resources/ui/src/utils/CSV/CSV.utils.tsx | 11 +- .../ui/src/utils/CSV/CSVUtilsClassBase.tsx | 7 +- 7 files changed, 372 insertions(+), 54 deletions(-) create mode 100644 openmetadata-ui/src/main/resources/ui/playwright/utils/importUtils.ts diff --git a/openmetadata-ui/src/main/resources/ui/playwright/e2e/Pages/GlossaryImportExport.spec.ts b/openmetadata-ui/src/main/resources/ui/playwright/e2e/Pages/GlossaryImportExport.spec.ts index 0e782585822f..b93c554629a9 100644 --- a/openmetadata-ui/src/main/resources/ui/playwright/e2e/Pages/GlossaryImportExport.spec.ts +++ b/openmetadata-ui/src/main/resources/ui/playwright/e2e/Pages/GlossaryImportExport.spec.ts @@ -14,8 +14,18 @@ import { expect, test } from '@playwright/test'; import { SidebarItem } from '../../constant/sidebar'; import { Glossary } from '../../support/glossary/Glossary'; import { GlossaryTerm } from '../../support/glossary/GlossaryTerm'; -import { createNewPage, redirectToHomePage } from '../../utils/common'; +import { UserClass } from '../../support/user/UserClass'; +import { + createNewPage, + redirectToHomePage, + toastNotification, +} from '../../utils/common'; import { selectActiveGlossary } from '../../utils/glossary'; +import { + createGlossaryTermRowDetails, + fillGlossaryRowDetails, + validateImportStatus, +} from '../../utils/importUtils'; import { sidebarClick } from '../../utils/sidebar'; // use the admin user to login @@ -23,17 +33,25 @@ test.use({ storageState: 'playwright/.auth/admin.json', }); -const glossary = new Glossary(); -const glossaryTerm1 = new GlossaryTerm(glossary); +const user1 = new UserClass(); +const user2 = new UserClass(); +const glossary1 = new Glossary(); +const glossary2 = new Glossary(); +const glossaryTerm1 = new GlossaryTerm(glossary1); +const glossaryTerm2 = new GlossaryTerm(glossary2); -test.describe('Bulk Import Export', () => { - test.slow(); +test.describe('Glossary Bulk Import Export', () => { + test.slow(true); test.beforeAll('setup pre-test', async ({ browser }) => { const { apiContext, afterAction } = await createNewPage(browser); - await glossary.create(apiContext); + await user1.create(apiContext); + await user2.create(apiContext); + await glossary1.create(apiContext); + await glossary2.create(apiContext); await glossaryTerm1.create(apiContext); + await glossaryTerm2.create(apiContext); await afterAction(); }); @@ -41,7 +59,10 @@ test.describe('Bulk Import Export', () => { test.afterAll('Cleanup', async ({ browser }) => { const { apiContext, afterAction } = await createNewPage(browser); - await glossary.delete(apiContext); + await user1.delete(apiContext); + await user2.delete(apiContext); + await glossary1.delete(apiContext); + await glossary2.delete(apiContext); await afterAction(); }); @@ -50,16 +71,16 @@ test.describe('Bulk Import Export', () => { await redirectToHomePage(page); }); - test('Import and Export Glossary', async ({ page }) => { - await test.step('Export data', async () => { + test('Glossary Bulk Import Export', async ({ page }) => { + await test.step('should export data glossary term details', async () => { await sidebarClick(page, SidebarItem.GLOSSARY); - await selectActiveGlossary(page, glossary.data.displayName); + await selectActiveGlossary(page, glossary1.data.displayName); const downloadPromise = page.waitForEvent('download'); await page.click('[data-testid="manage-button"]'); await page.click('[data-testid="export-button-description"]'); - await page.fill('#fileName', glossary.data.displayName); + await page.fill('#fileName', glossary1.data.displayName); await page.click('#submit-button'); const download = await downloadPromise; @@ -67,39 +88,84 @@ test.describe('Bulk Import Export', () => { await download.saveAs('downloads/' + download.suggestedFilename()); }); - await test.step('Import data', async () => { - await selectActiveGlossary(page, glossary.data.displayName); - await page.click('[data-testid="manage-button"]'); - await page.click('[data-testid="import-button-description"]'); - const fileInput = await page.$('[type="file"]'); - await fileInput?.setInputFiles([ - 'downloads/' + glossary.data.displayName + '.csv', - ]); - - // Adding manual wait for the file to load - await page.waitForTimeout(500); - - await expect( - page.getByText('Number of rows: 2 | Passed: 2') - ).toBeVisible(); - - await expect(page.getByTestId('import-result-table')).toBeVisible(); - - await expect(page.getByTestId('preview-cancel-button')).toBeVisible(); - - const glossaryImport = page.waitForResponse( - '/api/v1/glossaries/name/*/import?dryRun=false' - ); - await page.getByTestId('import-button').click(); - await glossaryImport; - - const glossaryResponse = page.waitForResponse('/api/v1/glossaryTerms?**'); - await page.getByTestId('preview-button').click(); - await glossaryResponse; - - await expect(page.getByTestId('entity-header-display-name')).toHaveText( - glossary.responseData.displayName - ); - }); + await test.step( + 'should import and edit with two additional database', + async () => { + await sidebarClick(page, SidebarItem.GLOSSARY); + await selectActiveGlossary(page, glossary1.data.displayName); + await page.click('[data-testid="manage-button"]'); + await page.click('[data-testid="import-button-description"]'); + const fileInput = await page.$('[type="file"]'); + await fileInput?.setInputFiles([ + 'downloads/' + glossary1.data.displayName + '.csv', + ]); + + // Adding manual wait for the file to load + await page.waitForTimeout(500); + + // Adding some assertion to make sure that CSV loaded correctly + await expect( + page.locator('.InovuaReactDataGrid__header-layout') + ).toBeVisible(); + await expect(page.getByTestId('add-row-btn')).toBeVisible(); + await expect(page.getByRole('button', { name: 'Next' })).toBeVisible(); + await expect( + page.getByRole('button', { name: 'Previous' }) + ).toBeVisible(); + + await page.click('[data-testid="add-row-btn"]'); + + // click on last row first cell + await page.click( + '.InovuaReactDataGrid__row--last > .InovuaReactDataGrid__row-cell-wrap > .InovuaReactDataGrid__cell--first' + ); + // Click on first cell and edit + await fillGlossaryRowDetails( + { + ...createGlossaryTermRowDetails(), + owners: [user1.responseData?.['displayName']], + reviewers: [user2.responseData?.['displayName']], + relatedTerm: { + parent: glossary2.data.name, + name: glossaryTerm2.data.name, + }, + }, + page + ); + + await page.getByRole('button', { name: 'Next' }).click(); + const loader = page.locator( + '.inovua-react-toolkit-load-mask__background-layer' + ); + + await loader.waitFor({ state: 'hidden' }); + + await validateImportStatus(page, { + passed: '3', + processed: '3', + failed: '0', + }); + + await page.waitForSelector('.InovuaReactDataGrid__header-layout', { + state: 'visible', + }); + + const rowStatus = ['Entity updated', 'Entity created']; + + await expect(page.locator('[data-props-id="details"]')).toHaveText( + rowStatus + ); + + await page.getByRole('button', { name: 'Update' }).click(); + await page + .locator('.inovua-react-toolkit-load-mask__background-layer') + .waitFor({ state: 'detached' }); + + await toastNotification( + page, + `Glossaryterm ${glossary1.responseData.fullyQualifiedName} details updated successfully` + ); + } + ); }); }); diff --git a/openmetadata-ui/src/main/resources/ui/playwright/utils/importUtils.ts b/openmetadata-ui/src/main/resources/ui/playwright/utils/importUtils.ts new file mode 100644 index 000000000000..e54e056a5675 --- /dev/null +++ b/openmetadata-ui/src/main/resources/ui/playwright/utils/importUtils.ts @@ -0,0 +1,234 @@ +/* + * Copyright 2024 Collate. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * http://www.apache.org/licenses/LICENSE-2.0 + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import { expect, Page } from '@playwright/test'; +import { uuid } from './common'; + +export const createGlossaryTermRowDetails = () => { + return { + name: `playwright,glossaryTerm ${uuid()}`, + displayName: 'Playwright,Glossary Term', + description: `Playwright GlossaryTerm description. + Neque porro quisquam est qui dolorem ipsum quia dolor sit amet, consectetur, adipisci velit... + There is no one who loves pain itself, who seeks after it and wants to have it, simply because it is pain..`, + tag: 'PII.Sensitive', + synonyms: 'playwright,glossaryTerm,testing', + references: 'data;http:sandbox.com', + }; +}; + +export const fillTextInputDetails = async (page: Page, text: string) => { + await page.locator('.InovuaReactDataGrid__cell--cell-active').press('Enter'); + + await page.locator('.ant-layout-content').getByRole('textbox').fill(text); + await page + .locator('.ant-layout-content') + .getByRole('textbox') + .press('Enter', { delay: 100 }); +}; + +export const fillDescriptionDetails = async ( + page: Page, + description: string +) => { + await page.locator('.InovuaReactDataGrid__cell--cell-active').press('Enter'); + await page.click( + '.toastui-editor-md-container > .toastui-editor > .ProseMirror' + ); + + await page.fill( + '.toastui-editor-md-container > .toastui-editor > .ProseMirror', + description + ); + + await page.click('[data-testid="save"]'); + await page.click('.InovuaReactDataGrid__cell--cell-active'); +}; + +export const fillOwnerDetails = async (page: Page, owners: string[]) => { + await page + .locator('.InovuaReactDataGrid__cell--cell-active') + .press('Enter', { delay: 100 }); + + const userListResponse = page.waitForResponse( + '/api/v1/users?limit=*&isBot=false*' + ); + await page.getByRole('tab', { name: 'Users' }).click(); + await userListResponse; + await page.waitForSelector('[data-testid="loader"]', { state: 'detached' }); + + await page.click('[data-testid="owner-select-users-search-bar"]'); + + for (const owner of owners) { + await page.locator('[data-testid="owner-select-users-search-bar"]').clear(); + await page.keyboard.type(owner); + await page.waitForResponse( + `/api/v1/search/query?q=*${owner}*%20AND%20isBot:false&from=0&size=25&index=user_search_index` + ); + + await page.getByRole('listitem', { name: owner }).click(); + } + + await page.getByTestId('selectable-list-update-btn').click(); + + await page.click('.InovuaReactDataGrid__cell--cell-active'); +}; + +export const fillTagDetails = async (page: Page, tag: string) => { + await page + .locator('.InovuaReactDataGrid__cell--cell-active') + .press('Enter', { delay: 100 }); + + await page.click('[data-testid="tag-selector"]'); + await page.locator('[data-testid="tag-selector"] input').fill(tag); + await page.click(`[data-testid="tag-${tag}"]`); + await page.click('[data-testid="inline-save-btn"]'); + await page.click('.InovuaReactDataGrid__cell--cell-active'); +}; + +export const fillGlossaryTermDetails = async ( + page: Page, + glossary: { parent: string; name: string } +) => { + await page + .locator('.InovuaReactDataGrid__cell--cell-active') + .press('Enter', { delay: 100 }); + + await page.waitForSelector('[data-testid="loader"]', { state: 'detached' }); + + await page.click('[data-testid="tag-selector"]'); + await page.locator('[data-testid="tag-selector"] input').fill(glossary.name); + await page.getByTestId(`tag-"${glossary.parent}"."${glossary.name}"`).click(); + await page.click('[data-testid="saveAssociatedTag"]'); + await page.click('.InovuaReactDataGrid__cell--cell-active'); +}; + +export const fillDomainDetails = async ( + page: Page, + domains: { name: string; displayName: string } +) => { + await page + .locator('.InovuaReactDataGrid__cell--cell-active') + .press('Enter', { delay: 100 }); + + await page.click('[data-testid="selectable-list"] [data-testid="searchbar"]'); + await page + .locator('[data-testid="selectable-list"] [data-testid="searchbar"]') + .fill(domains.name); + + await page.click(`.ant-popover [title="${domains.displayName}"]`); + await page.waitForTimeout(100); +}; + +export const fillGlossaryRowDetails = async ( + row: { + name: string; + displayName: string; + description: string; + synonyms: string; + relatedTerm: { + name: string; + parent: string; + }; + references: string; + tag: string; + reviewers: string[]; + owners: string[]; + }, + page: Page +) => { + await page + .locator('.InovuaReactDataGrid__cell--cell-active') + .press('ArrowRight'); + + await fillTextInputDetails(page, row.name); + + await page + .locator('.InovuaReactDataGrid__cell--cell-active') + .press('ArrowRight'); + + await fillTextInputDetails(page, row.displayName); + + // Navigate to next cell and make cell editable + await page + .locator('.InovuaReactDataGrid__cell--cell-active') + .press('ArrowRight', { delay: 100 }); + + await fillDescriptionDetails(page, row.description); + + await page + .locator('.InovuaReactDataGrid__cell--cell-active') + .press('ArrowRight', { delay: 100 }); + + await fillTextInputDetails(page, row.synonyms); + + await page + .locator('.InovuaReactDataGrid__cell--cell-active') + .press('ArrowRight', { delay: 100 }); + + await fillGlossaryTermDetails(page, row.relatedTerm); + + await page + .locator('.InovuaReactDataGrid__cell--cell-active') + .press('ArrowRight', { delay: 100 }); + + await fillTextInputDetails(page, row.references); + + await page + .locator('.InovuaReactDataGrid__cell--cell-active') + .press('ArrowRight', { delay: 100 }); + + await fillTagDetails(page, row.tag); + + await page + .locator('.InovuaReactDataGrid__cell--cell-active') + .press('ArrowRight', { delay: 100 }); + + await fillOwnerDetails(page, row.reviewers); + + await page + .locator('.InovuaReactDataGrid__cell--cell-active') + .press('ArrowRight', { delay: 100 }); + + await fillOwnerDetails(page, row.owners); +}; + +export const validateImportStatus = async ( + page: Page, + status: { passed: string; failed: string; processed: string } +) => { + await page.waitForSelector('[data-testid="processed-row"]'); + const processedRow = await page.$eval( + '[data-testid="processed-row"]', + (el) => el.textContent + ); + + expect(processedRow).toBe(status.processed); + + const passedRow = await page.$eval( + '[data-testid="passed-row"]', + (el) => el.textContent + ); + + expect(passedRow).toBe(status.passed); + + const failedRow = await page.$eval( + '[data-testid="failed-row"]', + (el) => el.textContent + ); + + expect(failedRow).toBe(status.failed); + + await page.waitForSelector('.InovuaReactDataGrid__header-layout', { + state: 'visible', + }); +}; diff --git a/openmetadata-ui/src/main/resources/ui/src/components/BulkImport/BulkEntityImport.component.tsx b/openmetadata-ui/src/main/resources/ui/src/components/BulkImport/BulkEntityImport.component.tsx index 072b046c9d62..d1067f96632b 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/BulkImport/BulkEntityImport.component.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/components/BulkImport/BulkEntityImport.component.tsx @@ -18,7 +18,7 @@ import { } from '@inovua/reactdatagrid-community/types'; import { Button, Card, Col, Row, Space, Typography } from 'antd'; import { AxiosError } from 'axios'; -import React, { MutableRefObject, useCallback, useState } from 'react'; +import React, { MutableRefObject, useCallback, useMemo, useState } from 'react'; import { useTranslation } from 'react-i18next'; import { usePapaParse } from 'react-papaparse'; @@ -32,6 +32,7 @@ import { getCSVStringFromColumnsAndDataSource, getEntityColumnsAndDataSourceFromCSV, } from '../../utils/CSV/CSV.utils'; +import csvUtilsClassBase from '../../utils/CSV/CSVUtilsClassBase'; import { showErrorToast, showSuccessToast } from '../../utils/ToastUtils'; import { ImportStatus } from '../common/EntityImport/ImportStatus/ImportStatus.component'; import Stepper from '../Settings/Services/Ingestion/IngestionStepper/IngestionStepper.component'; @@ -62,6 +63,15 @@ const BulkEntityImport = ({ MutableRefObject >({ current: null }); + const filterColumns = useMemo( + () => + columns?.filter( + (col) => + !csvUtilsClassBase.hideImportsColumnList().includes(col.name ?? '') + ), + [columns] + ); + const focusToGrid = useCallback(() => { setGridRef((ref) => { ref.current?.focus(); @@ -319,7 +329,7 @@ const BulkEntityImport = ({ {activeStep === 1 && ( (false); const [isSaveLoading, setIsSaveLoading] = useState(false); const [customPropertyValue, setCustomPropertyValue] = - useState({}); + useState(); const [customPropertyTypes, setCustomPropertyTypes] = useState(); const fetchTypeDetail = async () => { diff --git a/openmetadata-ui/src/main/resources/ui/src/components/Modals/ModalWithCustomProperty/ModalWithMarkdownEditor.interface.ts b/openmetadata-ui/src/main/resources/ui/src/components/Modals/ModalWithCustomProperty/ModalWithMarkdownEditor.interface.ts index 91e0ce67f40b..6696df303564 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/Modals/ModalWithCustomProperty/ModalWithMarkdownEditor.interface.ts +++ b/openmetadata-ui/src/main/resources/ui/src/components/Modals/ModalWithCustomProperty/ModalWithMarkdownEditor.interface.ts @@ -28,7 +28,7 @@ export type ModalWithCustomPropertyEditorProps = { entityType: EntityType; header: string; value?: string; - onSave: (extension: string) => Promise; + onSave: (extension?: string) => Promise; onCancel?: () => void; visible: boolean; }; diff --git a/openmetadata-ui/src/main/resources/ui/src/utils/CSV/CSV.utils.tsx b/openmetadata-ui/src/main/resources/ui/src/utils/CSV/CSV.utils.tsx index 78b16fb0e082..057f2a434a0c 100644 --- a/openmetadata-ui/src/main/resources/ui/src/utils/CSV/CSV.utils.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/utils/CSV/CSV.utils.tsx @@ -134,12 +134,15 @@ export const getCSVStringFromColumnsAndDataSource = ( .columnsWithMultipleValuesEscapeNeeded() .includes(colName) ) { - return `"${value.replaceAll(new RegExp('"', 'g'), '""')}"`; + return isEmpty(value) + ? '' + : `"${value.replaceAll(new RegExp('"', 'g'), '""')}"`; } else if ( value.includes(',') || value.includes('\n') || colName.includes('tags') || colName.includes('glossaryTerms') || + colName.includes('relatedTerms') || colName.includes('domain') ) { return isEmpty(value) ? '' : `"${value}"`; @@ -277,11 +280,11 @@ export const convertCustomPropertyStringToEntityExtension = ( }; export const convertEntityExtensionToCustomPropertyString = ( - value: ExtensionDataProps, + value?: ExtensionDataProps, customPropertyType?: Type ) => { - if (isUndefined(customPropertyType)) { - return ''; + if (isUndefined(customPropertyType) || isUndefined(value)) { + return; } const keyAndValueTypes: Record = {}; diff --git a/openmetadata-ui/src/main/resources/ui/src/utils/CSV/CSVUtilsClassBase.tsx b/openmetadata-ui/src/main/resources/ui/src/utils/CSV/CSVUtilsClassBase.tsx index e2c9711c4519..48b49342d631 100644 --- a/openmetadata-ui/src/main/resources/ui/src/utils/CSV/CSVUtilsClassBase.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/utils/CSV/CSVUtilsClassBase.tsx @@ -32,6 +32,10 @@ import Fqn from '../Fqn'; import { EditorProps } from './CSV.utils'; class CSVUtilsClassBase { + public hideImportsColumnList() { + return ['glossaryStatus']; + } + public columnsWithMultipleValuesEscapeNeeded() { return ['extension', 'synonyms']; } @@ -138,6 +142,7 @@ class CSVUtilsClassBase { ); }; case 'glossaryTerms': + case 'relatedTerms': return ({ value, ...props }) => { const tags = value ? value?.split(';') : []; @@ -281,7 +286,7 @@ class CSVUtilsClassBase { }; case 'extension': return ({ value, ...props }: EditorProps) => { - const handleSave = async (extension: string) => { + const handleSave = async (extension?: string) => { props.onChange(extension); setTimeout(() => { From 01ec59d0f75e1445c44b2d0dfa6fa67246c52033 Mon Sep 17 00:00:00 2001 From: Ashish Gupta Date: Sat, 28 Sep 2024 21:16:20 +0530 Subject: [PATCH 21/39] added unit test around csv util logic --- .../main/resources/ui/src/mocks/CSV.mock.ts | 413 ++++++++++++++++++ .../ui/src/utils/CSV/CSV.utils.test.tsx | 58 +++ .../resources/ui/src/utils/CSV/CSV.utils.tsx | 2 +- .../src/utils/CSV/CSVUtilsClassBase.test.tsx | 7 + 4 files changed, 479 insertions(+), 1 deletion(-) create mode 100644 openmetadata-ui/src/main/resources/ui/src/mocks/CSV.mock.ts diff --git a/openmetadata-ui/src/main/resources/ui/src/mocks/CSV.mock.ts b/openmetadata-ui/src/main/resources/ui/src/mocks/CSV.mock.ts new file mode 100644 index 000000000000..1aa529cd30d4 --- /dev/null +++ b/openmetadata-ui/src/main/resources/ui/src/mocks/CSV.mock.ts @@ -0,0 +1,413 @@ +/* + * Copyright 2024 Collate. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * http://www.apache.org/licenses/LICENSE-2.0 + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +/* eslint-disable max-len */ + +import { ExtensionDataProps } from '../components/Modals/ModalWithCustomProperty/ModalWithMarkdownEditor.interface'; +import { Category, Type } from '../generated/entity/type'; +import { EntityReference } from '../generated/tests/testCase'; + +export const MOCK_GLOSSARY_TERM_CUSTOM_PROPERTIES_EXTENSION_CSV_STRING = `dateCp:2024-09-18;dateTimeCp:15-09-2024 22:09:57;durationCp:PH23723D;emailCp:john.david@email.com;expert:user:"aaron.singh2";expertListPanel:databaseSchema:Glue.default.information_schema|glossaryTerm:"PW%40606600.Quick073437a4"."PW.ec0bbdf3%Bear5c6a56cc"|dashboard:sample_superset.11|user:angel_smith0|team:Legal Admin|user:anna_parker9;integerCp:2244;"markdownCp:# Project Title + +## Overview +This project is designed to **simplify** and *automate* daily tasks. It aims to: +- Increase productivity +- Reduce manual effort +- Provide real-time data insights + +## Features +1. **Task Management**: Organize tasks efficiently with custom tags. +2. **Real-Time Analytics**: Get up-to-date insights on task progress. +3. **Automation**: Automate repetitive workflows using custom scripts. + +## Installation +To install the project, follow these steps: + +1. Clone the repository: +";multiEnumCp:multiCp1;numberCp:4422;singleEnumCp:single1;"sqlQueryCp:SELECT * FROM ALL_DATA;";"stringCp:select * from table where id="23";;";timeCp:03:04:06;timerIntervalCp:1727532807278:1727532820197;timeStampCp:1727532807278`; + +export const MOCK_GLOSSARY_TERM_CUSTOM_PROPERTIES_CONVERTED_EXTENSION_CSV_STRING = `dateCp:2024-09-18;dateTimeCp:15-09-2024 22:09:57;durationCp:PH23723D;emailCp:john.david@email.com;expert:user:"aaron.singh2";expertListPanel:databaseSchema:Glue.default.information_schema|glossaryTerm:"PW%40606600.Quick073437a4"."PW.ec0bbdf3%Bear5c6a56cc"|dashboard:sample_superset.11|user:angel_smith0|team:Legal Admin|user:anna_parker9;integerCp:2244;"markdownCp:# Project Title + +## Overview +This project is designed to **simplify** and *automate* daily tasks. It aims to: +- Increase productivity +- Reduce manual effort +- Provide real-time data insights + +## Features +1. **Task Management**: Organize tasks efficiently with custom tags. +2. **Real-Time Analytics**: Get up-to-date insights on task progress. +3. **Automation**: Automate repetitive workflows using custom scripts. + +## Installation +To install the project, follow these steps: + +1. Clone the repository:";multiEnumCp:multiCp1;numberCp:4422;singleEnumCp:single1;"sqlQueryCp:SELECT * FROM ALL_DATA;";"stringCp:select * from table where id="23";;";timeCp:03:04:06;timerIntervalCp:1727532807278:1727532820197;timeStampCp:1727532807278`; + +export const MOCK_GLOSSARY_TERM_CUSTOM_PROPERTIES_EXTENSION_OBJECT: ExtensionDataProps = + { + dateCp: '2024-09-18', + dateTimeCp: '15-09-2024 22:09:57', + durationCp: 'PH23723D', + emailCp: 'john.david@email.com', + expert: { + type: 'user', + fullyQualifiedName: '"aaron.singh2"', + name: 'aaron.singh2', + } as EntityReference, + expertListPanel: [ + { + fullyQualifiedName: 'Glue.default.information_schema', + name: 'Glue.default.information_schema', + type: 'databaseSchema', + } as EntityReference, + { + fullyQualifiedName: + '"PW%40606600.Quick073437a4"."PW.ec0bbdf3%Bear5c6a56cc"', + name: 'PW%40606600.Quick073437a4"."PW.ec0bbdf3%Bear5c6a56cc', + type: 'glossaryTerm', + } as EntityReference, + { + fullyQualifiedName: 'sample_superset.11', + name: 'sample_superset.11', + type: 'dashboard', + } as EntityReference, + { + fullyQualifiedName: 'angel_smith0', + name: 'angel_smith0', + type: 'user', + } as EntityReference, + { + fullyQualifiedName: 'Legal Admin', + name: 'Legal Admin', + type: 'team', + } as EntityReference, + { + fullyQualifiedName: 'anna_parker9', + name: 'anna_parker9', + type: 'user', + } as EntityReference, + ], + integerCp: '2244', + markdownCp: + '# Project Title\n' + + '\n' + + '## Overview\n' + + 'This project is designed to **simplify** and *automate* daily tasks. It aims to:\n' + + '- Increase productivity\n' + + '- Reduce manual effort\n' + + '- Provide real-time data insights\n' + + '\n' + + '## Features\n' + + '1. **Task Management**: Organize tasks efficiently with custom tags.\n' + + '2. **Real-Time Analytics**: Get up-to-date insights on task progress.\n' + + '3. **Automation**: Automate repetitive workflows using custom scripts.\n' + + '\n' + + '## Installation\n' + + 'To install the project, follow these steps:\n' + + '\n' + + '1. Clone the repository:', + multiEnumCp: ['multiCp1'], + numberCp: '4422', + singleEnumCp: ['single1'], + sqlQueryCp: 'SELECT * FROM ALL_DATA;', + stringCp: 'select * from table where id="23";;', + timeCp: '03:04:06', + timerIntervalCp: { + end: '1727532820197', + start: '1727532807278', + }, + timeStampCp: '1727532807278', + }; + +export const MOCK_GLOSSARY_TERM_CUSTOM_PROPERTIES: Type = { + id: '84b20f4c-6d7d-4294-a7d4-2f8adcbcdbf8', + name: 'glossaryTerm', + fullyQualifiedName: 'glossaryTerm', + displayName: 'glossaryTerm', + description: '"This schema defines te Glossary term entities."', + category: Category.Entity, + nameSpace: 'data', + schema: '', + customProperties: [ + { + name: 'dateCp', + description: 'dateCp', + propertyType: { + id: '2a960268-51dc-45aa-9c6b-217f62552a77', + type: 'type', + name: 'date-cp', + fullyQualifiedName: 'date-cp', + description: '"Date as defined in custom property."', + displayName: 'date-cp', + href: 'http://localhost:8585/api/v1/metadata/types/2a960268-51dc-45aa-9c6b-217f62552a77', + }, + customPropertyConfig: { + config: 'yyyy-MM-dd', + }, + }, + { + name: 'dateTimeCp', + description: 'dateTimeCp', + propertyType: { + id: '8b17270b-4191-41cc-971b-4d5baaa8791a', + type: 'type', + name: 'dateTime-cp', + fullyQualifiedName: 'dateTime-cp', + description: '"Date and time as defined in custom property."', + displayName: 'dateTime-cp', + href: 'http://localhost:8585/api/v1/metadata/types/8b17270b-4191-41cc-971b-4d5baaa8791a', + }, + customPropertyConfig: { + config: 'dd-MM-yyyy HH:mm:ss', + }, + }, + { + name: 'durationCp', + description: 'durationCp', + propertyType: { + id: '21dc01b3-5dd5-4534-a72c-a8badc519954', + type: 'type', + name: 'duration', + fullyQualifiedName: 'duration', + description: + '"Duration in ISO 8601 format in UTC. Example - \'P23DT23H\'."', + displayName: 'duration', + href: 'http://localhost:8585/api/v1/metadata/types/21dc01b3-5dd5-4534-a72c-a8badc519954', + }, + }, + { + name: 'emailCp', + description: 'emailCp', + propertyType: { + id: '176103aa-f539-46cc-a173-fd7e03cc1c76', + type: 'type', + name: 'email', + fullyQualifiedName: 'email', + description: '"Email address of a user or other entities."', + displayName: 'email', + href: 'http://localhost:8585/api/v1/metadata/types/176103aa-f539-46cc-a173-fd7e03cc1c76', + }, + }, + { + name: 'expert', + description: 'user', + propertyType: { + id: 'e05cc88f-a324-4a83-bb66-d332540095d1', + type: 'type', + name: 'entityReference', + fullyQualifiedName: 'entityReference', + description: '"Entity Reference for Custom Property."', + displayName: 'entityReference', + href: 'http://localhost:8585/api/v1/metadata/types/e05cc88f-a324-4a83-bb66-d332540095d1', + }, + customPropertyConfig: { + config: ['user'], + }, + }, + { + name: 'expertListPanel', + description: 'expertListPanel', + propertyType: { + id: 'ae86b69c-6296-43ab-a532-9aa5ecf85c09', + type: 'type', + name: 'entityReferenceList', + fullyQualifiedName: 'entityReferenceList', + description: '"Entity Reference List for Custom Property."', + displayName: 'entityReferenceList', + href: 'http://localhost:8585/api/v1/metadata/types/ae86b69c-6296-43ab-a532-9aa5ecf85c09', + }, + customPropertyConfig: { + config: [ + 'table', + 'pipeline', + 'team', + 'user', + 'searchIndex', + 'topic', + 'container', + 'glossaryTerm', + 'mlmodel', + 'tag', + 'dashboardDataModel', + 'dashboard', + 'database', + 'databaseSchema', + 'storedProcedure', + ], + }, + }, + { + name: 'integerCp', + description: 'integerCp', + propertyType: { + id: '34ec7caa-26d8-43ec-8189-60cd6c4bf80f', + type: 'type', + name: 'integer', + fullyQualifiedName: 'integer', + description: '"An integer type."', + displayName: 'integer', + href: 'http://localhost:8585/api/v1/metadata/types/34ec7caa-26d8-43ec-8189-60cd6c4bf80f', + }, + }, + { + name: 'markdownCp', + description: 'markdownCp', + propertyType: { + id: '7306754e-f4bb-4a52-8bef-cc183977f65f', + type: 'type', + name: 'markdown', + fullyQualifiedName: 'markdown', + description: '"Text in Markdown format."', + displayName: 'markdown', + href: 'http://localhost:8585/api/v1/metadata/types/7306754e-f4bb-4a52-8bef-cc183977f65f', + }, + }, + { + name: 'multiEnumCp', + description: 'multiEnumCp', + propertyType: { + id: '7fc6eb17-0179-428e-8d9f-3770d4648547', + type: 'type', + name: 'enum', + fullyQualifiedName: 'enum', + description: '"List of values in Enum."', + displayName: 'enum', + href: 'http://localhost:8585/api/v1/metadata/types/7fc6eb17-0179-428e-8d9f-3770d4648547', + }, + customPropertyConfig: { + config: { + values: ['multiCp1', 'm;l\'t"i:p,leCp'], + multiSelect: true, + }, + }, + }, + { + name: 'numberCp', + description: 'numberCp', + propertyType: { + id: '9e4a15d5-726f-487a-9424-94e1ca9057be', + type: 'type', + name: 'number', + fullyQualifiedName: 'number', + description: + '"A numeric type that includes integer or floating point numbers."', + displayName: 'number', + href: 'http://localhost:8585/api/v1/metadata/types/9e4a15d5-726f-487a-9424-94e1ca9057be', + }, + }, + { + name: 'singleEnumCp', + description: 'singleEnumCp', + propertyType: { + id: '7fc6eb17-0179-428e-8d9f-3770d4648547', + type: 'type', + name: 'enum', + fullyQualifiedName: 'enum', + description: '"List of values in Enum."', + displayName: 'enum', + href: 'http://localhost:8585/api/v1/metadata/types/7fc6eb17-0179-428e-8d9f-3770d4648547', + }, + customPropertyConfig: { + config: { + values: ['single1', 's;i"n,g:l\'e'], + multiSelect: false, + }, + }, + }, + { + name: 'sqlQueryCp', + description: 'sqlQueryCp', + propertyType: { + id: '9b105f9e-0d43-4282-8e51-6ec2dc23570c', + type: 'type', + name: 'sqlQuery', + fullyQualifiedName: 'sqlQuery', + description: + '"SQL query statement. Example - \'select * from orders\'."', + displayName: 'sqlQuery', + href: 'http://localhost:8585/api/v1/metadata/types/9b105f9e-0d43-4282-8e51-6ec2dc23570c', + }, + }, + { + name: 'stringCp', + description: 'stringCp', + propertyType: { + id: 'a173abc7-893b-4af9-a1b3-18ec62c6d44b', + type: 'type', + name: 'string', + fullyQualifiedName: 'string', + description: '"A String type."', + displayName: 'string', + href: 'http://localhost:8585/api/v1/metadata/types/a173abc7-893b-4af9-a1b3-18ec62c6d44b', + }, + }, + { + name: 'timeCp', + description: 'timeCp', + propertyType: { + id: 'b83e462e-58f4-409f-acf8-11ee1a107240', + type: 'type', + name: 'time-cp', + fullyQualifiedName: 'time-cp', + description: '"Time as defined in custom property."', + displayName: 'time-cp', + href: 'http://localhost:8585/api/v1/metadata/types/b83e462e-58f4-409f-acf8-11ee1a107240', + }, + customPropertyConfig: { + config: 'HH:mm:ss', + }, + }, + { + name: 'timeStampCp', + description: 'timeStampCp', + propertyType: { + id: '68cc48c3-8fc9-4ea8-a5e4-cfb679ff2b07', + type: 'type', + name: 'timestamp', + fullyQualifiedName: 'timestamp', + description: '"Timestamp in Unix epoch time milliseconds."', + displayName: 'timestamp', + href: 'http://localhost:8585/api/v1/metadata/types/68cc48c3-8fc9-4ea8-a5e4-cfb679ff2b07', + }, + }, + { + name: 'timerIntervalCp', + description: 'timerIntervalCp', + propertyType: { + id: '5e070252-d6e3-47dd-9ff3-c4a5862fad22', + type: 'type', + name: 'timeInterval', + fullyQualifiedName: 'timeInterval', + description: '"Time interval in unixTimeMillis."', + displayName: 'timeInterval', + href: 'http://localhost:8585/api/v1/metadata/types/5e070252-d6e3-47dd-9ff3-c4a5862fad22', + }, + }, + ], + version: 1.7, + updatedAt: 1727532551691, + updatedBy: 'admin', + href: 'http://localhost:8585/api/v1/metadata/types/84b20f4c-6d7d-4294-a7d4-2f8adcbcdbf8', + changeDescription: { + fieldsAdded: [ + { + name: 'customProperties', + newValue: + '[{"name":"timeStampCp","description":"timeStampCp","propertyType":{"id":"68cc48c3-8fc9-4ea8-a5e4-cfb679ff2b07","type":"type","name":"timestamp","fullyQualifiedName":"timestamp","description":"\\"Timestamp in Unix epoch time milliseconds.\\"","displayName":"timestamp"}}]', + }, + ], + fieldsUpdated: [], + fieldsDeleted: [], + previousVersion: 1.6, + }, +}; diff --git a/openmetadata-ui/src/main/resources/ui/src/utils/CSV/CSV.utils.test.tsx b/openmetadata-ui/src/main/resources/ui/src/utils/CSV/CSV.utils.test.tsx index 0c014da2b2f2..be4e97297d55 100644 --- a/openmetadata-ui/src/main/resources/ui/src/utils/CSV/CSV.utils.test.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/utils/CSV/CSV.utils.test.tsx @@ -12,6 +12,14 @@ */ import { EntityType } from '../../enums/entity.enum'; import { + MOCK_GLOSSARY_TERM_CUSTOM_PROPERTIES, + MOCK_GLOSSARY_TERM_CUSTOM_PROPERTIES_CONVERTED_EXTENSION_CSV_STRING, + MOCK_GLOSSARY_TERM_CUSTOM_PROPERTIES_EXTENSION_CSV_STRING, + MOCK_GLOSSARY_TERM_CUSTOM_PROPERTIES_EXTENSION_OBJECT, +} from '../../mocks/CSV.mock'; +import { + convertCustomPropertyStringToEntityExtension, + convertEntityExtensionToCustomPropertyString, getColumnConfig, getCSVStringFromColumnsAndDataSource, getEntityColumnsAndDataSourceFromCSV, @@ -106,4 +114,54 @@ describe('CSVUtils', () => { ); }); }); + + describe('convertCustomPropertyStringToEntityExtension', () => { + it('should return empty object if customProperty type is empty', () => { + const convertedCSVEntities = + convertCustomPropertyStringToEntityExtension('dateCp:2021-09-01'); + + expect(convertedCSVEntities).toStrictEqual({}); + }); + + it('should return object correctly which contains dot and percentage in it', () => { + const convertedCSVEntities = convertCustomPropertyStringToEntityExtension( + MOCK_GLOSSARY_TERM_CUSTOM_PROPERTIES_EXTENSION_CSV_STRING, + MOCK_GLOSSARY_TERM_CUSTOM_PROPERTIES + ); + + expect(convertedCSVEntities).toStrictEqual( + MOCK_GLOSSARY_TERM_CUSTOM_PROPERTIES_EXTENSION_OBJECT + ); + }); + }); + + describe('convertEntityExtensionToCustomPropertyString', () => { + it('should return empty object if customProperty type is empty', () => { + const convertedCSVEntities = convertEntityExtensionToCustomPropertyString( + MOCK_GLOSSARY_TERM_CUSTOM_PROPERTIES_EXTENSION_OBJECT + ); + + expect(convertedCSVEntities).toBeUndefined(); + }); + + it('should return empty object if value is empty', () => { + const convertedCSVEntities = convertEntityExtensionToCustomPropertyString( + undefined, + MOCK_GLOSSARY_TERM_CUSTOM_PROPERTIES + ); + + expect(convertedCSVEntities).toBeUndefined(); + }); + + it('should return object correctly which contains dot and percentage in it', () => { + const convertedCSVEntities = convertEntityExtensionToCustomPropertyString( + MOCK_GLOSSARY_TERM_CUSTOM_PROPERTIES_EXTENSION_OBJECT, + MOCK_GLOSSARY_TERM_CUSTOM_PROPERTIES + ); + + expect(convertedCSVEntities).toStrictEqual( + MOCK_GLOSSARY_TERM_CUSTOM_PROPERTIES_CONVERTED_EXTENSION_CSV_STRING + ); + }); + }); }); diff --git a/openmetadata-ui/src/main/resources/ui/src/utils/CSV/CSV.utils.tsx b/openmetadata-ui/src/main/resources/ui/src/utils/CSV/CSV.utils.tsx index 057f2a434a0c..ca0022b5edef 100644 --- a/openmetadata-ui/src/main/resources/ui/src/utils/CSV/CSV.utils.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/utils/CSV/CSV.utils.tsx @@ -243,7 +243,7 @@ const convertCustomPropertyValueExtensionToStringBasedOnType = ( export const convertCustomPropertyStringToEntityExtension = ( value: string, - customPropertyType: Type + customPropertyType?: Type ) => { if (isUndefined(customPropertyType)) { return {}; diff --git a/openmetadata-ui/src/main/resources/ui/src/utils/CSV/CSVUtilsClassBase.test.tsx b/openmetadata-ui/src/main/resources/ui/src/utils/CSV/CSVUtilsClassBase.test.tsx index 4c8719f75fd6..3ef4c396b244 100644 --- a/openmetadata-ui/src/main/resources/ui/src/utils/CSV/CSVUtilsClassBase.test.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/utils/CSV/CSVUtilsClassBase.test.tsx @@ -129,5 +129,12 @@ describe('CSV utils ClassBase', () => { expect(editor).toBeDefined(); }); + + it('should return the editor component for the "relatedTerms" column', () => { + const column = 'relatedTerms'; + const editor = csvUtilsClassBase.getEditor(column, EntityType.GLOSSARY); + + expect(editor).toBeDefined(); + }); }); }); From eaa9e86983df303263f0971bd6ac9630164d8894 Mon Sep 17 00:00:00 2001 From: sonikashah Date: Sun, 29 Sep 2024 12:15:40 +0530 Subject: [PATCH 22/39] resolve conflicts --- .../resources/metadata/TypeResourceTest.java | 174 ++++++++++++++++++ 1 file changed, 174 insertions(+) diff --git a/openmetadata-service/src/test/java/org/openmetadata/service/resources/metadata/TypeResourceTest.java b/openmetadata-service/src/test/java/org/openmetadata/service/resources/metadata/TypeResourceTest.java index 4e64ebf07c03..18a371eb20e2 100644 --- a/openmetadata-service/src/test/java/org/openmetadata/service/resources/metadata/TypeResourceTest.java +++ b/openmetadata-service/src/test/java/org/openmetadata/service/resources/metadata/TypeResourceTest.java @@ -45,6 +45,8 @@ import org.openmetadata.schema.type.CustomPropertyConfig; import org.openmetadata.schema.type.EntityReference; import org.openmetadata.schema.type.customproperties.EnumConfig; +import org.openmetadata.schema.type.customproperties.EnumWithDescriptionsConfig; +import org.openmetadata.schema.type.customproperties.Value; import org.openmetadata.service.Entity; import org.openmetadata.service.resources.EntityResourceTest; import org.openmetadata.service.resources.types.TypeResource; @@ -71,6 +73,7 @@ public void setupTypes() throws HttpResponseException { STRING_TYPE = getEntityByName("string", "", ADMIN_AUTH_HEADERS); EMAIL_TYPE = getEntityByName("email", "", ADMIN_AUTH_HEADERS); ENUM_TYPE = getEntityByName("enum", "", ADMIN_AUTH_HEADERS); + ENUM_WITH_DESCRIPTIONS_TYPE = getEntityByName("enumWithDescriptions", "", ADMIN_AUTH_HEADERS); DATECP_TYPE = getEntityByName("date-cp", "", ADMIN_AUTH_HEADERS); DATETIMECP_TYPE = getEntityByName("dateTime-cp", "", ADMIN_AUTH_HEADERS); TIMECP_TYPE = getEntityByName("time-cp", "", ADMIN_AUTH_HEADERS); @@ -292,6 +295,177 @@ void put_patch_customProperty_enum_200() throws IOException { new ArrayList<>(List.of(fieldA, fieldB)), tableEntity.getCustomProperties());*/ } + @Test + void put_patch_customProperty_enumWithDescriptions_200() throws IOException { + Type databaseEntity = getEntityByName("database", "customProperties", ADMIN_AUTH_HEADERS); + + // Add a custom property of type enumWithDescriptions with PUT + CustomProperty enumWithDescriptionsFieldA = + new CustomProperty() + .withName("enumWithDescriptionsTest") + .withDescription("enumWithDescriptionsTest") + .withPropertyType(ENUM_WITH_DESCRIPTIONS_TYPE.getEntityReference()); + ChangeDescription change = getChangeDescription(databaseEntity, MINOR_UPDATE); + fieldAdded(change, "customProperties", new ArrayList<>(List.of(enumWithDescriptionsFieldA))); + Type finalDatabaseEntity = databaseEntity; + ChangeDescription finalChange = change; + assertResponseContains( + () -> + addCustomPropertyAndCheck( + finalDatabaseEntity.getId(), + enumWithDescriptionsFieldA, + ADMIN_AUTH_HEADERS, + MINOR_UPDATE, + finalChange), + Status.BAD_REQUEST, + "EnumWithDescriptions Custom Property Type must have customPropertyConfig."); + enumWithDescriptionsFieldA.setCustomPropertyConfig( + new CustomPropertyConfig().withConfig(new EnumWithDescriptionsConfig())); + ChangeDescription change1 = getChangeDescription(databaseEntity, MINOR_UPDATE); + Type databaseEntity1 = databaseEntity; + assertResponseContains( + () -> + addCustomPropertyAndCheck( + databaseEntity1.getId(), + enumWithDescriptionsFieldA, + ADMIN_AUTH_HEADERS, + MINOR_UPDATE, + change1), + Status.BAD_REQUEST, + "EnumWithDescriptions Custom Property Type must have customPropertyConfig populated with values."); + + List valuesWithDuplicateKey = + List.of( + new Value().withKey("A").withDescription("Description A"), + new Value().withKey("B").withDescription("Description B"), + new Value().withKey("C").withDescription("Description C"), + new Value().withKey("C").withDescription("Description C")); + + enumWithDescriptionsFieldA.setCustomPropertyConfig( + new CustomPropertyConfig() + .withConfig(new EnumWithDescriptionsConfig().withValues(valuesWithDuplicateKey))); + ChangeDescription change7 = getChangeDescription(databaseEntity, MINOR_UPDATE); + Type databaseEntity2 = databaseEntity; + assertResponseContains( + () -> + addCustomPropertyAndCheck( + databaseEntity2.getId(), + enumWithDescriptionsFieldA, + ADMIN_AUTH_HEADERS, + MINOR_UPDATE, + change7), + Status.BAD_REQUEST, + "EnumWithDescriptions Custom Property key cannot have duplicates."); + List valuesWithUniqueKey = + List.of( + new Value().withKey("A").withDescription("Description A"), + new Value().withKey("B").withDescription("Description B"), + new Value().withKey("C").withDescription("Description C")); + enumWithDescriptionsFieldA.setCustomPropertyConfig( + new CustomPropertyConfig() + .withConfig(new EnumWithDescriptionsConfig().withValues(valuesWithUniqueKey))); + databaseEntity = + addCustomPropertyAndCheck( + databaseEntity.getId(), + enumWithDescriptionsFieldA, + ADMIN_AUTH_HEADERS, + MINOR_UPDATE, + change); + assertCustomProperties( + new ArrayList<>(List.of(enumWithDescriptionsFieldA)), databaseEntity.getCustomProperties()); + CustomPropertyConfig prevConfig = enumWithDescriptionsFieldA.getCustomPropertyConfig(); + // Changing custom property description with PUT + enumWithDescriptionsFieldA.withDescription("updatedEnumWithDescriptionsTest"); + ChangeDescription change2 = getChangeDescription(databaseEntity, MINOR_UPDATE); + fieldUpdated( + change2, + EntityUtil.getCustomField(enumWithDescriptionsFieldA, "description"), + "enumWithDescriptionsTest", + "updatedEnumWithDescriptionsTest"); + databaseEntity = + addCustomPropertyAndCheck( + databaseEntity.getId(), + enumWithDescriptionsFieldA, + ADMIN_AUTH_HEADERS, + MINOR_UPDATE, + change2); + assertCustomProperties( + new ArrayList<>(List.of(enumWithDescriptionsFieldA)), databaseEntity.getCustomProperties()); + + List valuesWithUniqueKeyAB = + List.of( + new Value().withKey("A").withDescription("Description A"), + new Value().withKey("B").withDescription("Description B")); + enumWithDescriptionsFieldA.setCustomPropertyConfig( + new CustomPropertyConfig() + .withConfig(new EnumWithDescriptionsConfig().withValues(valuesWithUniqueKeyAB))); + ChangeDescription change3 = getChangeDescription(databaseEntity, MINOR_UPDATE); + assertResponseContains( + () -> + addCustomPropertyAndCheck( + databaseEntity1.getId(), + enumWithDescriptionsFieldA, + ADMIN_AUTH_HEADERS, + MINOR_UPDATE, + change3), + Status.BAD_REQUEST, + "Existing EnumWithDescriptions Custom Property values cannot be removed."); + + enumWithDescriptionsFieldA.setCustomPropertyConfig( + new CustomPropertyConfig() + .withConfig(new EnumWithDescriptionsConfig().withValues(valuesWithDuplicateKey))); + ChangeDescription change4 = getChangeDescription(databaseEntity, MINOR_UPDATE); + assertResponseContains( + () -> + addCustomPropertyAndCheck( + databaseEntity1.getId(), + enumWithDescriptionsFieldA, + ADMIN_AUTH_HEADERS, + MINOR_UPDATE, + change4), + Status.BAD_REQUEST, + "EnumWithDescriptions Custom Property key cannot have duplicates."); + valuesWithUniqueKey = + List.of( + new Value().withKey("A").withDescription("Description A"), + new Value().withKey("B").withDescription("Description B"), + new Value().withKey("C").withDescription("Description C"), + new Value().withKey("D").withDescription("Description D")); + ChangeDescription change5 = getChangeDescription(databaseEntity, MINOR_UPDATE); + enumWithDescriptionsFieldA.setCustomPropertyConfig( + new CustomPropertyConfig() + .withConfig(new EnumWithDescriptionsConfig().withValues(valuesWithUniqueKey))); + fieldUpdated( + change5, + EntityUtil.getCustomField(enumWithDescriptionsFieldA, "customPropertyConfig"), + prevConfig, + enumWithDescriptionsFieldA.getCustomPropertyConfig()); + databaseEntity = + addCustomPropertyAndCheck( + databaseEntity.getId(), + enumWithDescriptionsFieldA, + ADMIN_AUTH_HEADERS, + MINOR_UPDATE, + change5); + assertCustomProperties( + new ArrayList<>(List.of(enumWithDescriptionsFieldA)), databaseEntity.getCustomProperties()); + + // Changing custom property description with PATCH + // Changes from this PATCH is consolidated with the previous changes + enumWithDescriptionsFieldA.withDescription("updated2"); + String json = JsonUtils.pojoToJson(databaseEntity); + databaseEntity.setCustomProperties(List.of(enumWithDescriptionsFieldA)); + change = getChangeDescription(databaseEntity, CHANGE_CONSOLIDATED); + fieldUpdated( + change5, + EntityUtil.getCustomField(enumWithDescriptionsFieldA, "description"), + "updatedEnumWithDescriptionsTest", + "updated2"); + + databaseEntity = + patchEntityAndCheck(databaseEntity, json, ADMIN_AUTH_HEADERS, CHANGE_CONSOLIDATED, change5); + } + @Test void put_customPropertyToPropertyType_4xx() { // Adding a custom property to a property type is not allowed (only entity type is allowed) From c2457bbb6886d40a23ce82a89f6b2642d00a11a0 Mon Sep 17 00:00:00 2001 From: sonikashah Date: Sun, 29 Sep 2024 14:48:11 +0530 Subject: [PATCH 23/39] Backend - add support for enumWithDescriptions in bulk import --- .../java/org/openmetadata/csv/EntityCsv.java | 79 ++++++++++++++++++- 1 file changed, 78 insertions(+), 1 deletion(-) diff --git a/openmetadata-service/src/main/java/org/openmetadata/csv/EntityCsv.java b/openmetadata-service/src/main/java/org/openmetadata/csv/EntityCsv.java index 50fc7807e45d..b8a7c7ba8052 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/csv/EntityCsv.java +++ b/openmetadata-service/src/main/java/org/openmetadata/csv/EntityCsv.java @@ -46,6 +46,8 @@ import java.util.Optional; import java.util.Set; import java.util.UUID; +import java.util.stream.Collectors; +import java.util.stream.StreamSupport; import javax.ws.rs.core.Response; import lombok.extern.slf4j.Slf4j; import org.apache.commons.csv.CSVFormat; @@ -66,6 +68,7 @@ import org.openmetadata.schema.type.csv.CsvFile; import org.openmetadata.schema.type.csv.CsvHeader; import org.openmetadata.schema.type.csv.CsvImportResult; +import org.openmetadata.schema.type.customproperties.EnumWithDescriptionsConfig; import org.openmetadata.service.Entity; import org.openmetadata.service.TypeRegistry; import org.openmetadata.service.jdbi3.EntityRepository; @@ -393,7 +396,7 @@ private void validateExtension( fieldValue.toString(), customPropertyType, propertyConfig); - case "enum", "enumList" -> { + case "enum" -> { List enumKeys = listOrEmpty(fieldToInternalArray(fieldValue.toString())); fieldValue = enumKeys.isEmpty() ? null : enumKeys; } @@ -411,6 +414,16 @@ private void validateExtension( fieldValue = null; } } + case "enumWithDescriptions" -> { + fieldValue = + parseEnumWithDescriptions( + printer, + csvRecord, + fieldNumber, + fieldName, + fieldValue.toString(), + propertyConfig); + } default -> {} } // Validate the field against the JSON schema @@ -456,6 +469,70 @@ private Object parseEntityReferences( return isList ? entityReferences : entityReferences.isEmpty() ? null : entityReferences.get(0); } + private Object parseEnumWithDescriptions( + CSVPrinter printer, + CSVRecord csvRecord, + int fieldNumber, + String fieldName, + String fieldValue, + String propertyConfig) + throws IOException { + List enumKeys = listOrEmpty(fieldToInternalArray(fieldValue)); + List enumObjects = new ArrayList<>(); + + JsonNode propertyConfigNode = JsonUtils.readTree(propertyConfig); + if (propertyConfigNode == null) { + importFailure( + printer, + invalidCustomPropertyFieldFormat( + fieldNumber, + fieldName, + "enumWithDescriptions", + "Invalid propertyConfig of enumWithDescriptions: " + fieldValue), + csvRecord); + return null; + } + + Map keyToObjectMap = + StreamSupport.stream(propertyConfigNode.get("values").spliterator(), false) + .collect(Collectors.toMap(node -> node.get("key").asText(), node -> node)); + EnumWithDescriptionsConfig config = + JsonUtils.treeToValue(propertyConfigNode, EnumWithDescriptionsConfig.class); + if (!config.getMultiSelect() && enumKeys.size() > 1) { + importFailure( + printer, + invalidCustomPropertyFieldFormat( + fieldNumber, + fieldName, + "enumWithDescriptions", + "only one key is allowed for non-multiSelect enumWithDescriptions"), + csvRecord); + return null; + } + + for (String key : enumKeys) { + try { + JsonNode valueObject = keyToObjectMap.get(key); + if (valueObject == null) { + importFailure( + printer, + invalidCustomPropertyValue( + fieldNumber, + fieldName, + "enumWithDescriptions", + key + " not found in propertyConfig of " + fieldName), + csvRecord); + return null; + } + enumObjects.add(valueObject); + } catch (Exception e) { + importFailure(printer, e.getMessage(), csvRecord); + } + } + + return enumObjects; + } + protected String getFormattedDateTimeField( CSVPrinter printer, CSVRecord csvRecord, From e4fb9d73148e4b114089483d23c737103a866992 Mon Sep 17 00:00:00 2001 From: sonikashah Date: Sun, 29 Sep 2024 19:17:35 +0530 Subject: [PATCH 24/39] add tests and other error handling improvements related to enumWithDescriptions --- .../java/org/openmetadata/csv/CsvUtil.java | 13 ++- .../java/org/openmetadata/csv/EntityCsv.java | 19 +++-- .../service/jdbi3/GlossaryRepository.java | 4 +- .../glossary/GlossaryResourceTest.java | 79 ++++++++++++++++++- 4 files changed, 103 insertions(+), 12 deletions(-) diff --git a/openmetadata-service/src/main/java/org/openmetadata/csv/CsvUtil.java b/openmetadata-service/src/main/java/org/openmetadata/csv/CsvUtil.java index 09ca7beaba92..aebd60fa679b 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/csv/CsvUtil.java +++ b/openmetadata-service/src/main/java/org/openmetadata/csv/CsvUtil.java @@ -327,8 +327,13 @@ private static String formatListValue(List list) { return ""; } - if (list.get(0) instanceof Map) { - // Handle a list of entity references or maps + if (list.get(0) instanceof Map && isEnumWithDescriptions((Map) list.get(0))) { + // Handle a list of maps with keys and descriptions + return list.stream() + .map(item -> ((Map) item).get("key").toString()) + .collect(Collectors.joining(INTERNAL_ARRAY_SEPARATOR)); + } else if (list.get(0) instanceof Map) { + // Handle a list of entity references or other maps return list.stream() .map(item -> formatMapValue((Map) item)) .collect(Collectors.joining(INTERNAL_ARRAY_SEPARATOR)); @@ -348,6 +353,10 @@ private static boolean isTimeInterval(Map valueMap) { return valueMap.containsKey("start") && valueMap.containsKey("end"); } + private static boolean isEnumWithDescriptions(Map valueMap) { + return valueMap.containsKey("key") && valueMap.containsKey("description"); + } + private static String formatEntityReference(Map valueMap) { return valueMap.get("type") + ENTITY_TYPE_SEPARATOR + valueMap.get("fullyQualifiedName"); } diff --git a/openmetadata-service/src/main/java/org/openmetadata/csv/EntityCsv.java b/openmetadata-service/src/main/java/org/openmetadata/csv/EntityCsv.java index b8a7c7ba8052..7260eb48addd 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/csv/EntityCsv.java +++ b/openmetadata-service/src/main/java/org/openmetadata/csv/EntityCsv.java @@ -578,16 +578,23 @@ private Map handleTimeInterval( List timestampValues = fieldToEntities(fieldValue.toString()); Map timestampMap = new HashMap<>(); if (timestampValues.size() == 2) { - timestampMap.put("start", Long.parseLong(timestampValues.get(0))); - timestampMap.put("end", Long.parseLong(timestampValues.get(1))); + try { + timestampMap.put("start", Long.parseLong(timestampValues.get(0))); + timestampMap.put("end", Long.parseLong(timestampValues.get(1))); + } catch (NumberFormatException e) { + importFailure( + printer, + invalidCustomPropertyValue( + fieldNumber, fieldName, "timeInterval", fieldValue.toString()), + csvRecord); + return null; + } } else { importFailure( printer, - invalidField( - fieldNumber, - invalidCustomPropertyFieldFormat( - fieldNumber, fieldName, "timeInterval", "start:end")), + invalidCustomPropertyFieldFormat(fieldNumber, fieldName, "timeInterval", "start:end"), csvRecord); + return null; } return timestampMap; } diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/GlossaryRepository.java b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/GlossaryRepository.java index 2c9dc64b0468..1ff9c3a879cd 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/GlossaryRepository.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/GlossaryRepository.java @@ -221,7 +221,9 @@ private List getTermReferences(CSVPrinter printer, CSVRecord csvR if (termRefList.size() % 2 != 0) { // List should have even numbered terms - termName and endPoint importFailure( - printer, invalidField(6, "Term references should termName;endpoint"), csvRecord); + printer, + invalidField(6, "References should be given in the format referenceName:endpoint url."), + csvRecord); processRecord = false; return null; } diff --git a/openmetadata-service/src/test/java/org/openmetadata/service/resources/glossary/GlossaryResourceTest.java b/openmetadata-service/src/test/java/org/openmetadata/service/resources/glossary/GlossaryResourceTest.java index ac9579e86be3..a8a38abe80d3 100644 --- a/openmetadata-service/src/test/java/org/openmetadata/service/resources/glossary/GlossaryResourceTest.java +++ b/openmetadata-service/src/test/java/org/openmetadata/service/resources/glossary/GlossaryResourceTest.java @@ -28,6 +28,7 @@ import static org.openmetadata.csv.EntityCsv.invalidCustomPropertyKey; import static org.openmetadata.csv.EntityCsv.invalidCustomPropertyValue; import static org.openmetadata.csv.EntityCsv.invalidExtension; +import static org.openmetadata.csv.EntityCsv.invalidField; import static org.openmetadata.csv.EntityCsvTest.assertRows; import static org.openmetadata.csv.EntityCsvTest.assertSummary; import static org.openmetadata.csv.EntityCsvTest.createCsv; @@ -533,6 +534,21 @@ record = "invalidParent,g1,dsp1,dsc1,h1;h2;h3,,term1;http://term1,Tier.Tier1,,,, }; assertRows(result, expectedRows); + // Create glossaryTerm with Invalid references + record = ",g1,dsp1,dsc1,h1;h2;h3,,term1:http://term1,,,,,"; + csv = createCsv(GlossaryCsv.HEADERS, listOf(record), null); + result = importCsv(glossaryName, csv, false); + assertSummary(result, ApiStatus.PARTIAL_SUCCESS, 2, 1, 1); + expectedRows = + new String[] { + resultsHeader, + getFailedRecord( + record, + invalidField( + 6, "References should be given in the format referenceName:endpoint url.")) + }; + assertRows(result, expectedRows); + // Create glossaryTerm with invalid tags field record = ",g1,dsp1,dsc1,h1;h2;h3,,term1;http://term1,Tag.invalidTag,,,,"; csv = createCsv(GlossaryCsv.HEADERS, listOf(record), null); @@ -627,6 +643,49 @@ invalidCustomPropertyKeyRecord, invalidCustomPropertyKey(11, "invalidCustomPrope 11, "glossaryTermDateCp", DATECP_TYPE.getDisplayName(), "dd-MM-yyyy")) }; assertRows(result, expectedRows); + + // Create glossaryTerm with Invalid custom property of type enumWithDescriptions + CustomProperty glossaryTermEnumCp = + new CustomProperty() + .withName("glossaryTermEnumWithDescriptionsCp") + .withDescription("enumWithDescriptions type custom property with multiselect = true") + .withPropertyType(ENUM_WITH_DESCRIPTIONS_TYPE.getEntityReference()) + .withCustomPropertyConfig( + new CustomPropertyConfig() + .withConfig( + Map.of( + "values", + List.of( + Map.of("key", "key1", "description", "description1"), + Map.of("key", "key2", "description", "description2")), + "multiSelect", + true))); + entityType = + typeResourceTest.getEntityByName( + Entity.GLOSSARY_TERM, "customProperties", ADMIN_AUTH_HEADERS); + entityType = + typeResourceTest.addAndCheckCustomProperty( + entityType.getId(), glossaryTermEnumCp, OK, ADMIN_AUTH_HEADERS); + String invalidEnumWithDescriptionRecord = + ",g1,dsp1,dsc1,h1;h2;h3,,term1;http://term1,PII.None,,,,glossaryTermEnumWithDescriptionsCp:key1|key3"; + String invalidEnumWithDescriptionValue = + ",g1,dsp1,dsc1,h1;h2;h3,,term1;http://term1,PII.None,,,,glossaryTermEnumWithDescriptionsCp:key1|key3"; + csv = createCsv(GlossaryCsv.HEADERS, listOf(invalidEnumWithDescriptionValue), null); + result = importCsv(glossaryName, csv, false); + Awaitility.await().atMost(4, TimeUnit.SECONDS).until(() -> true); + assertSummary(result, ApiStatus.PARTIAL_SUCCESS, 2, 1, 1); + expectedRows = + new String[] { + resultsHeader, + getFailedRecord( + invalidEnumWithDescriptionRecord, + invalidCustomPropertyValue( + 11, + "glossaryTermEnumWithDescriptionsCp", + ENUM_WITH_DESCRIPTIONS_TYPE.getDisplayName(), + "key3 not found in propertyConfig of glossaryTermEnumWithDescriptionsCp")) + }; + assertRows(result, expectedRows); } @Test @@ -751,7 +810,21 @@ void testGlossaryImportExport() throws IOException { .withName("glossaryTermEnumCpMulti") .withDescription("enum type custom property with multiselect = true") .withPropertyType(ENUM_TYPE.getEntityReference()) - .withCustomPropertyConfig(enumConfig) + .withCustomPropertyConfig(enumConfig), + new CustomProperty() + .withName("glossaryTermEnumWithDescriptionsCp") + .withDescription("enumWithDescriptions type custom property with multiselect = true") + .withPropertyType(ENUM_WITH_DESCRIPTIONS_TYPE.getEntityReference()) + .withCustomPropertyConfig( + new CustomPropertyConfig() + .withConfig( + Map.of( + "values", + List.of( + Map.of("key", "key1", "description", "description1"), + Map.of("key", "key2", "description", "description2")), + "multiSelect", + true))) }; for (CustomProperty customProperty : customProperties) { @@ -772,7 +845,7 @@ void testGlossaryImportExport() throws IOException { ",g2,dsp2,dsc3,h1;h3;h3,,term2;https://term2,PII.NonSensitive,,user:%s,%s,\"glossaryTermEnumCpMulti:val3|val2|val1|val4|val5;glossaryTermEnumCpSingle:singleVal1;glossaryTermIntegerCp:7777;glossaryTermMarkdownCp:# Sample Markdown Text;glossaryTermNumberCp:123456;\"\"glossaryTermQueryCp:select col,row from table where id ='30';\"\";glossaryTermStringCp:sample string content;glossaryTermTimeCp:10:08:45;glossaryTermTimeIntervalCp:1726142300000:17261420000;glossaryTermTimestampCp:1726142400000\"", user1, "Approved"), String.format( - "importExportTest.g1,g11,dsp2,dsc11,h1;h3;h3,,,,user:%s,team:%s,%s,", + "importExportTest.g1,g11,dsp2,dsc11,h1;h3;h3,,,,user:%s,team:%s,%s,glossaryTermEnumWithDescriptionsCp:key1|key2", reviewerRef.get(0), team11, "Draft")); // Update terms with change in description @@ -785,7 +858,7 @@ void testGlossaryImportExport() throws IOException { ",g2,dsp2,new-dsc3,h1;h3;h3,,term2;https://term2,PII.NonSensitive,user:%s,user:%s,%s,\"glossaryTermEnumCpMulti:val3|val2|val1|val4|val5;glossaryTermEnumCpSingle:singleVal1;glossaryTermIntegerCp:7777;glossaryTermMarkdownCp:# Sample Markdown Text;glossaryTermNumberCp:123456;\"\"glossaryTermQueryCp:select col,row from table where id ='30';\"\";glossaryTermStringCp:sample string content;glossaryTermTimeCp:10:08:45;glossaryTermTimeIntervalCp:1726142300000:17261420000;glossaryTermTimestampCp:1726142400000\"", user1, user2, "Approved"), String.format( - "importExportTest.g1,g11,dsp2,new-dsc11,h1;h3;h3,,,,user:%s,team:%s,%s,", + "importExportTest.g1,g11,dsp2,new-dsc11,h1;h3;h3,,,,user:%s,team:%s,%s,glossaryTermEnumWithDescriptionsCp:key1|key2", reviewerRef.get(0), team11, "Draft")); // Add new row to existing rows From 1e25a5ca1c2ce94426f86a9f14605e1e9cd441d7 Mon Sep 17 00:00:00 2001 From: Ashish Gupta Date: Sun, 29 Sep 2024 20:06:55 +0530 Subject: [PATCH 25/39] fix the custom property modal header and render the layout as per right panel in entities --- .../ModalWithCustomPropertyEditor.component.tsx | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/openmetadata-ui/src/main/resources/ui/src/components/Modals/ModalWithCustomProperty/ModalWithCustomPropertyEditor.component.tsx b/openmetadata-ui/src/main/resources/ui/src/components/Modals/ModalWithCustomProperty/ModalWithCustomPropertyEditor.component.tsx index d85b28493648..1a56cb8e5978 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/Modals/ModalWithCustomProperty/ModalWithCustomPropertyEditor.component.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/components/Modals/ModalWithCustomProperty/ModalWithCustomPropertyEditor.component.tsx @@ -10,7 +10,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -import { Button, Modal } from 'antd'; +import { Button, Modal, Typography } from 'antd'; import React, { useEffect, useState } from 'react'; import { useTranslation } from 'react-i18next'; @@ -32,6 +32,7 @@ import { } from './ModalWithMarkdownEditor.interface'; export const ModalWithCustomPropertyEditor = ({ + header, entityType, value, onSave, @@ -106,6 +107,7 @@ export const ModalWithCustomPropertyEditor = ({ ]} maskClosable={false} open={visible} + title={{header}} width="90%" onCancel={onCancel}> {isLoading ? ( @@ -114,6 +116,7 @@ export const ModalWithCustomPropertyEditor = ({ Date: Sun, 29 Sep 2024 21:02:46 +0530 Subject: [PATCH 26/39] parese enumWithDescription for the customProperty modal while editable --- .../ModalWithMarkdownEditor.interface.ts | 3 ++- .../main/resources/ui/src/utils/CSV/CSV.utils.tsx | 12 ++++++++++++ 2 files changed, 14 insertions(+), 1 deletion(-) diff --git a/openmetadata-ui/src/main/resources/ui/src/components/Modals/ModalWithCustomProperty/ModalWithMarkdownEditor.interface.ts b/openmetadata-ui/src/main/resources/ui/src/components/Modals/ModalWithCustomProperty/ModalWithMarkdownEditor.interface.ts index 6696df303564..5fa33f3b60b4 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/Modals/ModalWithCustomProperty/ModalWithMarkdownEditor.interface.ts +++ b/openmetadata-ui/src/main/resources/ui/src/components/Modals/ModalWithCustomProperty/ModalWithMarkdownEditor.interface.ts @@ -11,13 +11,14 @@ * limitations under the License. */ import { EntityType } from '../../../enums/entity.enum'; -import { EntityReference } from '../../../generated/entity/type'; +import { EntityReference, ValueClass } from '../../../generated/entity/type'; export type ExtensionDataTypes = | string | string[] | EntityReference | EntityReference[] + | ValueClass[] | { start: string; end: string }; export interface ExtensionDataProps { diff --git a/openmetadata-ui/src/main/resources/ui/src/utils/CSV/CSV.utils.tsx b/openmetadata-ui/src/main/resources/ui/src/utils/CSV/CSV.utils.tsx index ca0022b5edef..53bc9acf9528 100644 --- a/openmetadata-ui/src/main/resources/ui/src/utils/CSV/CSV.utils.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/utils/CSV/CSV.utils.tsx @@ -25,6 +25,7 @@ import { CustomProperty, EntityReference, Type, + ValueClass, } from '../../generated/entity/type'; import { Status } from '../../generated/type/csvImportResult'; import { removeOuterEscapes } from '../CommonUtils'; @@ -192,6 +193,12 @@ const convertCustomPropertyStringToValueExtensionBasedOnType = ( } } + case 'enumWithDescriptions': + return value.split('|').map((item) => ({ + key: item, + description: item, + })); + case 'timeInterval': { const [start, end] = value.split(':'); @@ -230,6 +237,11 @@ const convertCustomPropertyValueExtensionToStringBasedOnType = ( case 'enum': return (value as unknown as string[]).map((item) => item).join('|'); + case 'enumWithDescriptions': + return (value as unknown as ValueClass[]) + .map((item) => item.key) + .join('|'); + case 'timeInterval': { const interval = value as { start: string; end: string }; From 94b85159b4966657a513ce1312871ff7915dc795 Mon Sep 17 00:00:00 2001 From: Ashish Gupta Date: Sun, 29 Sep 2024 21:21:34 +0530 Subject: [PATCH 27/39] fix description data in enumWithDescription one --- .../resources/ui/src/utils/CSV/CSV.utils.tsx | 17 ++++++++++++++--- 1 file changed, 14 insertions(+), 3 deletions(-) diff --git a/openmetadata-ui/src/main/resources/ui/src/utils/CSV/CSV.utils.tsx b/openmetadata-ui/src/main/resources/ui/src/utils/CSV/CSV.utils.tsx index 53bc9acf9528..e22840a397de 100644 --- a/openmetadata-ui/src/main/resources/ui/src/utils/CSV/CSV.utils.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/utils/CSV/CSV.utils.tsx @@ -19,11 +19,13 @@ import { ExtensionDataProps, ExtensionDataTypes, } from '../../components/Modals/ModalWithCustomProperty/ModalWithMarkdownEditor.interface'; +import { NO_DATA_PLACEHOLDER } from '../../constants/constants'; import { SEMICOLON_SPLITTER } from '../../constants/regex.constants'; import { EntityType } from '../../enums/entity.enum'; import { CustomProperty, EntityReference, + EnumConfig, Type, ValueClass, } from '../../generated/entity/type'; @@ -159,7 +161,7 @@ export const getCSVStringFromColumnsAndDataSource = ( const convertCustomPropertyStringToValueExtensionBasedOnType = ( value: string, - customProperty: CustomProperty + customProperty?: CustomProperty ) => { switch (customProperty?.propertyType.name) { case 'entityReference': { @@ -193,11 +195,20 @@ const convertCustomPropertyStringToValueExtensionBasedOnType = ( } } - case 'enumWithDescriptions': + case 'enumWithDescriptions': { + const propertyEnumValues = + ((customProperty?.customPropertyConfig?.config as EnumConfig) + .values as ValueClass[]) ?? []; + + const keyAndValue: Record = {}; + + propertyEnumValues.forEach((cp) => (keyAndValue[cp.key] = cp)); + return value.split('|').map((item) => ({ key: item, - description: item, + description: keyAndValue[item].description ?? NO_DATA_PLACEHOLDER, })); + } case 'timeInterval': { const [start, end] = value.split(':'); From 6ad168a46e6a3c28cdefbf326d7c877a25fba7a6 Mon Sep 17 00:00:00 2001 From: sonikashah Date: Sun, 29 Sep 2024 23:35:17 +0530 Subject: [PATCH 28/39] fix: Handle NullPointerException when adding custom properties to ensure loop continues for other schemas of the same type for addToRegistry --- .../org/openmetadata/service/TypeRegistry.java | 15 +++++++++++---- .../service/jdbi3/GlossaryRepository.java | 3 ++- .../resources/glossary/GlossaryResourceTest.java | 2 +- 3 files changed, 14 insertions(+), 6 deletions(-) diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/TypeRegistry.java b/openmetadata-service/src/main/java/org/openmetadata/service/TypeRegistry.java index 96cd9d143007..a294a79cc51b 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/TypeRegistry.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/TypeRegistry.java @@ -69,10 +69,17 @@ private void addCustomProperty( String customPropertyFQN = getCustomPropertyFQN(entityType, propertyName); CUSTOM_PROPERTIES.put(customPropertyFQN, customProperty); - JsonSchema jsonSchema = - JsonUtils.getJsonSchema(TYPES.get(customProperty.getPropertyType().getName()).getSchema()); - CUSTOM_PROPERTY_SCHEMAS.put(customPropertyFQN, jsonSchema); - LOG.info("Adding custom property {} with JSON schema {}", customPropertyFQN, jsonSchema); + try { + JsonSchema jsonSchema = + JsonUtils.getJsonSchema( + TYPES.get(customProperty.getPropertyType().getName()).getSchema()); + CUSTOM_PROPERTY_SCHEMAS.put(customPropertyFQN, jsonSchema); + LOG.info("Adding custom property {} with JSON schema {}", customPropertyFQN, jsonSchema); + + } catch (Exception e) { + CUSTOM_PROPERTIES.remove(customPropertyFQN); + LOG.info("Failed to add custom property {}: {}", customPropertyFQN, e.getMessage()); + } } public JsonSchema getSchema(String entityType, String propertyName) { diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/GlossaryRepository.java b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/GlossaryRepository.java index 1ff9c3a879cd..1bfd75e94635 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/GlossaryRepository.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/GlossaryRepository.java @@ -222,7 +222,8 @@ private List getTermReferences(CSVPrinter printer, CSVRecord csvR // List should have even numbered terms - termName and endPoint importFailure( printer, - invalidField(6, "References should be given in the format referenceName:endpoint url."), + invalidField( + 6, "Term References should be given in the format referenceName:endpoint url."), csvRecord); processRecord = false; return null; diff --git a/openmetadata-service/src/test/java/org/openmetadata/service/resources/glossary/GlossaryResourceTest.java b/openmetadata-service/src/test/java/org/openmetadata/service/resources/glossary/GlossaryResourceTest.java index a8a38abe80d3..45d3efdc6387 100644 --- a/openmetadata-service/src/test/java/org/openmetadata/service/resources/glossary/GlossaryResourceTest.java +++ b/openmetadata-service/src/test/java/org/openmetadata/service/resources/glossary/GlossaryResourceTest.java @@ -545,7 +545,7 @@ record = ",g1,dsp1,dsc1,h1;h2;h3,,term1:http://term1,,,,,"; getFailedRecord( record, invalidField( - 6, "References should be given in the format referenceName:endpoint url.")) + 6, "Term References should be given in the format referenceName:endpoint url.")) }; assertRows(result, expectedRows); From 1986cf01116b2ce2004c04b9c1daaf4d7ca2a896 Mon Sep 17 00:00:00 2001 From: Ashish Gupta Date: Mon, 30 Sep 2024 00:46:01 +0530 Subject: [PATCH 29/39] added extension playwrigth test and fix enumWithDescription object failure --- .../constant/glossaryImportExport.ts | 21 ++++ .../e2e/Pages/GlossaryImportExport.spec.ts | 48 +++++++- .../ui/playwright/utils/importUtils.ts | 110 +++++++++++++++++- ...odalWithCustomPropertyEditor.component.tsx | 55 ++++++++- .../CustomPropertyTable/PropertyValue.tsx | 2 +- .../resources/ui/src/utils/CommonUtils.tsx | 2 +- 6 files changed, 228 insertions(+), 10 deletions(-) create mode 100644 openmetadata-ui/src/main/resources/ui/playwright/constant/glossaryImportExport.ts diff --git a/openmetadata-ui/src/main/resources/ui/playwright/constant/glossaryImportExport.ts b/openmetadata-ui/src/main/resources/ui/playwright/constant/glossaryImportExport.ts new file mode 100644 index 000000000000..54d6db070312 --- /dev/null +++ b/openmetadata-ui/src/main/resources/ui/playwright/constant/glossaryImportExport.ts @@ -0,0 +1,21 @@ +export const CUSTOM_PROPERTIES_TYPES = { + STRING: 'String', + MARKDOWN: 'Markdown', + SQL_QUERY: 'Sql Query', + ENUM_WITH_DESCRIPTIONS: 'Enum With Descriptions', +}; + +export const FIELD_VALUES_CUSTOM_PROPERTIES = { + STRING: 'This is "testing" string;', + MARKDOWN: `## Overview +This project is designed to **simplify** and *automate* daily tasks. It aims to: +- Increase productivity +- Reduce manual effort +- Provide real-time data insights + +## Features +1. **Task Management**: Organize tasks efficiently with custom tags. +2. **Real-Time Analytics**: Get up-to-date insights on task progress. +3. **Automation**: Automate repetitive workflows using custom scripts.`, + SQL_QUERY: 'SELECT * FROM table_name WHERE id="20";', +}; diff --git a/openmetadata-ui/src/main/resources/ui/playwright/e2e/Pages/GlossaryImportExport.spec.ts b/openmetadata-ui/src/main/resources/ui/playwright/e2e/Pages/GlossaryImportExport.spec.ts index b93c554629a9..c28eeba14f22 100644 --- a/openmetadata-ui/src/main/resources/ui/playwright/e2e/Pages/GlossaryImportExport.spec.ts +++ b/openmetadata-ui/src/main/resources/ui/playwright/e2e/Pages/GlossaryImportExport.spec.ts @@ -11,6 +11,9 @@ * limitations under the License. */ import { expect, test } from '@playwright/test'; +import { CUSTOM_PROPERTIES_ENTITIES } from '../../constant/customProperty'; +import { CUSTOM_PROPERTIES_TYPES } from '../../constant/glossaryImportExport'; +import { GlobalSettingOptions } from '../../constant/settings'; import { SidebarItem } from '../../constant/sidebar'; import { Glossary } from '../../support/glossary/Glossary'; import { GlossaryTerm } from '../../support/glossary/GlossaryTerm'; @@ -19,14 +22,19 @@ import { createNewPage, redirectToHomePage, toastNotification, + uuid, } from '../../utils/common'; +import { + addCustomPropertiesForEntity, + deleteCreatedProperty, +} from '../../utils/customProperty'; import { selectActiveGlossary } from '../../utils/glossary'; import { createGlossaryTermRowDetails, fillGlossaryRowDetails, validateImportStatus, } from '../../utils/importUtils'; -import { sidebarClick } from '../../utils/sidebar'; +import { settingClick, sidebarClick } from '../../utils/sidebar'; // use the admin user to login test.use({ @@ -39,6 +47,9 @@ const glossary1 = new Glossary(); const glossary2 = new Glossary(); const glossaryTerm1 = new GlossaryTerm(glossary1); const glossaryTerm2 = new GlossaryTerm(glossary2); +const propertiesList = Object.values(CUSTOM_PROPERTIES_TYPES); + +const propertyListName: Record = {}; test.describe('Glossary Bulk Import Export', () => { test.slow(true); @@ -72,6 +83,24 @@ test.describe('Glossary Bulk Import Export', () => { }); test('Glossary Bulk Import Export', async ({ page }) => { + await test.step('create custom properties for extension edit', async () => { + for (const property of propertiesList) { + const entity = CUSTOM_PROPERTIES_ENTITIES.entity_glossaryTerm; + const propertyName = `pwcustomproperty${entity.name}test${uuid()}`; + propertyListName[property] = propertyName; + + await settingClick(page, GlobalSettingOptions.GLOSSARY_TERM, true); + + await addCustomPropertiesForEntity({ + page, + propertyName, + customPropertyData: entity, + customType: property, + enumWithDescriptionConfig: entity.enumWithDescriptionConfig, + }); + } + }); + await test.step('should export data glossary term details', async () => { await sidebarClick(page, SidebarItem.GLOSSARY); await selectActiveGlossary(page, glossary1.data.displayName); @@ -89,7 +118,7 @@ test.describe('Glossary Bulk Import Export', () => { }); await test.step( - 'should import and edit with two additional database', + 'should import and edit with one additional glossaryTerm', async () => { await sidebarClick(page, SidebarItem.GLOSSARY); await selectActiveGlossary(page, glossary1.data.displayName); @@ -130,7 +159,8 @@ test.describe('Glossary Bulk Import Export', () => { name: glossaryTerm2.data.name, }, }, - page + page, + propertyListName ); await page.getByRole('button', { name: 'Next' }).click(); @@ -167,5 +197,17 @@ test.describe('Glossary Bulk Import Export', () => { ); } ); + + await test.step('delete custom properties', async () => { + for (const propertyName of Object.values(propertyListName)) { + await settingClick(page, GlobalSettingOptions.GLOSSARY_TERM, true); + + await page.waitForLoadState('networkidle'); + + await page.getByTestId('loader').waitFor({ state: 'detached' }); + + await deleteCreatedProperty(page, propertyName); + } + }); }); }); diff --git a/openmetadata-ui/src/main/resources/ui/playwright/utils/importUtils.ts b/openmetadata-ui/src/main/resources/ui/playwright/utils/importUtils.ts index e54e056a5675..1003eda7f7bf 100644 --- a/openmetadata-ui/src/main/resources/ui/playwright/utils/importUtils.ts +++ b/openmetadata-ui/src/main/resources/ui/playwright/utils/importUtils.ts @@ -11,7 +11,12 @@ * limitations under the License. */ import { expect, Page } from '@playwright/test'; -import { uuid } from './common'; +import { CUSTOM_PROPERTIES_ENTITIES } from '../constant/customProperty'; +import { + CUSTOM_PROPERTIES_TYPES, + FIELD_VALUES_CUSTOM_PROPERTIES, +} from '../constant/glossaryImportExport'; +import { descriptionBox, uuid } from './common'; export const createGlossaryTermRowDetails = () => { return { @@ -129,6 +134,100 @@ export const fillDomainDetails = async ( await page.waitForTimeout(100); }; +const editGlossaryCustomProperty = async ( + page: Page, + propertyName: string, + type: string +) => { + await page + .locator( + `[data-testid=${propertyName}] [data-testid='edit-icon-right-panel']` + ) + .click(); + + if (type === CUSTOM_PROPERTIES_TYPES.STRING) { + await page + .getByTestId('value-input') + .fill(FIELD_VALUES_CUSTOM_PROPERTIES.STRING); + await page.getByTestId('inline-save-btn').click(); + } + + if (type === CUSTOM_PROPERTIES_TYPES.MARKDOWN) { + await page.waitForSelector(descriptionBox, { state: 'visible' }); + + await page + .locator(descriptionBox) + .fill(FIELD_VALUES_CUSTOM_PROPERTIES.MARKDOWN); + + await page.getByTestId('markdown-editor').getByTestId('save').click(); + + await page.waitForSelector(descriptionBox, { + state: 'detached', + }); + } + + if (type === CUSTOM_PROPERTIES_TYPES.SQL_QUERY) { + await page + .getByTestId('code-mirror-container') + .getByRole('textbox') + .fill(FIELD_VALUES_CUSTOM_PROPERTIES.SQL_QUERY); + + await page.getByTestId('inline-save-btn').click(); + } + + if (type === CUSTOM_PROPERTIES_TYPES.ENUM_WITH_DESCRIPTIONS) { + await page.getByTestId('enum-with-description-select').click(); + + await page.waitForSelector('.ant-select-dropdown', { + state: 'visible', + }); + + // await page + // .getByRole('option', { + // name: CUSTOM_PROPERTIES_ENTITIES.entity_glossaryTerm + // .enumWithDescriptionConfig.values[0].key, + // }) + // .click(); + + await page + .locator('span') + .filter({ + hasText: + CUSTOM_PROPERTIES_ENTITIES.entity_glossaryTerm + .enumWithDescriptionConfig.values[0].key, + }) + .click(); + + await page.getByTestId('inline-save-btn').click(); + } +}; + +export const fillCustomPropertyDetails = async ( + page: Page, + propertyListName: Record +) => { + await page + .locator('.InovuaReactDataGrid__cell--cell-active') + .press('Enter', { delay: 100 }); + + // Wait for the loader to disappear + await page.waitForSelector('.ant-skeleton-content', { state: 'hidden' }); + + for (const propertyName of Object.values(CUSTOM_PROPERTIES_TYPES)) { + await editGlossaryCustomProperty( + page, + propertyListName[propertyName], + propertyName + ); + } + + await page.getByTestId('save').click(); + + await expect(page.locator('.ant-modal-wrap')).not.toBeVisible(); + + await page.click('.InovuaReactDataGrid__cell--cell-active'); +}; + export const fillGlossaryRowDetails = async ( row: { name: string; @@ -144,7 +243,8 @@ export const fillGlossaryRowDetails = async ( reviewers: string[]; owners: string[]; }, - page: Page + page: Page, + propertyListName: Record ) => { await page .locator('.InovuaReactDataGrid__cell--cell-active') @@ -200,6 +300,12 @@ export const fillGlossaryRowDetails = async ( .press('ArrowRight', { delay: 100 }); await fillOwnerDetails(page, row.owners); + + await page + .locator('.InovuaReactDataGrid__cell--cell-active') + .press('ArrowRight', { delay: 100 }); + + await fillCustomPropertyDetails(page, propertyListName); }; export const validateImportStatus = async ( diff --git a/openmetadata-ui/src/main/resources/ui/src/components/Modals/ModalWithCustomProperty/ModalWithCustomPropertyEditor.component.tsx b/openmetadata-ui/src/main/resources/ui/src/components/Modals/ModalWithCustomProperty/ModalWithCustomPropertyEditor.component.tsx index 1a56cb8e5978..567bbf2da9da 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/Modals/ModalWithCustomProperty/ModalWithCustomPropertyEditor.component.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/components/Modals/ModalWithCustomProperty/ModalWithCustomPropertyEditor.component.tsx @@ -11,13 +11,14 @@ * limitations under the License. */ import { Button, Modal, Typography } from 'antd'; -import React, { useEffect, useState } from 'react'; +import React, { useCallback, useEffect, useMemo, useState } from 'react'; import { useTranslation } from 'react-i18next'; import { AxiosError } from 'axios'; +import { isObject } from 'lodash'; import { EntityType } from '../../../enums/entity.enum'; import { GlossaryTerm } from '../../../generated/entity/data/glossaryTerm'; -import { Type } from '../../../generated/entity/type'; +import { EnumConfig, Type, ValueClass } from '../../../generated/entity/type'; import { getTypeByFQN } from '../../../rest/metadataTypeAPI'; import { convertCustomPropertyStringToEntityExtension, @@ -46,6 +47,20 @@ export const ModalWithCustomPropertyEditor = ({ useState(); const [customPropertyTypes, setCustomPropertyTypes] = useState(); + const enumWithDescriptionsKeyPairValues = useMemo(() => { + const valuesWithEnumKey: Record = {}; + + customPropertyTypes?.customProperties?.forEach((property) => { + if (property.propertyType.name === 'enumWithDescriptions') { + valuesWithEnumKey[property.name] = ( + property.customPropertyConfig?.config as EnumConfig + ).values as ValueClass[]; + } + }); + + return valuesWithEnumKey; + }, [customPropertyTypes]); + const fetchTypeDetail = async () => { setIsLoading(true); try { @@ -72,8 +87,42 @@ export const ModalWithCustomPropertyEditor = ({ setIsSaveLoading(false); }; + // EnumWithDescriptions values are change only contain keys, + // so we need to modify the extension data to include descriptions for them to display in the table + const modifyExtensionData = useCallback( + (extension: ExtensionDataProps) => { + const modifiedExtension = Object.entries(extension).reduce( + (acc, [key, value]) => { + if (enumWithDescriptionsKeyPairValues[key]) { + return { + ...acc, + [key]: (value as string[] | ValueClass[]).map((item) => { + if (isObject(item)) { + return item; + } + + return { + key: item, + description: enumWithDescriptionsKeyPairValues[key].find( + (val) => val.key === item + )?.description, + }; + }), + }; + } + + return { ...acc, [key]: value }; + }, + {} + ); + + return modifiedExtension; + }, + [enumWithDescriptionsKeyPairValues] + ); + const onExtensionUpdate = async (data: GlossaryTerm) => { - setCustomPropertyValue(data.extension); + setCustomPropertyValue(modifyExtensionData(data.extension)); }; useEffect(() => { diff --git a/openmetadata-ui/src/main/resources/ui/src/components/common/CustomPropertyTable/PropertyValue.tsx b/openmetadata-ui/src/main/resources/ui/src/components/common/CustomPropertyTable/PropertyValue.tsx index 72add4ef09e4..ab0eb32376d6 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/common/CustomPropertyTable/PropertyValue.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/components/common/CustomPropertyTable/PropertyValue.tsx @@ -1002,7 +1002,7 @@ export const PropertyValue: FC = ({ }, [property, extension, contentRef, value]); const customPropertyElement = ( - + diff --git a/openmetadata-ui/src/main/resources/ui/src/utils/CommonUtils.tsx b/openmetadata-ui/src/main/resources/ui/src/utils/CommonUtils.tsx index 264eba154b4c..ac20fe39f757 100644 --- a/openmetadata-ui/src/main/resources/ui/src/utils/CommonUtils.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/utils/CommonUtils.tsx @@ -934,5 +934,5 @@ export const removeOuterEscapes = (input: string) => { const match = input.match(VALIDATE_ESCAPE_START_END_REGEX); // Return the middle part without the outer escape characters or the original input if no match - return match ? match[2] : input; + return match && match.length > 3 ? match[2] : input; }; From 354ffb3a5009a40c937b7af2f811a1ef3b089c66 Mon Sep 17 00:00:00 2001 From: Ashish Gupta Date: Mon, 30 Sep 2024 01:05:47 +0530 Subject: [PATCH 30/39] descrease the size of extension modal --- .../ModalWithCustomPropertyEditor.component.tsx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/openmetadata-ui/src/main/resources/ui/src/components/Modals/ModalWithCustomProperty/ModalWithCustomPropertyEditor.component.tsx b/openmetadata-ui/src/main/resources/ui/src/components/Modals/ModalWithCustomProperty/ModalWithCustomPropertyEditor.component.tsx index 567bbf2da9da..e98d9324e1c8 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/Modals/ModalWithCustomProperty/ModalWithCustomPropertyEditor.component.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/components/Modals/ModalWithCustomProperty/ModalWithCustomPropertyEditor.component.tsx @@ -157,7 +157,7 @@ export const ModalWithCustomPropertyEditor = ({ maskClosable={false} open={visible} title={{header}} - width="90%" + width={650} onCancel={onCancel}> {isLoading ? ( From 24dba51b3d5d7d4072825ca9ae5c1e860badb8b1 Mon Sep 17 00:00:00 2001 From: sonikashah Date: Mon, 30 Sep 2024 05:50:38 +0530 Subject: [PATCH 31/39] remove additional comments --- .../java/org/openmetadata/csv/CsvUtil.java | 12 +-------- .../java/org/openmetadata/csv/EntityCsv.java | 25 +++++-------------- 2 files changed, 7 insertions(+), 30 deletions(-) diff --git a/openmetadata-service/src/main/java/org/openmetadata/csv/CsvUtil.java b/openmetadata-service/src/main/java/org/openmetadata/csv/CsvUtil.java index aebd60fa679b..1d6ed073a1a9 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/csv/CsvUtil.java +++ b/openmetadata-service/src/main/java/org/openmetadata/csv/CsvUtil.java @@ -124,16 +124,14 @@ public static List fieldToExtensionStrings(String field) throws IOExcept return List.of(); } - // Step 1: Replace semicolons within quoted strings with a placeholder + // Replace semicolons within quoted strings with a placeholder String preprocessedField = Pattern.compile("\"([^\"]*)\"") // Matches content inside double quotes .matcher(field) .replaceAll(mr -> "\"" + mr.group(1).replace(";", "__SEMICOLON__") + "\""); - // Step 2: Escape newlines and double quotes for CSV parsing preprocessedField = preprocessedField.replace("\n", "\\n").replace("\"", "\\\""); - // Step 3: Define CSV format with semicolon as the delimiter and proper handling of quotes CSVFormat format = CSVFormat.DEFAULT .withDelimiter(';') @@ -143,7 +141,6 @@ public static List fieldToExtensionStrings(String field) throws IOExcept .withIgnoreEmptyLines(true) .withEscape('\\'); // Use backslash for escaping special characters - // Step 4: Parse the CSV and process the records try (CSVParser parser = CSVParser.parse(new StringReader(preprocessedField), format)) { return parser.getRecords().stream() .flatMap(CSVRecord::stream) @@ -297,17 +294,14 @@ public static List addExtension(List csvRecord, Object extension } private static String formatValue(Object value) { - // Handle Map (e.g., entity reference or date interval) if (value instanceof Map) { return formatMapValue((Map) value); } - // Handle List (e.g., Entity Reference List or multi-select Enum List) if (value instanceof List) { return formatListValue((List) value); } - // Fallback for simple types return value != null ? value.toString() : ""; } @@ -318,7 +312,6 @@ private static String formatMapValue(Map valueMap) { return formatTimeInterval(valueMap); } - // If no specific format, return the raw map string return valueMap.toString(); } @@ -328,17 +321,14 @@ private static String formatListValue(List list) { } if (list.get(0) instanceof Map && isEnumWithDescriptions((Map) list.get(0))) { - // Handle a list of maps with keys and descriptions return list.stream() .map(item -> ((Map) item).get("key").toString()) .collect(Collectors.joining(INTERNAL_ARRAY_SEPARATOR)); } else if (list.get(0) instanceof Map) { - // Handle a list of entity references or other maps return list.stream() .map(item -> formatMapValue((Map) item)) .collect(Collectors.joining(INTERNAL_ARRAY_SEPARATOR)); } else { - // Handle a simple list of strings or numbers return list.stream() .map(Object::toString) .collect(Collectors.joining(INTERNAL_ARRAY_SEPARATOR)); diff --git a/openmetadata-service/src/main/java/org/openmetadata/csv/EntityCsv.java b/openmetadata-service/src/main/java/org/openmetadata/csv/EntityCsv.java index 7260eb48addd..c169465526e9 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/csv/EntityCsv.java +++ b/openmetadata-service/src/main/java/org/openmetadata/csv/EntityCsv.java @@ -335,7 +335,6 @@ public Map getExtension(CSVPrinter printer, CSVRecord csvRecord, return null; } - // Parse the extension string into a map of key-value pairs Map extensionMap = new HashMap<>(); for (String extensions : fieldToExtensionStrings(extensionString)) { @@ -343,19 +342,17 @@ public Map getExtension(CSVPrinter printer, CSVRecord csvRecord, int separatorIndex = extensions.indexOf(ENTITY_TYPE_SEPARATOR); if (separatorIndex == -1) { - // No separator found, invalid entry importFailure(printer, invalidExtension(fieldNumber, extensions, "null"), csvRecord); continue; } - String key = extensions.substring(0, separatorIndex); // Get the key part - String value = extensions.substring(separatorIndex + 1); // Get the value part + String key = extensions.substring(0, separatorIndex); + String value = extensions.substring(separatorIndex + 1); - // Validate that the key and value are present if (key.isEmpty() || value.isEmpty()) { importFailure(printer, invalidExtension(fieldNumber, key, value), csvRecord); } else { - extensionMap.put(key, value); // Add to the map + extensionMap.put(key, value); } } @@ -370,7 +367,6 @@ private void validateExtension( String fieldName = entry.getKey(); Object fieldValue = entry.getValue(); - // Fetch the JSON schema and property type for the given field name JsonSchema jsonSchema = TypeRegistry.instance().getSchema(entityType, fieldName); if (jsonSchema == null) { importFailure(printer, invalidCustomPropertyKey(fieldNumber, fieldName), csvRecord); @@ -379,8 +375,6 @@ private void validateExtension( String customPropertyType = TypeRegistry.getCustomPropertyType(entityType, fieldName); String propertyConfig = TypeRegistry.getCustomPropertyConfig(entityType, fieldName); - // Validate field based on the custom property type - switch (customPropertyType) { case "entityReference", "entityReferenceList" -> { boolean isList = "entityReferenceList".equals(customPropertyType); @@ -444,13 +438,11 @@ private Object parseEntityReferences( throws IOException { List entityReferences = new ArrayList<>(); - // Split the field into individual references or handle as single entity List entityRefStrings = isList - ? listOrEmpty(fieldToInternalArray(fieldValue)) // Split by 'INTERNAL_ARRAY_SEPARATOR' - : Collections.singletonList(fieldValue); // Single entity reference + ? listOrEmpty(fieldToInternalArray(fieldValue)) + : Collections.singletonList(fieldValue); - // Process each entity reference string for (String entityRefStr : entityRefStrings) { List entityRefTypeAndValue = listOrEmpty(fieldToEntities(entityRefStr)); @@ -549,17 +541,14 @@ protected String getFormattedDateTimeField( case "date-cp" -> { TemporalAccessor date = formatter.parse(fieldValue); yield formatter.format(date); - // Parse and format as date } case "dateTime-cp" -> { LocalDateTime dateTime = LocalDateTime.parse(fieldValue, formatter); yield dateTime.format(formatter); - // Parse and format as LocalDateTime } case "time-cp" -> { LocalTime time = LocalTime.parse(fieldValue, formatter); yield time.format(formatter); - // Parse and format as LocalTime } default -> throw new IllegalStateException("Unexpected value: " + fieldType); }; @@ -609,11 +598,9 @@ private void validateAndUpdateExtension( Map extensionMap, JsonSchema jsonSchema) throws IOException { - // Convert the field value into a JsonNode if (fieldValue != null) { JsonNode jsonNodeValue = JsonUtils.convertValue(fieldValue, JsonNode.class); - // Validate the field value using the JSON schema Set validationMessages = jsonSchema.validate(jsonNodeValue); if (!validationMessages.isEmpty()) { importFailure( @@ -622,7 +609,7 @@ private void validateAndUpdateExtension( fieldNumber, fieldName, customPropertyType, validationMessages.toString()), csvRecord); } else { - extensionMap.put(fieldName, fieldValue); // Add to extensionMap if valid + extensionMap.put(fieldName, fieldValue); } } } From addac1aa6da3ced2b482bc9e3ef89260ecd534c5 Mon Sep 17 00:00:00 2001 From: Ashish Gupta Date: Mon, 30 Sep 2024 12:45:45 +0530 Subject: [PATCH 32/39] fix the escape in parent key --- .../src/main/resources/ui/src/utils/CSV/CSV.utils.tsx | 6 +++--- .../main/resources/ui/src/utils/CSV/CSVUtilsClassBase.tsx | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/openmetadata-ui/src/main/resources/ui/src/utils/CSV/CSV.utils.tsx b/openmetadata-ui/src/main/resources/ui/src/utils/CSV/CSV.utils.tsx index e22840a397de..26b1212b943b 100644 --- a/openmetadata-ui/src/main/resources/ui/src/utils/CSV/CSV.utils.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/utils/CSV/CSV.utils.tsx @@ -306,18 +306,18 @@ export const convertEntityExtensionToCustomPropertyString = ( value?: ExtensionDataProps, customPropertyType?: Type ) => { - if (isUndefined(customPropertyType) || isUndefined(value)) { + if (isEmpty(customPropertyType) || isEmpty(value)) { return; } const keyAndValueTypes: Record = {}; - customPropertyType.customProperties?.forEach( + customPropertyType?.customProperties?.forEach( (cp) => (keyAndValueTypes[cp.name] = cp) ); let convertedString = ''; - const objectArray = Object.entries(value); + const objectArray = Object.entries(value ?? {}); objectArray.forEach(([key, value], index) => { const isLastElement = objectArray.length - 1 === index; diff --git a/openmetadata-ui/src/main/resources/ui/src/utils/CSV/CSVUtilsClassBase.tsx b/openmetadata-ui/src/main/resources/ui/src/utils/CSV/CSVUtilsClassBase.tsx index 48b49342d631..669c43692dad 100644 --- a/openmetadata-ui/src/main/resources/ui/src/utils/CSV/CSVUtilsClassBase.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/utils/CSV/CSVUtilsClassBase.tsx @@ -37,7 +37,7 @@ class CSVUtilsClassBase { } public columnsWithMultipleValuesEscapeNeeded() { - return ['extension', 'synonyms']; + return ['parent', 'extension', 'synonyms']; } public getEditor( From 779bed76089fa3c744a657ba2292d8fe4e303374 Mon Sep 17 00:00:00 2001 From: Sachin Chaurasiya Date: Mon, 30 Sep 2024 13:38:25 +0530 Subject: [PATCH 33/39] improve custom property layout --- .../APIEndpointDetails/APIEndpointDetails.tsx | 7 ++-- .../DashboardDetails.component.tsx | 7 ++-- .../DataModels/DataModelDetails.component.tsx | 7 ++-- .../DataProductsTab.component.tsx | 7 ++-- .../DocumentationTab.component.tsx | 7 ++-- .../GlossaryDetails.component.tsx | 7 ++-- .../tabs/GlossaryOverviewTab.component.tsx | 7 ++-- .../Metric/MetricDetails/MetricDetails.tsx | 7 ++-- .../MlModelDetail/MlModelDetail.component.tsx | 7 ++-- .../PipelineDetails.component.tsx | 7 ++-- .../TopicDetails/TopicDetails.component.tsx | 7 ++-- .../CustomPropertyTable.tsx | 35 ++++++++++++------- .../CustomPropertyTable/PropertyValue.tsx | 6 ++-- .../custom-property-table.less | 18 ++++++++++ .../CustomPropertyTable/property-value.less | 6 ++++ .../src/constants/ResizablePanel.constants.ts | 10 ++++++ .../APICollectionPage/APICollectionPage.tsx | 7 ++-- .../src/pages/ContainerPage/ContainerPage.tsx | 7 ++-- .../DatabaseDetailsPage.tsx | 7 ++-- .../DatabaseSchemaPage.component.tsx | 7 ++-- .../SearchIndexDetailsPage.tsx | 7 ++-- .../ServiceMainTabContent.tsx | 7 ++-- .../StoredProcedure/StoredProcedurePage.tsx | 7 ++-- .../TableDetailsPageV1/TableDetailsPageV1.tsx | 7 ++-- 24 files changed, 117 insertions(+), 91 deletions(-) create mode 100644 openmetadata-ui/src/main/resources/ui/src/components/common/CustomPropertyTable/custom-property-table.less create mode 100644 openmetadata-ui/src/main/resources/ui/src/constants/ResizablePanel.constants.ts diff --git a/openmetadata-ui/src/main/resources/ui/src/components/APIEndpoint/APIEndpointDetails/APIEndpointDetails.tsx b/openmetadata-ui/src/main/resources/ui/src/components/APIEndpoint/APIEndpointDetails/APIEndpointDetails.tsx index b3a65ac6d1e8..b0c3036d2aae 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/APIEndpoint/APIEndpointDetails/APIEndpointDetails.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/components/APIEndpoint/APIEndpointDetails/APIEndpointDetails.tsx @@ -19,6 +19,7 @@ import { useTranslation } from 'react-i18next'; import { useHistory, useParams } from 'react-router-dom'; import { getEntityDetailsPath } from '../../../constants/constants'; import { FEED_COUNT_INITIAL_DATA } from '../../../constants/entity.constants'; +import { COMMON_RESIZABLE_PANEL_CONFIG } from '../../../constants/ResizablePanel.constants'; import LineageProvider from '../../../context/LineageProvider/LineageProvider'; import { EntityTabs, EntityType } from '../../../enums/entity.enum'; import { Tag } from '../../../generated/entity/classification/tag'; @@ -325,8 +326,7 @@ const APIEndpointDetails: React.FC = ({ /> ), - minWidth: 800, - flex: 0.87, + ...COMMON_RESIZABLE_PANEL_CONFIG.LEFT_PANEL, }} secondPanel={{ children: ( @@ -350,8 +350,7 @@ const APIEndpointDetails: React.FC = ({ /> ), - minWidth: 320, - flex: 0.13, + ...COMMON_RESIZABLE_PANEL_CONFIG.RIGHT_PANEL, className: 'entity-resizable-right-panel-container entity-resizable-panel-container', }} diff --git a/openmetadata-ui/src/main/resources/ui/src/components/Dashboard/DashboardDetails/DashboardDetails.component.tsx b/openmetadata-ui/src/main/resources/ui/src/components/Dashboard/DashboardDetails/DashboardDetails.component.tsx index 7410e12f75fa..43403585cfdd 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/Dashboard/DashboardDetails/DashboardDetails.component.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/components/Dashboard/DashboardDetails/DashboardDetails.component.tsx @@ -28,6 +28,7 @@ import { getEntityDetailsPath, } from '../../../constants/constants'; import { FEED_COUNT_INITIAL_DATA } from '../../../constants/entity.constants'; +import { COMMON_RESIZABLE_PANEL_CONFIG } from '../../../constants/ResizablePanel.constants'; import LineageProvider from '../../../context/LineageProvider/LineageProvider'; import { usePermissionProvider } from '../../../context/PermissionProvider/PermissionProvider'; import { ResourceEntity } from '../../../context/PermissionProvider/PermissionProvider.interface'; @@ -652,8 +653,7 @@ const DashboardDetails = ({ )} ), - minWidth: 800, - flex: 0.87, + ...COMMON_RESIZABLE_PANEL_CONFIG.LEFT_PANEL, }} secondPanel={{ children: ( @@ -677,8 +677,7 @@ const DashboardDetails = ({ /> ), - minWidth: 320, - flex: 0.13, + ...COMMON_RESIZABLE_PANEL_CONFIG.RIGHT_PANEL, className: 'entity-resizable-right-panel-container entity-resizable-panel-container', }} diff --git a/openmetadata-ui/src/main/resources/ui/src/components/Dashboard/DataModel/DataModels/DataModelDetails.component.tsx b/openmetadata-ui/src/main/resources/ui/src/components/Dashboard/DataModel/DataModels/DataModelDetails.component.tsx index 1d9a808c9b8a..e844a0d696c1 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/Dashboard/DataModel/DataModels/DataModelDetails.component.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/components/Dashboard/DataModel/DataModels/DataModelDetails.component.tsx @@ -23,6 +23,7 @@ import { getVersionPath, } from '../../../../constants/constants'; import { FEED_COUNT_INITIAL_DATA } from '../../../../constants/entity.constants'; +import { COMMON_RESIZABLE_PANEL_CONFIG } from '../../../../constants/ResizablePanel.constants'; import LineageProvider from '../../../../context/LineageProvider/LineageProvider'; import { CSMode } from '../../../../enums/codemirror.enum'; import { EntityTabs, EntityType } from '../../../../enums/entity.enum'; @@ -259,8 +260,7 @@ const DataModelDetails = ({ /> ), - minWidth: 800, - flex: 0.87, + ...COMMON_RESIZABLE_PANEL_CONFIG.LEFT_PANEL, }} secondPanel={{ children: ( @@ -286,8 +286,7 @@ const DataModelDetails = ({ /> ), - minWidth: 320, - flex: 0.13, + ...COMMON_RESIZABLE_PANEL_CONFIG.RIGHT_PANEL, className: 'entity-resizable-right-panel-container entity-resizable-panel-container', }} diff --git a/openmetadata-ui/src/main/resources/ui/src/components/Domain/DomainTabs/DataProductsTab/DataProductsTab.component.tsx b/openmetadata-ui/src/main/resources/ui/src/components/Domain/DomainTabs/DataProductsTab/DataProductsTab.component.tsx index 22a50405b386..9a807da1c622 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/Domain/DomainTabs/DataProductsTab/DataProductsTab.component.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/components/Domain/DomainTabs/DataProductsTab/DataProductsTab.component.tsx @@ -22,6 +22,7 @@ import React, { } from 'react'; import { useTranslation } from 'react-i18next'; import { PAGE_SIZE_LARGE } from '../../../../constants/constants'; +import { COMMON_RESIZABLE_PANEL_CONFIG } from '../../../../constants/ResizablePanel.constants'; import { ERROR_PLACEHOLDER_TYPE } from '../../../../enums/common.enum'; import { EntityType } from '../../../../enums/entity.enum'; import { SearchIndex } from '../../../../enums/search.enum'; @@ -142,8 +143,7 @@ const DataProductsTab = forwardRef( ))} ), - minWidth: 800, - flex: 0.87, + ...COMMON_RESIZABLE_PANEL_CONFIG.LEFT_PANEL, }} pageTitle={t('label.domain')} secondPanel={{ @@ -158,8 +158,7 @@ const DataProductsTab = forwardRef( handleClosePanel={() => setSelectedCard(undefined)} /> ), - minWidth: 320, - flex: 0.13, + ...COMMON_RESIZABLE_PANEL_CONFIG.RIGHT_PANEL, className: 'entity-summary-resizable-right-panel-container domain-resizable-panel-container', }} diff --git a/openmetadata-ui/src/main/resources/ui/src/components/Domain/DomainTabs/DocumentationTab/DocumentationTab.component.tsx b/openmetadata-ui/src/main/resources/ui/src/components/Domain/DomainTabs/DocumentationTab/DocumentationTab.component.tsx index dedea541f7cc..2378134a3d78 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/Domain/DomainTabs/DocumentationTab/DocumentationTab.component.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/components/Domain/DomainTabs/DocumentationTab/DocumentationTab.component.tsx @@ -22,6 +22,7 @@ import { UserTeamSelectableList } from '../../../../components/common/UserTeamSe import DomainTypeSelectForm from '../../../../components/Domain/DomainTypeSelectForm/DomainTypeSelectForm.component'; import { DE_ACTIVE_COLOR } from '../../../../constants/constants'; import { EntityField } from '../../../../constants/Feeds.constants'; +import { COMMON_RESIZABLE_PANEL_CONFIG } from '../../../../constants/ResizablePanel.constants'; import { usePermissionProvider } from '../../../../context/PermissionProvider/PermissionProvider'; import { ResourceEntity } from '../../../../context/PermissionProvider/PermissionProvider.interface'; import { EntityType, TabSpecificField } from '../../../../enums/entity.enum'; @@ -191,8 +192,7 @@ const DocumentationTab = ({ /> ), - minWidth: 800, - flex: 0.75, + ...COMMON_RESIZABLE_PANEL_CONFIG.LEFT_PANEL, }} secondPanel={{ children: ( @@ -374,8 +374,7 @@ const DocumentationTab = ({ )} ), - minWidth: 320, - flex: 0.25, + ...COMMON_RESIZABLE_PANEL_CONFIG.RIGHT_PANEL, className: 'entity-resizable-right-panel-container domain-resizable-panel-container', }} diff --git a/openmetadata-ui/src/main/resources/ui/src/components/Glossary/GlossaryDetails/GlossaryDetails.component.tsx b/openmetadata-ui/src/main/resources/ui/src/components/Glossary/GlossaryDetails/GlossaryDetails.component.tsx index 6f3ede3e481f..0ffba4c955a2 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/Glossary/GlossaryDetails/GlossaryDetails.component.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/components/Glossary/GlossaryDetails/GlossaryDetails.component.tsx @@ -20,6 +20,7 @@ import { useHistory, useParams } from 'react-router-dom'; import { getGlossaryTermDetailsPath } from '../../../constants/constants'; import { FEED_COUNT_INITIAL_DATA } from '../../../constants/entity.constants'; import { EntityField } from '../../../constants/Feeds.constants'; +import { COMMON_RESIZABLE_PANEL_CONFIG } from '../../../constants/ResizablePanel.constants'; import { EntityType } from '../../../enums/entity.enum'; import { Glossary } from '../../../generated/entity/data/glossary'; import { ChangeDescription } from '../../../generated/entity/type'; @@ -179,8 +180,7 @@ const GlossaryDetails = ({ ), - minWidth: 800, - flex: 0.75, + ...COMMON_RESIZABLE_PANEL_CONFIG.LEFT_PANEL, }} secondPanel={{ children: ( @@ -195,8 +195,7 @@ const GlossaryDetails = ({ onUpdate={handleGlossaryUpdate} /> ), - minWidth: 320, - flex: 0.25, + ...COMMON_RESIZABLE_PANEL_CONFIG.RIGHT_PANEL, className: 'entity-resizable-right-panel-container glossary-resizable-panel-container', }} diff --git a/openmetadata-ui/src/main/resources/ui/src/components/Glossary/GlossaryTerms/tabs/GlossaryOverviewTab.component.tsx b/openmetadata-ui/src/main/resources/ui/src/components/Glossary/GlossaryTerms/tabs/GlossaryOverviewTab.component.tsx index 4df714ceccce..4a85cb9f81e2 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/Glossary/GlossaryTerms/tabs/GlossaryOverviewTab.component.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/components/Glossary/GlossaryTerms/tabs/GlossaryOverviewTab.component.tsx @@ -13,6 +13,7 @@ import { Col, Row, Space } from 'antd'; import React, { useMemo, useState } from 'react'; import { EntityField } from '../../../../constants/Feeds.constants'; +import { COMMON_RESIZABLE_PANEL_CONFIG } from '../../../../constants/ResizablePanel.constants'; import { OperationPermission } from '../../../../context/PermissionProvider/PermissionProvider.interface'; import { EntityType } from '../../../../enums/entity.enum'; import { Glossary } from '../../../../generated/entity/data/glossary'; @@ -188,8 +189,7 @@ const GlossaryOverviewTab = ({ ), - minWidth: 800, - flex: 0.75, + ...COMMON_RESIZABLE_PANEL_CONFIG.LEFT_PANEL, }} secondPanel={{ children: ( @@ -205,8 +205,7 @@ const GlossaryOverviewTab = ({ onUpdate={onUpdate} /> ), - minWidth: 320, - flex: 0.25, + ...COMMON_RESIZABLE_PANEL_CONFIG.RIGHT_PANEL, className: 'entity-resizable-right-panel-container', }} /> diff --git a/openmetadata-ui/src/main/resources/ui/src/components/Metric/MetricDetails/MetricDetails.tsx b/openmetadata-ui/src/main/resources/ui/src/components/Metric/MetricDetails/MetricDetails.tsx index a8bb7ef12d69..dd4fbc326dfb 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/Metric/MetricDetails/MetricDetails.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/components/Metric/MetricDetails/MetricDetails.tsx @@ -19,6 +19,7 @@ import { useTranslation } from 'react-i18next'; import { useHistory, useParams } from 'react-router-dom'; import { getEntityDetailsPath, ROUTES } from '../../../constants/constants'; import { FEED_COUNT_INITIAL_DATA } from '../../../constants/entity.constants'; +import { COMMON_RESIZABLE_PANEL_CONFIG } from '../../../constants/ResizablePanel.constants'; import LineageProvider from '../../../context/LineageProvider/LineageProvider'; import { EntityTabs, EntityType } from '../../../enums/entity.enum'; import { Tag } from '../../../generated/entity/classification/tag'; @@ -312,8 +313,7 @@ const MetricDetails: React.FC = ({ /> ), - minWidth: 800, - flex: 0.87, + ...COMMON_RESIZABLE_PANEL_CONFIG.LEFT_PANEL, }} secondPanel={{ children: ( @@ -346,8 +346,7 @@ const MetricDetails: React.FC = ({ /> ), - minWidth: 320, - flex: 0.13, + ...COMMON_RESIZABLE_PANEL_CONFIG.RIGHT_PANEL, className: 'entity-resizable-right-panel-container entity-resizable-panel-container', }} diff --git a/openmetadata-ui/src/main/resources/ui/src/components/MlModel/MlModelDetail/MlModelDetail.component.tsx b/openmetadata-ui/src/main/resources/ui/src/components/MlModel/MlModelDetail/MlModelDetail.component.tsx index 590684e9d731..77fc45d9b2f6 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/MlModel/MlModelDetail/MlModelDetail.component.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/components/MlModel/MlModelDetail/MlModelDetail.component.tsx @@ -21,6 +21,7 @@ import { useTranslation } from 'react-i18next'; import { useHistory, useParams } from 'react-router-dom'; import { getEntityDetailsPath } from '../../../constants/constants'; import { FEED_COUNT_INITIAL_DATA } from '../../../constants/entity.constants'; +import { COMMON_RESIZABLE_PANEL_CONFIG } from '../../../constants/ResizablePanel.constants'; import LineageProvider from '../../../context/LineageProvider/LineageProvider'; import { usePermissionProvider } from '../../../context/PermissionProvider/PermissionProvider'; import { ResourceEntity } from '../../../context/PermissionProvider/PermissionProvider.interface'; @@ -426,8 +427,7 @@ const MlModelDetail: FC = ({ /> ), - minWidth: 800, - flex: 0.87, + ...COMMON_RESIZABLE_PANEL_CONFIG.LEFT_PANEL, }} secondPanel={{ children: ( @@ -451,8 +451,7 @@ const MlModelDetail: FC = ({ /> ), - minWidth: 320, - flex: 0.13, + ...COMMON_RESIZABLE_PANEL_CONFIG.RIGHT_PANEL, className: 'entity-resizable-right-panel-container entity-resizable-panel-container', }} diff --git a/openmetadata-ui/src/main/resources/ui/src/components/Pipeline/PipelineDetails/PipelineDetails.component.tsx b/openmetadata-ui/src/main/resources/ui/src/components/Pipeline/PipelineDetails/PipelineDetails.component.tsx index ba430f9804b9..7ea1244a54d9 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/Pipeline/PipelineDetails/PipelineDetails.component.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/components/Pipeline/PipelineDetails/PipelineDetails.component.tsx @@ -30,6 +30,7 @@ import { } from '../../../constants/constants'; import { FEED_COUNT_INITIAL_DATA } from '../../../constants/entity.constants'; import { PIPELINE_TASK_TABS } from '../../../constants/pipeline.constants'; +import { COMMON_RESIZABLE_PANEL_CONFIG } from '../../../constants/ResizablePanel.constants'; import LineageProvider from '../../../context/LineageProvider/LineageProvider'; import { usePermissionProvider } from '../../../context/PermissionProvider/PermissionProvider'; import { ResourceEntity } from '../../../context/PermissionProvider/PermissionProvider.interface'; @@ -648,8 +649,7 @@ const PipelineDetails = ({ ), - minWidth: 800, - flex: 0.87, + ...COMMON_RESIZABLE_PANEL_CONFIG.LEFT_PANEL, }} secondPanel={{ children: ( @@ -673,8 +673,7 @@ const PipelineDetails = ({ /> ), - minWidth: 320, - flex: 0.13, + ...COMMON_RESIZABLE_PANEL_CONFIG.RIGHT_PANEL, className: 'entity-resizable-right-panel-container entity-resizable-panel-container', }} diff --git a/openmetadata-ui/src/main/resources/ui/src/components/Topic/TopicDetails/TopicDetails.component.tsx b/openmetadata-ui/src/main/resources/ui/src/components/Topic/TopicDetails/TopicDetails.component.tsx index cca6405f5121..809a5c9397cb 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/Topic/TopicDetails/TopicDetails.component.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/components/Topic/TopicDetails/TopicDetails.component.tsx @@ -20,6 +20,7 @@ import { useTranslation } from 'react-i18next'; import { useHistory, useParams } from 'react-router-dom'; import { getEntityDetailsPath } from '../../../constants/constants'; import { FEED_COUNT_INITIAL_DATA } from '../../../constants/entity.constants'; +import { COMMON_RESIZABLE_PANEL_CONFIG } from '../../../constants/ResizablePanel.constants'; import LineageProvider from '../../../context/LineageProvider/LineageProvider'; import { ERROR_PLACEHOLDER_TYPE } from '../../../enums/common.enum'; import { EntityTabs, EntityType } from '../../../enums/entity.enum'; @@ -337,8 +338,7 @@ const TopicDetails: React.FC = ({ /> ), - minWidth: 800, - flex: 0.87, + ...COMMON_RESIZABLE_PANEL_CONFIG.LEFT_PANEL, }} secondPanel={{ children: ( @@ -362,8 +362,7 @@ const TopicDetails: React.FC = ({ /> ), - minWidth: 320, - flex: 0.13, + ...COMMON_RESIZABLE_PANEL_CONFIG.RIGHT_PANEL, className: 'entity-resizable-right-panel-container entity-resizable-panel-container', }} diff --git a/openmetadata-ui/src/main/resources/ui/src/components/common/CustomPropertyTable/CustomPropertyTable.tsx b/openmetadata-ui/src/main/resources/ui/src/components/common/CustomPropertyTable/CustomPropertyTable.tsx index 4bf4cb047006..b82da885bb1b 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/common/CustomPropertyTable/CustomPropertyTable.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/components/common/CustomPropertyTable/CustomPropertyTable.tsx @@ -13,6 +13,7 @@ import { Col, Divider, Row, Skeleton, Typography } from 'antd'; import { AxiosError } from 'axios'; +import classNames from 'classnames'; import { isEmpty, isUndefined } from 'lodash'; import React, { Fragment, @@ -43,6 +44,7 @@ import { } from '../../../utils/EntityVersionUtils'; import { showErrorToast } from '../../../utils/ToastUtils'; import ErrorPlaceHolder from '../ErrorWithPlaceholder/ErrorPlaceHolder'; +import './custom-property-table.less'; import { CustomPropertyProps, ExtentionEntities, @@ -252,25 +254,32 @@ export const CustomPropertyTable = ({ {isRenderedInRightPanel ? ( - <> +
{dataSource.map((record, index) => ( - +
+ +
{index !== dataSource.length - 1 && ( - + )} ))} - +
) : ( {dataSource.map((record) => ( diff --git a/openmetadata-ui/src/main/resources/ui/src/components/common/CustomPropertyTable/PropertyValue.tsx b/openmetadata-ui/src/main/resources/ui/src/components/common/CustomPropertyTable/PropertyValue.tsx index ab0eb32376d6..b7adbe1ee9d3 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/common/CustomPropertyTable/PropertyValue.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/components/common/CustomPropertyTable/PropertyValue.tsx @@ -617,6 +617,7 @@ export const PropertyValue: FC = ({ return ( = ({ default: return ( {value} @@ -1030,6 +1031,7 @@ export const PropertyValue: FC = ({ @@ -1039,7 +1041,7 @@ export const PropertyValue: FC = ({ { /> ), - minWidth: 800, - flex: 0.87, + ...COMMON_RESIZABLE_PANEL_CONFIG.LEFT_PANEL, }} secondPanel={{ children: ( @@ -613,8 +613,7 @@ const APICollectionPage: FunctionComponent = () => { /> ), - minWidth: 320, - flex: 0.13, + ...COMMON_RESIZABLE_PANEL_CONFIG.RIGHT_PANEL, className: 'entity-resizable-right-panel-container entity-resizable-panel-container', }} diff --git a/openmetadata-ui/src/main/resources/ui/src/pages/ContainerPage/ContainerPage.tsx b/openmetadata-ui/src/main/resources/ui/src/pages/ContainerPage/ContainerPage.tsx index fd87e53dfb1b..81fc520f577a 100644 --- a/openmetadata-ui/src/main/resources/ui/src/pages/ContainerPage/ContainerPage.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/pages/ContainerPage/ContainerPage.tsx @@ -44,6 +44,7 @@ import { ROUTES, } from '../../constants/constants'; import { FEED_COUNT_INITIAL_DATA } from '../../constants/entity.constants'; +import { COMMON_RESIZABLE_PANEL_CONFIG } from '../../constants/ResizablePanel.constants'; import LineageProvider from '../../context/LineageProvider/LineageProvider'; import { usePermissionProvider } from '../../context/PermissionProvider/PermissionProvider'; import { @@ -619,8 +620,7 @@ const ContainerPage = () => { )} ), - minWidth: 800, - flex: 0.87, + ...COMMON_RESIZABLE_PANEL_CONFIG.LEFT_PANEL, }} secondPanel={{ children: ( @@ -646,8 +646,7 @@ const ContainerPage = () => { /> ), - minWidth: 320, - flex: 0.13, + ...COMMON_RESIZABLE_PANEL_CONFIG.RIGHT_PANEL, className: 'entity-resizable-right-panel-container entity-resizable-panel-container', }} diff --git a/openmetadata-ui/src/main/resources/ui/src/pages/DatabaseDetailsPage/DatabaseDetailsPage.tsx b/openmetadata-ui/src/main/resources/ui/src/pages/DatabaseDetailsPage/DatabaseDetailsPage.tsx index 1364c5853c7b..9a26c17aebe4 100644 --- a/openmetadata-ui/src/main/resources/ui/src/pages/DatabaseDetailsPage/DatabaseDetailsPage.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/pages/DatabaseDetailsPage/DatabaseDetailsPage.tsx @@ -51,6 +51,7 @@ import { ROUTES, } from '../../constants/constants'; import { FEED_COUNT_INITIAL_DATA } from '../../constants/entity.constants'; +import { COMMON_RESIZABLE_PANEL_CONFIG } from '../../constants/ResizablePanel.constants'; import { usePermissionProvider } from '../../context/PermissionProvider/PermissionProvider'; import { OperationPermission, @@ -548,8 +549,7 @@ const DatabaseDetails: FunctionComponent = () => { ), - minWidth: 800, - flex: 0.87, + ...COMMON_RESIZABLE_PANEL_CONFIG.LEFT_PANEL, }} secondPanel={{ children: ( @@ -573,8 +573,7 @@ const DatabaseDetails: FunctionComponent = () => { /> ), - minWidth: 320, - flex: 0.13, + ...COMMON_RESIZABLE_PANEL_CONFIG.RIGHT_PANEL, className: 'entity-resizable-right-panel-container entity-resizable-panel-container', }} diff --git a/openmetadata-ui/src/main/resources/ui/src/pages/DatabaseSchemaPage/DatabaseSchemaPage.component.tsx b/openmetadata-ui/src/main/resources/ui/src/pages/DatabaseSchemaPage/DatabaseSchemaPage.component.tsx index 6017820fd9c9..461b0e32e641 100644 --- a/openmetadata-ui/src/main/resources/ui/src/pages/DatabaseSchemaPage/DatabaseSchemaPage.component.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/pages/DatabaseSchemaPage/DatabaseSchemaPage.component.tsx @@ -50,6 +50,7 @@ import { ROUTES, } from '../../constants/constants'; import { FEED_COUNT_INITIAL_DATA } from '../../constants/entity.constants'; +import { COMMON_RESIZABLE_PANEL_CONFIG } from '../../constants/ResizablePanel.constants'; import { usePermissionProvider } from '../../context/PermissionProvider/PermissionProvider'; import { OperationPermission, @@ -608,8 +609,7 @@ const DatabaseSchemaPage: FunctionComponent = () => { /> ), - minWidth: 800, - flex: 0.87, + ...COMMON_RESIZABLE_PANEL_CONFIG.LEFT_PANEL, }} secondPanel={{ children: ( @@ -633,8 +633,7 @@ const DatabaseSchemaPage: FunctionComponent = () => { /> ), - minWidth: 320, - flex: 0.13, + ...COMMON_RESIZABLE_PANEL_CONFIG.RIGHT_PANEL, className: 'entity-resizable-right-panel-container entity-resizable-panel-container', }} diff --git a/openmetadata-ui/src/main/resources/ui/src/pages/SearchIndexDetailsPage/SearchIndexDetailsPage.tsx b/openmetadata-ui/src/main/resources/ui/src/pages/SearchIndexDetailsPage/SearchIndexDetailsPage.tsx index 2465d3da5a79..4ffc13f2a324 100644 --- a/openmetadata-ui/src/main/resources/ui/src/pages/SearchIndexDetailsPage/SearchIndexDetailsPage.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/pages/SearchIndexDetailsPage/SearchIndexDetailsPage.tsx @@ -44,6 +44,7 @@ import { getVersionPath, } from '../../constants/constants'; import { FEED_COUNT_INITIAL_DATA } from '../../constants/entity.constants'; +import { COMMON_RESIZABLE_PANEL_CONFIG } from '../../constants/ResizablePanel.constants'; import LineageProvider from '../../context/LineageProvider/LineageProvider'; import { usePermissionProvider } from '../../context/PermissionProvider/PermissionProvider'; import { @@ -413,8 +414,7 @@ function SearchIndexDetailsPage() { /> ), - minWidth: 800, - flex: 0.87, + ...COMMON_RESIZABLE_PANEL_CONFIG.LEFT_PANEL, }} secondPanel={{ children: ( @@ -438,8 +438,7 @@ function SearchIndexDetailsPage() { /> ), - minWidth: 320, - flex: 0.13, + ...COMMON_RESIZABLE_PANEL_CONFIG.RIGHT_PANEL, className: 'entity-resizable-right-panel-container entity-resizable-panel-container', }} diff --git a/openmetadata-ui/src/main/resources/ui/src/pages/ServiceDetailsPage/ServiceMainTabContent.tsx b/openmetadata-ui/src/main/resources/ui/src/pages/ServiceDetailsPage/ServiceMainTabContent.tsx index 6444734e1f0c..66022e73746f 100644 --- a/openmetadata-ui/src/main/resources/ui/src/pages/ServiceDetailsPage/ServiceMainTabContent.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/pages/ServiceDetailsPage/ServiceMainTabContent.tsx @@ -26,6 +26,7 @@ import { NextPreviousProps } from '../../components/common/NextPrevious/NextPrev import ResizablePanels from '../../components/common/ResizablePanels/ResizablePanels'; import EntityRightPanel from '../../components/Entity/EntityRightPanel/EntityRightPanel'; import { PAGE_SIZE } from '../../constants/constants'; +import { COMMON_RESIZABLE_PANEL_CONFIG } from '../../constants/ResizablePanel.constants'; import { OperationPermission } from '../../context/PermissionProvider/PermissionProvider.interface'; import { EntityType } from '../../enums/entity.enum'; import { DatabaseService } from '../../generated/entity/services/databaseService'; @@ -223,8 +224,7 @@ function ServiceMainTabContent({ ), - minWidth: 800, - flex: 0.87, + ...COMMON_RESIZABLE_PANEL_CONFIG.LEFT_PANEL, }} secondPanel={{ children: ( @@ -247,8 +247,7 @@ function ServiceMainTabContent({ /> ), - minWidth: 320, - flex: 0.13, + ...COMMON_RESIZABLE_PANEL_CONFIG.RIGHT_PANEL, className: 'entity-resizable-right-panel-container entity-resizable-panel-container', }} diff --git a/openmetadata-ui/src/main/resources/ui/src/pages/StoredProcedure/StoredProcedurePage.tsx b/openmetadata-ui/src/main/resources/ui/src/pages/StoredProcedure/StoredProcedurePage.tsx index 49c272a3bbb5..9b0af54406e9 100644 --- a/openmetadata-ui/src/main/resources/ui/src/pages/StoredProcedure/StoredProcedurePage.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/pages/StoredProcedure/StoredProcedurePage.tsx @@ -43,6 +43,7 @@ import { ROUTES, } from '../../constants/constants'; import { FEED_COUNT_INITIAL_DATA } from '../../constants/entity.constants'; +import { COMMON_RESIZABLE_PANEL_CONFIG } from '../../constants/ResizablePanel.constants'; import LineageProvider from '../../context/LineageProvider/LineageProvider'; import { usePermissionProvider } from '../../context/PermissionProvider/PermissionProvider'; import { @@ -578,8 +579,7 @@ const StoredProcedurePage = () => { ), - minWidth: 800, - flex: 0.87, + ...COMMON_RESIZABLE_PANEL_CONFIG.LEFT_PANEL, }} secondPanel={{ children: ( @@ -603,8 +603,7 @@ const StoredProcedurePage = () => { /> ), - minWidth: 320, - flex: 0.13, + ...COMMON_RESIZABLE_PANEL_CONFIG.RIGHT_PANEL, className: 'entity-resizable-right-panel-container entity-resizable-panel-container', }} diff --git a/openmetadata-ui/src/main/resources/ui/src/pages/TableDetailsPageV1/TableDetailsPageV1.tsx b/openmetadata-ui/src/main/resources/ui/src/pages/TableDetailsPageV1/TableDetailsPageV1.tsx index 5253827e1d34..6d0c540a99f5 100644 --- a/openmetadata-ui/src/main/resources/ui/src/pages/TableDetailsPageV1/TableDetailsPageV1.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/pages/TableDetailsPageV1/TableDetailsPageV1.tsx @@ -53,6 +53,7 @@ import { } from '../../constants/constants'; import { FEED_COUNT_INITIAL_DATA } from '../../constants/entity.constants'; import { mockDatasetData } from '../../constants/mockTourData.constants'; +import { COMMON_RESIZABLE_PANEL_CONFIG } from '../../constants/ResizablePanel.constants'; import LineageProvider from '../../context/LineageProvider/LineageProvider'; import { usePermissionProvider } from '../../context/PermissionProvider/PermissionProvider'; import { @@ -585,8 +586,7 @@ const TableDetailsPageV1: React.FC = () => { /> ), - minWidth: 800, - flex: 0.87, + ...COMMON_RESIZABLE_PANEL_CONFIG.LEFT_PANEL, }} secondPanel={{ children: ( @@ -626,8 +626,7 @@ const TableDetailsPageV1: React.FC = () => { /> ), - minWidth: 320, - flex: 0.13, + ...COMMON_RESIZABLE_PANEL_CONFIG.RIGHT_PANEL, className: 'entity-resizable-panel-container entity-resizable-right-panel-container ', }} From 21622177edd22283787772c0d3e9c6308ac5e64f Mon Sep 17 00:00:00 2001 From: Sachin Chaurasiya Date: Mon, 30 Sep 2024 16:05:26 +0530 Subject: [PATCH 34/39] improve ui for inline properties --- .../CustomPropertyTable/PropertyValue.tsx | 81 ++++++++++++++----- .../CustomPropertyTable/property-value.less | 5 ++ 2 files changed, 68 insertions(+), 18 deletions(-) diff --git a/openmetadata-ui/src/main/resources/ui/src/components/common/CustomPropertyTable/PropertyValue.tsx b/openmetadata-ui/src/main/resources/ui/src/components/common/CustomPropertyTable/PropertyValue.tsx index b7adbe1ee9d3..08012c9d1779 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/common/CustomPropertyTable/PropertyValue.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/components/common/CustomPropertyTable/PropertyValue.tsx @@ -56,7 +56,10 @@ import { ICON_DIMENSION, VALIDATION_MESSAGES, } from '../../../constants/constants'; -import { ENUM_WITH_DESCRIPTION } from '../../../constants/CustomProperty.constants'; +import { + ENUM_WITH_DESCRIPTION, + INLINE_PROPERTY_TYPES, +} from '../../../constants/CustomProperty.constants'; import { TIMESTAMP_UNIX_IN_MILLISECONDS_REGEX } from '../../../constants/regex.constants'; import { CSMode } from '../../../enums/codemirror.enum'; import { SearchIndex } from '../../../enums/search.enum'; @@ -91,18 +94,23 @@ export const PropertyValue: FC = ({ property, isRenderedInRightPanel = false, }) => { - const { propertyName, propertyType, value } = useMemo(() => { - const propertyName = property.name; - const propertyType = property.propertyType; + const { propertyName, propertyType, value, isInlineProperty } = + useMemo(() => { + const propertyName = property.name; + const propertyType = property.propertyType; + const isInlineProperty = INLINE_PROPERTY_TYPES.includes( + propertyType.name ?? '' + ); - const value = extension?.[propertyName]; + const value = extension?.[propertyName]; - return { - propertyName, - propertyType, - value, - }; - }, [property, extension]); + return { + propertyName, + propertyType, + value, + isInlineProperty, + }; + }, [property, extension]); const [showInput, setShowInput] = useState(false); const [isLoading, setIsLoading] = useState(false); @@ -857,7 +865,7 @@ export const PropertyValue: FC = ({ isTeam={item.type === 'team'} name={item.name ?? ''} type="circle" - width="24" + width="18" /> ) : ( searchClassBase.getEntityIcon(item.type) @@ -901,14 +909,14 @@ export const PropertyValue: FC = ({ icon={
+ style={{ width: '18px', display: 'flex' }}> {['user', 'team'].includes(item.type) ? ( ) : ( searchClassBase.getEntityIcon(item.type) @@ -1002,6 +1010,41 @@ export const PropertyValue: FC = ({ setIsOverflowing(isOverflowing); }, [property, extension, contentRef, value]); + const customPropertyInlineElement = ( +
+
+ + {getEntityName(property)} + + +
+ +
+ {showInput ? getPropertyInput() : getValueElement()} + {hasEditPermissions && !showInput && ( + + + + )} +
+
+ ); + const customPropertyElement = (
@@ -1044,7 +1087,7 @@ export const PropertyValue: FC = ({ span={isOverflowing && !showInput ? 22 : 24} style={{ height: isExpanded || showInput ? 'auto' : '30px', - overflow: isExpanded ? 'visible' : 'hidden', + overflow: isExpanded || showInput ? 'visible' : 'hidden', }}> {showInput ? getPropertyInput() : getValueElement()} @@ -1067,15 +1110,17 @@ export const PropertyValue: FC = ({ if (isRenderedInRightPanel) { return ( -
- {customPropertyElement} +
+ {isInlineProperty ? customPropertyInlineElement : customPropertyElement}
); } return ( {customPropertyElement} diff --git a/openmetadata-ui/src/main/resources/ui/src/components/common/CustomPropertyTable/property-value.less b/openmetadata-ui/src/main/resources/ui/src/components/common/CustomPropertyTable/property-value.less index db2d2ae8ab3d..9ae2c82ddcd9 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/common/CustomPropertyTable/property-value.less +++ b/openmetadata-ui/src/main/resources/ui/src/components/common/CustomPropertyTable/property-value.less @@ -55,3 +55,8 @@ width: 100%; } } +.custom-property-card { + .ant-card-body { + overflow-x: scroll; + } +} From 97346491a947684a66f540ffd6d924ba066661a1 Mon Sep 17 00:00:00 2001 From: Ashish Gupta Date: Mon, 30 Sep 2024 16:39:28 +0530 Subject: [PATCH 35/39] fix description, glossary and relatedTerm escape char issue --- .../resources/ui/src/utils/CSV/CSV.utils.tsx | 2 -- .../ui/src/utils/CSV/CSVUtilsClassBase.tsx | 24 ++++++++++--------- 2 files changed, 13 insertions(+), 13 deletions(-) diff --git a/openmetadata-ui/src/main/resources/ui/src/utils/CSV/CSV.utils.tsx b/openmetadata-ui/src/main/resources/ui/src/utils/CSV/CSV.utils.tsx index 26b1212b943b..28d4d1bc9f81 100644 --- a/openmetadata-ui/src/main/resources/ui/src/utils/CSV/CSV.utils.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/utils/CSV/CSV.utils.tsx @@ -144,8 +144,6 @@ export const getCSVStringFromColumnsAndDataSource = ( value.includes(',') || value.includes('\n') || colName.includes('tags') || - colName.includes('glossaryTerms') || - colName.includes('relatedTerms') || colName.includes('domain') ) { return isEmpty(value) ? '' : `"${value}"`; diff --git a/openmetadata-ui/src/main/resources/ui/src/utils/CSV/CSVUtilsClassBase.tsx b/openmetadata-ui/src/main/resources/ui/src/utils/CSV/CSVUtilsClassBase.tsx index 669c43692dad..f79facd626ff 100644 --- a/openmetadata-ui/src/main/resources/ui/src/utils/CSV/CSVUtilsClassBase.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/utils/CSV/CSVUtilsClassBase.tsx @@ -37,7 +37,14 @@ class CSVUtilsClassBase { } public columnsWithMultipleValuesEscapeNeeded() { - return ['parent', 'extension', 'synonyms']; + return [ + 'parent', + 'extension', + 'synonyms', + 'description', + 'glossaryTerms', + 'relatedTerms', + ]; } public getEditor( @@ -151,16 +158,10 @@ class CSVUtilsClassBase { ) => { if (Array.isArray(option)) { props.onChange( - option - .map((tag) => - toString(tag.value)?.replace(new RegExp('"', 'g'), '""') - ) - .join(';') + option.map((tag) => toString(tag.value)).join(';') ); } else { - props.onChange( - toString(option.value)?.replace(new RegExp('"', 'g'), '""') - ); + props.onChange(toString(option.value)); } }; @@ -280,8 +281,9 @@ class CSVUtilsClassBase { popoverProps={{ open: true, }} - onUpdate={handleChange} - /> + onUpdate={handleChange}> + {' '} + ); }; case 'extension': From eec45d40c5b904fae6d77f29f3c4a4672e8e90c3 Mon Sep 17 00:00:00 2001 From: Ashish Gupta Date: Mon, 30 Sep 2024 16:57:21 +0530 Subject: [PATCH 36/39] fix some customProperty ui changes --- .../common/CustomPropertyTable/PropertyValue.tsx | 8 ++++---- .../common/CustomPropertyTable/property-value.less | 4 ++++ .../ui/src/constants/CustomProperty.constants.ts | 1 - 3 files changed, 8 insertions(+), 5 deletions(-) diff --git a/openmetadata-ui/src/main/resources/ui/src/components/common/CustomPropertyTable/PropertyValue.tsx b/openmetadata-ui/src/main/resources/ui/src/components/common/CustomPropertyTable/PropertyValue.tsx index 08012c9d1779..1e93b6ea479a 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/common/CustomPropertyTable/PropertyValue.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/components/common/CustomPropertyTable/PropertyValue.tsx @@ -865,7 +865,7 @@ export const PropertyValue: FC = ({ isTeam={item.type === 'team'} name={item.name ?? ''} type="circle" - width="18" + width="20" /> ) : ( searchClassBase.getEntityIcon(item.type) @@ -909,14 +909,14 @@ export const PropertyValue: FC = ({ icon={
+ style={{ width: '20px', display: 'flex' }}> {['user', 'team'].includes(item.type) ? ( ) : ( searchClassBase.getEntityIcon(item.type) @@ -1111,7 +1111,7 @@ export const PropertyValue: FC = ({ if (isRenderedInRightPanel) { return (
{isInlineProperty ? customPropertyInlineElement : customPropertyElement}
diff --git a/openmetadata-ui/src/main/resources/ui/src/components/common/CustomPropertyTable/property-value.less b/openmetadata-ui/src/main/resources/ui/src/components/common/CustomPropertyTable/property-value.less index 9ae2c82ddcd9..6dcbfa1f9cb4 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/common/CustomPropertyTable/property-value.less +++ b/openmetadata-ui/src/main/resources/ui/src/components/common/CustomPropertyTable/property-value.less @@ -60,3 +60,7 @@ overflow-x: scroll; } } + +.custom-property-card-right-panel { + overflow-x: scroll; +} diff --git a/openmetadata-ui/src/main/resources/ui/src/constants/CustomProperty.constants.ts b/openmetadata-ui/src/main/resources/ui/src/constants/CustomProperty.constants.ts index ba81a8631aa4..05af0f6cb6bc 100644 --- a/openmetadata-ui/src/main/resources/ui/src/constants/CustomProperty.constants.ts +++ b/openmetadata-ui/src/main/resources/ui/src/constants/CustomProperty.constants.ts @@ -134,7 +134,6 @@ export const INLINE_PROPERTY_TYPES = [ 'dateTime-cp', 'duration', 'email', - 'entityReference', 'integer', 'number', 'string', From fd2117089c7298763a102b143701a70dbc13bdc2 Mon Sep 17 00:00:00 2001 From: Ashish Gupta Date: Mon, 30 Sep 2024 17:29:33 +0530 Subject: [PATCH 37/39] fix sonar issue --- .../src/main/resources/ui/src/utils/CSV/CSV.utils.test.tsx | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/openmetadata-ui/src/main/resources/ui/src/utils/CSV/CSV.utils.test.tsx b/openmetadata-ui/src/main/resources/ui/src/utils/CSV/CSV.utils.test.tsx index be4e97297d55..4bb7a40a4373 100644 --- a/openmetadata-ui/src/main/resources/ui/src/utils/CSV/CSV.utils.test.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/utils/CSV/CSV.utils.test.tsx @@ -85,7 +85,7 @@ describe('CSVUtils', () => { ); expect(csvString).toBe( - 'tags,glossaryTerms,description,domain\n"value1","value2",something new,"domain1"' + 'tags,glossaryTerms,description,domain\n"value1","value2","something new","domain1"' ); }); @@ -110,7 +110,7 @@ describe('CSVUtils', () => { ); expect(csvString).toBe( - `tags,glossaryTerms,description,domain\n"value,1","value_2",something#new,"domain,1"` + `tags,glossaryTerms,description,domain\n"value,1","value_2","something#new","domain,1"` ); }); }); From 305b92d95c2b18a5fb244457dff26006385cec36 Mon Sep 17 00:00:00 2001 From: Sachin Chaurasiya Date: Mon, 30 Sep 2024 17:48:37 +0530 Subject: [PATCH 38/39] minor layout changes --- .../GlossaryDetailsRightPanel.component.tsx | 2 +- .../CustomPropertyTable/PropertyValue.tsx | 90 +++++++++---------- 2 files changed, 46 insertions(+), 46 deletions(-) diff --git a/openmetadata-ui/src/main/resources/ui/src/components/Glossary/GlossaryDetailsRightPanel/GlossaryDetailsRightPanel.component.tsx b/openmetadata-ui/src/main/resources/ui/src/components/Glossary/GlossaryDetailsRightPanel/GlossaryDetailsRightPanel.component.tsx index 9f23582e98ec..c753cc350695 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/Glossary/GlossaryDetailsRightPanel/GlossaryDetailsRightPanel.component.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/components/Glossary/GlossaryDetailsRightPanel/GlossaryDetailsRightPanel.component.tsx @@ -288,7 +288,7 @@ const GlossaryDetailsRightPanel = ({
)} -
+ {!isGlossary && selectedData && ( = ({ }, [property, extension, contentRef, value]); const customPropertyInlineElement = ( -
-
- - {getEntityName(property)} - - -
- -
- {showInput ? getPropertyInput() : getValueElement()} - {hasEditPermissions && !showInput && ( - - - - )} +
+
+
+ + {getEntityName(property)} + +
+ +
+ {showInput ? getPropertyInput() : getValueElement()} + {hasEditPermissions && !showInput && ( + + + + )} +
+
); @@ -1081,29 +1083,27 @@ export const PropertyValue: FC = ({
- - +
{showInput ? getPropertyInput() : getValueElement()} - +
{isOverflowing && !showInput && ( - - - + )} - + ); From 6da7ccb8e13d0297219f67b94e4b45c0598cd06f Mon Sep 17 00:00:00 2001 From: Sachin Chaurasiya Date: Mon, 30 Sep 2024 18:04:19 +0530 Subject: [PATCH 39/39] minor label improvements for entity ref and list --- .../components/common/CustomPropertyTable/PropertyValue.tsx | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/openmetadata-ui/src/main/resources/ui/src/components/common/CustomPropertyTable/PropertyValue.tsx b/openmetadata-ui/src/main/resources/ui/src/components/common/CustomPropertyTable/PropertyValue.tsx index 25a948bf520d..d8838d060dac 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/common/CustomPropertyTable/PropertyValue.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/components/common/CustomPropertyTable/PropertyValue.tsx @@ -874,7 +874,7 @@ export const PropertyValue: FC = ({ } type="text"> {getEntityName(item)} @@ -925,7 +925,7 @@ export const PropertyValue: FC = ({ } type="text"> {getEntityName(item)}