Feature: Allow Custom Property Update in Glossary Bulk Import/export (#…

…17919) * fix import issue * Feat : Allow Custom Property Update in Bulk Upload * Feat : Allow Custom Property Update in Bulk Upload * supported editable imports in glossary page * added remaning localizaion keys * update logic of fieldToExtensionStrings to use csvparser * update json and partialStatus condition * fix tests for partialSuccess status change * supported customProperty editable field * fix error in custom property edit modal on new line empty custom property * added entity type from root to support other bulk import entity as well * fix the quote removing due to the regex in the string type * Add backend tests , and error msg improvements * GlossaryStatus header change * fix unit test and dry run in case of synonyms having quotes in it * Remove extension column in CSVs for all entities except glossaryTerm * added editor for reviewers * unit test around csv utils * added escape for string too, in case of semicolon comes * added playwright test without extension and supported relatedTerm as editable * added unit test around csv util logic * resolve conflicts * Backend - add support for enumWithDescriptions in bulk import * add tests and other error handling improvements related to enumWithDescriptions * fix the custom property modal header and render the layout as per right panel in entities * parese enumWithDescription for the customProperty modal while editable * fix description data in enumWithDescription one * fix: Handle NullPointerException when adding custom properties to ensure loop continues for other schemas of the same type for addToRegistry * added extension playwrigth test and fix enumWithDescription object failure * descrease the size of extension modal * remove additional comments * fix the escape in parent key * improve custom property layout * improve ui for inline properties * fix description, glossary and relatedTerm escape char issue * fix some customProperty ui changes * fix sonar issue * minor layout changes * minor label improvements for entity ref and list --------- Co-authored-by: Chirag Madlani <12962843+chirag-madlani@users.noreply.github.com> Co-authored-by: Ashish Gupta <ashish@getcollate.io> Co-authored-by: mohitdeuex <mohit.y@deuexsolutions.com> Co-authored-by: Sachin Chaurasiya <sachinchaurasiyachotey87@gmail.com>
open-metadata · Sep 30, 2024 · 1d727d5 · 1d727d5
1 parent f1ae49e
commit 1d727d5
Show file tree

Hide file tree

Showing 77 changed files with 3,965 additions and 691 deletions.
diff --git a/openmetadata-service/src/main/java/org/openmetadata/csv/CsvUtil.java b/openmetadata-service/src/main/java/org/openmetadata/csv/CsvUtil.java
@@ -17,14 +17,20 @@
 import static org.openmetadata.common.utils.CommonUtil.listOrEmpty;
 import static org.openmetadata.common.utils.CommonUtil.nullOrEmpty;
 
+import com.fasterxml.jackson.databind.ObjectMapper;
 import java.io.IOException;
+import java.io.StringReader;
 import java.io.StringWriter;
 import java.util.ArrayList;
 import java.util.Arrays;
+import java.util.Collections;
 import java.util.List;
+import java.util.Map;
+import java.util.regex.Pattern;
 import java.util.stream.Collectors;
 import org.apache.commons.csv.CSVFormat;
 import org.apache.commons.csv.CSVFormat.Builder;
+import org.apache.commons.csv.CSVParser;
 import org.apache.commons.csv.CSVPrinter;
 import org.apache.commons.csv.CSVRecord;
 import org.openmetadata.schema.type.EntityReference;
@@ -39,6 +45,8 @@ public final class CsvUtil {
   public static final String ENTITY_TYPE_SEPARATOR = ":";
   public static final String LINE_SEPARATOR = "\r\n";
 
+  public static final String INTERNAL_ARRAY_SEPARATOR = "|";
+
   private CsvUtil() {
     // Utility class hides the constructor
   }
@@ -94,6 +102,62 @@ public static List<String> fieldToEntities(String field) {
     return field == null ? null : listOf(field.split(ENTITY_TYPE_SEPARATOR));
   }
 
+  public static List<String> fieldToInternalArray(String field) {
+    // Split a fieldValue that contains multiple elements of an array separated by
+    // INTERNAL_ARRAY_SEPARATOR
+    if (field == null || field.isBlank()) {
+      return Collections.emptyList();
+    }
+    return listOf(field.split(Pattern.quote(INTERNAL_ARRAY_SEPARATOR)));
+  }
+
+  /**
+   * Parses a field containing key-value pairs separated by semicolons, correctly handling quotes.
+   * Each key-value pair may also be enclosed in quotes, especially if it contains delimiter like (SEPARATOR , FIELD_SEPARATOR).
+   * Input Example:
+   * "key1:value1;key2:value2;\"key3:value;with;semicolon\""
+   * Output: [key1:value1, key2:value2, key3:value;with;semicolon]
+   *
+   */
+  public static List<String> fieldToExtensionStrings(String field) throws IOException {
+    if (field == null || field.isBlank()) {
+      return List.of();
+    }
+
+    // Replace semicolons within quoted strings with a placeholder
+    String preprocessedField =
+        Pattern.compile("\"([^\"]*)\"") // Matches content inside double quotes
+            .matcher(field)
+            .replaceAll(mr -> "\"" + mr.group(1).replace(";", "__SEMICOLON__") + "\"");
+
+    preprocessedField = preprocessedField.replace("\n", "\\n").replace("\"", "\\\"");
+
+    CSVFormat format =
+        CSVFormat.DEFAULT
+            .withDelimiter(';')
+            .withQuote('"')
+            .withRecordSeparator(null)
+            .withIgnoreSurroundingSpaces(true)
+            .withIgnoreEmptyLines(true)
+            .withEscape('\\'); // Use backslash for escaping special characters
+
+    try (CSVParser parser = CSVParser.parse(new StringReader(preprocessedField), format)) {
+      return parser.getRecords().stream()
+          .flatMap(CSVRecord::stream)
+          .map(
+              value ->
+                  value
+                      .replace("__SEMICOLON__", ";")
+                      .replace("\\n", "\n")) // Restore original semicolons and newlines
+          .map(
+              value ->
+                  value.startsWith("\"") && value.endsWith("\"") // Remove outer quotes if present
+                      ? value.substring(1, value.length() - 1)
+                      : value)
+          .toList();
+    }
+  }
+
   public static String quote(String field) {
     return String.format("\"%s\"", field);
   }
@@ -205,4 +269,89 @@ private static String quoteCsvField(String str) {
     }
     return str;
   }
+
+  public static List<String> addExtension(List<String> csvRecord, Object extension) {
+    if (extension == null) {
+      csvRecord.add(null);
+      return csvRecord;
+    }
+
+    ObjectMapper objectMapper = new ObjectMapper();
+    Map<String, Object> extensionMap = objectMapper.convertValue(extension, Map.class);
+
+    String extensionString =
+        extensionMap.entrySet().stream()
+            .map(
+                entry -> {
+                  String key = entry.getKey();
+                  Object value = entry.getValue();
+                  return CsvUtil.quoteCsvField(key + ENTITY_TYPE_SEPARATOR + formatValue(value));
+                })
+            .collect(Collectors.joining(FIELD_SEPARATOR));
+
+    csvRecord.add(extensionString);
+    return csvRecord;
+  }
+
+  private static String formatValue(Object value) {
+    if (value instanceof Map) {
+      return formatMapValue((Map<String, Object>) value);
+    }
+
+    if (value instanceof List) {
+      return formatListValue((List<?>) value);
+    }
+
+    return value != null ? value.toString() : "";
+  }
+
+  private static String formatMapValue(Map<String, Object> valueMap) {
+    if (isEntityReference(valueMap)) {
+      return formatEntityReference(valueMap);
+    } else if (isTimeInterval(valueMap)) {
+      return formatTimeInterval(valueMap);
+    }
+
+    return valueMap.toString();
+  }
+
+  private static String formatListValue(List<?> list) {
+    if (list.isEmpty()) {
+      return "";
+    }
+
+    if (list.get(0) instanceof Map && isEnumWithDescriptions((Map<String, Object>) list.get(0))) {
+      return list.stream()
+          .map(item -> ((Map<String, Object>) item).get("key").toString())
+          .collect(Collectors.joining(INTERNAL_ARRAY_SEPARATOR));
+    } else if (list.get(0) instanceof Map) {
+      return list.stream()
+          .map(item -> formatMapValue((Map<String, Object>) item))
+          .collect(Collectors.joining(INTERNAL_ARRAY_SEPARATOR));
+    } else {
+      return list.stream()
+          .map(Object::toString)
+          .collect(Collectors.joining(INTERNAL_ARRAY_SEPARATOR));
+    }
+  }
+
+  private static boolean isEntityReference(Map<String, Object> valueMap) {
+    return valueMap.containsKey("type") && valueMap.containsKey("fullyQualifiedName");
+  }
+
+  private static boolean isTimeInterval(Map<String, Object> valueMap) {
+    return valueMap.containsKey("start") && valueMap.containsKey("end");
+  }
+
+  private static boolean isEnumWithDescriptions(Map<String, Object> valueMap) {
+    return valueMap.containsKey("key") && valueMap.containsKey("description");
+  }
+
+  private static String formatEntityReference(Map<String, Object> valueMap) {
+    return valueMap.get("type") + ENTITY_TYPE_SEPARATOR + valueMap.get("fullyQualifiedName");
+  }
+
+  private static String formatTimeInterval(Map<String, Object> valueMap) {
+    return valueMap.get("start") + ENTITY_TYPE_SEPARATOR + valueMap.get("end");
+  }
 }