Merge branch 'main' into fix_rbac_for_multiple_rules

bootstrap/sql/migrations/native/1.5.6/mysql/schemaChanges.sql

@@ -0,0 +1,5 @@
+-- Delete Search Indexing Application 
+DELETE er FROM entity_relationship er JOIN installed_apps ia ON er.fromId = ia.id OR er.toId = ia.id WHERE ia.name = 'SearchIndexingApplication';
+DELETE er FROM entity_relationship er JOIN apps_marketplace ia ON er.fromId = ia.id OR er.toId = ia.id WHERE ia.name = 'SearchIndexingApplication';
+DELETE FROM installed_apps where name = 'SearchIndexingApplication';
+DELETE FROM apps_marketplace where name = 'SearchIndexingApplication';

bootstrap/sql/migrations/native/1.5.6/postgres/schemaChanges.sql

@@ -0,0 +1,5 @@
+-- Delete Search Indexing Application 
+DELETE FROM entity_relationship er USING installed_apps ia WHERE (er.fromId = ia.id OR er.toId = ia.id) AND ia.name = 'SearchIndexingApplication';
+DELETE FROM entity_relationship er USING apps_marketplace ia WHERE (er.fromId = ia.id OR er.toId = ia.id) AND ia.name = 'SearchIndexingApplication';
+DELETE FROM installed_apps where name = 'SearchIndexingApplication';
+DELETE FROM apps_marketplace where name = 'SearchIndexingApplication';

openmetadata-docs/content/v1.5.x/how-to-guides/getting-started/day-1/index.md

@@ -83,7 +83,21 @@ to import metadata into OpenMetadata on a regular schedule.
 
 - You can repeat these steps to ingest metadata from other data sources as needed.
 
-## Step 3: Invite Users
+## Step 3: Invite Users After Setting Up SMTP
+
+### SMTP Configuration
+
+To invite users you will need to ensure that you have an SMTP server available. With the information for your SMTP server you can configure OpenMetadata to send email alerts by updating the details from the UI.
+
+To update the details from the UI, navigate to Settings > Preferences > Email
+
+{% image
+src="/images/v1.5/how-to-guides/admin-guide/email.png"
+alt="Email Configuration"
+caption="Email Configuration"
+/%}
+
+### Inviting Users
 
 After metadata has been ingested into OpenMetadata, you can [invite users](/how-to-guides/admin-guide/teams-and-users/invite-users) to collaborate on the data and assign different roles.
 

openmetadata-docs/content/v1.6.x-SNAPSHOT/how-to-guides/getting-started/day-1/index.md

@@ -83,7 +83,21 @@ to import metadata into OpenMetadata on a regular schedule.
 
 - You can repeat these steps to ingest metadata from other data sources as needed.
 
-## Step 3: Invite Users
+## Step 3: Invite Users After Setting Up SMTP
+
+### SMTP Configuration
+
+To invite users you will need to ensure that you have an SMTP server available. With the information for your SMTP server you can configure OpenMetadata to send email alerts by updating the details from the UI.
+
+To update the details from the UI, navigate to Settings > Preferences > Email
+
+{% image
+src="/images/v1.6/how-to-guides/admin-guide/email.png"
+alt="Email Configuration"
+caption="Email Configuration"
+/%}
+
+### Inviting Users
 
 After metadata has been ingested into OpenMetadata, you can [invite users](/how-to-guides/admin-guide/teams-and-users/invite-users) to collaborate on the data and assign different roles.
 

openmetadata-service/src/main/java/org/openmetadata/csv/CsvUtil.java

@@ -17,14 +17,20 @@
 import static org.openmetadata.common.utils.CommonUtil.listOrEmpty;
 import static org.openmetadata.common.utils.CommonUtil.nullOrEmpty;
 
+import com.fasterxml.jackson.databind.ObjectMapper;
 import java.io.IOException;
+import java.io.StringReader;
 import java.io.StringWriter;
 import java.util.ArrayList;
 import java.util.Arrays;
+import java.util.Collections;
 import java.util.List;
+import java.util.Map;
+import java.util.regex.Pattern;
 import java.util.stream.Collectors;
 import org.apache.commons.csv.CSVFormat;
 import org.apache.commons.csv.CSVFormat.Builder;
+import org.apache.commons.csv.CSVParser;
 import org.apache.commons.csv.CSVPrinter;
 import org.apache.commons.csv.CSVRecord;
 import org.openmetadata.schema.type.EntityReference;
@@ -39,6 +45,8 @@ public final class CsvUtil {
  public static final String ENTITY_TYPE_SEPARATOR = ":";
  public static final String LINE_SEPARATOR = "\r\n";
 
+ public static final String INTERNAL_ARRAY_SEPARATOR = "|";
+
  private CsvUtil() {
  // Utility class hides the constructor
  }
@@ -94,6 +102,62 @@ public static List<String> fieldToEntities(String field) {
  return field == null ? null : listOf(field.split(ENTITY_TYPE_SEPARATOR));
  }
 
+ public static List<String> fieldToInternalArray(String field) {
+ // Split a fieldValue that contains multiple elements of an array separated by
+ // INTERNAL_ARRAY_SEPARATOR
+ if (field == null || field.isBlank()) {
+ return Collections.emptyList();
+ }
+ return listOf(field.split(Pattern.quote(INTERNAL_ARRAY_SEPARATOR)));
+ }
+
+ /**
+ * Parses a field containing key-value pairs separated by semicolons, correctly handling quotes.
+ * Each key-value pair may also be enclosed in quotes, especially if it contains delimiter like (SEPARATOR , FIELD_SEPARATOR).
+ * Input Example:
+ * "key1:value1;key2:value2;\"key3:value;with;semicolon\""
+ * Output: [key1:value1, key2:value2, key3:value;with;semicolon]
+ *
+ */
+ public static List<String> fieldToExtensionStrings(String field) throws IOException {
+ if (field == null || field.isBlank()) {
+ return List.of();
+ }
+
+ // Replace semicolons within quoted strings with a placeholder
+ String preprocessedField =
+ Pattern.compile("\"([^\"]*)\"") // Matches content inside double quotes
+ .matcher(field)
+ .replaceAll(mr -> "\"" + mr.group(1).replace(";", "__SEMICOLON__") + "\"");
+
+ preprocessedField = preprocessedField.replace("\n", "\\n").replace("\"", "\\\"");
+
+ CSVFormat format =
+ CSVFormat.DEFAULT
+ .withDelimiter(';')
+ .withQuote('"')
+ .withRecordSeparator(null)
+ .withIgnoreSurroundingSpaces(true)
+ .withIgnoreEmptyLines(true)
+ .withEscape('\\'); // Use backslash for escaping special characters
+
+ try (CSVParser parser = CSVParser.parse(new StringReader(preprocessedField), format)) {
+ return parser.getRecords().stream()
+ .flatMap(CSVRecord::stream)
+ .map(
+ value ->
+ value
+ .replace("__SEMICOLON__", ";")
+ .replace("\\n", "\n")) // Restore original semicolons and newlines
+ .map(
+ value ->
+ value.startsWith("\"") && value.endsWith("\"") // Remove outer quotes if present
+ ? value.substring(1, value.length() - 1)
+ : value)
+ .toList();
+ }
+ }
+
  public static String quote(String field) {
  return String.format("\"%s\"", field);
  }
@@ -205,4 +269,89 @@ private static String quoteCsvField(String str) {
  }
  return str;
  }
+
+ public static List<String> addExtension(List<String> csvRecord, Object extension) {
+ if (extension == null) {
+ csvRecord.add(null);
+ return csvRecord;
+ }
+
+ ObjectMapper objectMapper = new ObjectMapper();
+ Map<String, Object> extensionMap = objectMapper.convertValue(extension, Map.class);
+
+ String extensionString =
+ extensionMap.entrySet().stream()
+ .map(
+ entry -> {
+ String key = entry.getKey();
+ Object value = entry.getValue();
+ return CsvUtil.quoteCsvField(key + ENTITY_TYPE_SEPARATOR + formatValue(value));
+ })
+ .collect(Collectors.joining(FIELD_SEPARATOR));
+
+ csvRecord.add(extensionString);
+ return csvRecord;
+ }
+
+ private static String formatValue(Object value) {
+ if (value instanceof Map) {
+ return formatMapValue((Map<String, Object>) value);
+ }
+
+ if (value instanceof List) {
+ return formatListValue((List<?>) value);
+ }
+
+ return value != null ? value.toString() : "";
+ }
+
+ private static String formatMapValue(Map<String, Object> valueMap) {
+ if (isEntityReference(valueMap)) {
+ return formatEntityReference(valueMap);
+ } else if (isTimeInterval(valueMap)) {
+ return formatTimeInterval(valueMap);
+ }
+
+ return valueMap.toString();
+ }
+
+ private static String formatListValue(List<?> list) {
+ if (list.isEmpty()) {
+ return "";
+ }
+
+ if (list.get(0) instanceof Map && isEnumWithDescriptions((Map<String, Object>) list.get(0))) {
+ return list.stream()
+ .map(item -> ((Map<String, Object>) item).get("key").toString())
+ .collect(Collectors.joining(INTERNAL_ARRAY_SEPARATOR));
+ } else if (list.get(0) instanceof Map) {
+ return list.stream()
+ .map(item -> formatMapValue((Map<String, Object>) item))
+ .collect(Collectors.joining(INTERNAL_ARRAY_SEPARATOR));
+ } else {
+ return list.stream()
+ .map(Object::toString)
+ .collect(Collectors.joining(INTERNAL_ARRAY_SEPARATOR));
+ }
+ }
+
+ private static boolean isEntityReference(Map<String, Object> valueMap) {
+ return valueMap.containsKey("type") && valueMap.containsKey("fullyQualifiedName");
+ }
+
+ private static boolean isTimeInterval(Map<String, Object> valueMap) {
+ return valueMap.containsKey("start") && valueMap.containsKey("end");
+ }
+
+ private static boolean isEnumWithDescriptions(Map<String, Object> valueMap) {
+ return valueMap.containsKey("key") && valueMap.containsKey("description");
+ }
+
+ private static String formatEntityReference(Map<String, Object> valueMap) {
+ return valueMap.get("type") + ENTITY_TYPE_SEPARATOR + valueMap.get("fullyQualifiedName");
+ }
+
+ private static String formatTimeInterval(Map<String, Object> valueMap) {
+ return valueMap.get("start") + ENTITY_TYPE_SEPARATOR + valueMap.get("end");
+ }
 }