Evolveum · wadahiro · Oct 2, 2023
diff --git a/pom.xml b/pom.xml
@@ -84,6 +84,11 @@
             <artifactId>commons-csv</artifactId>
             <version>${commons.csv.version}</version>
         </dependency>
+        <dependency>
+            <groupId>com.google.code.gson</groupId>
+            <artifactId>gson</artifactId>
+            <version>2.10.1</version>
+        </dependency>
 
         <!-- TEST DEPENDENCIES -->
 

diff --git a/src/main/java/com/evolveum/polygon/connector/csv/CsvConfiguration.java b/src/main/java/com/evolveum/polygon/connector/csv/CsvConfiguration.java
@@ -197,6 +197,17 @@ public boolean isAuxiliary() {
         return config.isAuxiliary();
     }
 
+    @ConfigurationProperty(
+            displayMessageKey = "UI_CSV_GROUP_BY_ENABLED",
+            helpMessageKey = "UI_CSV_GROUP_BY_ENABLED_HELP")
+    public boolean isGroupByEnabled() {
+        return config.isGroupByEnabled();
+    }
+
+    public void setGroupByEnabled(boolean groupByEnabled) {
+        config.setGroupByEnabled(groupByEnabled);
+    }
+
     public void setContainer(boolean container) {
         config.setContainer(container);
     }

diff --git a/src/main/java/com/evolveum/polygon/connector/csv/ObjectClassHandler.java b/src/main/java/com/evolveum/polygon/connector/csv/ObjectClassHandler.java
@@ -4,6 +4,7 @@
 import com.evolveum.polygon.connector.csv.util.ConfigurationDetector;
 import com.evolveum.polygon.connector.csv.util.StringAccessor;
 import com.evolveum.polygon.connector.csv.util.Util;
+import com.google.gson.Gson;
 import org.apache.commons.csv.CSVFormat;
 import org.apache.commons.csv.CSVParser;
 import org.apache.commons.csv.CSVPrinter;
@@ -47,6 +48,7 @@ private enum Operation {
 	}
 
 	private static final Log LOG = Log.getLog(ObjectClassHandler.class);
+	private static final String ATTR_RAW_JSON = AttributeUtil.createSpecialName("RAW_JSON");
 
 	private ObjectClassHandlerConfiguration configuration;
 
@@ -260,6 +262,17 @@ private List<AttributeInfo> createAttributeInfo(Map<String, Column> columns) {
 			infos.add(builder.build());
 		}
 
+		if (configuration.isGroupByEnabled()) {
+			AttributeInfoBuilder builder = new AttributeInfoBuilder(ATTR_RAW_JSON);
+			builder.setType(String.class);
+			builder.setNativeName(ATTR_RAW_JSON);
+			builder.setCreateable(false);
+			builder.setUpdateable(false);
+			builder.setSubtype(AttributeInfo.Subtypes.STRING_JSON);
+
+			infos.add(builder.build());
+		}
+
 		return infos;
 	}
 
@@ -434,6 +447,11 @@ public void executeQuery(ObjectClass oc, Filter filter, ResultsHandler handler,
 		CSVFormat csv = Util.createCsvFormatReader(configuration);
 		try (Reader reader = Util.createReader(configuration)) {
 
+			boolean shouldContinue = true;
+			List<CSVRecord> recordGroup = new ArrayList<>();
+			int uidIndex = this.getHeader().get(configuration.getUniqueAttribute()).getIndex();
+			String uid = extractUidFromFilter(filter);
+
 			CSVParser parser = csv.parse(reader);
 			Iterator<CSVRecord> iterator = parser.iterator();
 			while (iterator.hasNext()) {
@@ -442,13 +460,36 @@ public void executeQuery(ObjectClass oc, Filter filter, ResultsHandler handler,
 					continue;
 				}
 
-				ConnectorObject obj = createConnectorObject(record);
+				ConnectorObject obj;
 
-				String uid = extractUidFromFilter(filter);
+				if (configuration.isGroupByEnabled()) {
+					// Record group is empty, so push it then check next record
+					if (recordGroup.isEmpty()) {
+						recordGroup.add(record);
+						continue;
+					}
+					// If the current record has same uid with the current record group, push it then check next record
+					String currentUid = record.get(uidIndex);
+					String currentUidOfRecordGroup = recordGroup.get(0).get(uidIndex);
+					if (uidMatches(currentUid, currentUidOfRecordGroup, configuration.isIgnoreIdentifierCase())) {
+						recordGroup.add(record);
+						continue;
+					}
+
+					obj = createConnectorObject(recordGroup);
+
+					// Reset record group for next
+					recordGroup.clear();
+					recordGroup.add(record);
+
+				} else {
+					obj = createConnectorObject(record);
+				}
 
 				if (uid == null) {
 					if (filter == null || filter.accept(obj)) {
-						if (!handler.handle(obj)) {
+						shouldContinue = handler.handle(obj);
+						if (!shouldContinue) {
 							break;
 						}
 					}
@@ -459,10 +500,31 @@ public void executeQuery(ObjectClass oc, Filter filter, ResultsHandler handler,
 					continue;
 				}
 
-				if (!handler.handle(obj)) {
+				shouldContinue = handler.handle(obj);
+				if (!shouldContinue) {
 					break;
 				}
 			}
+
+			if (configuration.isGroupByEnabled()) {
+				// Handle remaining record group
+				if (shouldContinue && !recordGroup.isEmpty()) {
+					ConnectorObject obj = createConnectorObject(recordGroup);
+
+					if (uid == null) {
+						if (filter == null || filter.accept(obj)) {
+							handler.handle(obj);
+						}
+						return;
+					}
+
+					if (!uidMatches(uid, obj.getUid().getUidValue(), configuration.isIgnoreIdentifierCase())) {
+						return;
+					}
+
+					handler.handle(obj);
+				}
+			}
 		} catch (Exception ex) {
 			handleGenericException(ex, "Error during query execution");
 		}
@@ -934,7 +996,59 @@ private Map<Integer, String> reverseHeaderMap() {
 		return reversed;
 	}
 
+	private ConnectorObject createConnectorObject(List<CSVRecord> recordGroup) {
+		Map<Integer, String> header = reverseHeaderMap();
+		Set<String> multivalueAttrs = StringUtil.isEmpty(configuration.getMultivalueAttributes()) || StringUtil.isEmpty(configuration.getMultivalueDelimiter()) ?
+				Collections.emptySet() : Set.of(configuration.getMultivalueAttributes().split(configuration.getMultivalueDelimiter()));
+
+		// Create base connectorObject using first record
+		ConnectorObjectBuilder builder = createConnectorObjectBuilder(recordGroup.get(0));
+
+		// Create rawJson and append to the connectorObject
+		List<Map<String, Object>> data = new ArrayList<>();
+		for (CSVRecord record : recordGroup) {
+			if (header.size() != record.size()) {
+				throw new ConnectorException("Number of columns in header (" + header.size()
+						+ ") doesn't match number of columns for record (" + record.size()
+						+ "). File row number: " + record.getRecordNumber());
+			}
+
+			Map<String, Object> recordMap = new HashMap<>();
+			for (int i = 0; i < record.size(); i++) {
+				String name = header.get(i);
+				String value = record.get(i);
+
+				if (multivalueAttrs.contains(name)) {
+					// Multiple value
+					String[] values = Arrays.stream(value.split(configuration.getMultivalueDelimiter()))
+							.filter(StringUtil::isNotEmpty)
+							.toArray(String[]::new);
+					recordMap.put(name, values);
+				} else {
+					// Single value
+					if (StringUtil.isEmpty(value)) {
+						// Include empty string for JSON if empty
+						recordMap.put(name, "");
+					} else {
+						recordMap.put(name, value);
+					}
+				}
+			}
+			data.add(recordMap);
+		}
+		if (!data.isEmpty()) {
+			String json = new Gson().toJson(data);
+			builder.addAttribute(ATTR_RAW_JSON, json);
+		}
+
+		return builder.build();
+	}
+
 	private ConnectorObject createConnectorObject(CSVRecord record) {
+		return createConnectorObjectBuilder(record).build();
+	}
+
+	private ConnectorObjectBuilder createConnectorObjectBuilder(CSVRecord record) {
 		ConnectorObjectBuilder builder = new ConnectorObjectBuilder();
 
 		Map<Integer, String> header = reverseHeaderMap();
@@ -974,7 +1088,7 @@ private ConnectorObject createConnectorObject(CSVRecord record) {
 			builder.addAttribute(name, createAttributeValues(value));
 		}
 
-		return builder.build();
+		return builder;
 	}
 
 	private boolean isUniqueAndNameAttributeEqual() {

diff --git a/src/main/java/com/evolveum/polygon/connector/csv/ObjectClassHandlerConfiguration.java b/src/main/java/com/evolveum/polygon/connector/csv/ObjectClassHandlerConfiguration.java
@@ -55,6 +55,8 @@ public class ObjectClassHandlerConfiguration {
     private boolean container = false;
     private boolean auxiliary = false;
 
+    private boolean groupByEnabled = false;
+
     public ObjectClassHandlerConfiguration() {
         this(ObjectClass.ACCOUNT, null);
     }
@@ -92,6 +94,8 @@ public ObjectClassHandlerConfiguration(ObjectClass oc, Map<String, Object> value
 
         setContainer(Util.getSafeValue(values, "container", false, Boolean.class));
         setAuxiliary(Util.getSafeValue(values, "auxiliary", false, Boolean.class));
+
+        setGroupByEnabled(Util.getSafeValue(values, "groupByEnabled", false, Boolean.class));
     }
 
     public void recompute() {
@@ -104,6 +108,14 @@ public void recompute() {
         }
     }
 
+    public boolean isGroupByEnabled() {
+        return groupByEnabled;
+    }
+
+    public void setGroupByEnabled(boolean groupByEnabled) {
+        this.groupByEnabled = groupByEnabled;
+    }
+
     public boolean isContainer() {
         return container;
     }

diff --git a/src/main/resources/com/evolveum/polygon/connector/csv/Messages.properties b/src/main/resources/com/evolveum/polygon/connector/csv/Messages.properties
@@ -50,4 +50,6 @@ UI_MULTIVALUE_ATTRIBUTES_HELP=List all attributes that can have multiple values
 UI_CONTAINER=Container
 UI_CONTAINER_HELP=Should this object class be marked as container? Default is false.
 UI_AUXILIARY=Auxiliary
-UI_AUXILIARY_HELP=Should this object class be marked as auxiliary? Default is false.
+UI_AUXILIARY_HELP=Should this object class be marked as auxiliary? Default is false.
+UI_CSV_GROUP_BY_ENABLED=Group by enabled
+UI_CSV_GROUP_BY_ENABLED_HELP=Enable group by unique attribute when reading csv (except for sync operation). The aggregated data is mapped to the "__RAW_JSON__" attribute with string type.
diff --git a/src/test/java/com/evolveum/polygon/connector/csv/SearchOpTest.java b/src/test/java/com/evolveum/polygon/connector/csv/SearchOpTest.java
@@ -3,6 +3,8 @@
 import com.evolveum.polygon.connector.csv.util.ListResultHandler;
 import org.identityconnectors.framework.api.ConnectorFacade;
 import org.identityconnectors.framework.common.exceptions.ConnectorException;
+import org.identityconnectors.framework.common.objects.Attribute;
+import org.identityconnectors.framework.common.objects.AttributeUtil;
 import org.identityconnectors.framework.common.objects.ConnectorObject;
 import org.identityconnectors.framework.common.objects.ObjectClass;
 import org.identityconnectors.framework.common.objects.Uid;
@@ -11,6 +13,7 @@
 import org.testng.annotations.Test;
 
 import java.io.File;
+import java.util.Collections;
 import java.util.List;
 
 /**
@@ -77,4 +80,102 @@ public void searchWrongNumberColumnCountInRow() throws Exception {
         ConnectorFacade connector = setupConnector("/search-wrong-column-count-row.csv", config);
         connector.search(ObjectClass.ACCOUNT, null, new ListResultHandler(), null);
     }
+
+    @Test
+    public void findOneWithGroupByEnabled() throws Exception {
+        CsvConfiguration config = new CsvConfiguration();
+        config.setFilePath(new File(CSV_FILE_PATH));
+        config.setUniqueAttribute("id");
+        config.setMultivalueAttributes("tel");
+        config.setMultivalueDelimiter(",");
+        config.setGroupByEnabled(true);
+        ConnectorFacade connector = setupConnector("/search-grouping.csv", config);
+
+        ListResultHandler handler = new ListResultHandler();
+        EqualsFilter ef = new EqualsFilter(new Uid("1"));
+        connector.search(ObjectClass.ACCOUNT, ef, handler, null);
+
+        List<ConnectorObject> objects = handler.getObjects();
+
+        AssertJUnit.assertEquals(1, objects.size());
+        ConnectorObject connectorObject = objects.get(0);
+        Attribute id = connectorObject.getAttributeByName(Uid.NAME);
+        AssertJUnit.assertEquals("1", AttributeUtil.getStringValue(id));
+        Attribute dept = connectorObject.getAttributeByName("dept");
+        AssertJUnit.assertEquals("abc", AttributeUtil.getStringValue(dept));
+        Attribute title = connectorObject.getAttributeByName("title");
+        AssertJUnit.assertEquals("engineer", AttributeUtil.getStringValue(title));
+        Attribute tel = connectorObject.getAttributeByName("tel");
+        AssertJUnit.assertEquals(List.of("1111"), tel.getValue());
+        Attribute rawJson = connectorObject.getAttributeByName("__RAW_JSON__");
+        AssertJUnit.assertNotNull(rawJson);
+        String jsonValue = AttributeUtil.getStringValue(rawJson);
+        AssertJUnit.assertEquals("[{\"name\":\"john\",\"tel\":[\"1111\"],\"id\":\"1\",\"dept\":\"abc\",\"title\":\"engineer\"},{\"name\":\"john\",\"tel\":[\"1111\"],\"id\":\"1\",\"dept\":\"efg\",\"title\":\"manager\"}]", jsonValue);
+    }
+
+    @Test
+    public void findAllWithGroupByEnabled() throws Exception {
+        CsvConfiguration config = new CsvConfiguration();
+        config.setFilePath(new File(CSV_FILE_PATH));
+        config.setUniqueAttribute("id");
+        config.setMultivalueAttributes("tel");
+        config.setMultivalueDelimiter(",");
+        config.setGroupByEnabled(true);
+        ConnectorFacade connector = setupConnector("/search-grouping.csv", config);
+
+        ListResultHandler handler = new ListResultHandler();
+        connector.search(ObjectClass.ACCOUNT, null, handler, null);
+
+        List<ConnectorObject> objects = handler.getObjects();
+
+        AssertJUnit.assertEquals(3, objects.size());
+
+        ConnectorObject connectorObject = objects.get(0);
+        Attribute id = connectorObject.getAttributeByName(Uid.NAME);
+        AssertJUnit.assertEquals("1", AttributeUtil.getStringValue(id));
+        Attribute name = connectorObject.getAttributeByName("name");
+        AssertJUnit.assertEquals("john", AttributeUtil.getStringValue(name));
+        Attribute dept = connectorObject.getAttributeByName("dept");
+        AssertJUnit.assertEquals("abc", AttributeUtil.getStringValue(dept));
+        Attribute title = connectorObject.getAttributeByName("title");
+        AssertJUnit.assertEquals("engineer", AttributeUtil.getStringValue(title));
+        Attribute tel = connectorObject.getAttributeByName("tel");
+        AssertJUnit.assertEquals(List.of("1111"), tel.getValue());
+        Attribute rawJson = connectorObject.getAttributeByName("__RAW_JSON__");
+        AssertJUnit.assertNotNull(rawJson);
+        String jsonValue = AttributeUtil.getStringValue(rawJson);
+        AssertJUnit.assertEquals("[{\"name\":\"john\",\"tel\":[\"1111\"],\"id\":\"1\",\"dept\":\"abc\",\"title\":\"engineer\"},{\"name\":\"john\",\"tel\":[\"1111\"],\"id\":\"1\",\"dept\":\"efg\",\"title\":\"manager\"}]", jsonValue);
+
+        connectorObject = objects.get(1);
+        id = connectorObject.getAttributeByName(Uid.NAME);
+        AssertJUnit.assertEquals("2", AttributeUtil.getStringValue(id));
+        name = connectorObject.getAttributeByName("name");
+        AssertJUnit.assertEquals("jack", AttributeUtil.getStringValue(name));
+        dept = connectorObject.getAttributeByName("dept");
+        AssertJUnit.assertEquals("abc", AttributeUtil.getStringValue(dept));
+        title = connectorObject.getAttributeByName("title");
+        AssertJUnit.assertEquals("manager", AttributeUtil.getStringValue(title));
+        tel = connectorObject.getAttributeByName("tel");
+        AssertJUnit.assertEquals(List.of("1111", "2222"), tel.getValue());
+        rawJson = connectorObject.getAttributeByName("__RAW_JSON__");
+        AssertJUnit.assertNotNull(rawJson);
+        jsonValue = AttributeUtil.getStringValue(rawJson);
+        AssertJUnit.assertEquals("[{\"name\":\"jack\",\"tel\":[\"1111\",\"2222\"],\"id\":\"2\",\"dept\":\"abc\",\"title\":\"manager\"}]", jsonValue);
+
+        connectorObject = objects.get(2);
+        id = connectorObject.getAttributeByName(Uid.NAME);
+        AssertJUnit.assertEquals("3", AttributeUtil.getStringValue(id));
+        name = connectorObject.getAttributeByName("name");
+        AssertJUnit.assertEquals("bob", AttributeUtil.getStringValue(name));
+        dept = connectorObject.getAttributeByName("dept");
+        AssertJUnit.assertNull(dept);
+        title = connectorObject.getAttributeByName("title");
+        AssertJUnit.assertNull(title);
+        tel = connectorObject.getAttributeByName("tel");
+        AssertJUnit.assertNull(tel);
+        rawJson = connectorObject.getAttributeByName("__RAW_JSON__");
+        AssertJUnit.assertNotNull(rawJson);
+        jsonValue = AttributeUtil.getStringValue(rawJson);
+        AssertJUnit.assertEquals("[{\"name\":\"bob\",\"tel\":[],\"id\":\"3\",\"dept\":\"\",\"title\":\"\"}]", jsonValue);
+    }
 }
diff --git a/src/test/resources/search-grouping.csv b/src/test/resources/search-grouping.csv
@@ -0,0 +1,5 @@
+id;name;dept;title;tel
+1;john;abc;engineer;1111
+1;john;efg;manager;1111
+2;jack;abc;manager;1111,2222
+3;bob;;;