YDA-5578: add duplicate group check

Verify that CSV file does not have duplicate group names, since this
can result in unexpected errors while importing (e.g. trying to create
a group that has already been created earlier during the import).

requirements.txt

@@ -2,6 +2,7 @@ python_irodsclient==1.1.1
 enum34
 six
 humanize>=0.5
+iteration_utilities==0.12.0
 dnspython>=2.2.0
 backports.functools-lru-cache>=1.6.4
 typing_extensions==4.1.1

setup.py

@@ -9,6 +9,7 @@
         'enum34',
         'six',
         'humanize>=0.5',
+        'iteration_utilities==0.12.0',
         'dnspython>=2.2.0',
         'backports.functools-lru-cache>=1.6.4',
         'PyYaml'

unit-tests/files/no-duplicates.csv

@@ -0,0 +1,3 @@
+category,subcategory,groupname,manager
+test-automation,initial,groupteama,m.manager@yoda.dev
+test-automation,initial,groupteamb,m.manager@yoda.dev

unit-tests/files/with-duplicates.csv

@@ -0,0 +1,5 @@
+category,subcategory,groupname,manager
+test-automation,initial,groupteama,m.manager@yoda.dev
+test-automation,initial,groupteamb,m.manager@yoda.dev
+test-automation,initial,data-duplicate,m.manager@yoda.dev
+test-automation,initial,data-duplicate,m.manager@yoda.dev

unit-tests/test_importgroups.py

@@ -13,7 +13,7 @@
 
 sys.path.append("../yclienttools")
 
-from importgroups import _get_duplicate_columns, _process_csv_line, parse_csv_file
+from importgroups import _get_duplicate_columns, _get_duplicate_groups, _process_csv_line, parse_csv_file
 
 
 class ImportGroupsTest(TestCase):
@@ -139,3 +139,13 @@ def test_parse_invalid_csv_file(self, mock_stderr):
         # csv that has too many items in the rows compared to the headers
         with self.assertRaises(SystemExit):
             parse_csv_file("files/more-entries-than-headers.csv", args, "1.9")
+
+    @patch('sys.stderr', new_callable=StringIO)
+    def test_parse_duplicate_groups(self, mock_stderr):
+        args = {"offline_check": True}
+
+        data_no_duplicates = parse_csv_file("files/no-duplicates.csv", args, "1.9")
+        self.assertEquals(_get_duplicate_groups(data_no_duplicates), [])
+
+        data_with_duplicates = parse_csv_file("files/with-duplicates.csv", args, "1.9")
+        self.assertEquals(_get_duplicate_groups(data_with_duplicates), ["research-data-duplicate"])

yclienttools/importgroups.py

@@ -3,6 +3,8 @@
 import csv
 import sys
 
+from iteration_utilities import duplicates, unique_everseen
+
 from yclienttools import common_args, common_config
 
 from yclienttools import session as s
@@ -110,6 +112,11 @@ def _get_duplicate_columns(fields_list, yoda_version):
     return duplicate_fields
 
 
+def _get_duplicate_groups(row_data):
+    group_names = list(map(lambda r: r[2], row_data))
+    return list(unique_everseen(duplicates(group_names)))
+
+
 def _process_csv_line(line, args, yoda_version):
     if ('category' not in line or not len(line['category'])
             or 'subcategory' not in line or not len(line['subcategory'])
@@ -387,6 +394,11 @@ def entry():
         _exit_with_error(
             "The --creator-user and --creator-zone options are only supported with Yoda versions 1.9 and higher.")
 
+    duplicate_groups = _get_duplicate_groups(data)
+    if duplicate_groups:
+        _exit_with_error(
+            "The group list has multiple rows with the same group name(s): " + ",".join(duplicate_groups))
+
     if args.offline_check:
         sys.exit(0)