From 0e2c5a3e54b0d3a18d7594b296877a9ac1b5dc69 Mon Sep 17 00:00:00 2001 From: Sietse Snel Date: Wed, 24 Jan 2024 10:52:52 +0100 Subject: [PATCH] YDA-5578: add duplicate group check Verify that CSV file does not have duplicate group names, since this can result in unexpected errors while importing (e.g. trying to create a group that has already been created earlier during the import). --- requirements.txt | 1 + setup.py | 1 + unit-tests/files/no-duplicates.csv | 3 +++ unit-tests/files/with-duplicates.csv | 5 +++++ unit-tests/test_importgroups.py | 12 +++++++++++- yclienttools/importgroups.py | 12 ++++++++++++ 6 files changed, 33 insertions(+), 1 deletion(-) create mode 100644 unit-tests/files/no-duplicates.csv create mode 100644 unit-tests/files/with-duplicates.csv diff --git a/requirements.txt b/requirements.txt index ecef885..8f52406 100644 --- a/requirements.txt +++ b/requirements.txt @@ -2,6 +2,7 @@ python_irodsclient==1.1.1 enum34 six humanize>=0.5 +iteration_utilities==0.12.0 dnspython>=2.2.0 backports.functools-lru-cache>=1.6.4 typing_extensions==4.1.1 diff --git a/setup.py b/setup.py index 8d14330..3a385f1 100644 --- a/setup.py +++ b/setup.py @@ -9,6 +9,7 @@ 'enum34', 'six', 'humanize>=0.5', + 'iteration_utilities==0.12.0', 'dnspython>=2.2.0', 'backports.functools-lru-cache>=1.6.4', 'PyYaml' diff --git a/unit-tests/files/no-duplicates.csv b/unit-tests/files/no-duplicates.csv new file mode 100644 index 0000000..9c644d7 --- /dev/null +++ b/unit-tests/files/no-duplicates.csv @@ -0,0 +1,3 @@ +category,subcategory,groupname,manager +test-automation,initial,groupteama,m.manager@yoda.dev +test-automation,initial,groupteamb,m.manager@yoda.dev diff --git a/unit-tests/files/with-duplicates.csv b/unit-tests/files/with-duplicates.csv new file mode 100644 index 0000000..7a08a0d --- /dev/null +++ b/unit-tests/files/with-duplicates.csv @@ -0,0 +1,5 @@ +category,subcategory,groupname,manager +test-automation,initial,groupteama,m.manager@yoda.dev +test-automation,initial,groupteamb,m.manager@yoda.dev +test-automation,initial,data-duplicate,m.manager@yoda.dev +test-automation,initial,data-duplicate,m.manager@yoda.dev diff --git a/unit-tests/test_importgroups.py b/unit-tests/test_importgroups.py index 6d0eddd..ab52980 100644 --- a/unit-tests/test_importgroups.py +++ b/unit-tests/test_importgroups.py @@ -13,7 +13,7 @@ sys.path.append("../yclienttools") -from importgroups import _get_duplicate_columns, _process_csv_line, parse_csv_file +from importgroups import _get_duplicate_columns, _get_duplicate_groups, _process_csv_line, parse_csv_file class ImportGroupsTest(TestCase): @@ -139,3 +139,13 @@ def test_parse_invalid_csv_file(self, mock_stderr): # csv that has too many items in the rows compared to the headers with self.assertRaises(SystemExit): parse_csv_file("files/more-entries-than-headers.csv", args, "1.9") + + @patch('sys.stderr', new_callable=StringIO) + def test_parse_duplicate_groups(self, mock_stderr): + args = {"offline_check": True} + + data_no_duplicates = parse_csv_file("files/no-duplicates.csv", args, "1.9") + self.assertEquals(_get_duplicate_groups(data_no_duplicates), []) + + data_with_duplicates = parse_csv_file("files/with-duplicates.csv", args, "1.9") + self.assertEquals(_get_duplicate_groups(data_with_duplicates), ["research-data-duplicate"]) diff --git a/yclienttools/importgroups.py b/yclienttools/importgroups.py index 9e9670d..de34018 100644 --- a/yclienttools/importgroups.py +++ b/yclienttools/importgroups.py @@ -3,6 +3,8 @@ import csv import sys +from iteration_utilities import duplicates, unique_everseen + from yclienttools import common_args, common_config from yclienttools import session as s @@ -110,6 +112,11 @@ def _get_duplicate_columns(fields_list, yoda_version): return duplicate_fields +def _get_duplicate_groups(row_data): + group_names = list(map(lambda r: r[2], row_data)) + return list(unique_everseen(duplicates(group_names))) + + def _process_csv_line(line, args, yoda_version): if ('category' not in line or not len(line['category']) or 'subcategory' not in line or not len(line['subcategory']) @@ -387,6 +394,11 @@ def entry(): _exit_with_error( "The --creator-user and --creator-zone options are only supported with Yoda versions 1.9 and higher.") + duplicate_groups = _get_duplicate_groups(data) + if duplicate_groups: + _exit_with_error( + "The group list has multiple rows with the same group name(s): " + ",".join(duplicate_groups)) + if args.offline_check: sys.exit(0)