diff --git a/src/tests/ftest/harness/unit.py b/src/tests/ftest/harness/unit.py index 263087639de..318a29406ed 100644 --- a/src/tests/ftest/harness/unit.py +++ b/src/tests/ftest/harness/unit.py @@ -3,9 +3,12 @@ SPDX-License-Identifier: BSD-2-Clause-Patent """ +from ClusterShell.NodeSet import NodeSet + from apricot import TestWithoutServers from data_utils import list_unique, list_flatten, list_stats, \ dict_extract_values, dict_subtract +from run_utils import run_remote, ResultData class HarnessUnitTest(TestWithoutServers): @@ -14,6 +17,41 @@ class HarnessUnitTest(TestWithoutServers): :avocado: recursive """ + def _verify_remote_command_result(self, result, passed, expected, timeout, homogeneous, + passed_hosts, failed_hosts, all_stdout, all_stderr): + """Verify a RemoteCommandResult object. + + Args: + result (RemoteCommandResult): object to verify + passed (bool): expected passed command state + expected (list): expected list of ResultData objects + timeout (bool): expected command timeout state + homogeneous (bool): expected homogeneous command output state + passed_hosts (NodeSet): expected set of hosts on which the command passed + failed_hosts (NodeSet): expected set of hosts on which the command failed + all_stdout (dict): expected stdout str per host key + all_stderr (dict): expected stderr str per host key + """ + self.assertEqual(passed, result.passed, 'Incorrect RemoteCommandResult.passed') + self.assertEqual( + len(expected), len(result.output), 'Incorrect RemoteCommandResult.output count') + sorted_output = sorted(result.output) + for index, expect in enumerate(sorted(expected)): + actual = sorted_output[index] + for key in ('command', 'returncode', 'hosts', 'stdout', 'stderr', 'timeout'): + self.assertEqual( + getattr(expect, key), getattr(actual, key), + 'Incorrect ResultData.{}'.format(key)) + self.assertEqual(timeout, result.timeout, 'Incorrect RemoteCommandResult.timeout') + self.assertEqual( + homogeneous, result.homogeneous, 'Incorrect RemoteCommandResult.homogeneous') + self.assertEqual( + passed_hosts, result.passed_hosts, 'Incorrect RemoteCommandResult.passed_hosts') + self.assertEqual( + failed_hosts, result.failed_hosts, 'Incorrect RemoteCommandResult.failed_hosts') + self.assertEqual(all_stdout, result.all_stdout, 'Incorrect RemoteCommandResult.all_stdout') + self.assertEqual(all_stderr, result.all_stderr, 'Incorrect RemoteCommandResult.all_stderr') + def test_harness_unit_list_unique(self): """Verify list_unique(). @@ -22,6 +60,7 @@ def test_harness_unit_list_unique(self): :avocado: tags=harness,dict_utils :avocado: tags=HarnessUnitTest,test_harness_unit_list_unique """ + self.log_step('Verify list_unique()') self.assertEqual( list_unique([1, 2, 3]), [1, 2, 3]) @@ -37,6 +76,7 @@ def test_harness_unit_list_unique(self): self.assertEqual( list_unique([{0: 1}, {2: 3}, {2: 3}]), [{0: 1}, {2: 3}]) + self.log_step('Unit Test Passed') def test_harness_unit_list_flatten(self): """Verify list_flatten(). @@ -46,6 +86,7 @@ def test_harness_unit_list_flatten(self): :avocado: tags=harness,dict_utils :avocado: tags=HarnessUnitTest,test_harness_unit_list_flatten """ + self.log_step('Verify list_flatten()') self.assertEqual( list_flatten([1, 2, 3]), [1, 2, 3]) @@ -67,6 +108,7 @@ def test_harness_unit_list_flatten(self): self.assertEqual( list_flatten([1, 2, 3, {'foo': 'bar'}]), [1, 2, 3, {'foo': 'bar'}]) + self.log_step('Unit Test Passed') def test_harness_unit_list_stats(self): """Verify list_stats(). @@ -76,6 +118,7 @@ def test_harness_unit_list_stats(self): :avocado: tags=harness,dict_utils :avocado: tags=HarnessUnitTest,test_harness_unit_list_stats """ + self.log_step('Verify list_stats()') self.assertEqual( list_stats([100, 200]), { @@ -90,6 +133,7 @@ def test_harness_unit_list_stats(self): 'min': -100, 'max': 200 }) + self.log_step('Unit Test Passed') def test_harness_unit_dict_extract_values(self): """Verify dict_extract_values(). @@ -99,6 +143,7 @@ def test_harness_unit_dict_extract_values(self): :avocado: tags=harness,dict_utils :avocado: tags=HarnessUnitTest,test_harness_unit_dict_extract_values """ + self.log_step('Verify dict_extract_values()') dict1 = { 'key1': { 'key1.1': { @@ -151,6 +196,7 @@ def test_harness_unit_dict_extract_values(self): self.assertEqual( dict_extract_values(dict2, ['a']), [{'b': {'a': 0}}, 0]) + self.log_step('Unit Test Passed') def test_harness_unit_dict_subtract(self): """Verify dict_subtract(). @@ -160,6 +206,7 @@ def test_harness_unit_dict_subtract(self): :avocado: tags=harness,dict_utils :avocado: tags=HarnessUnitTest,test_harness_unit_dict_subtract """ + self.log_step('Verify dict_subtract()') dict1 = { 'key1': { 'key2': { @@ -186,3 +233,220 @@ def test_harness_unit_dict_subtract(self): } } }) + self.log_step('Unit Test Passed') + + def test_harness_unit_run_remote_single(self): + """Verify run_remote() with a single host. + + :avocado: tags=all + :avocado: tags=vm + :avocado: tags=harness,run_utils + :avocado: tags=HarnessUnitTest,test_harness_unit_run_remote_single + """ + hosts = self.get_hosts_from_yaml('test_clients', 'partition', 'reservation', '/run/hosts/*') + command = 'uname -o' + self.log_step('Verify run_remote() w/ single host') + self._verify_remote_command_result( + result=run_remote(self.log, NodeSet(hosts[0]), command), + passed=True, + expected=[ResultData(command, 0, NodeSet(hosts[0]), ['GNU/Linux'], [], False)], + timeout=False, + homogeneous=True, + passed_hosts=NodeSet(hosts[0]), + failed_hosts=NodeSet(), + all_stdout={hosts[0]: 'GNU/Linux'}, + all_stderr={hosts[0]: ''} + ) + self.log_step('Unit Test Passed') + + def test_harness_unit_run_remote_homogeneous(self): + """Verify run_remote() with homogeneous output. + + :avocado: tags=all + :avocado: tags=vm + :avocado: tags=harness,run_utils + :avocado: tags=HarnessUnitTest,test_harness_unit_run_remote_homogeneous + """ + hosts = self.get_hosts_from_yaml('test_clients', 'partition', 'reservation', '/run/hosts/*') + command = 'uname -o' + self.log_step('Verify run_remote() w/ homogeneous output') + self._verify_remote_command_result( + result=run_remote(self.log, hosts, command), + passed=True, + expected=[ResultData(command, 0, hosts, ['GNU/Linux'], [], False)], + timeout=False, + homogeneous=True, + passed_hosts=hosts, + failed_hosts=NodeSet(), + all_stdout={str(hosts): 'GNU/Linux'}, + all_stderr={str(hosts): ''} + ) + self.log_step('Unit Test Passed') + + def test_harness_unit_run_remote_heterogeneous(self): + """Verify run_remote() with heterogeneous output. + + :avocado: tags=all + :avocado: tags=vm + :avocado: tags=harness,run_utils + :avocado: tags=HarnessUnitTest,test_harness_unit_run_remote_heterogeneous + """ + hosts = self.get_hosts_from_yaml('test_clients', 'partition', 'reservation', '/run/hosts/*') + command = 'hostname -s' + self.log_step('Verify run_remote() w/ heterogeneous output') + self._verify_remote_command_result( + result=run_remote(self.log, hosts, command), + passed=True, + expected=[ + ResultData(command, 0, NodeSet(hosts[0]), [hosts[0]], [], False), + ResultData(command, 0, NodeSet(hosts[1]), [hosts[1]], [], False), + ], + timeout=False, + homogeneous=False, + passed_hosts=hosts, + failed_hosts=NodeSet(), + all_stdout={ + hosts[0]: hosts[0], + hosts[1]: hosts[1] + }, + all_stderr={ + hosts[0]: '', + hosts[1]: '' + }, + ) + self.log_step('Unit Test Passed') + + def test_harness_unit_run_remote_combined(self): + """Verify run_remote() with combined stdout and stderr. + + :avocado: tags=all + :avocado: tags=vm + :avocado: tags=harness,run_utils + :avocado: tags=HarnessUnitTest,test_harness_unit_run_remote_combined + """ + hosts = self.get_hosts_from_yaml('test_clients', 'partition', 'reservation', '/run/hosts/*') + command = 'echo stdout; if [ $(hostname -s) == \'{}\' ]; then echo stderr 1>&2; fi'.format( + hosts[1]) + self.log_step('Verify run_remote() w/ separated stdout and stderr') + self._verify_remote_command_result( + result=run_remote(self.log, hosts, command, stderr=False), + passed=True, + expected=[ + ResultData(command, 0, NodeSet(hosts[0]), ['stdout'], [], False), + ResultData(command, 0, NodeSet(hosts[1]), ['stdout', 'stderr'], [], False), + ], + timeout=False, + homogeneous=False, + passed_hosts=hosts, + failed_hosts=NodeSet(), + all_stdout={ + hosts[0]: 'stdout', + hosts[1]: 'stdout\nstderr' + }, + all_stderr={ + hosts[0]: '', + hosts[1]: '' + } + ) + self.log_step('Unit Test Passed') + + def test_harness_unit_run_remote_separated(self): + """Verify run_remote() with separated stdout and stderr. + + :avocado: tags=all + :avocado: tags=vm + :avocado: tags=harness,run_utils + :avocado: tags=HarnessUnitTest,test_harness_unit_run_remote_separated + """ + hosts = self.get_hosts_from_yaml('test_clients', 'partition', 'reservation', '/run/hosts/*') + command = 'echo stdout; if [ $(hostname -s) == \'{}\' ]; then echo stderr 1>&2; fi'.format( + hosts[1]) + self.log_step('Verify run_remote() w/ separated stdout and stderr') + self._verify_remote_command_result( + result=run_remote(self.log, hosts, command, stderr=True), + passed=True, + expected=[ + ResultData(command, 0, NodeSet(hosts[0]), ['stdout'], [], False), + ResultData(command, 0, NodeSet(hosts[1]), ['stdout'], ['stderr'], False), + ], + timeout=False, + homogeneous=False, + passed_hosts=hosts, + failed_hosts=NodeSet(), + all_stdout={ + hosts[0]: 'stdout', + hosts[1]: 'stdout' + }, + all_stderr={ + hosts[0]: '', + hosts[1]: 'stderr' + } + ) + self.log_step('Unit Test Passed') + + def test_harness_unit_run_remote_no_stdout(self): + """Verify run_remote() with separated stdout and stderr. + + :avocado: tags=all + :avocado: tags=vm + :avocado: tags=harness,run_utils + :avocado: tags=HarnessUnitTest,test_harness_unit_run_remote_separated + """ + hosts = self.get_hosts_from_yaml('test_clients', 'partition', 'reservation', '/run/hosts/*') + command = 'if [ $(hostname -s) == \'{}\' ]; then echo stderr 1>&2; fi'.format(hosts[1]) + self.log_step('Verify run_remote() w/ no stdout') + self._verify_remote_command_result( + result=run_remote(self.log, hosts, command, stderr=True), + passed=True, + expected=[ + ResultData(command, 0, NodeSet(hosts[0]), [], [], False), + ResultData(command, 0, NodeSet(hosts[1]), [], ['stderr'], False), + ], + timeout=False, + homogeneous=False, + passed_hosts=hosts, + failed_hosts=NodeSet(), + all_stdout={ + hosts[0]: '', + hosts[1]: '' + }, + all_stderr={ + hosts[0]: '', + hosts[1]: 'stderr' + } + ) + self.log_step('Unit Test Passed') + + def test_harness_unit_run_remote_failure(self): + """Verify run_remote() with separated stdout and stderr. + + :avocado: tags=all + :avocado: tags=vm + :avocado: tags=harness,run_utils + :avocado: tags=HarnessUnitTest,test_harness_unit_run_remote_separated + """ + hosts = self.get_hosts_from_yaml('test_clients', 'partition', 'reservation', '/run/hosts/*') + command = 'if [ $(hostname -s) == \'{}\' ]; then echo fail; exit 1; fi; echo pass'.format( + hosts[1]) + self.log_step('Verify run_remote() w/ a failure') + self._verify_remote_command_result( + result=run_remote(self.log, hosts, command, stderr=True), + passed=False, + expected=[ + ResultData(command, 0, NodeSet(hosts[0]), ['pass'], [], False), + ResultData(command, 1, NodeSet(hosts[1]), ['fail'], [], False), + ], + timeout=False, + homogeneous=False, + passed_hosts=NodeSet(hosts[0]), + failed_hosts=NodeSet(hosts[1]), + all_stdout={ + hosts[0]: 'pass', + hosts[1]: 'fail' + }, + all_stderr={ + hosts[0]: '', + hosts[1]: '' + } + ) + self.log_step('Unit Test Passed') diff --git a/src/tests/ftest/harness/unit.yaml b/src/tests/ftest/harness/unit.yaml index 9ca52a511a8..03a2c3a16e1 100644 --- a/src/tests/ftest/harness/unit.yaml +++ b/src/tests/ftest/harness/unit.yaml @@ -1,2 +1,3 @@ timeout: 10 -test_clients: 1 +hosts: + test_clients: 2 diff --git a/src/tests/ftest/util/run_utils.py b/src/tests/ftest/util/run_utils.py index b1cd471756c..730dbb3bbd1 100644 --- a/src/tests/ftest/util/run_utils.py +++ b/src/tests/ftest/util/run_utils.py @@ -15,38 +15,64 @@ class RunException(Exception): """Base exception for this module.""" -class RemoteCommandResult(): - """Stores the command result from a Task object.""" +class ResultData(): + # pylint: disable=too-few-public-methods + """Command result data for the set of hosts.""" + + def __init__(self, command, returncode, hosts, stdout, stderr, timeout): + """Initialize a ResultData object. + + Args: + command (str): the executed command + returncode (int): the return code of the executed command + hosts (NodeSet): the host(s) on which the executed command yielded this result + stdout (list): the result of the executed command split by newlines + timeout (bool): indicator for a command timeout + """ + self.command = command + self.returncode = returncode + self.hosts = hosts + self.stdout = stdout + self.stderr = stderr + self.timeout = timeout - class ResultData(): - # pylint: disable=too-few-public-methods - """Command result data for the set of hosts.""" + def __lt__(self, other): + """Determine if another ResultData object is less than this one. - def __init__(self, command, returncode, hosts, stdout, timeout): - """Initialize a ResultData object. + Args: + other (NodeSet): the other NodSet to compare - Args: - command (str): the executed command - returncode (int): the return code of the executed command - hosts (NodeSet): the host(s) on which the executed command yielded this result - stdout (list): the result of the executed command split by newlines - timeout (bool): indicator for a command timeout - """ - self.command = command - self.returncode = returncode - self.hosts = hosts - self.stdout = stdout - self.timeout = timeout + Returns: + bool: True if this object is less than the other ResultData object; False otherwise + """ + if not isinstance(other, ResultData): + raise NotImplementedError + return str(self.hosts) < str(other.hosts) - @property - def passed(self): - """Did the command pass. + def __gt__(self, other): + """Determine if another ResultData object is greater than this one. - Returns: - bool: if the command was successful + Args: + other (NodeSet): the other NodSet to compare - """ - return self.returncode == 0 + Returns: + bool: True if this object is greater than the other ResultData object; False otherwise + """ + return not self.__lt__(other) + + @property + def passed(self): + """Did the command pass. + + Returns: + bool: if the command was successful + + """ + return self.returncode == 0 + + +class RemoteCommandResult(): + """Stores the command result from a Task object.""" def __init__(self, command, task): """Create a RemoteCommandResult object. @@ -122,6 +148,19 @@ def all_stdout(self): stdout[str(data.hosts)] = '\n'.join(data.stdout) return stdout + @property + def all_stderr(self): + """Get all of the stderr from the issued command from each host. + + Returns: + dict: the stderr (the values) from each set of hosts (the keys, as a str of the NodeSet) + + """ + stderr = {} + for data in self.output: + stderr[str(data.hosts)] = '\n'.join(data.stderr) + return stderr + def _process_task(self, task, command): """Populate the output list and determine the passed result for the specified task. @@ -137,23 +176,67 @@ def _process_task(self, task, command): # Populate the a list of unique output for each NodeSet for code in sorted(results): - output_data = list(task.iter_buffers(results[code])) - if not output_data: - output_data = [["", results[code]]] - for output, output_hosts in output_data: + stdout_data = self._sanitize_iter_data( + results[code], list(task.iter_buffers(results[code])), '') + + for stdout_raw, stdout_hosts in stdout_data: # In run_remote(), task.run() is executed with the stderr=False default. # As a result task.iter_buffers() will return combined stdout and stderr. - stdout = [] - for line in output.splitlines(): - if isinstance(line, bytes): - stdout.append(line.decode("utf-8")) - else: - stdout.append(line) - self.output.append( - self.ResultData(command, code, NodeSet.fromlist(output_hosts), stdout, False)) + stdout = self._msg_tree_elem_to_list(stdout_raw) + stderr_data = self._sanitize_iter_data( + stdout_hosts, list(task.iter_errors(stdout_hosts)), '') + for stderr_raw, stderr_hosts in stderr_data: + stderr = self._msg_tree_elem_to_list(stderr_raw) + self.output.append( + ResultData( + command, code, NodeSet.fromlist(stderr_hosts), stdout, stderr, False)) if timed_out: self.output.append( - self.ResultData(command, 124, NodeSet.fromlist(timed_out), None, True)) + ResultData(command, 124, NodeSet.fromlist(timed_out), None, None, True)) + + @staticmethod + def _sanitize_iter_data(hosts, data, default_entry): + """Ensure the data generated from an iter function has entries for each host. + + Args: + hosts (list): lists of host which generated data + data (list): data from an iter function as a list + default_entry (object): entry to add to data for missing hosts in data + + Returns: + list: a list of tuples of entries and list of hosts + """ + if not data: + return [(default_entry, hosts)] + + source_keys = NodeSet.fromlist(hosts) + data_keys = NodeSet() + for _, keys in data: + data_keys.add(NodeSet.fromlist(keys)) + + sanitized_data = data.copy() + missing_keys = source_keys - data_keys + if missing_keys: + sanitized_data.append((default_entry, list(missing_keys))) + return sanitized_data + + @staticmethod + def _msg_tree_elem_to_list(msg_tree_elem): + """Convert a ClusterShell.MsgTree.MsgTreeElem to a list of strings. + + Args: + msg_tree_elem (MsgTreeElem): output from Task.iter_* method. + + Returns: + list: list of strings + """ + msg_tree_elem_list = [] + for line in msg_tree_elem.splitlines(): + if isinstance(line, bytes): + msg_tree_elem_list.append(line.decode("utf-8")) + else: + msg_tree_elem_list.append(line) + return msg_tree_elem_list def log_output(self, log): """Log the command result. @@ -174,14 +257,21 @@ def log_result_data(log, data): data (ResultData): command result common to a set of hosts """ info = " timed out" if data.timeout else "" - if not data.stdout: + if not data.stdout and not data.stderr: log.debug(" %s (rc=%s)%s: ", str(data.hosts), data.returncode, info) - elif len(data.stdout) == 1: + elif data.stdout and len(data.stdout) == 1 and not data.stderr: log.debug(" %s (rc=%s)%s: %s", str(data.hosts), data.returncode, info, data.stdout[0]) else: log.debug(" %s (rc=%s)%s:", str(data.hosts), data.returncode, info) + indent = 6 if data.stderr else 4 + if data.stdout and data.stderr: + log.debug(" :") for line in data.stdout: - log.debug(" %s", line) + log.debug("%s%s", " " * indent, line) + if data.stderr: + log.debug(" :") + for line in data.stderr: + log.debug("%s%s", " " * indent, line) def get_clush_command(hosts, args=None, command="", command_env=None, command_sudo=False): @@ -289,7 +379,7 @@ def run_local(log, command, capture_output=True, timeout=None, check=False, verb return result -def run_remote(log, hosts, command, verbose=True, timeout=120, task_debug=False): +def run_remote(log, hosts, command, verbose=True, timeout=120, task_debug=False, stderr=False): """Run the command on the remote hosts. Args: @@ -300,6 +390,7 @@ def run_remote(log, hosts, command, verbose=True, timeout=120, task_debug=False) timeout (int, optional): number of seconds to wait for the command to complete. Defaults to 120 seconds. task_debug (bool, optional): whether to enable debug for the task object. Defaults to False. + stderr (bool, optional): whether to enable stdout/stderr separation. Defaults to False. Returns: RemoteCommandResult: a grouping of the command results from the same hosts with the same @@ -307,8 +398,8 @@ def run_remote(log, hosts, command, verbose=True, timeout=120, task_debug=False) """ task = task_self() - if task_debug: - task.set_info('debug', True) + task.set_info('debug', task_debug) + task.set_default("stderr", stderr) # Enable forwarding of the ssh authentication agent connection task.set_info("ssh_options", "-oForwardAgent=yes") if verbose: