From 0a31f6eca93371ac019b401d78f559828c98547c Mon Sep 17 00:00:00 2001 From: Lukasz Mrugala Date: Wed, 21 Aug 2024 11:24:51 +0000 Subject: [PATCH] scripts: twister: Enchance TestCase/Instance info and presentation ExecutionCounter has been expanded and now hold i.a. more information on the statuses of TestCases. This information is now incorporated in relevant summaries - runner.py and reports.py. Layout of those was changed to present that and previous information in a clear and concise way. TestInstance execution counter now is more intuitive. Instances filtered out before running are no longer included there. Retries now properly reset the counter. TestCases with None and other incorrect final statuses are logged as errors, but do not exit Twister with a nonzero exit code. This is because None statuses, although incorrect, are currently common. Inconsistent spacing in ERROR and FAILED fixed. Signed-off-by: Lukasz Mrugala scripts: Dmitri fix Fix of a problem noticed by Dmitri Removed unnecessary additional spaces when printing FAILED and ERROR status. Now TwisterStatus.get_color is used more. Signed-off-by: Lukasz Mrugala --- scripts/pylib/twister/twisterlib/reports.py | 56 +-- scripts/pylib/twister/twisterlib/runner.py | 389 +++++++++++++++--- scripts/tests/twister/test_runner.py | 143 +++++-- scripts/tests/twister_blackbox/test_device.py | 3 +- scripts/tests/twister_blackbox/test_error.py | 3 +- .../tests/twister_blackbox/test_platform.py | 61 ++- scripts/tests/twister_blackbox/test_runner.py | 67 +-- 7 files changed, 549 insertions(+), 173 deletions(-) diff --git a/scripts/pylib/twister/twisterlib/reports.py b/scripts/pylib/twister/twisterlib/reports.py index e7160969ac9a87..058afce58c5479 100644 --- a/scripts/pylib/twister/twisterlib/reports.py +++ b/scripts/pylib/twister/twisterlib/reports.py @@ -584,39 +584,43 @@ def summary(self, results, ignore_unrecognized_sections, duration): pass_rate = 0 logger.info( - "{}{} of {}{} test configurations passed ({:.2%}), {} built (not run), {}{}{} failed, {}{}{} errored, {} skipped with {}{}{} warnings in {:.2f} seconds".format( - Fore.RED if failed else Fore.GREEN, - results.passed, - results.total, - Fore.RESET, - pass_rate, - results.notrun, - Fore.RED if results.failed else Fore.RESET, - results.failed, - Fore.RESET, - Fore.RED if results.error else Fore.RESET, - results.error, - Fore.RESET, - results.skipped_configs, - Fore.YELLOW if self.plan.warnings else Fore.RESET, - self.plan.warnings, - Fore.RESET, - duration)) + f"{TwisterStatus.get_color(TwisterStatus.FAIL) if failed else TwisterStatus.get_color(TwisterStatus.PASS)}{results.passed}" + f" of {results.total - results.skipped_configs}{Fore.RESET}" + f" executed test configurations passed ({pass_rate:.2%})," + f" {f'{TwisterStatus.get_color(TwisterStatus.NOTRUN)}{results.notrun}{Fore.RESET}' if results.notrun else f'{results.notrun}'} built (not run)," + f" {f'{TwisterStatus.get_color(TwisterStatus.FAIL)}{results.failed}{Fore.RESET}' if results.failed else f'{results.failed}'} failed," + f" {f'{TwisterStatus.get_color(TwisterStatus.ERROR)}{results.error}{Fore.RESET}' if results.error else f'{results.error}'} errored," + f" with {f'{Fore.YELLOW}{self.plan.warnings}{Fore.RESET}' if self.plan.warnings else 'no'} warnings" + f" in {duration:.2f} seconds." + ) total_platforms = len(self.platforms) # if we are only building, do not report about tests being executed. if self.platforms and not self.env.options.build_only: - logger.info("In total {} test cases were executed, {} skipped on {} out of total {} platforms ({:02.2f}%)".format( - results.cases - results.skipped_cases - results.notrun, - results.skipped_cases, - len(self.filtered_platforms), - total_platforms, - (100 * len(self.filtered_platforms) / len(self.platforms)) - )) + executed_cases = results.cases - results.filtered_cases - results.skipped_cases - results.notrun_cases + pass_rate = 100 * (float(results.passed_cases) / float(executed_cases)) \ + if executed_cases != 0 else 0 + platform_rate = (100 * len(self.filtered_platforms) / len(self.platforms)) + logger.info( + f'{results.passed_cases} of {executed_cases} executed test cases passed ({pass_rate:02.2f}%)' + f'{", " + str(results.blocked_cases) + " blocked" if results.blocked_cases else ""}' + f'{", " + str(results.failed_cases) + " failed" if results.failed_cases else ""}' + f'{", " + str(results.error_cases) + " errored" if results.error_cases else ""}' + f'{", " + str(results.none_cases) + " without a status" if results.none_cases else ""}' + f' on {len(self.filtered_platforms)} out of total {total_platforms} platforms ({platform_rate:02.2f}%).' + ) + if results.skipped_cases or results.filtered_cases or results.notrun_cases: + logger.info( + f'{results.skipped_cases + results.filtered_cases} selected test cases not executed:' \ + f'{" " + str(results.skipped_cases) + " skipped" if results.skipped_cases else ""}' \ + f'{(", " if results.skipped_cases else " ") + str(results.filtered_cases) + " filtered" if results.filtered_cases else ""}' \ + f'{(", " if results.skipped_cases or results.filtered_cases else " ") + str(results.notrun_cases) + " not run (built only)" if results.notrun_cases else ""}' \ + f'.' + ) built_only = results.total - run - results.skipped_configs logger.info(f"{Fore.GREEN}{run}{Fore.RESET} test configurations executed on platforms, \ -{Fore.RED}{built_only}{Fore.RESET} test configurations were only built.") +{TwisterStatus.get_color(TwisterStatus.NOTRUN)}{built_only}{Fore.RESET} test configurations were only built.") def save_reports(self, name, suffix, report_dir, no_update, platform_reports): if not self.instances: diff --git a/scripts/pylib/twister/twisterlib/runner.py b/scripts/pylib/twister/twisterlib/runner.py index f5e2cabf74dcb3..9dc81f08e4c688 100644 --- a/scripts/pylib/twister/twisterlib/runner.py +++ b/scripts/pylib/twister/twisterlib/runner.py @@ -5,6 +5,7 @@ # SPDX-License-Identifier: Apache-2.0 import logging +from math import log10 import multiprocessing import os import pickle @@ -62,12 +63,16 @@ class ExecutionCounter(object): def __init__(self, total=0): ''' Most of the stats are at test instance level - Except that "_cases" and "_skipped_cases" are for cases of ALL test instances + Except that case statistics are for cases of ALL test instances - total complete = done + skipped_filter total = yaml test scenarios * applicable platforms - complete perctenage = (done + skipped_filter) / total + done := instances that reached report_out stage of the pipeline + done = skipped_configs + passed + failed + error + completed = done - skipped_filter + skipped_configs = skipped_runtime + skipped_filter + pass rate = passed / (total - skipped_configs) + case pass rate = passed_cases / (cases - filtered_cases - skipped_cases) ''' # instances that go through the pipeline # updated by report_out() @@ -92,13 +97,10 @@ def __init__(self, total=0): # updated by report_out() self._skipped_runtime = Value('i', 0) - # staic filtered at yaml parsing time + # static filtered at yaml parsing time # updated by update_counting_before_pipeline() self._skipped_filter = Value('i', 0) - # updated by update_counting_before_pipeline() and report_out() - self._skipped_cases = Value('i', 0) - # updated by report_out() in pipeline self._error = Value('i', 0) self._failed = Value('i', 0) @@ -106,25 +108,83 @@ def __init__(self, total=0): # initialized to number of test instances self._total = Value('i', total) + ####################################### + # TestCase counters for all instances # + ####################################### # updated in report_out self._cases = Value('i', 0) + + # updated by update_counting_before_pipeline() and report_out() + self._skipped_cases = Value('i', 0) + self._filtered_cases = Value('i', 0) + + # updated by report_out() in pipeline + self._passed_cases = Value('i', 0) + self._notrun_cases = Value('i', 0) + self._failed_cases = Value('i', 0) + self._error_cases = Value('i', 0) + self._blocked_cases = Value('i', 0) + + # Incorrect statuses + self._none_cases = Value('i', 0) + self._started_cases = Value('i', 0) + + self.lock = Lock() + @staticmethod + def _find_number_length(n): + if n > 0: + length = int(log10(n))+1 + elif n == 0: + length = 1 + else: + length = int(log10(-n))+2 + return length + def summary(self): - print("--------------------------------") - print(f"Total test suites: {self.total}") # actually test instances - print(f"Total test cases: {self.cases}") - print(f"Executed test cases: {self.cases - self.skipped_cases}") - print(f"Skipped test cases: {self.skipped_cases}") - print(f"Completed test suites: {self.done}") - print(f"Passing test suites: {self.passed}") - print(f"Built only test suites: {self.notrun}") - print(f"Failing test suites: {self.failed}") - print(f"Skipped test suites: {self.skipped_configs}") - print(f"Skipped test suites (runtime): {self.skipped_runtime}") - print(f"Skipped test suites (filter): {self.skipped_filter}") - print(f"Errors: {self.error}") - print("--------------------------------") + executed_cases = self.cases - self.skipped_cases - self.filtered_cases + completed_configs = self.done - self.skipped_filter + + # Find alignment length for aesthetic printing + suites_n_length = self._find_number_length(self.total if self.total > self.done else self.done) + processed_suites_n_length = self._find_number_length(self.done) + completed_suites_n_length = self._find_number_length(completed_configs) + skipped_suites_n_length = self._find_number_length(self.skipped_configs) + total_cases_n_length = self._find_number_length(self.cases) + executed_cases_n_length = self._find_number_length(executed_cases) + + print("--------------------------------------------------") + print(f"{'Total test suites: ':<23}{self.total:>{suites_n_length}}") # actually test instances + print(f"{'Processed test suites: ':<23}{self.done:>{suites_n_length}}") + print(f"├─ {'Filtered test suites (static): ':<37}{self.skipped_filter:>{processed_suites_n_length}}") + print(f"└─ {'Completed test suites: ':<37}{completed_configs:>{processed_suites_n_length}}") + print(f" ├─ {'Filtered test suites (at runtime): ':<37}{self.skipped_runtime:>{completed_suites_n_length}}") + print(f" ├─ {'Passed test suites: ':<37}{self.passed:>{completed_suites_n_length}}") + print(f" ├─ {'Built only test suites: ':<37}{self.notrun:>{completed_suites_n_length}}") + print(f" ├─ {'Failed test suites: ':<37}{self.failed:>{completed_suites_n_length}}") + print(f" └─ {'Errors in test suites: ':<37}{self.error:>{completed_suites_n_length}}") + print(f"") + print(f"{'Filtered test suites: ':<21}{self.skipped_configs}") + print(f"├─ {'Filtered test suites (static): ':<37}{self.skipped_filter:>{skipped_suites_n_length}}") + print(f"└─ {'Filtered test suites (at runtime): ':<37}{self.skipped_runtime:>{skipped_suites_n_length}}") + print("---------------------- ----------------------") + print(f"{'Total test cases: ':<18}{self.cases}") + print(f"├─ {'Filtered test cases: ':<21}{self.filtered_cases:>{total_cases_n_length}}") + print(f"├─ {'Skipped test cases: ':<21}{self.skipped_cases:>{total_cases_n_length}}") + print(f"└─ {'Executed test cases: ':<21}{executed_cases:>{total_cases_n_length}}") + print(f" ├─ {'Passed test cases: ':<25}{self.passed_cases:>{executed_cases_n_length}}") + print(f" ├─ {'Built only test cases: ':<25}{self.notrun_cases:>{executed_cases_n_length}}") + print(f" ├─ {'Blocked test cases: ':<25}{self.blocked_cases:>{executed_cases_n_length}}") + print(f" ├─ {'Failed test cases: ':<25}{self.failed_cases:>{executed_cases_n_length}}") + print(f" {'├' if self.none_cases or self.started_cases else '└'}─ {'Errors in test cases: ':<25}{self.error_cases:>{executed_cases_n_length}}") + if self.none_cases or self.started_cases: + print(f" ├──── The following test case statuses should not appear in a proper execution ───") + if self.none_cases: + print(f" {'├' if self.started_cases else '└'}─ {'Statusless test cases: ':<25}{self.none_cases:>{executed_cases_n_length}}") + if self.started_cases: + print(f" └─ {'Test cases only started: ':<25}{self.started_cases:>{executed_cases_n_length}}") + print("--------------------------------------------------") @property def cases(self): @@ -136,6 +196,10 @@ def cases(self, value): with self._cases.get_lock(): self._cases.value = value + def cases_increment(self, value=1): + with self._cases.get_lock(): + self._cases.value += value + @property def skipped_cases(self): with self._skipped_cases.get_lock(): @@ -146,6 +210,122 @@ def skipped_cases(self, value): with self._skipped_cases.get_lock(): self._skipped_cases.value = value + def skipped_cases_increment(self, value=1): + with self._skipped_cases.get_lock(): + self._skipped_cases.value += value + + @property + def filtered_cases(self): + with self._filtered_cases.get_lock(): + return self._filtered_cases.value + + @filtered_cases.setter + def filtered_cases(self, value): + with self._filtered_cases.get_lock(): + self._filtered_cases.value = value + + def filtered_cases_increment(self, value=1): + with self._filtered_cases.get_lock(): + self._filtered_cases.value += value + + @property + def passed_cases(self): + with self._passed_cases.get_lock(): + return self._passed_cases.value + + @passed_cases.setter + def passed_cases(self, value): + with self._passed_cases.get_lock(): + self._passed_cases.value = value + + def passed_cases_increment(self, value=1): + with self._passed_cases.get_lock(): + self._passed_cases.value += value + + @property + def notrun_cases(self): + with self._notrun_cases.get_lock(): + return self._notrun_cases.value + + @notrun_cases.setter + def notrun_cases(self, value): + with self._notrun.get_lock(): + self._notrun.value = value + + def notrun_cases_increment(self, value=1): + with self._notrun_cases.get_lock(): + self._notrun_cases.value += value + + @property + def failed_cases(self): + with self._failed_cases.get_lock(): + return self._failed_cases.value + + @failed_cases.setter + def failed_cases(self, value): + with self._failed_cases.get_lock(): + self._failed_cases.value = value + + def failed_cases_increment(self, value=1): + with self._failed_cases.get_lock(): + self._failed_cases.value += value + + @property + def error_cases(self): + with self._error_cases.get_lock(): + return self._error_cases.value + + @error_cases.setter + def error_cases(self, value): + with self._error_cases.get_lock(): + self._error_cases.value = value + + def error_cases_increment(self, value=1): + with self._error_cases.get_lock(): + self._error_cases.value += value + + @property + def blocked_cases(self): + with self._blocked_cases.get_lock(): + return self._blocked_cases.value + + @blocked_cases.setter + def blocked_cases(self, value): + with self._blocked_cases.get_lock(): + self._blocked_cases.value = value + + def blocked_cases_increment(self, value=1): + with self._blocked_cases.get_lock(): + self._blocked_cases.value += value + + @property + def none_cases(self): + with self._none_cases.get_lock(): + return self._none_cases.value + + @none_cases.setter + def none_cases(self, value): + with self._none_cases.get_lock(): + self._none_cases.value = value + + def none_cases_increment(self, value=1): + with self._none_cases.get_lock(): + self._none_cases.value += value + + @property + def started_cases(self): + with self._started_cases.get_lock(): + return self._started_cases.value + + @started_cases.setter + def started_cases(self, value): + with self._started_cases.get_lock(): + self._started_cases.value = value + + def started_cases_increment(self, value=1): + with self._started_cases.get_lock(): + self._started_cases.value += value + @property def error(self): with self._error.get_lock(): @@ -156,6 +336,10 @@ def error(self, value): with self._error.get_lock(): self._error.value = value + def error_increment(self, value=1): + with self._error.get_lock(): + self._error.value += value + @property def iteration(self): with self._iteration.get_lock(): @@ -166,6 +350,10 @@ def iteration(self, value): with self._iteration.get_lock(): self._iteration.value = value + def iteration_increment(self, value=1): + with self._iteration.get_lock(): + self._iteration.value += value + @property def done(self): with self._done.get_lock(): @@ -176,6 +364,10 @@ def done(self, value): with self._done.get_lock(): self._done.value = value + def done_increment(self, value=1): + with self._done.get_lock(): + self._done.value += value + @property def passed(self): with self._passed.get_lock(): @@ -186,6 +378,10 @@ def passed(self, value): with self._passed.get_lock(): self._passed.value = value + def passed_increment(self, value=1): + with self._passed.get_lock(): + self._passed.value += value + @property def notrun(self): with self._notrun.get_lock(): @@ -196,6 +392,10 @@ def notrun(self, value): with self._notrun.get_lock(): self._notrun.value = value + def notrun_increment(self, value=1): + with self._notrun.get_lock(): + self._notrun.value += value + @property def skipped_configs(self): with self._skipped_configs.get_lock(): @@ -206,6 +406,10 @@ def skipped_configs(self, value): with self._skipped_configs.get_lock(): self._skipped_configs.value = value + def skipped_configs_increment(self, value=1): + with self._skipped_configs.get_lock(): + self._skipped_configs.value += value + @property def skipped_filter(self): with self._skipped_filter.get_lock(): @@ -216,6 +420,10 @@ def skipped_filter(self, value): with self._skipped_filter.get_lock(): self._skipped_filter.value = value + def skipped_filter_increment(self, value=1): + with self._skipped_filter.get_lock(): + self._skipped_filter.value += value + @property def skipped_runtime(self): with self._skipped_runtime.get_lock(): @@ -226,6 +434,10 @@ def skipped_runtime(self, value): with self._skipped_runtime.get_lock(): self._skipped_runtime.value = value + def skipped_runtime_increment(self, value=1): + with self._skipped_runtime.get_lock(): + self._skipped_runtime.value += value + @property def failed(self): with self._failed.get_lock(): @@ -236,11 +448,24 @@ def failed(self, value): with self._failed.get_lock(): self._failed.value = value + def failed_increment(self, value=1): + with self._failed.get_lock(): + self._failed.value += value + @property def total(self): with self._total.get_lock(): return self._total.value + @total.setter + def total(self, value): + with self._total.get_lock(): + self._total.value = value + + def total_increment(self, value=1): + with self._total.get_lock(): + self._total.value += value + class CMake: config_re = re.compile('(CONFIG_[A-Za-z0-9_]+)[=]\"?([^\"]*)\"?$') dt_re = re.compile('([A-Za-z0-9_]+)[=]\"?([^\"]*)\"?$') @@ -652,7 +877,7 @@ def process(self, pipeline, done, message, lock, results): logger.debug("filtering %s" % self.instance.name) self.instance.status = TwisterStatus.FILTER self.instance.reason = "runtime filter" - results.skipped_runtime += 1 + results.skipped_runtime_increment() self.instance.add_missing_case_status(TwisterStatus.SKIP) next_op = 'report' else: @@ -683,7 +908,7 @@ def process(self, pipeline, done, message, lock, results): logger.debug("filtering %s" % self.instance.name) self.instance.status = TwisterStatus.FILTER self.instance.reason = "runtime filter" - results.skipped_runtime += 1 + results.skipped_runtime_increment() self.instance.add_missing_case_status(TwisterStatus.SKIP) next_op = 'report' else: @@ -710,7 +935,7 @@ def process(self, pipeline, done, message, lock, results): # Count skipped cases during build, for example # due to ram/rom overflow. if self.instance.status == TwisterStatus.SKIP: - results.skipped_runtime += 1 + results.skipped_runtime_increment() self.instance.add_missing_case_status(TwisterStatus.SKIP, self.instance.reason) if ret.get('returncode', 1) > 0: @@ -1068,52 +1293,84 @@ def _sanitize_zephyr_base_from_files(self): with open(file_path, "wt") as file: file.write(data) + @staticmethod + def _add_instance_testcases_to_status_counts(instance, results, decrement=False): + increment_value = -1 if decrement else 1 + for tc in instance.testcases: + match tc.status: + case TwisterStatus.PASS: + results.passed_cases_increment(increment_value) + case TwisterStatus.NOTRUN: + results.notrun_cases_increment(increment_value) + case TwisterStatus.BLOCK: + results.blocked_cases_increment(increment_value) + case TwisterStatus.SKIP: + results.skipped_cases_increment(increment_value) + case TwisterStatus.FILTER: + results.filtered_cases_increment(increment_value) + case TwisterStatus.ERROR: + results.error_cases_increment(increment_value) + case TwisterStatus.FAIL: + results.failed_cases_increment(increment_value) + # Statuses that should not appear. + # Crashing Twister at this point would be counterproductive, + # but having those statuses in this part of processing is an error. + case TwisterStatus.NONE: + results.none_cases_increment(increment_value) + logger.error(f'A None status detected in instance {instance.name},' + f' test case {tc.name}.') + case TwisterStatus.STARTED: + results.started_cases_increment(increment_value) + logger.error(f'A started status detected in instance {instance.name},' + f' test case {tc.name}.') + case _: + logger.error(f'An unknown status "{tc.status}" detected in instance {instance.name},' + f' test case {tc.name}.') + + def report_out(self, results): - total_to_do = results.total + total_to_do = results.total - results.skipped_filter total_tests_width = len(str(total_to_do)) - results.done += 1 + results.done_increment() instance = self.instance if results.iteration == 1: - results.cases += len(instance.testcases) + results.cases_increment(len(instance.testcases)) + + self._add_instance_testcases_to_status_counts(instance, results) + + status = f'{TwisterStatus.get_color(instance.status)}{str.upper(instance.status)}{Fore.RESET}' if instance.status in [TwisterStatus.ERROR, TwisterStatus.FAIL]: if instance.status == TwisterStatus.ERROR: - results.error += 1 - txt = " ERROR " + results.error_increment() else: - results.failed += 1 - txt = " FAILED " + results.failed_increment() if self.options.verbose: - status = Fore.RED + txt + Fore.RESET + instance.reason + status += " " + instance.reason else: logger.error( - "{:<25} {:<50} {}{}{}: {}".format( + "{:<25} {:<50} {}: {}".format( instance.platform.name, instance.testsuite.name, - Fore.RED, - txt, - Fore.RESET, + status, instance.reason)) if not self.options.verbose: self.log_info_file(self.options.inline_logs) - elif instance.status in [TwisterStatus.SKIP, TwisterStatus.FILTER]: - status = Fore.YELLOW + "SKIPPED" + Fore.RESET - results.skipped_configs += 1 - # test cases skipped at the test instance level - results.skipped_cases += len(instance.testsuite.testcases) + elif instance.status == TwisterStatus.SKIP: + results.skipped_configs_increment() + elif instance.status == TwisterStatus.FILTER: + results.skipped_configs_increment() elif instance.status == TwisterStatus.PASS: - status = Fore.GREEN + "PASSED" + Fore.RESET - results.passed += 1 + results.passed_increment() for case in instance.testcases: # test cases skipped at the test case level if case.status == TwisterStatus.SKIP: - results.skipped_cases += 1 + results.skipped_cases_increment() elif instance.status == TwisterStatus.NOTRUN: - status = Fore.CYAN + "NOT RUN" + Fore.RESET - results.notrun += 1 + results.notrun_increment() for case in instance.testcases: if case.status == TwisterStatus.SKIP: - results.skipped_cases += 1 + results.skipped_cases_increment() else: logger.debug(f"Unknown status = {instance.status}") status = Fore.YELLOW + "UNKNOWN" + Fore.RESET @@ -1139,7 +1396,7 @@ def report_out(self, results): and self.instance.handler.seed is not None ): more_info += "/seed: " + str(self.options.seed) logger.info("{:>{}}/{} {:<25} {:<50} {} ({})".format( - results.done, total_tests_width, total_to_do , instance.platform.name, + results.done - results.skipped_filter, total_tests_width, total_to_do , instance.platform.name, instance.testsuite.name, status, more_info)) if self.options.verbose > 1: @@ -1154,22 +1411,24 @@ def report_out(self, results): else: completed_perc = 0 if total_to_do > 0: - completed_perc = int((float(results.done) / total_to_do) * 100) + completed_perc = int((float(results.done - results.skipped_filter) / total_to_do) * 100) - sys.stdout.write("INFO - Total complete: %s%4d/%4d%s %2d%% built (not run): %4d, skipped: %s%4d%s, failed: %s%4d%s, error: %s%4d%s\r" % ( - Fore.GREEN, - results.done, + sys.stdout.write("INFO - Total complete: %s%4d/%4d%s %2d%% built (not run): %s%4d%s, filtered: %s%4d%s, failed: %s%4d%s, error: %s%4d%s\r" % ( + TwisterStatus.get_color(TwisterStatus.PASS), + results.done - results.skipped_filter, total_to_do, Fore.RESET, completed_perc, + TwisterStatus.get_color(TwisterStatus.NOTRUN), results.notrun, - Fore.YELLOW if results.skipped_configs > 0 else Fore.RESET, + Fore.RESET, + TwisterStatus.get_color(TwisterStatus.SKIP) if results.skipped_configs > 0 else Fore.RESET, results.skipped_configs, Fore.RESET, - Fore.RED if results.failed > 0 else Fore.RESET, + TwisterStatus.get_color(TwisterStatus.FAIL) if results.failed > 0 else Fore.RESET, results.failed, Fore.RESET, - Fore.RED if results.error > 0 else Fore.RESET, + TwisterStatus.get_color(TwisterStatus.ERROR) if results.error > 0 else Fore.RESET, results.error, Fore.RESET ) @@ -1376,15 +1635,16 @@ def run(self): self.update_counting_before_pipeline() while True: - self.results.iteration += 1 + self.results.iteration_increment() if self.results.iteration > 1: logger.info("%d Iteration:" % (self.results.iteration)) time.sleep(self.options.retry_interval) # waiting for the system to settle down - self.results.done = self.results.total - self.results.failed - self.results.error + self.results.done = self.results.total - self.results.failed self.results.failed = 0 if self.options.retry_build_errors: self.results.error = 0 + self.results.done -= self.results.error else: self.results.done = self.results.skipped_filter @@ -1421,16 +1681,16 @@ def update_counting_before_pipeline(self): ''' for instance in self.instances.values(): if instance.status == TwisterStatus.FILTER and not instance.reason == 'runtime filter': - self.results.skipped_filter += 1 - self.results.skipped_configs += 1 - self.results.skipped_cases += len(instance.testsuite.testcases) - self.results.cases += len(instance.testsuite.testcases) + self.results.skipped_filter_increment() + self.results.skipped_configs_increment() + self.results.filtered_cases_increment(len(instance.testsuite.testcases)) + self.results.cases_increment(len(instance.testsuite.testcases)) elif instance.status == TwisterStatus.ERROR: - self.results.error += 1 + self.results.error_increment() def show_brief(self): logger.info("%d test scenarios (%d test instances) selected, " - "%d configurations skipped (%d by static filter, %d at runtime)." % + "%d configurations filtered (%d by static filter, %d at runtime)." % (len(self.suites), len(self.instances), self.results.skipped_configs, self.results.skipped_filter, @@ -1450,6 +1710,9 @@ def add_tasks_to_queue(self, pipeline, build_only=False, test_only=False, retry_ if instance.status != TwisterStatus.NONE: instance.retries += 1 instance.status = TwisterStatus.NONE + # Previous states should be removed from the stats + if self.results.iteration > 1: + ProjectBuilder._add_instance_testcases_to_status_counts(instance, self.results, decrement=True) # Check if cmake package_helper script can be run in advance. instance.filter_stages = [] diff --git a/scripts/tests/twister/test_runner.py b/scripts/tests/twister/test_runner.py index 1aecba6dbfc372..6455592e4aa060 100644 --- a/scripts/tests/twister/test_runner.py +++ b/scripts/tests/twister/test_runner.py @@ -206,20 +206,31 @@ def test_executioncounter(capfd): sys.stderr.write(err) assert ( - f'--------------------------------\n' - f'Total test suites: 12\n' - f'Total test cases: 25\n' - f'Executed test cases: 19\n' - f'Skipped test cases: 6\n' - f'Completed test suites: 9\n' - f'Passing test suites: 6\n' - f'Built only test suites: 0\n' - f'Failing test suites: 1\n' - f'Skipped test suites: 3\n' - f'Skipped test suites (runtime): 1\n' - f'Skipped test suites (filter): 2\n' - f'Errors: 2\n' - f'--------------------------------' + '--------------------------------------------------\n' + 'Total test suites: 12\n' + 'Processed test suites: 9\n' + '├─ Filtered test suites (static): 2\n' + '└─ Completed test suites: 7\n' + ' ├─ Filtered test suites (at runtime): 1\n' + ' ├─ Passed test suites: 6\n' + ' ├─ Built only test suites: 0\n' + ' ├─ Failed test suites: 1\n' + ' └─ Errors in test suites: 2\n' + '\n' + 'Filtered test suites: 3\n' + '├─ Filtered test suites (static): 2\n' + '└─ Filtered test suites (at runtime): 1\n' + '---------------------- ----------------------\n' + 'Total test cases: 25\n' + '├─ Filtered test cases: 0\n' + '├─ Skipped test cases: 6\n' + '└─ Executed test cases: 19\n' + ' ├─ Passed test cases: 0\n' + ' ├─ Built only test cases: 0\n' + ' ├─ Blocked test cases: 0\n' + ' ├─ Failed test cases: 0\n' + ' └─ Errors in test cases: 0\n' + '--------------------------------------------------\n' ) in out assert ec.cases == 25 @@ -1547,7 +1558,7 @@ def mock_determine_testcases(res): assert pb.instance.status == expected_status assert pb.instance.reason == expected_reason - assert results_mock.skipped_runtime == expected_skipped + assert results_mock.skipped_runtime_increment.call_args_list == [mock.call()] * expected_skipped if expected_missing: pb.instance.add_missing_case_status.assert_called_with(*expected_missing) @@ -1936,16 +1947,16 @@ def mock_open(fname, *args, **kwargs): TwisterStatus.ERROR, True, True, False, ['INFO 20/25 dummy platform' \ ' dummy.testsuite.name' \ - ' ERROR dummy reason (cmake)'], + ' ERROR dummy reason (cmake)'], None ), ( TwisterStatus.FAIL, False, False, False, ['ERROR dummy platform' \ ' dummy.testsuite.name' \ - ' FAILED : dummy reason'], + ' FAILED: dummy reason'], 'INFO - Total complete: 20/ 25 80%' \ - ' built (not run): 0, skipped: 3, failed: 3, error: 1' + ' built (not run): 0, filtered: 3, failed: 3, error: 1' ), ( TwisterStatus.SKIP, True, False, False, @@ -1958,7 +1969,7 @@ def mock_open(fname, *args, **kwargs): TwisterStatus.FILTER, False, False, False, [], 'INFO - Total complete: 20/ 25 80%' \ - ' built (not run): 0, skipped: 4, failed: 2, error: 1' + ' built (not run): 0, filtered: 4, failed: 2, error: 1' ), ( TwisterStatus.PASS, True, False, True, @@ -1979,7 +1990,7 @@ def mock_open(fname, *args, **kwargs): 'unknown status', False, False, False, ['Unknown status = unknown status'], 'INFO - Total complete: 20/ 25 80%' - ' built (not run): 0, skipped: 3, failed: 2, error: 1\r' + ' built (not run): 0, filtered: 3, failed: 2, error: 1\r' ) ] @@ -2026,21 +2037,49 @@ def test_projectbuilder_report_out( pb.options.seed = 123 pb.log_info_file = mock.Mock() - results_mock = mock.Mock() + results_mock = mock.Mock( + total = 25, + done = 19, + passed = 17, + notrun = 0, + failed = 2, + skipped_configs = 3, + skipped_runtime = 0, + skipped_filter = 0, + error = 1, + cases = 0, + filtered_cases = 0, + skipped_cases = 4, + failed_cases = 0, + error_cases = 0, + blocked_cases = 0, + passed_cases = 0, + none_cases = 0, + started_cases = 0 + ) results_mock.iteration = 1 - results_mock.total = 25 - results_mock.done = 19 - results_mock.passed = 17 - results_mock.notrun = 0 - results_mock.skipped_configs = 3 - results_mock.skipped_cases = 4 - results_mock.failed = 2 - results_mock.error = 1 - results_mock.cases = 0 + def results_done_increment(value=1, decrement=False): + results_mock.done += value * (-1 if decrement else 1) + results_mock.done_increment = results_done_increment + def skipped_configs_increment(value=1, decrement=False): + results_mock.skipped_configs += value * (-1 if decrement else 1) + results_mock.skipped_configs_increment = skipped_configs_increment + def skipped_filter_increment(value=1, decrement=False): + results_mock.skipped_filter += value * (-1 if decrement else 1) + results_mock.skipped_filter_increment = skipped_filter_increment + def skipped_runtime_increment(value=1, decrement=False): + results_mock.skipped_runtime += value * (-1 if decrement else 1) + results_mock.skipped_runtime_increment = skipped_runtime_increment + def failed_increment(value=1, decrement=False): + results_mock.failed += value * (-1 if decrement else 1) + results_mock.failed_increment = failed_increment + def notrun_increment(value=1, decrement=False): + results_mock.notrun += value * (-1 if decrement else 1) + results_mock.notrun_increment = notrun_increment pb.report_out(results_mock) - assert results_mock.cases == 25 + assert results_mock.cases_increment.call_args_list == [mock.call(25)] trim_actual_log = re.sub( r'\x1B(?:[@-Z\\-_]|\[[0-?]*[ -/]*[@-~])', @@ -2449,6 +2488,10 @@ def mock_client_from_environ(jobs): results_mock().failed = 2 results_mock().total = 9 + def iteration_increment(value=1, decrement=False): + results_mock().iteration += value * (-1 if decrement else 1) + results_mock().iteration_increment = iteration_increment + with mock.patch('twisterlib.runner.ExecutionCounter', results_mock), \ mock.patch('twisterlib.runner.BaseManager', manager_mock), \ mock.patch('twisterlib.runner.GNUMakeJobClient.from_environ', @@ -2519,18 +2562,45 @@ def test_twisterrunner_update_counting_before_pipeline(): tr = TwisterRunner(instances, suites, env=env_mock) tr.results = mock.Mock( - skipped_filter = 0, + total = 0, + done = 0, + passed = 0, + failed = 0, skipped_configs = 0, - skipped_cases = 0, + skipped_runtime = 0, + skipped_filter = 0, + error = 0, cases = 0, - error = 0 + filtered_cases = 0, + skipped_cases = 0, + failed_cases = 0, + error_cases = 0, + blocked_cases = 0, + passed_cases = 0, + none_cases = 0, + started_cases = 0 ) + def skipped_configs_increment(value=1, decrement=False): + tr.results.skipped_configs += value * (-1 if decrement else 1) + tr.results.skipped_configs_increment = skipped_configs_increment + def skipped_filter_increment(value=1, decrement=False): + tr.results.skipped_filter += value * (-1 if decrement else 1) + tr.results.skipped_filter_increment = skipped_filter_increment + def error_increment(value=1, decrement=False): + tr.results.error += value * (-1 if decrement else 1) + tr.results.error_increment = error_increment + def cases_increment(value=1, decrement=False): + tr.results.cases += value * (-1 if decrement else 1) + tr.results.cases_increment = cases_increment + def filtered_cases_increment(value=1, decrement=False): + tr.results.filtered_cases += value * (-1 if decrement else 1) + tr.results.filtered_cases_increment = filtered_cases_increment tr.update_counting_before_pipeline() assert tr.results.skipped_filter == 1 assert tr.results.skipped_configs == 1 - assert tr.results.skipped_cases == 4 + assert tr.results.filtered_cases == 4 assert tr.results.cases == 4 assert tr.results.error == 1 @@ -2558,7 +2628,7 @@ def test_twisterrunner_show_brief(caplog): tr.show_brief() log = '2 test scenarios (5 test instances) selected,' \ - ' 4 configurations skipped (3 by static filter, 1 at runtime).' + ' 4 configurations filtered (3 by static filter, 1 at runtime).' assert log in caplog.text @@ -2609,6 +2679,7 @@ def mock_get_cmake_filter_stages(filter, keys): tr.get_cmake_filter_stages = mock.Mock( side_effect=mock_get_cmake_filter_stages ) + tr.results = mock.Mock(iteration=0) pipeline_mock = mock.Mock() diff --git a/scripts/tests/twister_blackbox/test_device.py b/scripts/tests/twister_blackbox/test_device.py index 0ec9ae0322fe74..b6e78acc28b7ac 100644 --- a/scripts/tests/twister_blackbox/test_device.py +++ b/scripts/tests/twister_blackbox/test_device.py @@ -13,6 +13,7 @@ import sys import re +# pylint: disable=no-name-in-module from conftest import ZEPHYR_BASE, TEST_DATA, testsuite_filename_mock from twisterlib.testplan import TestPlan @@ -71,5 +72,5 @@ def test_seed(self, capfd, out_path, seed): assert str(sys_exit.value) == '1' - expected_line = r'seed_native_sim.dummy FAILED Failed \(native (\d+\.\d+)s/seed: {}\)'.format(seed[0]) + expected_line = r'seed_native_sim.dummy FAILED Failed \(native (\d+\.\d+)s/seed: {}\)'.format(seed[0]) assert re.search(expected_line, err) diff --git a/scripts/tests/twister_blackbox/test_error.py b/scripts/tests/twister_blackbox/test_error.py index 3c18d8398bab37..ba43731e097343 100644 --- a/scripts/tests/twister_blackbox/test_error.py +++ b/scripts/tests/twister_blackbox/test_error.py @@ -13,6 +13,7 @@ import sys import re +# pylint: disable=no-name-in-module from conftest import ZEPHYR_BASE, TEST_DATA, testsuite_filename_mock from twisterlib.testplan import TestPlan from twisterlib.error import TwisterRuntimeError @@ -45,7 +46,7 @@ class TestError: ), ( '--overflow-as-errors', - r'always_overflow.dummy ERROR Build failure \(build\)' + r'always_overflow.dummy ERROR Build failure \(build\)' ) ] diff --git a/scripts/tests/twister_blackbox/test_platform.py b/scripts/tests/twister_blackbox/test_platform.py index d9d3b145b6c192..75a6cda909309b 100644 --- a/scripts/tests/twister_blackbox/test_platform.py +++ b/scripts/tests/twister_blackbox/test_platform.py @@ -28,6 +28,7 @@ class TestPlatform: { 'selected_test_scenarios': 3, 'selected_test_instances': 9, + 'executed_test_instances': 6, 'skipped_configurations': 3, 'skipped_by_static_filter': 3, 'skipped_at_runtime': 0, @@ -48,6 +49,7 @@ class TestPlatform: { 'selected_test_scenarios': 1, 'selected_test_instances': 3, + 'executed_test_instances': 0, 'skipped_configurations': 3, 'skipped_by_static_filter': 3, 'skipped_at_runtime': 0, @@ -190,8 +192,9 @@ def test_any_platform(self, capfd, out_path, test_path, test_platforms, flag): os.path.join(TEST_DATA, 'tests', 'dummy', 'agnostic'), ['qemu_x86', 'qemu_x86_64'], { - 'passed_configurations': 2, 'selected_test_instances': 6, + 'passed_configurations': 2, + 'executed_test_instances': 3, 'executed_on_platform': 2, 'only_built': 1, } @@ -217,7 +220,7 @@ def test_exclude_platform(self, capfd, out_path, test_path, test_platforms, expe sys.stderr.write(err) pass_regex = r'^INFO - (?P[0-9]+) of' \ - r' (?P[0-9]+) test configurations passed' + r' (?P[0-9]+) executed test configurations passed' built_regex = r'^INFO - (?P[0-9]+)' \ r' test configurations executed on platforms, (?P[0-9]+)' \ @@ -229,7 +232,7 @@ def test_exclude_platform(self, capfd, out_path, test_path, test_platforms, expe assert int(pass_search.group('passed_configurations')) == \ expected['passed_configurations'] assert int(pass_search.group('test_instances')) == \ - expected['selected_test_instances'] + expected['executed_test_instances'] built_search = re.search(built_regex, err, re.MULTILINE) @@ -262,22 +265,31 @@ def test_emulation_only(self, capfd, out_path, test_path, test_platforms, expect select_regex = r'^INFO - (?P[0-9]+) test scenarios' \ r' \((?P[0-9]+) test instances\) selected,' \ - r' (?P[0-9]+) configurations skipped' \ + r' (?P[0-9]+) configurations filtered' \ r' \((?P[0-9]+) by static filter,' \ r' (?P[0-9]+) at runtime\)\.$' pass_regex = r'^INFO - (?P[0-9]+) of' \ - r' (?P[0-9]+) test configurations passed' \ + r' (?P[0-9]+) executed test configurations passed' \ r' \([0-9]+\.[0-9]+%\), (?P[0-9]+) built \(not run\),' \ r' (?P[0-9]+) failed,' \ r' (?P[0-9]+) errored,' \ - r' (?P[0-9]+) skipped with' \ - r' [0-9]+ warnings in [0-9]+\.[0-9]+ seconds$' - - case_regex = r'^INFO - In total (?P[0-9]+)' \ - r' test cases were executed, (?P[0-9]+) skipped' \ - r' on (?P[0-9]+) out of total [0-9]+ platforms' \ - r' \([0-9]+\.[0-9]+%\)$' + r' with (?:[0-9]+|no) warnings in [0-9]+\.[0-9]+ seconds.$' + + case_regex = r'^INFO - (?P[0-9]+) of' \ + r' (?P[0-9]+) executed test cases passed' \ + r' \([0-9]+\.[0-9]+%\)' \ + r'(?:, (?P[0-9]+) blocked)?' \ + r'(?:, (?P[0-9]+) failed)?' \ + r'(?:, (?P[0-9]+) errored)?' \ + r'(?:, (?P[0-9]+) without a status)?' \ + r' on (?P[0-9]+) out of total' \ + r' (?P[0-9]+) platforms \([0-9]+\.[0-9]+%\)' + + skip_regex = r'(?P[0-9]+) selected test cases not executed:' \ + r'(?: (?P[0-9]+) skipped)?' \ + r'(?:, (?P[0-9]+) filtered)?' \ + r'.' built_regex = r'^INFO - (?P[0-9]+)' \ r' test configurations executed on platforms, (?P[0-9]+)' \ @@ -306,27 +318,32 @@ def test_emulation_only(self, capfd, out_path, test_path, test_platforms, expect assert pass_search assert int(pass_search.group('passed_configurations')) == \ expected['passed_configurations'] - assert int(pass_search.group('test_instances')) == \ - expected['selected_test_instances'] assert int(pass_search.group('built_configurations')) == \ expected['built_configurations'] - assert int(pass_search.group('failed_configurations')) == \ - expected['failed_configurations'] - assert int(pass_search.group('errored_configurations')) == \ - expected['errored_configurations'] - assert int(pass_search.group('skipped_configurations')) == \ - expected['skipped_configurations'] + assert int(pass_search.group('executed_test_instances')) == \ + expected['executed_test_instances'] + if expected['failed_configurations']: + assert int(pass_search.group('failed_configurations')) == \ + expected['failed_configurations'] + if expected['errored_configurations']: + assert int(pass_search.group('errored_configurations')) == \ + expected['errored_configurations'] case_search = re.search(case_regex, err, re.MULTILINE) assert case_search assert int(case_search.group('executed_test_cases')) == \ expected['executed_test_cases'] - assert int(case_search.group('skipped_test_cases')) == \ - expected['skipped_test_cases'] assert int(case_search.group('platform_count')) == \ expected['platform_count'] + if expected['skipped_test_cases']: + skip_search = re.search(skip_regex, err, re.MULTILINE) + + assert skip_search + assert int(skip_search.group('skipped_test_cases')) == \ + expected['skipped_test_cases'] + built_search = re.search(built_regex, err, re.MULTILINE) assert built_search diff --git a/scripts/tests/twister_blackbox/test_runner.py b/scripts/tests/twister_blackbox/test_runner.py index e414e24fe25aa1..865826741adc27 100644 --- a/scripts/tests/twister_blackbox/test_runner.py +++ b/scripts/tests/twister_blackbox/test_runner.py @@ -131,7 +131,7 @@ class TestRunner: ['dummy.agnostic.group2 SKIPPED: Command line testsuite tag filter', 'dummy.agnostic.group1.subgroup2 SKIPPED: Command line testsuite tag filter', 'dummy.agnostic.group1.subgroup1 SKIPPED: Command line testsuite tag filter', - r'0 of 4 test configurations passed \(0.00%\), 0 built \(not run\), 0 failed, 0 errored, 4 skipped' + r'0 of 0 executed test configurations passed \(0.00%\), 0 built \(not run\), 0 failed, 0 errored' ] ), ( @@ -139,14 +139,14 @@ class TestRunner: ['qemu_x86/atom'], ['subgrouped'], ['dummy.agnostic.group2 SKIPPED: Command line testsuite tag filter', - r'1 of 4 test configurations passed \(50.00%\), 1 built \(not run\), 0 failed, 0 errored, 2 skipped' + r'1 of 2 executed test configurations passed \(50.00%\), 1 built \(not run\), 0 failed, 0 errored' ] ), ( os.path.join(TEST_DATA, 'tests', 'dummy'), ['qemu_x86/atom'], ['agnostic', 'device'], - [r'2 of 4 test configurations passed \(66.67%\), 1 built \(not run\), 0 failed, 0 errored, 1 skipped'] + [r'2 of 3 executed test configurations passed \(66.67%\), 1 built \(not run\), 0 failed, 0 errored'] ), ] TESTDATA_10 = [ @@ -261,22 +261,32 @@ def test_runtest_only(self, capfd, out_path, test_path, test_platforms, expected select_regex = r'^INFO - (?P[0-9]+) test scenarios' \ r' \((?P[0-9]+) test instances\) selected,' \ - r' (?P[0-9]+) configurations skipped' \ + r' (?P[0-9]+) configurations filtered' \ r' \((?P[0-9]+) by static filter,' \ r' (?P[0-9]+) at runtime\)\.$' pass_regex = r'^INFO - (?P[0-9]+) of' \ - r' (?P[0-9]+) test configurations passed' \ + r' (?P[0-9]+) executed test configurations passed' \ r' \([0-9]+\.[0-9]+%\), (?P[0-9]+) built \(not run\),' \ r' (?P[0-9]+) failed,' \ - r' (?P[0-9]+) errored,' \ - r' (?P[0-9]+) skipped with' \ - r' [0-9]+ warnings in [0-9]+\.[0-9]+ seconds$' - - case_regex = r'^INFO - In total (?P[0-9]+)' \ - r' test cases were executed, (?P[0-9]+) skipped' \ - r' on (?P[0-9]+) out of total [0-9]+ platforms' \ - r' \([0-9]+\.[0-9]+%\)$' + r' (?P[0-9]+) errored, with' \ + r' (?:[0-9]+|no) warnings in [0-9]+\.[0-9]+ seconds.$' + + case_regex = r'^INFO - (?P[0-9]+) of' \ + r' (?P[0-9]+) executed test cases passed' \ + r' \([0-9]+\.[0-9]+%\)' \ + r'(?:, (?P[0-9]+) blocked)?' \ + r'(?:, (?P[0-9]+) failed)?' \ + r'(?:, (?P[0-9]+) errored)?' \ + r'(?:, (?P[0-9]+) without a status)?' \ + r' on (?P[0-9]+) out of total' \ + r' (?P[0-9]+) platforms \([0-9]+\.[0-9]+%\)' + + skip_regex = r'(?P[0-9]+) selected test cases not executed:' \ + r'(?: (?P[0-9]+) skipped)?' \ + r'(?:, (?P[0-9]+) filtered)?' \ + r'.' + built_regex = r'^INFO - (?P[0-9]+)' \ r' test configurations executed on platforms, (?P[0-9]+)' \ r' test configurations were only built.$' @@ -312,19 +322,21 @@ def test_runtest_only(self, capfd, out_path, test_path, test_platforms, expected expected['failed_configurations'] assert int(pass_search.group('errored_configurations')) == \ expected['errored_configurations'] - assert int(pass_search.group('skipped_configurations')) == \ - expected['skipped_configurations'] case_search = re.search(case_regex, err, re.MULTILINE) assert case_search assert int(case_search.group('executed_test_cases')) == \ expected['executed_test_cases'] - assert int(case_search.group('skipped_test_cases')) == \ - expected['skipped_test_cases'] assert int(case_search.group('platform_count')) == \ expected['platform_count'] + if expected['skipped_test_cases']: + skip_search = re.search(skip_regex, err, re.MULTILINE) + assert skip_search + assert int(skip_search.group('skipped_test_cases')) == \ + expected['skipped_test_cases'] + built_search = re.search(built_regex, err, re.MULTILINE) assert built_search @@ -384,7 +396,7 @@ def test_cmake_only(self, capfd, out_path, test_path, test_platforms, expected): sys.stderr.write(err) pass_regex = r'^INFO - (?P[0-9]+) of' \ - r' (?P[0-9]+) test configurations passed' + r' (?P[0-9]+) executed test configurations passed' built_regex = r'^INFO - (?P[0-9]+)' \ r' test configurations executed on platforms, (?P[0-9]+)' \ @@ -614,14 +626,18 @@ def test_only_failed(self, capfd, out_path, test_path, test_platforms, expected) pytest.raises(SystemExit) as sys_exit: self.loader.exec_module(self.twister_module) + select_regex = r'^INFO - (?P[0-9]+) test scenarios' \ + r' \((?P[0-9]+) test instances\) selected,' \ + r' (?P[0-9]+) configurations filtered' \ + r' \((?P[0-9]+) by static filter,' \ + r' (?P[0-9]+) at runtime\)\.$' pass_regex = r'^INFO - (?P[0-9]+) of' \ - r' (?P[0-9]+) test configurations passed' \ + r' (?P[0-9]+) executed test configurations passed' \ r' \([0-9]+\.[0-9]+%\), (?P[0-9]+) built \(not run\),' \ r' (?P[0-9]+) failed,' \ - r' (?P[0-9]+) errored,' \ - r' (?P[0-9]+) skipped with' \ - r' [0-9]+ warnings in [0-9]+\.[0-9]+ seconds$' + r' (?P[0-9]+) errored, with' \ + r' (?:[0-9]+|no) warnings in [0-9]+\.[0-9]+ seconds.$' out, err = capfd.readouterr() sys.stdout.write(out) @@ -631,6 +647,11 @@ def test_only_failed(self, capfd, out_path, test_path, test_platforms, expected) assert re.search( r'one_fail_one_pass.agnostic.group1.subgroup2 on qemu_x86/atom failed \(.*\)', err) + + select_search = re.search(select_regex, err, re.MULTILINE) + assert int(select_search.group('skipped_configurations')) == \ + expected['skipped_configurations'] + pass_search = re.search(pass_regex, err, re.MULTILINE) assert pass_search @@ -644,8 +665,6 @@ def test_only_failed(self, capfd, out_path, test_path, test_platforms, expected) expected['failed_configurations'] assert int(pass_search.group('errored_configurations')) == \ expected['errored_configurations'] - assert int(pass_search.group('skipped_configurations')) == \ - expected['skipped_configurations'] assert str(sys_exit.value) == '1'