Skip to content

Commit

Permalink
Merge pull request #477 from MarekVCodasip/test-exclusion
Browse files Browse the repository at this point in the history
Add a way to exclude tests by specifying an exclusion file
  • Loading branch information
timsifive authored Jun 27, 2023
2 parents d0d1b08 + 0009a26 commit 271655f
Show file tree
Hide file tree
Showing 4 changed files with 52 additions and 1 deletion.
3 changes: 2 additions & 1 deletion debug/Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -23,7 +23,8 @@ run.%:
--isolate \
--print-failures \
--sim_cmd $(RISCV)/bin/$(RISCV_SIM) \
--server_cmd $(RISCV)/bin/openocd
--server_cmd $(RISCV)/bin/openocd \
$(if $(EXCLUDE_TESTS),--exclude-tests $(EXCLUDE_TESTS))

# Target to check all the multicore options.
multi-tests: spike32-2 spike32-2-hwthread
Expand Down
17 changes: 17 additions & 0 deletions debug/excluded.yaml.example
Original file line number Diff line number Diff line change
@@ -0,0 +1,17 @@
# This is an example file showing the exclusion list format.
# It can be passed to gdb_server.py using the --exclude-tests argument.


# Tests excluded for all targets
all:
- SimpleF18Test

# Tests excluded for the "spike32" target only
spike32:
- MemTest32
- PrivRw
- Sv32Test

# Tests excluded for the "spike64" target only
spike64:
- UserInterrupt
1 change: 1 addition & 0 deletions debug/requirements.txt
Original file line number Diff line number Diff line change
@@ -1 +1,2 @@
pexpect>=4.0.0
pyyaml>=3.12
32 changes: 32 additions & 0 deletions debug/testlib.py
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,7 @@
import traceback

import pexpect
import yaml

print_log_names = False
real_stdout = sys.stdout
Expand Down Expand Up @@ -944,6 +945,32 @@ def __enter__(self):
def __exit__(self, _type, _value, _traceback):
self.gdb.pop_state()


def load_excluded_tests(excluded_tests_file, target_name):
result = []
if excluded_tests_file is None or len(excluded_tests_file) == 0:
return result

target_excludes = {}
with open(excluded_tests_file) as file:
raw_data = yaml.safe_load(file)
for (target, test_list) in raw_data.items():
if not isinstance(test_list, list):
raise ValueError(f"Target {target!r} does not contain a test list", excluded_tests_file, test_list)
if not all(isinstance(s, str) for s in test_list):
raise ValueError(f"Not every element in the target test list {target!r} is a string",
excluded_tests_file, test_list)

target_excludes.update(raw_data)

if target_name in target_excludes:
result += target_excludes[target_name]
if "all" in target_excludes:
result += target_excludes["all"]

return result


def run_all_tests(module, target, parsed):
todo = []
for name in dir(module):
Expand Down Expand Up @@ -982,6 +1009,9 @@ def run_all_tests(module, target, parsed):
todo.insert(0, ("ExamineTarget", ExamineTarget, None))
examine_added = True

excluded_tests = load_excluded_tests(parsed.exclude_tests, target.name)
target.skip_tests += excluded_tests

results, count = run_tests(parsed, target, todo)

header(f"ran {count} tests in {time.time() - overall_start:.0f}s", dash=':')
Expand Down Expand Up @@ -1068,6 +1098,8 @@ def add_test_run_options(parser):
parser.add_argument("--misaval",
help="Don't run ExamineTarget, just assume the misa value which is "
"specified.")
parser.add_argument("--exclude-tests",
help="Specify yaml file listing tests to exclude")

def header(title, dash='-', length=78):
if title:
Expand Down

0 comments on commit 271655f

Please sign in to comment.