From f8fbebe5e712cd27e93941f828653c7c47d20454 Mon Sep 17 00:00:00 2001 From: YAMAMOTO Takashi Date: Mon, 19 Dec 2022 14:54:59 +0900 Subject: [PATCH] Implement a simple test filter cf. https://github.com/WebAssembly/wasi-testsuite/issues/29 --- README.md | 10 +++++++ examples/skip.json | 5 ++++ test-runner/wasi_test_runner/__main__.py | 15 ++++++++++ test-runner/wasi_test_runner/filters.py | 21 ++++++++++++++ test-runner/wasi_test_runner/harness.py | 6 +++- .../wasi_test_runner/reporters/console.py | 1 + .../wasi_test_runner/test_suite_runner.py | 28 +++++++++++++++++-- 7 files changed, 83 insertions(+), 3 deletions(-) create mode 100644 examples/skip.json create mode 100644 test-runner/wasi_test_runner/filters.py diff --git a/README.md b/README.md index b94bf3a6..d04dde8d 100644 --- a/README.md +++ b/README.md @@ -38,6 +38,16 @@ python3 test-runner/wasi_test_runner.py -r adapters/wasmtime.sh # path to a runtime adapter ``` +Optianally you can specify test cases to skip with the `--filter` option. + +```bash +python3 test-runner/wasi_test_runner.py \ + -t ./tests/assemblyscript/testsuite/ `# path to folders containing .wasm test files` \ + ./tests/c/testsuite/ \ + --filter examples/skip.json + -r adapters/wasmtime.sh # path to a runtime adapter +``` + ## Contributing All contributions are very welcome. Contributors can help with: diff --git a/examples/skip.json b/examples/skip.json new file mode 100644 index 00000000..daa4827b --- /dev/null +++ b/examples/skip.json @@ -0,0 +1,5 @@ +{ + "WASI C tests": { + "stat-dev-ino": "d_ino is known broken in this combination of engine and wasi-sdk version." + } +} diff --git a/test-runner/wasi_test_runner/__main__.py b/test-runner/wasi_test_runner/__main__.py index 1eef0e1f..6d20fc50 100644 --- a/test-runner/wasi_test_runner/__main__.py +++ b/test-runner/wasi_test_runner/__main__.py @@ -5,6 +5,8 @@ from .runtime_adapter import RuntimeAdapter from .harness import run_all_tests +from .filters import TestFilter +from .filters import JSONTestFilter from .reporters import TestReporter from .reporters.console import ConsoleTestReporter from .reporters.json import JSONTestReporter @@ -23,6 +25,14 @@ def main() -> int: nargs="+", help="Locations of suites (directories with *.wasm test files).", ) + parser.add_argument( + "-f", + "--filter", + required=False, + nargs="+", + default=[], + help="Locations of test filters (JSON files).", + ) parser.add_argument( "-r", "--runtime-adapter", required=True, help="Path to a runtime adapter." ) @@ -45,11 +55,16 @@ def main() -> int: validators: List[Validator] = [exit_code_validator, stdout_validator] + filters: List[TestFilter] = [] + for filter in options.filter: + filters.append(JSONTestFilter(filter)) + return run_all_tests( RuntimeAdapter(options.runtime_adapter), options.test_suite, validators, reporters, + filters, ) diff --git a/test-runner/wasi_test_runner/filters.py b/test-runner/wasi_test_runner/filters.py new file mode 100644 index 00000000..79d6b6db --- /dev/null +++ b/test-runner/wasi_test_runner/filters.py @@ -0,0 +1,21 @@ +from abc import ABC + +import json + + +class TestFilter(ABC): + def should_skip(self, test_suite_name: str, test_name: str) -> (bool, str): + pass + + +class JSONTestFilter(TestFilter): + def __init__(self, filename): + with open(filename) as fp: + self.filter_dict = json.load(fp) + + def should_skip(self, test_suite_name, test_name): + d = self.filter_dict.get(test_suite_name) + if d is None: + return False, None + why = d.get(test_name) + return why is not None, why diff --git a/test-runner/wasi_test_runner/harness.py b/test-runner/wasi_test_runner/harness.py index d440b30a..d8d5a17c 100644 --- a/test-runner/wasi_test_runner/harness.py +++ b/test-runner/wasi_test_runner/harness.py @@ -1,5 +1,6 @@ from typing import List +from .filters import TestFilter from .reporters import TestReporter from .test_suite_runner import run_tests_from_test_suite from .runtime_adapter import RuntimeAdapter @@ -11,17 +12,20 @@ def run_all_tests( test_suite_paths: List[str], validators: List[Validator], reporters: List[TestReporter], + filters: List[TestFilter], ) -> int: ret = 0 for test_suite_path in test_suite_paths: test_suite = run_tests_from_test_suite( - test_suite_path, runtime, validators, reporters + test_suite_path, runtime, validators, reporters, filters, ) for reporter in reporters: reporter.report_test_suite(test_suite) if test_suite.fail_count > 0: ret = 1 + for test_case in test_suite.test_cases: + print(f"{test_suite.name} {test_case.name}"); for reporter in reporters: reporter.finalize(runtime.get_version()) diff --git a/test-runner/wasi_test_runner/reporters/console.py b/test-runner/wasi_test_runner/reporters/console.py index 9bbae086..2eb6ed96 100644 --- a/test-runner/wasi_test_runner/reporters/console.py +++ b/test-runner/wasi_test_runner/reporters/console.py @@ -54,6 +54,7 @@ def finalize(self, version: RuntimeVersion) -> None: print(f" Total: {suite.test_count}") self._print_pass(f" Passed: {suite.pass_count}") self._print_fail(f" Failed: {suite.fail_count}") + print(f" Skipped: {suite.skip_count}") print("") print( diff --git a/test-runner/wasi_test_runner/test_suite_runner.py b/test-runner/wasi_test_runner/test_suite_runner.py index 47a83c9e..eb27b920 100644 --- a/test-runner/wasi_test_runner/test_suite_runner.py +++ b/test-runner/wasi_test_runner/test_suite_runner.py @@ -8,6 +8,7 @@ from datetime import datetime from typing import List, cast +from .filters import TestFilter from .runtime_adapter import RuntimeAdapter from .test_case import ( Result, @@ -25,14 +26,26 @@ def run_tests_from_test_suite( runtime: RuntimeAdapter, validators: List[Validator], reporters: List[TestReporter], + filters: List[TestFilter], ) -> TestSuite: test_cases: List[TestCase] = [] test_start = datetime.now() _cleanup_test_output(test_suite_path) + test_suite_name=_read_manifest(test_suite_path) + for test_path in glob.glob(os.path.join(test_suite_path, "*.wasm")): - test_case = _execute_single_test(runtime, validators, test_path) + test_name = os.path.splitext(os.path.basename(test_path))[0] + for filter in filters: + # for now, just drop the skip reason string. it might be + # useful to make reporters report it. + skip, _ = filter.should_skip(test_suite_name, test_name) + if skip: + test_case = _skip_single_test(runtime, validators, test_path) + break + else: + test_case = _execute_single_test(runtime, validators, test_path) test_cases.append(test_case) for reporter in reporters: reporter.report_test(test_case) @@ -40,13 +53,24 @@ def run_tests_from_test_suite( elapsed = (datetime.now() - test_start).total_seconds() return TestSuite( - name=_read_manifest(test_suite_path), + name=test_suite_name, time=test_start, duration_s=elapsed, test_cases=test_cases, ) +def _skip_single_test( + _: RuntimeAdapter, validators: List[Validator], test_path: str +) -> TestCase: + config = _read_test_config(test_path) + return TestCase( + name=os.path.splitext(os.path.basename(test_path))[0], + config=config, + result=Result(output=None,is_executed=False,failures=[]), + duration_s=0, + ) + def _execute_single_test( runtime: RuntimeAdapter, validators: List[Validator], test_path: str ) -> TestCase: