Implement a simple test filter

README.md

@@ -38,6 +38,16 @@ python3 test-runner/wasi_test_runner.py
  -r adapters/wasmtime.sh # path to a runtime adapter
 ```
 
+Optionally you can specify test cases to skip with the `--exclude-filter` option.
+
+```bash
+python3 test-runner/wasi_test_runner.py \
+ -t ./tests/assemblyscript/testsuite/ `# path to folders containing .wasm test files` \
+ ./tests/c/testsuite/ \
+ --exclude-filter examples/skip.json \
+ -r adapters/wasmtime.sh # path to a runtime adapter
+```
+
 ## Contributing
 
 All contributions are very welcome. Contributors can help with:

examples/skip.json

@@ -0,0 +1,5 @@
+{
+ "WASI C tests": {
+ "stat-dev-ino": "d_ino is known broken in this combination of engine and wasi-sdk version."
+ }
+}

test-runner/.pylintrc

@@ -3,5 +3,6 @@ disable=
  C0114, # Missing module docstring
  C0115, # Missing class docstring
  C0116, # Missing function or method docstring
+ R0903, # Too few public methods
 [FORMAT]
 max-line-length=120

test-runner/requirements/dev.txt

@@ -1,6 +1,6 @@
 -r common.txt
 flake8==5.0.4
-mypy==0.910
+mypy==0.991
 pylint==2.14.3
 pytest==6.2.5
-coverage==6.3.3
+coverage==6.3.3

test-runner/tests/test_test_suite_runner.py

@@ -62,8 +62,12 @@ def test_runner_end_to_end() -> None:
 
  reporters = [Mock(), Mock()]
 
+ filt = Mock()
+ filt.should_skip.return_value = (False, None)
+ filters = [filt]
+
  with patch("glob.glob", return_value=test_paths):
- suite = tsr.run_tests_from_test_suite("my-path", runtime, validators, reporters) # type: ignore
+ suite = tsr.run_tests_from_test_suite("my-path", runtime, validators, reporters, filters) # type: ignore
 
  # Assert manifest was read correctly
  assert suite.name == "test-suite"
@@ -91,9 +95,15 @@ def test_runner_end_to_end() -> None:
  for config, output in zip(expected_config, outputs):
  validator.assert_any_call(config, output)
 
+ # Assert filter calls
+ for filt in filters:
+ assert filt.should_skip.call_count == 3
+ for test_case in expected_test_cases:
+ filt.should_skip.assert_any_call(suite.name, test_case.name)
+
 
 @patch("os.path.exists", Mock(return_value=False))
 def test_runner_should_use_path_for_name_if_manifest_does_not_exist() -> None:
- suite = tsr.run_tests_from_test_suite("my-path", Mock(), [], [])
+ suite = tsr.run_tests_from_test_suite("my-path", Mock(), [], [], [])
 
  assert suite.name == "my-path"

test-runner/wasi_test_runner/__main__.py

@@ -5,6 +5,8 @@
 
 from .runtime_adapter import RuntimeAdapter
 from .harness import run_all_tests
+from .filters import TestFilter
+from .filters import JSONTestExcludeFilter
 from .reporters import TestReporter
 from .reporters.console import ConsoleTestReporter
 from .reporters.json import JSONTestReporter
@@ -23,6 +25,14 @@ def main() -> int:
  nargs="+",
  help="Locations of suites (directories with *.wasm test files).",
  )
+ parser.add_argument(
+ "-f",
+ "--exclude-filter",
+ required=False,
+ nargs="+",
+ default=[],
+ help="Locations of test exclude filters (JSON files).",
+ )
  parser.add_argument(
  "-r", "--runtime-adapter", required=True, help="Path to a runtime adapter."
  )
@@ -45,11 +55,16 @@ def main() -> int:
 
  validators: List[Validator] = [exit_code_validator, stdout_validator]
 
+ filters: List[TestFilter] = []
+ for filt in options.exclude_filter:
+ filters.append(JSONTestExcludeFilter(filt))
+
  return run_all_tests(
  RuntimeAdapter(options.runtime_adapter),
  options.test_suite,
  validators,
  reporters,
+ filters,
  )
 
 

test-runner/wasi_test_runner/filters.py

@@ -0,0 +1,30 @@
+from typing import Tuple, Union, Literal
+from abc import ABC
+from abc import abstractmethod
+
+import json
+
+
+class TestFilter(ABC):
+ @abstractmethod
+ def should_skip(
+ self, test_suite_name: str, test_name: str
+ ) -> Union[Tuple[Literal[True], str], Tuple[Literal[False], Literal[None]]]:
+ pass
+
+
+class JSONTestExcludeFilter(TestFilter):
+ def __init__(self, filename: str) -> None:
+ with open(filename, encoding="utf-8") as file:
+ self.filter_dict = json.load(file)
+
+ def should_skip(
+ self, test_suite_name: str, test_name: str
+ ) -> Union[Tuple[Literal[True], str], Tuple[Literal[False], Literal[None]]]:
+ test_suite_filter = self.filter_dict.get(test_suite_name)
+ if test_suite_filter is None:
+ return False, None
+ why = test_suite_filter.get(test_name)
+ if why is not None:
+ return True, why
+ return False, None

test-runner/wasi_test_runner/harness.py

@@ -1,5 +1,6 @@
 from typing import List
 
+from .filters import TestFilter
 from .reporters import TestReporter
 from .test_suite_runner import run_tests_from_test_suite
 from .runtime_adapter import RuntimeAdapter
@@ -11,12 +12,13 @@ def run_all_tests(
  test_suite_paths: List[str],
  validators: List[Validator],
  reporters: List[TestReporter],
+ filters: List[TestFilter],
 ) -> int:
  ret = 0
 
  for test_suite_path in test_suite_paths:
  test_suite = run_tests_from_test_suite(
- test_suite_path, runtime, validators, reporters
+ test_suite_path, runtime, validators, reporters, filters,
  )
  for reporter in reporters:
  reporter.report_test_suite(test_suite)

test-runner/wasi_test_runner/reporters/console.py

@@ -54,6 +54,7 @@ def finalize(self, version: RuntimeVersion) -> None:
  print(f" Total: {suite.test_count}")
  self._print_pass(f" Passed: {suite.pass_count}")
  self._print_fail(f" Failed: {suite.fail_count}")
+ self._print_skip(f" Skipped: {suite.skip_count}")
  print("")
 
  print(

test-runner/wasi_test_runner/test_suite_runner.py

@@ -8,6 +8,7 @@
 from datetime import datetime
 from typing import List, cast
 
+from .filters import TestFilter
 from .runtime_adapter import RuntimeAdapter
 from .test_case import (
  Result,
@@ -25,28 +26,52 @@ def run_tests_from_test_suite(
  runtime: RuntimeAdapter,
  validators: List[Validator],
  reporters: List[TestReporter],
+ filters: List[TestFilter],
 ) -> TestSuite:
  test_cases: List[TestCase] = []
  test_start = datetime.now()
 
  _cleanup_test_output(test_suite_path)
 
+ test_suite_name = _read_manifest(test_suite_path)
+
  for test_path in glob.glob(os.path.join(test_suite_path, "*.wasm")):
- test_case = _execute_single_test(runtime, validators, test_path)
+ test_name = os.path.splitext(os.path.basename(test_path))[0]
+ for filt in filters:
+ # for now, just drop the skip reason string. it might be
+ # useful to make reporters report it.
+ skip, _ = filt.should_skip(test_suite_name, test_name)
+ if skip:
+ test_case = _skip_single_test(runtime, validators, test_path)
+ break
+ else:
+ test_case = _execute_single_test(runtime, validators, test_path)
  test_cases.append(test_case)
  for reporter in reporters:
  reporter.report_test(test_case)
 
  elapsed = (datetime.now() - test_start).total_seconds()
 
  return TestSuite(
- name=_read_manifest(test_suite_path),
+ name=test_suite_name,
  time=test_start,
  duration_s=elapsed,
  test_cases=test_cases,
  )
 
 
+def _skip_single_test(
+ _runtime: RuntimeAdapter, _validators: List[Validator], test_path: str
+) -> TestCase:
+ config = _read_test_config(test_path)
+ return TestCase(
+ name=os.path.splitext(os.path.basename(test_path))[0],
+ config=config,
+ result=Result(output=Output(0, "", ""), is_executed=False, failures=[]),
+ duration_s=0,
+ )
+
+
 def _execute_single_test(
  runtime: RuntimeAdapter, validators: List[Validator], test_path: str
 ) -> TestCase: