Refactor Result class

test-runner/wasi_test_runner/reporters/console.py

@@ -2,7 +2,7 @@
 from colorama import Fore, init
 
 from . import TestReporter
-from ..test_case import TestCase
+from ..test_case import TestCase, Skipped
 from ..test_suite import TestSuite
 from ..runtime_adapter import RuntimeVersion
 
@@ -19,18 +19,18 @@ def __init__(self, colored: bool = True) -> None:
  self._colored = colored
 
  def report_test(self, test: TestCase) -> None:
- if test.result.failed:
+ if isinstance(test.result, Skipped):
+ self._print_skip(f"Test {test.name} skipped: {test.result.reason}")
+ elif test.result.failures:
  self._print_fail(f"Test {test.name} failed")
  for reason in test.result.failures:
  self._print_fail(f" [{reason.type}] {reason.message}")
  print("STDOUT:")
  print(test.result.output.stdout)
  print("STDERR:")
  print(test.result.output.stderr)
- elif test.result.is_executed:
- self._print_pass(f"Test {test.name} passed")
  else:
- self._print_skip(f"Test {test.name} skipped")
+ self._print_pass(f"Test {test.name} passed")
 
  def report_test_suite(self, test_suite: TestSuite) -> None:
  self._test_suites.append(test_suite)

test-runner/wasi_test_runner/test_case.py

@@ -1,6 +1,6 @@
 import logging
 import json
-from typing import List, NamedTuple, TypeVar, Type, Dict, Any, Optional
+from typing import List, NamedTuple, TypeVar, Type, Dict, Any, Optional, Union
 
 
 class Output(NamedTuple):
@@ -14,14 +14,16 @@ class Failure(NamedTuple):
  message: str
 
 
-class Result(NamedTuple):
+class Executed(NamedTuple):
  output: Output
- is_executed: bool
  failures: List[Failure]
 
- @property
- def failed(self) -> bool:
- return len(self.failures) > 0
+
+class Skipped(NamedTuple):
+ reason: str
+
+
+Result = Union[Executed, Skipped]
 
 
 T = TypeVar("T", bound="Config")

test-runner/wasi_test_runner/test_suite.py

@@ -1,6 +1,6 @@
 from typing import NamedTuple, List
 from datetime import datetime
-from .test_case import TestCase
+from .test_case import TestCase, Executed, Skipped
 
 
 class TestSuite(NamedTuple):
@@ -19,7 +19,7 @@ def pass_count(self) -> int:
  [
  1
  for test in self.test_cases
- if test.result.is_executed and test.result.failed is False
+ if isinstance(test.result, Executed) and not test.result.failures
  ]
  )
 
@@ -29,10 +29,10 @@ def fail_count(self) -> int:
  [
  1
  for test in self.test_cases
- if test.result.is_executed and test.result.failed
+ if isinstance(test.result, Executed) and test.result.failures
  ]
  )
 
  @property
  def skip_count(self) -> int:
- return len([1 for test in self.test_cases if not test.result.is_executed])
+ return len([1 for test in self.test_cases if isinstance(test.result, Skipped)])

test-runner/wasi_test_runner/test_suite_runner.py

@@ -11,7 +11,8 @@
 from .filters import TestFilter
 from .runtime_adapter import RuntimeAdapter
 from .test_case import (
- Result,
+ Executed,
+ Skipped,
  Config,
  Output,
  TestCase,
@@ -38,11 +39,13 @@ def run_tests_from_test_suite(
  for test_path in glob.glob(os.path.join(test_suite_path, "*.wasm")):
  test_name = os.path.splitext(os.path.basename(test_path))[0]
  for filt in filters:
- # for now, just drop the skip reason string. it might be
- # useful to make reporters report it.
- skip, _ = filt.should_skip(test_suite_name, test_name)
+ skip, reason = filt.should_skip(test_suite_name, test_name)
  if skip:
- test_case = _skip_single_test(runtime, validators, test_path)
+ # Given the type of should_skip, "if skip" implies reason
+ # is str. But mypy doesn't seem to recognize it. Use an
+ # explicit cast for now.
+ reason = cast(str, reason)
+ test_case = _skip_single_test(runtime, validators, test_path, reason)
  break
  else:
  test_case = _execute_single_test(runtime, validators, test_path)
@@ -61,13 +64,13 @@ def run_tests_from_test_suite(
 
 
 def _skip_single_test(
- _runtime: RuntimeAdapter, _validators: List[Validator], test_path: str
+ _runtime: RuntimeAdapter, _validators: List[Validator], test_path: str, reason: str
 ) -> TestCase:
  config = _read_test_config(test_path)
  return TestCase(
  name=os.path.splitext(os.path.basename(test_path))[0],
  config=config,
- result=Result(output=Output(0, "", ""), is_executed=False, failures=[]),
+ result=Skipped(reason),
  duration_s=0,
  )
 
@@ -88,14 +91,14 @@ def _execute_single_test(
  )
 
 
-def _validate(validators: List[Validator], config: Config, output: Output) -> Result:
+def _validate(validators: List[Validator], config: Config, output: Output) -> Executed:
  failures = [
  result
  for result in [validator(config, output) for validator in validators]
  if result is not None
  ]
 
- return Result(failures=failures, is_executed=True, output=output)
+ return Executed(failures=failures, output=output)
 
 
 def _read_test_config(test_path: str) -> Config: