Add --fail-fast flag

Closes #1841

Add a `--fail-fast` flag which stops running test cases after the first
failure.

- In the Engine, check for failed tests immediately after running them.
  When fast failures are enabled `close()` the engine after any failure.
  Cleanups teardowns and cleanups will run, and any concurrently running
  tests in other suits will finish, but no further tests will start.
- Add the argument in `Configuration` and parse it with the args.

pkgs/test/test/utils.dart

@@ -348,6 +348,7 @@ Configuration configuration(
  tags: tags,
  onPlatform: onPlatform,
  testRandomizeOrderingSeed: testRandomizeOrderingSeed,
+ stopOnFirstFailure: false,
  timeout: timeout,
  verboseTrace: verboseTrace,
  chainStackTraces: chainStackTraces,

pkgs/test_core/lib/src/runner.dart

@@ -75,9 +75,11 @@ class Runner {
  /// Creates a new runner based on [configuration].
  factory Runner(Configuration config) => config.asCurrent(() {
  var engine = Engine(
- concurrency: config.concurrency,
- coverage: config.coverage,
- testRandomizeOrderingSeed: config.testRandomizeOrderingSeed);
+ concurrency: config.concurrency,
+ coverage: config.coverage,
+ testRandomizeOrderingSeed: config.testRandomizeOrderingSeed,
+ stopOnFirstFailure: config.stopOnFirstFailure,
+ );
 
  var sinks = <IOSink>[];
  Reporter createFileReporter(String reporterName, String filepath) {

pkgs/test_core/lib/src/runner/configuration.dart

@@ -207,6 +207,11 @@ class Configuration {
  /// The same seed will shuffle the tests in the same way every time.
  final int? testRandomizeOrderingSeed;
 
+ final bool? _stopOnFirstFailure;
+
+ /// Whether to stop running subsequent tests after a test fails.
+ bool get stopOnFirstFailure => _stopOnFirstFailure ?? false;
+
  /// Returns the current configuration, or a default configuration if no
  /// current configuration is set.
  ///
@@ -270,6 +275,7 @@ class Configuration {
  required Map<String, CustomRuntime>? defineRuntimes,
  required bool? noRetry,
  required int? testRandomizeOrderingSeed,
+ required bool? stopOnFirstFailure,
 
  // Suite-level configuration
  required bool? allowDuplicateTestNames,
@@ -322,6 +328,7 @@ class Configuration {
  defineRuntimes: defineRuntimes,
  noRetry: noRetry,
  testRandomizeOrderingSeed: testRandomizeOrderingSeed,
+ stopOnFirstFailure: stopOnFirstFailure,
  includeTags: includeTags,
  excludeTags: excludeTags,
  globalPatterns: globalPatterns,
@@ -379,6 +386,7 @@ class Configuration {
  Map<String, CustomRuntime>? defineRuntimes,
  bool? noRetry,
  int? testRandomizeOrderingSeed,
+ bool? stopOnFirstFailure,
 
  // Suite-level configuration
  bool? allowDuplicateTestNames,
@@ -430,6 +438,7 @@ class Configuration {
  defineRuntimes: defineRuntimes,
  noRetry: noRetry,
  testRandomizeOrderingSeed: testRandomizeOrderingSeed,
+ stopOnFirstFailure: stopOnFirstFailure,
  allowDuplicateTestNames: allowDuplicateTestNames,
  allowTestRandomization: allowTestRandomization,
  jsTrace: jsTrace,
@@ -496,6 +505,7 @@ class Configuration {
  defineRuntimes: null,
  noRetry: null,
  testRandomizeOrderingSeed: null,
+ stopOnFirstFailure: null,
  ignoreTimeouts: null,
  allowDuplicateTestNames: null,
  allowTestRandomization: null,
@@ -563,6 +573,7 @@ class Configuration {
  defineRuntimes: null,
  noRetry: null,
  testRandomizeOrderingSeed: null,
+ stopOnFirstFailure: null,
  jsTrace: null,
  runSkipped: null,
  dart2jsArgs: null,
@@ -627,6 +638,7 @@ class Configuration {
  defineRuntimes: null,
  noRetry: null,
  testRandomizeOrderingSeed: null,
+ stopOnFirstFailure: null,
  allowDuplicateTestNames: null,
  allowTestRandomization: null,
  jsTrace: null,
@@ -689,6 +701,7 @@ class Configuration {
  overrideRuntimes: null,
  noRetry: null,
  testRandomizeOrderingSeed: null,
+ stopOnFirstFailure: null,
  allowDuplicateTestNames: null,
  allowTestRandomization: null,
  jsTrace: null,
@@ -754,6 +767,7 @@ class Configuration {
  required Map<String, CustomRuntime>? defineRuntimes,
  required bool? noRetry,
  required this.testRandomizeOrderingSeed,
+ required bool? stopOnFirstFailure,
  required BooleanSelector? includeTags,
  required BooleanSelector? excludeTags,
  required Iterable<Pattern>? globalPatterns,
@@ -786,6 +800,7 @@ class Configuration {
  globalPatterns = globalPatterns == null
  ? const {}
  : UnmodifiableSetView(globalPatterns.toSet()),
+ _stopOnFirstFailure = stopOnFirstFailure,
  suiteDefaults = (() {
  var config = suiteDefaults ?? SuiteConfiguration.empty;
  if (pauseAfterLoad == true) {
@@ -839,6 +854,7 @@ class Configuration {
  defineRuntimes: null,
  noRetry: null,
  testRandomizeOrderingSeed: null,
+ stopOnFirstFailure: null,
  includeTags: null,
  excludeTags: null,
  );
@@ -944,6 +960,7 @@ class Configuration {
  noRetry: other._noRetry ?? _noRetry,
  testRandomizeOrderingSeed:
  other.testRandomizeOrderingSeed ?? testRandomizeOrderingSeed,
+ stopOnFirstFailure: other._stopOnFirstFailure ?? _stopOnFirstFailure,
  includeTags: includeTags.intersection(other.includeTags),
  excludeTags: excludeTags.union(other.excludeTags),
  globalPatterns: globalPatterns.union(other.globalPatterns),
@@ -1034,6 +1051,7 @@ class Configuration {
  noRetry: noRetry ?? _noRetry,
  testRandomizeOrderingSeed:
  testRandomizeOrderingSeed ?? this.testRandomizeOrderingSeed,
+ stopOnFirstFailure: _stopOnFirstFailure,
  includeTags: includeTags,
  excludeTags: excludeTags,
  globalPatterns: globalPatterns,

pkgs/test_core/lib/src/runner/configuration/args.dart

@@ -134,6 +134,8 @@ final ArgParser _parser = (() {
  'Must be a 32bit unsigned integer or "random".\n'
  'If "random", pick a random seed to use.\n'
  'If not passed, do not randomize test case execution order.');
+ parser.addFlag('fail-fast',
+ help: 'Stop running tests after the first failure.\n');
 
  var reporterDescriptions = <String, String>{};
  for (var reporter in allReporters.keys) {
@@ -348,6 +350,7 @@ class _Parser {
  noRetry: _ifParsed('no-retry'),
  testRandomizeOrderingSeed: testRandomizeOrderingSeed,
  ignoreTimeouts: _ifParsed('ignore-timeouts'),
+ stopOnFirstFailure: _ifParsed('fail-fast'),
  // Config that isn't supported on the command line
  addTags: null,
  allowTestRandomization: null,

pkgs/test_core/lib/src/runner/engine.dart

@@ -69,6 +69,9 @@ class Engine {
  /// The same seed will shuffle the tests in the same way every time.
  int? testRandomizeOrderingSeed;
 
+ /// Whether to stop running tests after a failure.
+ bool _stopOnFirstFailure;
+
  /// A pool that limits the number of test suites running concurrently.
  final Pool _runPool;
 
@@ -202,8 +205,16 @@ class Engine {
  /// Omitting this argument or passing `0` disables shuffling.
  ///
  /// [coverage] specifies a directory to output coverage information.
- Engine({int? concurrency, String? coverage, this.testRandomizeOrderingSeed})
- : _runPool = Pool(concurrency ?? 1),
+ ///
+ /// If [stopOnFirstFailure] then a single failing test will cause the engine
+ /// to [close] and stop ruunning further tests.
+ Engine({
+ int? concurrency,
+ String? coverage,
+ this.testRandomizeOrderingSeed,
+ bool stopOnFirstFailure = false,
+ }) : _runPool = Pool(concurrency ?? 1),
+ _stopOnFirstFailure = stopOnFirstFailure,
  _coverage = coverage {
  _group.future.then((_) {
  _onTestStartedGroup.close();
@@ -371,7 +382,10 @@ class Engine {
  // loop pump to avoid starving non-microtask events.
  await Future(() {});
 
- if (!_restarted.contains(liveTest)) return;
+ if (!_restarted.contains(liveTest)) {
+ if (_stopOnFirstFailure && liveTest.state.result.isFailing) close();
+ return;
+ }
  await _runLiveTest(suiteController, liveTest.copy(),
  countSuccess: countSuccess);
  _restarted.remove(liveTest);