From cdfdfb25d7b3347281c1099c4a129b8782b46ee6 Mon Sep 17 00:00:00 2001 From: Nate Bosch Date: Tue, 20 Jun 2023 23:29:32 +0000 Subject: [PATCH] Add --fail-fast flag Closes #1841 Add a `--fail-fast` flag which stops running test cases after the first failure. - In the Engine, check for failed tests immediately after running them. When fast failures are enabled `close()` the engine after any failure. Cleanups teardowns and cleanups will run, and any concurrently running tests in other suits will finish, but no further tests will start. - Add the argument in `Configuration` and parse it with the args. --- pkgs/test/test/utils.dart | 1 + pkgs/test_core/lib/src/runner.dart | 8 +++++--- .../lib/src/runner/configuration.dart | 18 +++++++++++++++++ .../lib/src/runner/configuration/args.dart | 3 +++ pkgs/test_core/lib/src/runner/engine.dart | 20 ++++++++++++++++--- 5 files changed, 44 insertions(+), 6 deletions(-) diff --git a/pkgs/test/test/utils.dart b/pkgs/test/test/utils.dart index 3c3ef0165..2b247f14f 100644 --- a/pkgs/test/test/utils.dart +++ b/pkgs/test/test/utils.dart @@ -348,6 +348,7 @@ Configuration configuration( tags: tags, onPlatform: onPlatform, testRandomizeOrderingSeed: testRandomizeOrderingSeed, + stopOnFirstFailure: false, timeout: timeout, verboseTrace: verboseTrace, chainStackTraces: chainStackTraces, diff --git a/pkgs/test_core/lib/src/runner.dart b/pkgs/test_core/lib/src/runner.dart index 05531368a..c458b5d23 100644 --- a/pkgs/test_core/lib/src/runner.dart +++ b/pkgs/test_core/lib/src/runner.dart @@ -75,9 +75,11 @@ class Runner { /// Creates a new runner based on [configuration]. factory Runner(Configuration config) => config.asCurrent(() { var engine = Engine( - concurrency: config.concurrency, - coverage: config.coverage, - testRandomizeOrderingSeed: config.testRandomizeOrderingSeed); + concurrency: config.concurrency, + coverage: config.coverage, + testRandomizeOrderingSeed: config.testRandomizeOrderingSeed, + stopOnFirstFailure: config.stopOnFirstFailure, + ); var sinks = []; Reporter createFileReporter(String reporterName, String filepath) { diff --git a/pkgs/test_core/lib/src/runner/configuration.dart b/pkgs/test_core/lib/src/runner/configuration.dart index b8d568c0a..12d3aa665 100644 --- a/pkgs/test_core/lib/src/runner/configuration.dart +++ b/pkgs/test_core/lib/src/runner/configuration.dart @@ -207,6 +207,11 @@ class Configuration { /// The same seed will shuffle the tests in the same way every time. final int? testRandomizeOrderingSeed; + final bool? _stopOnFirstFailure; + + /// Whether to stop running subsequent tests after a test fails. + bool get stopOnFirstFailure => _stopOnFirstFailure ?? false; + /// Returns the current configuration, or a default configuration if no /// current configuration is set. /// @@ -270,6 +275,7 @@ class Configuration { required Map? defineRuntimes, required bool? noRetry, required int? testRandomizeOrderingSeed, + required bool? stopOnFirstFailure, // Suite-level configuration required bool? allowDuplicateTestNames, @@ -322,6 +328,7 @@ class Configuration { defineRuntimes: defineRuntimes, noRetry: noRetry, testRandomizeOrderingSeed: testRandomizeOrderingSeed, + stopOnFirstFailure: stopOnFirstFailure, includeTags: includeTags, excludeTags: excludeTags, globalPatterns: globalPatterns, @@ -379,6 +386,7 @@ class Configuration { Map? defineRuntimes, bool? noRetry, int? testRandomizeOrderingSeed, + bool? stopOnFirstFailure, // Suite-level configuration bool? allowDuplicateTestNames, @@ -430,6 +438,7 @@ class Configuration { defineRuntimes: defineRuntimes, noRetry: noRetry, testRandomizeOrderingSeed: testRandomizeOrderingSeed, + stopOnFirstFailure: stopOnFirstFailure, allowDuplicateTestNames: allowDuplicateTestNames, allowTestRandomization: allowTestRandomization, jsTrace: jsTrace, @@ -496,6 +505,7 @@ class Configuration { defineRuntimes: null, noRetry: null, testRandomizeOrderingSeed: null, + stopOnFirstFailure: null, ignoreTimeouts: null, allowDuplicateTestNames: null, allowTestRandomization: null, @@ -563,6 +573,7 @@ class Configuration { defineRuntimes: null, noRetry: null, testRandomizeOrderingSeed: null, + stopOnFirstFailure: null, jsTrace: null, runSkipped: null, dart2jsArgs: null, @@ -627,6 +638,7 @@ class Configuration { defineRuntimes: null, noRetry: null, testRandomizeOrderingSeed: null, + stopOnFirstFailure: null, allowDuplicateTestNames: null, allowTestRandomization: null, jsTrace: null, @@ -689,6 +701,7 @@ class Configuration { overrideRuntimes: null, noRetry: null, testRandomizeOrderingSeed: null, + stopOnFirstFailure: null, allowDuplicateTestNames: null, allowTestRandomization: null, jsTrace: null, @@ -754,6 +767,7 @@ class Configuration { required Map? defineRuntimes, required bool? noRetry, required this.testRandomizeOrderingSeed, + required bool? stopOnFirstFailure, required BooleanSelector? includeTags, required BooleanSelector? excludeTags, required Iterable? globalPatterns, @@ -786,6 +800,7 @@ class Configuration { globalPatterns = globalPatterns == null ? const {} : UnmodifiableSetView(globalPatterns.toSet()), + _stopOnFirstFailure = stopOnFirstFailure, suiteDefaults = (() { var config = suiteDefaults ?? SuiteConfiguration.empty; if (pauseAfterLoad == true) { @@ -839,6 +854,7 @@ class Configuration { defineRuntimes: null, noRetry: null, testRandomizeOrderingSeed: null, + stopOnFirstFailure: null, includeTags: null, excludeTags: null, ); @@ -944,6 +960,7 @@ class Configuration { noRetry: other._noRetry ?? _noRetry, testRandomizeOrderingSeed: other.testRandomizeOrderingSeed ?? testRandomizeOrderingSeed, + stopOnFirstFailure: other._stopOnFirstFailure ?? _stopOnFirstFailure, includeTags: includeTags.intersection(other.includeTags), excludeTags: excludeTags.union(other.excludeTags), globalPatterns: globalPatterns.union(other.globalPatterns), @@ -1034,6 +1051,7 @@ class Configuration { noRetry: noRetry ?? _noRetry, testRandomizeOrderingSeed: testRandomizeOrderingSeed ?? this.testRandomizeOrderingSeed, + stopOnFirstFailure: _stopOnFirstFailure, includeTags: includeTags, excludeTags: excludeTags, globalPatterns: globalPatterns, diff --git a/pkgs/test_core/lib/src/runner/configuration/args.dart b/pkgs/test_core/lib/src/runner/configuration/args.dart index b8baf483c..5faa52da7 100644 --- a/pkgs/test_core/lib/src/runner/configuration/args.dart +++ b/pkgs/test_core/lib/src/runner/configuration/args.dart @@ -134,6 +134,8 @@ final ArgParser _parser = (() { 'Must be a 32bit unsigned integer or "random".\n' 'If "random", pick a random seed to use.\n' 'If not passed, do not randomize test case execution order.'); + parser.addFlag('fail-fast', + help: 'Stop running tests after the first failure.\n'); var reporterDescriptions = {}; for (var reporter in allReporters.keys) { @@ -348,6 +350,7 @@ class _Parser { noRetry: _ifParsed('no-retry'), testRandomizeOrderingSeed: testRandomizeOrderingSeed, ignoreTimeouts: _ifParsed('ignore-timeouts'), + stopOnFirstFailure: _ifParsed('fail-fast'), // Config that isn't supported on the command line addTags: null, allowTestRandomization: null, diff --git a/pkgs/test_core/lib/src/runner/engine.dart b/pkgs/test_core/lib/src/runner/engine.dart index 578a6bf4d..42f764abe 100644 --- a/pkgs/test_core/lib/src/runner/engine.dart +++ b/pkgs/test_core/lib/src/runner/engine.dart @@ -69,6 +69,9 @@ class Engine { /// The same seed will shuffle the tests in the same way every time. int? testRandomizeOrderingSeed; + /// Whether to stop running tests after a failure. + bool _stopOnFirstFailure; + /// A pool that limits the number of test suites running concurrently. final Pool _runPool; @@ -202,8 +205,16 @@ class Engine { /// Omitting this argument or passing `0` disables shuffling. /// /// [coverage] specifies a directory to output coverage information. - Engine({int? concurrency, String? coverage, this.testRandomizeOrderingSeed}) - : _runPool = Pool(concurrency ?? 1), + /// + /// If [stopOnFirstFailure] then a single failing test will cause the engine + /// to [close] and stop ruunning further tests. + Engine({ + int? concurrency, + String? coverage, + this.testRandomizeOrderingSeed, + bool stopOnFirstFailure = false, + }) : _runPool = Pool(concurrency ?? 1), + _stopOnFirstFailure = stopOnFirstFailure, _coverage = coverage { _group.future.then((_) { _onTestStartedGroup.close(); @@ -371,7 +382,10 @@ class Engine { // loop pump to avoid starving non-microtask events. await Future(() {}); - if (!_restarted.contains(liveTest)) return; + if (!_restarted.contains(liveTest)) { + if (_stopOnFirstFailure && liveTest.state.result.isFailing) close(); + return; + } await _runLiveTest(suiteController, liveTest.copy(), countSuccess: countSuccess); _restarted.remove(liveTest);