diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 618fab32..6ed09d26 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -160,6 +160,7 @@ jobs: install cargo-mutants ~/.cargo/bin/ - name: Mutants in-diff # Normally this would have --in-place, but for the sake of exercising more cases, it does not. + # TODO: Pass --profile=mutants when supported run: > cargo mutants --no-shuffle -vV --in-diff git.diff --test-tool=${{matrix.test_tool}} --timeout=500 --build-timeout=500 - name: Archive mutants.out @@ -200,6 +201,7 @@ jobs: cargo mutants --no-shuffle -vV --shard ${{ matrix.shard }}/10 --test-tool ${{ matrix.test_tool }} --baseline=skip --timeout=500 --build-timeout=500 --in-place + --cargo-arg=--profile=mutants - name: Archive mutants.out uses: actions/upload-artifact@v4 if: always() diff --git a/Cargo.lock b/Cargo.lock index ae8d8084..346bf2b2 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -41,7 +41,7 @@ version = "0.12.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d52a9bb7ec0cf484c551830a7ce27bd20d67eac647e1befb56b0be4ee39a55d2" dependencies = [ - "winapi 0.3.9", + "winapi", ] [[package]] @@ -126,7 +126,7 @@ checksum = "d9b39be18770d11421cdb1b9947a45dd3f37e93092cbf377614828a319d5fee8" dependencies = [ "hermit-abi 0.1.19", "libc", - "winapi 0.3.9", + "winapi", ] [[package]] @@ -227,7 +227,6 @@ dependencies = [ "serde_json", "similar", "strum", - "subprocess", "syn 2.0.46", "tempfile", "time", @@ -384,7 +383,7 @@ dependencies = [ "regex", "terminal_size 0.1.17", "unicode-width", - "winapi 0.3.9", + "winapi", ] [[package]] @@ -434,7 +433,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a19c6cedffdc8c03a3346d723eb20bd85a13362bb96dc2ac000842c6381ec7bf" dependencies = [ "nix 0.23.2", - "winapi 0.3.9", + "winapi", ] [[package]] @@ -527,13 +526,12 @@ checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" [[package]] name = "fs2" -version = "0.4.0" +version = "0.4.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "237b7991317b8d94391c0a4813b1f74fd81c11352440cd598d2b763ed288bfc1" +checksum = "9564fc758e15025b46aa6643b1b77d047d1a56a1aea6e01002ac0c7026876213" dependencies = [ - "kernel32-sys", "libc", - "winapi 0.2.8", + "winapi", ] [[package]] @@ -718,16 +716,6 @@ dependencies = [ "wasm-bindgen", ] -[[package]] -name = "kernel32-sys" -version = "0.2.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7507624b29483431c0ba2d82aece8ca6cdba9382bff4ddd0f7490560c056098d" -dependencies = [ - "winapi 0.2.8", - "winapi-build", -] - [[package]] name = "lazy_static" version = "1.4.0" @@ -864,7 +852,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "77a8165726e8236064dbb45459242600304b42a5ea24ee2948e18e023bf7ba84" dependencies = [ "overload", - "winapi 0.3.9", + "winapi", ] [[package]] @@ -906,7 +894,7 @@ version = "0.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "628223faebab4e3e40667ee0b2336d34a5b960ff60ea743ddfdbcf7770bcfb66" dependencies = [ - "winapi 0.3.9", + "winapi", ] [[package]] @@ -1274,16 +1262,6 @@ dependencies = [ "syn 2.0.46", ] -[[package]] -name = "subprocess" -version = "0.2.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "055cf3ebc2981ad8f0a5a17ef6652f652d87831f79fddcba2ac57bcb9a0aa407" -dependencies = [ - "libc", - "winapi 0.3.9", -] - [[package]] name = "syn" version = "1.0.109" @@ -1326,7 +1304,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "633c1a546cee861a1a6d0dc69ebeca693bf4296661ba7852b9d21d159e0506df" dependencies = [ "libc", - "winapi 0.3.9", + "winapi", ] [[package]] @@ -1632,12 +1610,6 @@ dependencies = [ "web-sys", ] -[[package]] -name = "winapi" -version = "0.2.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "167dc9d6949a9b857f3451275e911c3f44255842c1f7a76f33c55103a909087a" - [[package]] name = "winapi" version = "0.3.9" @@ -1648,12 +1620,6 @@ dependencies = [ "winapi-x86_64-pc-windows-gnu", ] -[[package]] -name = "winapi-build" -version = "0.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2d315eee3b34aca4797b2da6b13ed88266e6d612562a0c46390af8299fc699bc" - [[package]] name = "winapi-i686-pc-windows-gnu" version = "0.4.0" diff --git a/Cargo.toml b/Cargo.toml index 87edf484..9e144814 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -48,7 +48,6 @@ regex = "1.10" serde_json = "1.0.117" similar = "2.1" strum = { version = "0.26", features = ["derive"] } -subprocess = "0.2.8" tempfile = "3.4" time = "0.3" toml = "0.8" @@ -158,6 +157,10 @@ pr-run-mode = "plan" inherits = "release" lto = "thin" +[profile.mutants] +inherits = "test" +debug = "none" + # Config for [workspace.metadata.release] pre-release-replacements = [ diff --git a/NEWS.md b/NEWS.md index 7a7cb6c0..a88edbd9 100644 --- a/NEWS.md +++ b/NEWS.md @@ -6,6 +6,12 @@ - Fixed: The auto-set timeout for building mutants is now 2 times the baseline build time times the number of jobs, with a minimum of 20 seconds. This was changed because builds of mutants contend with each other for access to CPUs and may be slower than the baseline build. +- Changed: No build timeouts by default. Previously, cargo-mutants set a default build timeout based on the baseline build, but experience showed that this would sometimes make builds flaky, because build times can be quite variable. If mutants cause builds to hang, then you can still set a timeout using `--build-timeout` or `--build-timeout-multiplier`. + +- Fixed: Don't error if the `--in-diff` file is empty. + +- Changed: cargo-mutants no longer passes `--cap-lints=allow` to rustc. This was previously done so that mutants would not unnecessarily be unviable due to triggering compiler warnings in trees configured to deny some lints, but it had the undesirable effect of disabling rustc's detection of long running const evaluations. If your tree treats some lints as errors then the previous behavior can be restored with `--cap-lints=true` (or the equivalent config option), or you can use `cfg_attr` and a feature flag to accept those warnings when testing under cargo-mutants. + ## 24.5.0 - Fixed: Follow `path` attributes on `mod` statements. diff --git a/book/src/SUMMARY.md b/book/src/SUMMARY.md index 92d96035..d9d1a1ed 100644 --- a/book/src/SUMMARY.md +++ b/book/src/SUMMARY.md @@ -21,6 +21,7 @@ - [Baseline tests](baseline.md) - [Testing in-place](in-place.md) - [Iterating on missed mutants](iterate.md) + - [Strict lints](lints.md) - [Generating mutants](mutants.md) - [Error values](error-values.md) - [Improving performance](performance.md) diff --git a/book/src/lints.md b/book/src/lints.md new file mode 100644 index 00000000..97a89017 --- /dev/null +++ b/book/src/lints.md @@ -0,0 +1,12 @@ +# Strict lints + +Because cargo-mutants builds versions of your tree with many heuristically injected errors, it may not work well in trees that are configured to treat warnings as errors. + +For example, mutants that delete code are likely to cause some parameters to be seen as unused, which will cause problems with trees that configure `#[deny(unused)]`. This will manifest as an excessive number of mutants being reported as "unviable". + +There are a few possible solutions: + +1. Define a feature flag for mutation testing, and use `cfg_attr` to enable strict warnings only when not testing mutants. +2. Use the `cargo mutants --cap-lints=true` command line option, or the `cap_lints = true` config option. + +`--cap_lints=true` also disables rustc's detection of long-running const expression evaluation, so may cause some builds to fail. If that happens in your tree, you can set a [build timeout](timeouts.md). diff --git a/book/src/timeouts.md b/book/src/timeouts.md index 1e3c9c31..bff23255 100644 --- a/book/src/timeouts.md +++ b/book/src/timeouts.md @@ -22,32 +22,31 @@ continue to the next mutant. By default, the timeouts are set automatically, relative to the times taken to build and test the unmodified tree (baseline). -The default timeouts are: - -- `cargo build`/`cargo check`: 2 times the baseline build time times the number of jobs (with a minimum of 20 seconds) -- `cargo test`: 5 times baseline test time (with a minimum of 20 seconds) - -The build timeout scales with the number of jobs to reflect that cargo often spawns many jobs, and so builds run in parallel are likely to take longer than the baseline, which has no external parallelism. +The default test timeout is 5 times the baseline test time, with a minimum of 20 seconds. The minimum of 20 seconds for the test timeout can be overridden by the `--minimum-test-timeout` option or the `CARGO_MUTANTS_MINIMUM_TEST_TIMEOUT` environment variable, measured in seconds. -You can set explicit timeouts with the `--build-timeout`, and `--timeout` -options, also measured in seconds. If these options are specified then they -are applied to the baseline build and test as well. +You can set an explicit timeouts with the `--timeout` option, also measured in seconds. + +You can also set the test timeout as a multiple of the duration of the baseline test, with the `--timeout-multiplier` option and the `timeout_multiplier` configuration key. +The multiplier only has an effect if the baseline is not skipped and if `--timeout` is not specified. + +## Build timeouts + +`const` expressions may be evaluated at compile time. In the same way that mutations can cause tests to hang, mutations to const code may potentially cause the compiler to enter an infinite loop. + +rustc imposes a time limit on evaluation of const expressions. This is controlled by the `long_running_const_eval` lint, which by default will interrupt compilation: as a result the mutants will be seen as unviable. + +If this lint is configured off in your program, or if you use the `--cap-lints=true` option to turn off all lints, then the compiler may hang when constant expressions are mutated. + +In this case you can use the `--build-timeout` or `--build-timeout-multiplier` options, or their corresponding configuration keys, to impose a limit on overall build time. However, because build time can be quite variable there's some risk of this causing builds to be flaky, and so it's off by default. -You can set timeout multipliers that are relative to the duration of the -baseline build or test with `--build-timeout-multiplier` and -`--timeout-multiplier`, respectively. Additionally, these can be configured -with `build_timeout_multiplier` and `timeout_multiplier` in -`.cargo/mutants.toml` (e.g. `timeout-multiplier = 1.5`). These options are only -applied if the baseline is not skipped and no corresponding -`--build-timeout`/`--timeout` option is specified, otherwise they are ignored. +You might also choose to skip mutants that can cause long-running const evaluation. ## Exceptions The multiplier timeout options cannot be used when the baseline is skipped (`--baseline=skip`), or when the build is in-place (`--in-place`). If no -explicit timeouts is provided in these cases, then a default of 300 seconds -will be used. +explicit timeouts is provided in these cases, then there is no build timeout and the test timeout default of 300 seconds will be used. diff --git a/src/cargo.rs b/src/cargo.rs index fb844054..c32dc30e 100644 --- a/src/cargo.rs +++ b/src/cargo.rs @@ -17,7 +17,7 @@ pub fn run_cargo( build_dir: &BuildDir, packages: Option<&[&Package]>, phase: Phase, - timeout: Duration, + timeout: Option, log_file: &mut LogFile, options: &Options, console: &Console, @@ -26,7 +26,7 @@ pub fn run_cargo( let start = Instant::now(); let argv = cargo_argv(build_dir.path(), packages, phase, options); let env = vec![ - ("CARGO_ENCODED_RUSTFLAGS".to_owned(), rustflags()), + ("CARGO_ENCODED_RUSTFLAGS".to_owned(), rustflags(options)), // The tests might use Insta , and we don't want it to write // updates to the source tree, and we *certainly* don't want it to write // updates and then let the test pass. @@ -101,6 +101,7 @@ fn cargo_argv( cargo_args.push("--tests".to_string()); } } + cargo_args.push("--verbose".to_string()); if let Some([package]) = packages { // Use the unambiguous form for this case; it works better when the same // package occurs multiple times in the tree with different versions? @@ -140,7 +141,7 @@ fn cargo_argv( /// /// See /// -fn rustflags() -> String { +fn rustflags(options: &Options) -> String { let mut rustflags: Vec = if let Some(rustflags) = env::var_os("CARGO_ENCODED_RUSTFLAGS") { rustflags @@ -163,7 +164,9 @@ fn rustflags() -> String { // TODO: build.rustflags config value. Vec::new() }; - rustflags.push("--cap-lints=allow".to_owned()); + if options.cap_lints { + rustflags.push("--cap-lints=warn".to_owned()); + } // debug!("adjusted rustflags: {:?}", rustflags); rustflags.join("\x1f") } @@ -183,15 +186,15 @@ mod test { let build_dir = Utf8Path::new("/tmp/buildXYZ"); assert_eq!( cargo_argv(build_dir, None, Phase::Check, &options)[1..], - ["check", "--tests", "--workspace"] + ["check", "--tests", "--verbose", "--workspace"] ); assert_eq!( cargo_argv(build_dir, None, Phase::Build, &options)[1..], - ["test", "--no-run", "--workspace"] + ["test", "--no-run", "--verbose", "--workspace"] ); assert_eq!( cargo_argv(build_dir, None, Phase::Test, &options)[1..], - ["test", "--workspace"] + ["test", "--verbose", "--workspace"] ); } @@ -214,6 +217,7 @@ mod test { [ "check", "--tests", + "--verbose", "--manifest-path", build_manifest_path.as_str(), ] @@ -223,6 +227,7 @@ mod test { [ "test", "--no-run", + "--verbose", "--manifest-path", build_manifest_path.as_str(), ] @@ -231,6 +236,7 @@ mod test { cargo_argv(build_dir, Some(&[&package]), Phase::Test, &options)[1..], [ "test", + "--verbose", "--manifest-path", build_manifest_path.as_str(), "--lib", @@ -251,16 +257,17 @@ mod test { .extend(["--release".to_owned()]); assert_eq!( cargo_argv(build_dir, None, Phase::Check, &options)[1..], - ["check", "--tests", "--workspace", "--release"] + ["check", "--tests", "--verbose", "--workspace", "--release"] ); assert_eq!( cargo_argv(build_dir, None, Phase::Build, &options)[1..], - ["test", "--no-run", "--workspace", "--release"] + ["test", "--no-run", "--verbose", "--workspace", "--release"] ); assert_eq!( cargo_argv(build_dir, None, Phase::Test, &options)[1..], [ "test", + "--verbose", "--workspace", "--release", "--lib", @@ -276,7 +283,13 @@ mod test { let build_dir = Utf8Path::new("/tmp/buildXYZ"); assert_eq!( cargo_argv(build_dir, None, Phase::Check, &options)[1..], - ["check", "--tests", "--workspace", "--no-default-features"] + [ + "check", + "--tests", + "--verbose", + "--workspace", + "--no-default-features" + ] ); } @@ -287,7 +300,24 @@ mod test { let build_dir = Utf8Path::new("/tmp/buildXYZ"); assert_eq!( cargo_argv(build_dir, None, Phase::Check, &options)[1..], - ["check", "--tests", "--workspace", "--all-features"] + [ + "check", + "--tests", + "--verbose", + "--workspace", + "--all-features" + ] + ); + } + + #[test] + fn cap_lints_passed_to_cargo() { + let args = Args::try_parse_from(["mutants", "--cap-lints=true"].as_slice()).unwrap(); + let options = Options::from_args(&args).unwrap(); + let build_dir = Utf8Path::new("/tmp/buildXYZ"); + assert_eq!( + cargo_argv(build_dir, None, Phase::Check, &options)[1..], + ["check", "--tests", "--verbose", "--workspace",] ); } @@ -304,6 +334,7 @@ mod test { [ "check", "--tests", + "--verbose", "--workspace", "--features=foo", "--features=bar,baz" @@ -316,21 +347,34 @@ mod test { fn rustflags_with_no_environment_variables() { env::remove_var("RUSTFLAGS"); env::remove_var("CARGO_ENCODED_RUSTFLAGS"); - assert_eq!(rustflags(), "--cap-lints=allow"); + assert_eq!( + rustflags(&Options { + cap_lints: true, + ..Default::default() + }), + "--cap-lints=warn" + ); } #[test] fn rustflags_added_to_existing_encoded_rustflags() { env::set_var("RUSTFLAGS", "--something\x1f--else"); env::remove_var("CARGO_ENCODED_RUSTFLAGS"); - assert_eq!(rustflags(), "--something\x1f--else\x1f--cap-lints=allow"); + let options = Options { + cap_lints: true, + ..Default::default() + }; + assert_eq!(rustflags(&options), "--something\x1f--else\x1f--cap-lints=warn"); } #[test] fn rustflags_added_to_existing_rustflags() { env::set_var("RUSTFLAGS", "-Dwarnings"); env::remove_var("CARGO_ENCODED_RUSTFLAGS"); - assert_eq!(rustflags(), "-Dwarnings\x1f--cap-lints=allow"); + assert_eq!(rustflags(&Options { + cap_lints: true, + ..Default::default() + }), "-Dwarnings\x1f--cap-lints=warn"); } } } diff --git a/src/config.rs b/src/config.rs index 96cdd71b..5209f3b5 100644 --- a/src/config.rs +++ b/src/config.rs @@ -27,6 +27,8 @@ use crate::Result; #[derive(Debug, Default, Clone, Deserialize)] #[serde(default, deny_unknown_fields)] pub struct Config { + /// Pass `--cap-lints` to rustc. + pub cap_lints: bool, /// Generate these error values from functions returning Result. pub error_values: Vec, /// Generate mutants from source files matching these globs. diff --git a/src/in_diff.rs b/src/in_diff.rs index b7be4e98..755d32db 100644 --- a/src/in_diff.rs +++ b/src/in_diff.rs @@ -10,7 +10,7 @@ use camino::Utf8Path; use indoc::formatdoc; use itertools::Itertools; use patch::{Line, Patch}; -use tracing::{trace, warn}; +use tracing::{info, trace, warn}; use crate::mutate::Mutant; use crate::source::SourceFile; @@ -19,6 +19,10 @@ use crate::Result; /// Return only mutants to functions whose source was touched by this diff. pub fn diff_filter(mutants: Vec, diff_text: &str) -> Result> { // Flatten the error to a string because otherwise it references the diff, and can't be returned. + if diff_text.trim().is_empty() { + info!("diff file is empty; no mutants will match"); + return Ok(Vec::new()); + } let patches = Patch::from_multiple(diff_text).map_err(|err| anyhow!("Failed to parse diff: {err}"))?; check_diff_new_text_matches(&patches, &mutants)?; diff --git a/src/main.rs b/src/main.rs index f25cf4e5..52f577af 100644 --- a/src/main.rs +++ b/src/main.rs @@ -117,6 +117,10 @@ pub struct Args { #[arg(long, value_enum, default_value_t = BaselineStrategy::Run, help_heading = "Execution")] baseline: BaselineStrategy, + /// Turn off all rustc lints, so that denied warnings won't make mutants unviable. + #[arg(long, action = ArgAction::Set, help_heading = "Build")] + cap_lints: Option, + /// Print mutants that were caught by tests. #[arg(long, short = 'v', help_heading = "Output")] caught: bool, diff --git a/src/options.rs b/src/options.rs index 6902eccd..170851a7 100644 --- a/src/options.rs +++ b/src/options.rs @@ -24,6 +24,9 @@ pub struct Options { /// Run tests in an unmutated tree? pub baseline: BaselineStrategy, + /// Turn off all lints. + pub cap_lints: bool, + /// Don't run the tests, just see if each mutant builds. pub check_only: bool, @@ -196,6 +199,7 @@ impl Options { &config.additional_cargo_test_args, ), baseline: args.baseline, + cap_lints: args.cap_lints.unwrap_or(config.cap_lints), check_only: args.check, colors: args.colors, emit_json: args.json, @@ -270,6 +274,7 @@ mod test { let options = Options::new(&args, &Config::default()).unwrap(); assert!(!options.check_only); assert_eq!(options.test_tool, TestTool::Cargo); + assert!(!options.cap_lints); } #[test] @@ -358,9 +363,10 @@ mod test { } #[test] - fn test_tool_from_config() { + fn from_config() { let config = indoc! { r#" test_tool = "nextest" + cap_lints = true "#}; let mut config_file = NamedTempFile::new().unwrap(); config_file.write_all(config.as_bytes()).unwrap(); @@ -368,6 +374,7 @@ mod test { let config = Config::read_file(config_file.path()).unwrap(); let options = Options::new(&args, &config).unwrap(); assert_eq!(options.test_tool, TestTool::Nextest); + assert!(options.cap_lints); } #[test] diff --git a/src/process.rs b/src/process.rs index 3a6d5d75..8fc66828 100644 --- a/src/process.rs +++ b/src/process.rs @@ -2,21 +2,19 @@ //! Manage a subprocess, with polling, timeouts, termination, and so on. //! -//! This module is above the external `subprocess` crate, but has no -//! knowledge of whether it's running Cargo or potentially something else. -//! //! On Unix, the subprocess runs as its own process group, so that any //! grandchild processes are also signalled if it's interrupted. -use std::ffi::OsString; -use std::io::Read; +use std::ffi::OsStr; +#[cfg(unix)] +use std::os::unix::process::{CommandExt, ExitStatusExt}; +use std::process::{Child, Command, Stdio}; use std::thread::sleep; use std::time::{Duration, Instant}; -use anyhow::{anyhow, Context}; +use anyhow::{bail, Context}; use camino::Utf8Path; use serde::Serialize; -use subprocess::{ExitStatus, Popen, PopenConfig, Redirection}; use tracing::{debug, debug_span, error, span, trace, warn, Level}; use crate::console::Console; @@ -24,16 +22,13 @@ use crate::interrupt::check_interrupted; use crate::log_file::LogFile; use crate::Result; -/// How long to wait for metadata-only Cargo commands. -const METADATA_TIMEOUT: Duration = Duration::from_secs(20); - /// How frequently to check if a subprocess finished. const WAIT_POLL_INTERVAL: Duration = Duration::from_millis(50); pub struct Process { - child: Popen, + child: Child, start: Instant, - timeout: Duration, + timeout: Option, } impl Process { @@ -43,7 +38,7 @@ impl Process { argv: &[String], env: &[(String, String)], cwd: &Utf8Path, - timeout: Duration, + timeout: Option, log_file: &mut LogFile, console: &Console, ) -> Result { @@ -65,30 +60,27 @@ impl Process { argv: &[String], env: &[(String, String)], cwd: &Utf8Path, - timeout: Duration, + timeout: Option, log_file: &mut LogFile, ) -> Result { let start = Instant::now(); let quoted_argv = cheap_shell_quote(argv); log_file.message("ed_argv); debug!(%quoted_argv, "start process"); - let mut os_env = PopenConfig::current_env(); - os_env.extend( - env.iter() - .map(|(k, v)| (OsString::from(k), OsString::from(v))), - ); - let child = Popen::create( - argv, - PopenConfig { - stdin: Redirection::None, - stdout: Redirection::File(log_file.open_append()?), - stderr: Redirection::Merge, - cwd: Some(cwd.as_os_str().to_owned()), - env: Some(os_env), - ..setpgid_on_unix() - }, - ) - .with_context(|| format!("failed to spawn {}", argv.join(" ")))?; + let os_env = env.iter().map(|(k, v)| (OsStr::new(k), OsStr::new(v))); + let mut child = Command::new(&argv[0]); + child + .args(&argv[1..]) + .envs(os_env) + .stdin(Stdio::null()) + .stdout(log_file.open_append()?) + .stderr(log_file.open_append()?) + .current_dir(cwd); + #[cfg(unix)] + child.process_group(0); + let child = child + .spawn() + .with_context(|| format!("failed to spawn {}", argv.join(" ")))?; Ok(Process { child, start, @@ -99,22 +91,27 @@ impl Process { /// Check if the child process has finished; if so, return its status. #[mutants::skip] // It's hard to avoid timeouts if this never works... pub fn poll(&mut self) -> Result> { - let elapsed = self.start.elapsed(); - if elapsed > self.timeout { - debug!(?elapsed, "timeout, terminating child process...",); + if self.timeout.map_or(false, |t| self.start.elapsed() > t) { + debug!("timeout, terminating child process...",); self.terminate()?; Ok(Some(ProcessStatus::Timeout)) } else if let Err(e) = check_interrupted() { debug!("interrupted, terminating child process..."); self.terminate()?; Err(e) - } else if let Some(status) = self.child.poll() { - match status { - _ if status.success() => Ok(Some(ProcessStatus::Success)), - ExitStatus::Exited(code) => Ok(Some(ProcessStatus::Failure(code))), - ExitStatus::Signaled(signal) => Ok(Some(ProcessStatus::Signalled(signal))), - ExitStatus::Undetermined | ExitStatus::Other(_) => Ok(Some(ProcessStatus::Other)), + } else if let Some(status) = self.child.try_wait()? { + if let Some(code) = status.code() { + if code == 0 { + return Ok(Some(ProcessStatus::Success)); + } else { + return Ok(Some(ProcessStatus::Failure(code as u32))); + } } + #[cfg(unix)] + if let Some(signal) = status.signal() { + return Ok(Some(ProcessStatus::Signalled(signal as u8))); + } + Ok(Some(ProcessStatus::Other)) } else { Ok(None) } @@ -125,32 +122,15 @@ impl Process { /// Blocks until the subprocess is terminated and then returns the exit status. /// /// The status might not be Timeout if this raced with a normal exit. + #[mutants::skip] // would leak processes from tests if skipped fn terminate(&mut self) -> Result<()> { - let _span = span!(Level::DEBUG, "terminate_child", pid = self.child.pid()).entered(); + let _span = span!(Level::DEBUG, "terminate_child", pid = self.child.id()).entered(); debug!("terminating child process"); terminate_child_impl(&mut self.child)?; trace!("wait for child after termination"); - if let Some(exit_status) = self - .child - .wait_timeout(Duration::from_secs(10)) - .context("wait for child after terminating pgroup")? - { - debug!("terminated child exit status {exit_status:?}"); - } else { - warn!("child did not exit after termination"); - let kill_result = self.child.kill(); - warn!("force kill child: {:?}", kill_result); - if kill_result.is_ok() { - if let Ok(Some(exit_status)) = self - .child - .wait_timeout(Duration::from_secs(10)) - .context("wait for child after force kill") - { - debug!("force kill child exit status {exit_status:?}"); - } else { - warn!("child did not exit after force kill"); - } - } + match self.child.wait() { + Err(err) => debug!(?err, "Failed to wait for child after termination"), + Ok(exit) => debug!("terminated child exit status {exit:?}"), } Ok(()) } @@ -159,11 +139,11 @@ impl Process { #[cfg(unix)] #[allow(unknown_lints, clippy::needless_pass_by_ref_mut)] // To match Windows #[mutants::skip] // hard to exercise the ESRCH edge case -fn terminate_child_impl(child: &mut Popen) -> Result<()> { +fn terminate_child_impl(child: &mut Child) -> Result<()> { use nix::errno::Errno; use nix::sys::signal::{killpg, Signal}; - let pid = nix::unistd::Pid::from_raw(child.pid().expect("child has a pid").try_into().unwrap()); + let pid = nix::unistd::Pid::from_raw(child.id().try_into().unwrap()); match killpg(pid, Signal::SIGTERM) { Ok(()) => Ok(()), Err(Errno::ESRCH) => { @@ -175,22 +155,15 @@ fn terminate_child_impl(child: &mut Popen) -> Result<()> { Err(errno) => { let message = format!("failed to terminate child: {errno}"); warn!("{}", message); - Err(anyhow!(message)) + bail!(message); } } } -// We do not yet have a way to mutate this only on Windows, and I mostly test on Unix, so it's just skipped for now. -#[mutants::skip] -#[cfg(not(unix))] -fn terminate_child_impl(child: &mut Popen) -> Result<()> { - if let Err(e) = child.terminate() { - // most likely we raced and it's already gone - let message = format!("failed to terminate child: {}", e); - warn!("{}", message); - return Err(anyhow!(message)); - } - Ok(()) +#[cfg(windows)] +#[mutants::skip] // hard to exercise the ESRCH edge case +fn terminate_child_impl(child: &mut Child) -> Result<()> { + child.kill().context("Kill child") } /// The result of running a single child process. @@ -222,21 +195,6 @@ impl ProcessStatus { } } -#[cfg(unix)] -#[mutants::skip] // It's hard to observe if this is broken: we'd expect children to leak. -fn setpgid_on_unix() -> PopenConfig { - PopenConfig { - setpgid: true, - ..Default::default() - } -} - -#[mutants::skip] // Has no effect, so can't be tested. -#[cfg(not(unix))] -fn setpgid_on_unix() -> PopenConfig { - Default::default() -} - /// Run a command and return its stdout output as a string. /// /// If the command exits non-zero, the error includes any messages it wrote to stderr. @@ -246,46 +204,18 @@ pub fn get_command_output(argv: &[&str], cwd: &Utf8Path) -> Result { // TODO: Perhaps redirect to files so this doesn't jam if there's a lot of output. // For the commands we use this for today, which only produce small output, it's OK. let _span = debug_span!("get_command_output", argv = ?argv).entered(); - let mut child = Popen::create( - argv, - PopenConfig { - stdin: Redirection::None, - stdout: Redirection::Pipe, - stderr: Redirection::Pipe, - cwd: Some(cwd.as_os_str().to_owned()), - ..Default::default() - }, - ) - .with_context(|| format!("failed to spawn {argv:?}"))?; - match child.wait_timeout(METADATA_TIMEOUT) { - Err(e) => { - let message = format!("failed to wait for {argv:?}: {e}"); - return Err(anyhow!(message)); - } - Ok(None) => { - let message = format!("{argv:?} timed out",); - return Err(anyhow!(message)); - } - Ok(Some(status)) if status.success() => {} - Ok(Some(status)) => { - let mut stderr = String::new(); - let _ = child - .stderr - .take() - .expect("child has stderr") - .read_to_string(&mut stderr); - error!("child failed with status {status:?}: {stderr}"); - let message = format!("{argv:?} failed with status {status:?}: {stderr}"); - return Err(anyhow!(message)); - } + let output = Command::new(argv[0]) + .args(&argv[1..]) + .stderr(Stdio::inherit()) + .current_dir(cwd) + .output() + .with_context(|| format!("failed to spawn {argv:?}"))?; + let exit = output.status; + if !exit.success() { + error!(?exit, "Child failed"); + bail!("Child failed with status {exit:?}: {argv:?}"); } - let mut stdout = String::new(); - child - .stdout - .take() - .expect("child has stdout") - .read_to_string(&mut stdout) - .context("failed to read child stdout")?; + let stdout = String::from_utf8(output.stdout).context("Child output is not UTF-8")?; debug!("output: {}", stdout.trim()); Ok(stdout) } diff --git a/src/timeouts.rs b/src/timeouts.rs index 4685809f..2f4fa743 100644 --- a/src/timeouts.rs +++ b/src/timeouts.rs @@ -13,15 +13,15 @@ use crate::{ #[derive(Debug, Copy, Clone)] pub struct Timeouts { - pub build: Duration, - pub test: Duration, + pub build: Option, + pub test: Option, } impl Timeouts { pub fn for_baseline(options: &Options) -> Timeouts { Timeouts { - test: options.test_timeout.unwrap_or(Duration::MAX), - build: options.build_timeout.unwrap_or(Duration::MAX), + test: options.test_timeout, + build: None, } } @@ -51,67 +51,51 @@ fn warn_fallback_timeout(phase_name: &str, option: &str) { warn!("An explicit {phase_name} timeout is recommended when using {option}; using {FALLBACK_TIMEOUT_SECS} seconds by default"); } -fn phase_timeout( - phase: Phase, - explicit_timeout: Option, - baseline_duration: Option, - minimum: Duration, - multiplier: f64, - options: &Options, -) -> Duration { - if let Some(timeout) = explicit_timeout { - return timeout; - } - match baseline_duration { - Some(_) if options.in_place && phase != Phase::Test => { - warn_fallback_timeout(phase.name(), "--in-place"); - Duration::from_secs(FALLBACK_TIMEOUT_SECS) - } - Some(baseline_duration) => { - let timeout = max( - minimum, - Duration::from_secs((baseline_duration.as_secs_f64() * multiplier).ceil() as u64), +fn test_timeout(baseline_duration: Option, options: &Options) -> Option { + if let Some(explicit) = options.test_timeout { + Some(explicit) + } else if let Some(baseline_duration) = baseline_duration { + let timeout = max( + options.minimum_test_timeout, + Duration::from_secs( + (baseline_duration.as_secs_f64() * options.test_timeout_multiplier.unwrap_or(5.0)) + .ceil() as u64, + ), + ); + if options.show_times { + info!( + "Auto-set test timeout to {}", + humantime::format_duration(timeout) ); + } + Some(timeout) + } else { + warn_fallback_timeout("test", "--baseline=skip"); + Some(Duration::from_secs(FALLBACK_TIMEOUT_SECS)) + } +} + +fn build_timeout(baseline_duration: Option, options: &Options) -> Option { + if let Some(t) = options.build_timeout { + Some(t) + } else if let Some(baseline) = baseline_duration { + if let Some(multiplier) = options.build_timeout_multiplier { + let timeout = Duration::from_secs_f64(baseline.as_secs_f64() * multiplier); if options.show_times { info!( - "Auto-set {} timeout to {}", - phase.name(), + "Auto-set build timeout to {}", humantime::format_duration(timeout) ); } - timeout - } - None => { - warn_fallback_timeout(phase.name(), "--baseline=skip"); - Duration::from_secs(FALLBACK_TIMEOUT_SECS) + Some(timeout) + } else { + None } + } else { + None } } -fn test_timeout(baseline_duration: Option, options: &Options) -> Duration { - phase_timeout( - Phase::Test, - options.test_timeout, - baseline_duration, - options.minimum_test_timeout, - options.test_timeout_multiplier.unwrap_or(5.0), - options, - ) -} - -fn build_timeout(baseline_duration: Option, options: &Options) -> Duration { - phase_timeout( - Phase::Build, - options.build_timeout, - baseline_duration, - Duration::from_secs(20), - options - .build_timeout_multiplier - .unwrap_or(2.0 * options.jobs.unwrap_or(1) as f64), - options, - ) -} - #[cfg(test)] mod test { use std::str::FromStr; @@ -130,7 +114,7 @@ mod test { assert_eq!(options.test_timeout_multiplier, Some(1.5)); assert_eq!( test_timeout(Some(Duration::from_secs(40)), &options), - Duration::from_secs(60), + Some(Duration::from_secs(60)), ); } @@ -141,7 +125,7 @@ mod test { assert_eq!( test_timeout(Some(Duration::from_secs(40)), &options), - Duration::from_secs(60), + Some(Duration::from_secs(60)), ); } @@ -153,28 +137,18 @@ mod test { assert_eq!(options.build_timeout_multiplier, Some(1.5)); assert_eq!( build_timeout(Some(Duration::from_secs(40)), &options), - Duration::from_secs(60), - ); - } - - #[test] - fn mutant_build_timeout_with_multiple_jobs() { - let args = Args::parse_from(["mutants", "--jobs=4"]); - let options = Options::new(&args, &Config::default()).unwrap(); - assert_eq!( - build_timeout(Some(Duration::from_secs(40)), &options), - Duration::from_secs(40 * 2 * 4), + Some(Duration::from_secs(60)), ); } #[test] fn build_timeout_is_affected_by_in_place_build() { - let args = Args::parse_from(["mutants", "--build-timeout-multiplier", "1.5", "--in-place"]); + let args = Args::parse_from(["mutants", "--build-timeout-multiplier", "5", "--in-place"]); let options = Options::new(&args, &Config::default()).unwrap(); assert_eq!( build_timeout(Some(Duration::from_secs(40)), &options), - Duration::from_secs(300), + Some(Duration::from_secs(40 * 5)) ); } @@ -190,7 +164,7 @@ mod test { assert_eq!(options.test_timeout_multiplier, Some(2.0)); assert_eq!( test_timeout(Some(Duration::from_secs(42)), &options), - Duration::from_secs(42 * 2), + Some(Duration::from_secs(42 * 2)), ); } @@ -206,7 +180,7 @@ mod test { assert_eq!(options.build_timeout_multiplier, Some(2.0)); assert_eq!( build_timeout(Some(Duration::from_secs(42)), &options), - Duration::from_secs(42 * 2), + Some(Duration::from_secs(42 * 2)), ); } @@ -218,7 +192,7 @@ mod test { assert_eq!(options.test_timeout_multiplier, None); assert_eq!( test_timeout(Some(Duration::from_secs(42)), &options), - Duration::from_secs(42 * 5), + Some(Duration::from_secs(42 * 5)), ); } @@ -228,10 +202,7 @@ mod test { let options = Options::new(&args, &Config::default()).unwrap(); assert_eq!(options.build_timeout_multiplier, None); - assert_eq!( - build_timeout(Some(Duration::from_secs(42)), &options), - Duration::from_secs(42 * 2), - ); + assert_eq!(build_timeout(Some(Duration::from_secs(42)), &options), None,); } #[test] @@ -251,24 +222,22 @@ mod test { } #[test] - fn timeout_multiplier_default_with_baseline_skip() { - // The --baseline option is not used to set the timeout but it's - // indicative of the realistic situation. - let args = Args::parse_from(["mutants", "--baseline", "skip"]); + fn no_default_build_timeout() { + let args = Args::parse_from(["mutants"]); let options = Options::new(&args, &Config::default()).unwrap(); - assert_eq!(options.test_timeout_multiplier, None); - assert_eq!(test_timeout(None, &options), Duration::from_secs(300),); + assert_eq!(options.build_timeout, None); } #[test] - fn build_timeout_multiplier_default_with_baseline_skip() { + fn timeout_multiplier_default_with_baseline_skip() { // The --baseline option is not used to set the timeout but it's // indicative of the realistic situation. let args = Args::parse_from(["mutants", "--baseline", "skip"]); let options = Options::new(&args, &Config::default()).unwrap(); - assert_eq!(options.build_timeout_multiplier, None); - assert_eq!(build_timeout(None, &options), Duration::from_secs(300),); + assert_eq!(options.test_timeout_multiplier, None); + assert_eq!(test_timeout(None, &options), Some(Duration::from_secs(300))); + assert_eq!(build_timeout(None, &options), None); } } diff --git a/tests/in_diff.rs b/tests/in_diff.rs index 6f0af9cd..81a0eb51 100644 --- a/tests/in_diff.rs +++ b/tests/in_diff.rs @@ -4,6 +4,7 @@ use std::fs::read_to_string; use std::io::Write; use indoc::indoc; +use predicates::prelude::predicate; use similar::TextDiff; use tempfile::NamedTempFile; @@ -68,6 +69,22 @@ fn list_mutants_changed_in_diff1() { ); } +#[test] +fn empty_diff_is_not_an_error_and_matches_nothing() { + let diff_file = NamedTempFile::new().unwrap(); + let tmp = copy_of_testdata("diff1"); + run() + .args(["mutants", "--no-shuffle", "-d"]) + .arg(tmp.path()) + .arg("--in-diff") + .arg(diff_file.path()) + .arg("--list") + .assert() + .success() + .stdout("") + .stderr(predicate::str::contains("diff file is empty")); +} + /// If the text in the diff doesn't look like the tree then error out. #[test] fn mismatched_diff_causes_error() { diff --git a/tests/main.rs b/tests/main.rs index 9840e83c..e5f21280 100644 --- a/tests/main.rs +++ b/tests/main.rs @@ -4,7 +4,6 @@ use std::env; use std::fs::{self, read_dir, read_to_string}; -use std::io::Read; use std::path::Path; use std::thread::sleep; use std::time::Duration; @@ -15,7 +14,6 @@ use predicate::str::{contains, is_match}; use predicates::prelude::*; use pretty_assertions::assert_eq; -use subprocess::{Popen, PopenConfig, Redirection}; use tempfile::TempDir; mod util; @@ -164,9 +162,6 @@ fn test_small_well_tested_tree_with_baseline_skip() { predicate::str::contains( "An explicit test timeout is recommended when using --baseline=skip", ) - .and(predicate::str::contains( - "An explicit build timeout is recommended when using --baseline=skip", - )) .and(predicate::str::contains("Unmutated baseline in").not()), ); assert!(!tmp_src_dir @@ -409,11 +404,11 @@ fn small_well_tested_mutants_with_cargo_arg_release() { println!("{}", baseline_log_path.display()); let log_content = fs::read_to_string(baseline_log_path).unwrap(); println!("{log_content}"); - regex::Regex::new(r"cargo.* test --no-run --manifest-path .* --release") + regex::Regex::new(r"cargo.* test --no-run --verbose --manifest-path .* --release") .unwrap() .captures(&log_content) .unwrap(); - regex::Regex::new(r"cargo.* test --manifest-path .* --release") + regex::Regex::new(r"cargo.* test --verbose --manifest-path .* --release") .unwrap() .captures(&log_content) .unwrap(); @@ -639,25 +634,21 @@ fn timeout_when_unmutated_tree_test_hangs() { #[cfg(unix)] // Should in principle work on Windows, but does not at the moment. fn interrupt_caught_and_kills_children() { // Test a tree that has enough tests that we'll probably kill it before it completes. + + use std::process::{Command, Stdio}; + + use nix::libc::pid_t; + use nix::sys::signal::{kill, SIGTERM}; + use nix::unistd::Pid; + let tmp_src_dir = copy_of_testdata("well_tested"); // We can't use `assert_cmd` `timeout` here because that sends the child a `SIGKILL`, // which doesn't give it a chance to clean up. And, `std::process::Command` only - // has an abrupt kill. But `subprocess` has a gentle `terminate` method. + // has an abrupt kill. // Drop RUST_BACKTRACE because the traceback mentions "panic" handler functions // and we want to check that the process does not panic. - let env = Some( - env::vars_os() - .filter(|(k, _)| k != "RUST_BACKTRACE") - .collect::>(), - ); - let config = PopenConfig { - stdout: Redirection::Pipe, - stderr: Redirection::Pipe, - cwd: Some(tmp_src_dir.path().as_os_str().to_owned()), - env, - ..Default::default() - }; + // Skip baseline because firstly it should already pass but more importantly // #333 exhibited only during non-baseline scenarios. let args = [ @@ -669,43 +660,32 @@ fn interrupt_caught_and_kills_children() { ]; println!("Running: {args:?}"); - let mut child = Popen::create(&args, config).expect("spawn child"); - // TODO: Watch the output, maybe using `subprocess`, rather than just guessing how long it needs. + let mut child = Command::new(args[0]) + .args(&args[1..]) + .current_dir(&tmp_src_dir) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()) + .env_remove("RUST_BACKTRACE") + .spawn() + .expect("spawn child"); + sleep(Duration::from_secs(2)); // Let it get started - assert!(child.poll().is_none(), "child exited early"); + assert!( + child.try_wait().expect("try to wait for child").is_none(), + "child exited early" + ); - println!("Sending terminate to cargo-mutants..."); - child.terminate().expect("terminate child"); + println!("Sending SIGTERM to cargo-mutants..."); + kill(Pid::from_raw(child.id() as pid_t), SIGTERM).expect("send SIGTERM"); println!("Wait for cargo-mutants to exit..."); - match child.wait_timeout(Duration::from_secs(4)) { - Err(e) => panic!("failed to wait for child: {e}"), - Ok(None) => { - println!("child did not exit after interrupt"); - child.kill().expect("kill child"); - child.wait().expect("wait for child after kill"); - } - Ok(Some(status)) => { - println!("cargo-mutants exited with status: {status:?}"); - } - } + let output = child + .wait_with_output() + .expect("wait for child after SIGTERM"); - let mut stdout = String::new(); - child - .stdout - .as_mut() - .unwrap() - .read_to_string(&mut stdout) - .expect("read stdout"); + let stdout = String::from_utf8_lossy(&output.stdout); + let stderr = String::from_utf8_lossy(&output.stderr); println!("stdout:\n{stdout}"); - - let mut stderr = String::new(); - child - .stderr - .as_mut() - .unwrap() - .read_to_string(&mut stderr) - .expect("read stderr"); println!("stderr:\n{stderr}"); assert!(stderr.contains("interrupted")); @@ -746,14 +726,22 @@ fn interrupt_caught_and_kills_children() { fn mutants_causing_tests_to_hang_are_stopped_by_manual_timeout() { let tmp_src_dir = copy_of_testdata("hang_when_mutated"); // Also test that it accepts decimal seconds - run() + let out = run() .arg("mutants") - .args(["-t", "8.1"]) + .args(["-t", "8.1", "--build-timeout=15.5"]) .current_dir(tmp_src_dir.path()) .env_remove("RUST_BACKTRACE") .timeout(OUTER_TIMEOUT) .assert() .code(3); // exit_code::TIMEOUT + println!( + "output:\n{}", + String::from_utf8_lossy(&out.get_output().stdout) + ); + let unviable_txt = read_to_string(tmp_src_dir.path().join("mutants.out/unviable.txt")) + .expect("read timeout.txt"); + let caught_txt = read_to_string(tmp_src_dir.path().join("mutants.out/caught.txt")) + .expect("read timeout.txt"); let timeout_txt = read_to_string(tmp_src_dir.path().join("mutants.out/timeout.txt")) .expect("read timeout.txt"); assert!( @@ -761,10 +749,9 @@ fn mutants_causing_tests_to_hang_are_stopped_by_manual_timeout() { "expected text not found in:\n{timeout_txt}" ); assert!( - timeout_txt.contains("replace should_stop_const -> bool with false"), - "expected text not found in:\n{timeout_txt}" + unviable_txt.contains("replace should_stop_const -> bool with false"), + "expected text not found in:\n{unviable_txt}" ); - let caught_txt = read_to_string(tmp_src_dir.path().join("mutants.out/caught.txt")).unwrap(); assert!( caught_txt.contains("replace should_stop -> bool with true"), "expected text not found in:\n{caught_txt}" @@ -778,7 +765,7 @@ fn mutants_causing_tests_to_hang_are_stopped_by_manual_timeout() { .expect("read outcomes.json") .parse() .expect("parse outcomes.json"); - assert_eq!(outcomes_json["timeout"], 2); + assert_eq!(outcomes_json["timeout"], 1); let phases_for_const_fn = outcomes_json["outcomes"] .as_array() @@ -796,11 +783,15 @@ fn mutants_causing_tests_to_hang_are_stopped_by_manual_timeout() { } #[test] -fn mutants_causing_check_to_timeout_are_stopped_by_manual_timeout() { +fn hang_avoided_by_build_timeout_with_cap_lints() { let tmp_src_dir = copy_of_testdata("hang_when_mutated"); run() .arg("mutants") - .args(["--check", "--build-timeout=4"]) + .args([ + "--build-timeout-multiplier=4", + "--regex=const", + "--cap-lints=true", + ]) .current_dir(tmp_src_dir.path()) .env_remove("RUST_BACKTRACE") .timeout(OUTER_TIMEOUT) @@ -814,6 +805,20 @@ fn mutants_causing_check_to_timeout_are_stopped_by_manual_timeout() { ); } +#[test] +fn constfn_mutation_passes_check() { + let tmp_src_dir = copy_of_testdata("hang_when_mutated"); + let cmd = run() + .arg("mutants") + .args(["--check", "--build-timeout=4"]) + .current_dir(tmp_src_dir.path()) + .env_remove("RUST_BACKTRACE") + .timeout(OUTER_TIMEOUT) + .assert() + .code(0); + println!("{}", String::from_utf8_lossy(&cmd.get_output().stdout)); +} + #[test] fn log_file_names_are_short_and_dont_collide() { // The "well_tested" tree can generate multiple mutants from single lines. They get distinct file names. diff --git a/tests/workspace.rs b/tests/workspace.rs index 63b5fbce..824f5308 100644 --- a/tests/workspace.rs +++ b/tests/workspace.rs @@ -132,7 +132,7 @@ fn workspace_tree_is_well_tested() { assert_eq!(baseline_phases[0]["process_status"], "Success"); assert_eq!( baseline_phases[0]["argv"].as_array().unwrap().iter().map(|v| v.as_str().unwrap()).skip(1).collect_vec().join(" "), - "test --no-run --package cargo_mutants_testdata_workspace_utils --package main --package main2" + "test --no-run --verbose --package cargo_mutants_testdata_workspace_utils --package main --package main2" ); assert_eq!(baseline_phases[1]["process_status"], "Success"); assert_eq!( @@ -144,7 +144,7 @@ fn workspace_tree_is_well_tested() { .skip(1) .collect_vec() .join(" "), - "test --package cargo_mutants_testdata_workspace_utils --package main --package main2" + "test --verbose --package cargo_mutants_testdata_workspace_utils --package main --package main2" ); } @@ -158,13 +158,13 @@ fn workspace_tree_is_well_tested() { assert_eq!(mutant_phases.len(), 2); assert_eq!(mutant_phases[0]["process_status"], "Success"); assert_eq!( - mutant_phases[0]["argv"].as_array().unwrap()[1..=3], - ["test", "--no-run", "--manifest-path"] + mutant_phases[0]["argv"].as_array().unwrap()[1..=2], + ["test", "--no-run"] ); assert_eq!(mutant_phases[1]["process_status"], json!({"Failure": 101})); assert_eq!( - mutant_phases[1]["argv"].as_array().unwrap()[1..=2], - ["test", "--manifest-path"], + mutant_phases[1]["argv"].as_array().unwrap()[1..=3], + ["test", "--verbose", "--manifest-path"], ); } { @@ -176,7 +176,7 @@ fn workspace_tree_is_well_tested() { assert_eq!(baseline_phases[0]["process_status"], "Success"); assert_eq!( baseline_phases[0]["argv"].as_array().unwrap()[1..].iter().map(|v| v.as_str().unwrap()).join(" "), - "test --no-run --package cargo_mutants_testdata_workspace_utils --package main --package main2", + "test --no-run --verbose --package cargo_mutants_testdata_workspace_utils --package main --package main2", ); assert_eq!(baseline_phases[1]["process_status"], "Success"); assert_eq!( @@ -184,7 +184,7 @@ fn workspace_tree_is_well_tested() { .iter() .map(|v| v.as_str().unwrap()) .join(" "), - "test --package cargo_mutants_testdata_workspace_utils --package main --package main2", + "test --verbose --package cargo_mutants_testdata_workspace_utils --package main --package main2", ); } }