From d907440d331e966a0cc6fbb16661337102f2072f Mon Sep 17 00:00:00 2001 From: Martin Pool Date: Tue, 2 Jan 2024 10:16:36 -0800 Subject: [PATCH 01/18] Don't track vscode settings in git --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index ae711fe2..c2966ec9 100644 --- a/.gitignore +++ b/.gitignore @@ -3,3 +3,4 @@ mutants.out mutants.out.old .cargo/config.toml wiki +.vscode/ From f97da43289e474532d4f3d237199982176e32182 Mon Sep 17 00:00:00 2001 From: Martin Pool Date: Tue, 2 Jan 2024 14:58:27 -0800 Subject: [PATCH 02/18] Add --test-tool=nextest Fixes Support `cargo nextest` #85 (but still needs docs and an integration test) --- Cargo.lock | 29 ++++++++ Cargo.toml | 1 + NEWS.md | 2 + src/cargo.rs | 16 ++++- src/config.rs | 10 ++- src/main.rs | 16 +++-- src/options.rs | 71 +++++++++++++++++-- ..._expected_mutants_for_own_source_tree.snap | 4 -- 8 files changed, 127 insertions(+), 22 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 2b44ce51..976d0b96 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -191,6 +191,7 @@ dependencies = [ "serde", "serde_json", "similar", + "strum", "subprocess", "syn", "tempfile", @@ -958,6 +959,12 @@ dependencies = [ "windows-sys 0.48.0", ] +[[package]] +name = "rustversion" +version = "1.0.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7ffc183a10b4478d04cbbbfc96d0873219d962dd5accaff2ffbd4ceb7df837f4" + [[package]] name = "ryu" version = "1.0.15" @@ -1055,6 +1062,28 @@ version = "0.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "73473c0e59e6d5812c5dfe2a064a6444949f089e20eec9a2e5506596494e4623" +[[package]] +name = "strum" +version = "0.25.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "290d54ea6f91c969195bdbcd7442c8c2a2ba87da8bf60a7ee86a235d4bc1e125" +dependencies = [ + "strum_macros", +] + +[[package]] +name = "strum_macros" +version = "0.25.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "23dc1fa9ac9c169a78ba62f0b841814b7abae11bdd047b9c58f893439e309ea0" +dependencies = [ + "heck", + "proc-macro2", + "quote", + "rustversion", + "syn", +] + [[package]] name = "subprocess" version = "0.2.9" diff --git a/Cargo.toml b/Cargo.toml index 68886ef1..974708e8 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -60,6 +60,7 @@ quote = "1.0" regex = "1.10" serde_json = "1" similar = "2.0" +strum = { version = "0.25", features = ["derive"] } subprocess = "0.2.8" tempfile = "3.2" time = "0.3" diff --git a/NEWS.md b/NEWS.md index 05b670ba..a5977eaf 100644 --- a/NEWS.md +++ b/NEWS.md @@ -2,6 +2,8 @@ ## Unreleased +- New! `cargo mutants --test-tool nextest`, or `test_tool = "nextest"` in `.cargo/mutants.toml` runs tests under [Nextest](https://nextest.rs/). Some trees have tests that only work under Nextest, and this allows them to be tested. In other cases Nextest may be significantly faster, because it will exit soon after the first test failure. + - Fixed: Fixed spurious "Patch input contains repeated filenames" error when `--in-diff` is given a patch that deletes multiple files. ## 23.12.2 diff --git a/src/cargo.rs b/src/cargo.rs index b4ee142b..fae22e06 100644 --- a/src/cargo.rs +++ b/src/cargo.rs @@ -1,4 +1,4 @@ -// Copyright 2021-2023 Martin Pool +// Copyright 2021-2024 Martin Pool //! Run Cargo as a subprocess, including timeouts and propagating signals. @@ -10,6 +10,7 @@ use camino::Utf8Path; use itertools::Itertools; use tracing::{debug, debug_span}; +use crate::options::TestTool; use crate::outcome::PhaseResult; use crate::package::Package; use crate::process::Process; @@ -63,8 +64,17 @@ fn cargo_argv( phase: Phase, options: &Options, ) -> Vec { - let mut cargo_args = vec![cargo_bin(), phase.name().to_string()]; - if phase == Phase::Check || phase == Phase::Build { + let mut cargo_args = vec![cargo_bin()]; + if phase == Phase::Test { + match &options.test_tool { + TestTool::Cargo => cargo_args.push("test".to_string()), + TestTool::Nextest => { + cargo_args.push("nextest".to_string()); + cargo_args.push("run".to_string()); + } + } + } else { + cargo_args.push(phase.name().to_string()); cargo_args.push("--tests".to_string()); } if let Some([package]) = packages { diff --git a/src/config.rs b/src/config.rs index efcd4d62..b0cb8dfb 100644 --- a/src/config.rs +++ b/src/config.rs @@ -1,4 +1,4 @@ -// Copyright 2022-2023 Martin Pool. +// Copyright 2022-2024 Martin Pool. //! `.cargo/mutants.toml` configuration file. //! @@ -10,11 +10,13 @@ use std::default::Default; use std::fs::read_to_string; +use std::path::Path; use anyhow::Context; use camino::Utf8Path; use serde::Deserialize; +use crate::options::TestTool; use crate::Result; /// Configuration read from a config file. @@ -40,10 +42,12 @@ pub struct Config { pub additional_cargo_test_args: Vec, /// Minimum test timeout, in seconds, as a floor on the autoset value. pub minimum_test_timeout: Option, + /// Choice of test tool: cargo or nextest. + pub test_tool: Option, } impl Config { - pub fn read_file(path: &Utf8Path) -> Result { + pub fn read_file(path: &Path) -> Result { let toml = read_to_string(path).with_context(|| format!("read config {path:?}"))?; toml::de::from_str(&toml).with_context(|| format!("parse toml from {path:?}")) } @@ -53,7 +57,7 @@ impl Config { pub fn read_tree_config(workspace_dir: &Utf8Path) -> Result { let path = workspace_dir.join(".cargo").join("mutants.toml"); if path.exists() { - Config::read_file(&path) + Config::read_file(path.as_ref()) } else { Ok(Config::default()) } diff --git a/src/main.rs b/src/main.rs index a8966b89..4d44d5f5 100644 --- a/src/main.rs +++ b/src/main.rs @@ -1,4 +1,4 @@ -// Copyright 2021-2023 Martin Pool +// Copyright 2021-2024 Martin Pool //! `cargo-mutants`: Find test gaps by inserting bugs. @@ -51,7 +51,7 @@ use crate::list::{list_files, list_mutants, FmtToIoWrite}; use crate::log_file::LogFile; use crate::manifest::fix_manifest; use crate::mutate::{Genre, Mutant}; -use crate::options::Options; +use crate::options::{Options, TestTool}; use crate::outcome::{Phase, ScenarioOutcome}; use crate::scenario::Scenario; use crate::shard::Shard; @@ -190,6 +190,10 @@ struct Args { #[arg(long, short = 'D')] in_diff: Option, + /// minimum timeout for tests, in seconds, as a lower bound on the auto-set time. + #[arg(long, env = "CARGO_MUTANTS_MINIMUM_TEST_TIMEOUT")] + minimum_test_timeout: Option, + /// only test mutants from these packages. #[arg(id = "package", long, short = 'p')] mutate_packages: Vec, @@ -206,14 +210,14 @@ struct Args { #[arg(long)] shard: Option, + /// tool used to run test suites: cargo or nextest. + #[arg(long)] + test_tool: Option, + /// maximum run time for all cargo commands, in seconds. #[arg(long, short = 't')] timeout: Option, - /// minimum timeout for tests, in seconds, as a lower bound on the auto-set time. - #[arg(long, env = "CARGO_MUTANTS_MINIMUM_TEST_TIMEOUT")] - minimum_test_timeout: Option, - /// print mutations that failed to check or build. #[arg(long, short = 'V')] unviable: bool, diff --git a/src/options.rs b/src/options.rs index 0fcaf3f1..54a971f9 100644 --- a/src/options.rs +++ b/src/options.rs @@ -1,4 +1,4 @@ -// Copyright 2021-2023 Martin Pool +// Copyright 2021-2024 Martin Pool //! Global in-process options for experimenting on mutants. //! @@ -11,9 +11,12 @@ use anyhow::Context; use camino::Utf8PathBuf; use globset::{Glob, GlobSet, GlobSetBuilder}; use regex::RegexSet; +use serde::Deserialize; +use strum::{Display, EnumString}; use tracing::warn; -use crate::{config::Config, *}; +use crate::config::Config; +use crate::*; /// Options for mutation testing, based on both command-line arguments and the /// config file. @@ -90,8 +93,25 @@ pub struct Options { /// Emit diffs showing just what changed. pub emit_diffs: bool, + + /// The tool to use to run tests. + pub test_tool: TestTool, +} + +/// Choice of tool to use to run tests. +#[derive(Debug, Default, Clone, Copy, PartialEq, Eq, EnumString, Display, Deserialize)] +#[strum(serialize_all = "snake_case")] +#[serde(rename_all = "snake_case")] +pub enum TestTool { + /// Use `cargo test`, the default. + #[default] + Cargo, + + /// Use `cargo nextest`. + Nextest, } +/// Join two slices into a new vector. fn join_slices(a: &[String], b: &[String]) -> Vec { let mut v = Vec::with_capacity(a.len() + b.len()); v.extend_from_slice(a); @@ -119,6 +139,9 @@ impl Options { &config.additional_cargo_test_args, ), check_only: args.check, + colors: true, // TODO: An option for this and use CLICOLORS. + emit_json: args.json, + emit_diffs: args.diff, error_values: join_slices(&args.error, &config.error_values), examine_names: RegexSet::new(or_slices(&args.examine_re, &config.examine_re)) .context("Failed to compile examine_re regex")?, @@ -129,6 +152,7 @@ impl Options { gitignore: args.gitignore, jobs: args.jobs, leak_dirs: args.leak_dirs, + minimum_test_timeout, output_in_dir: args.output.clone(), print_caught: args.caught, print_unviable: args.unviable, @@ -137,10 +161,7 @@ impl Options { show_times: !args.no_times, show_all_logs: args.all_logs, test_timeout: args.timeout.map(Duration::from_secs_f64), - emit_json: args.json, - colors: true, // TODO: An option for this and use CLICOLORS. - emit_diffs: args.diff, - minimum_test_timeout, + test_tool: args.test_tool.or(config.test_tool).unwrap_or_default(), }; options.error_values.iter().for_each(|e| { if e.starts_with("Err(") { @@ -182,3 +203,41 @@ fn build_glob_set, I: IntoIterator>( } Ok(Some(builder.build()?)) } + +#[cfg(test)] +mod test { + use std::io::Write; + + use indoc::indoc; + use tempfile::NamedTempFile; + + use super::*; + + #[test] + fn default_options() { + let args = Args::parse_from(["mutants"]); + let options = Options::new(&args, &Config::default()).unwrap(); + assert!(!options.check_only); + assert_eq!(options.test_tool, TestTool::Cargo); + } + + #[test] + fn options_from_test_tool_arg() { + let args = Args::parse_from(["mutants", "--test-tool", "nextest"]); + let options = Options::new(&args, &Config::default()).unwrap(); + assert_eq!(options.test_tool, TestTool::Nextest); + } + + #[test] + fn test_tool_from_config() { + let config = indoc! { r#" + test_tool = "nextest" + "#}; + let mut config_file = NamedTempFile::new().unwrap(); + config_file.write_all(config.as_bytes()).unwrap(); + let args = Args::parse_from(["mutants"]); + let config = Config::read_file(config_file.path()).unwrap(); + let options = Options::new(&args, &config).unwrap(); + assert_eq!(options.test_tool, TestTool::Nextest); + } +} diff --git a/src/snapshots/cargo_mutants__visit__test__expected_mutants_for_own_source_tree.snap b/src/snapshots/cargo_mutants__visit__test__expected_mutants_for_own_source_tree.snap index 4db3dc38..813a96d9 100644 --- a/src/snapshots/cargo_mutants__visit__test__expected_mutants_for_own_source_tree.snap +++ b/src/snapshots/cargo_mutants__visit__test__expected_mutants_for_own_source_tree.snap @@ -14,10 +14,6 @@ src/cargo.rs: replace cargo_bin -> String with "xyzzy".into() src/cargo.rs: replace cargo_argv -> Vec with vec![] src/cargo.rs: replace cargo_argv -> Vec with vec![String::new()] src/cargo.rs: replace cargo_argv -> Vec with vec!["xyzzy".into()] -src/cargo.rs: replace || with && in cargo_argv -src/cargo.rs: replace || with == in cargo_argv -src/cargo.rs: replace || with != in cargo_argv -src/cargo.rs: replace == with != in cargo_argv src/cargo.rs: replace == with != in cargo_argv src/cargo.rs: replace == with != in cargo_argv src/cargo.rs: replace rustflags -> String with String::new() From ab4046bca238be1507d7b5cdae7f2228e9103241 Mon Sep 17 00:00:00 2001 From: Martin Pool Date: Tue, 2 Jan 2024 15:01:06 -0800 Subject: [PATCH 03/18] Mention --shard for testing --- book/src/shards.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/book/src/shards.md b/book/src/shards.md index f7a3e6e4..e93d2318 100644 --- a/book/src/shards.md +++ b/book/src/shards.md @@ -87,3 +87,7 @@ If your CI system offers a choice of VM sizes you might experiment with using sm You should also think about cost and capacity constraints in your CI system, and the risk of starving out other users. cargo-mutants has no internal scaling constraints to prevent you from setting `k` very large, if cost, efficiency and CI capacity are not a concern. + +## Sampling mutants + +An option like `--shard 1/100` can be used to run 1% of all the generated mutants for testing cargo-mutants, to get a sense of whether it works or to see how it performs on some tree. From 131388378bce7e66280326a652f9cf9ea17163d4 Mon Sep 17 00:00:00 2001 From: Martin Pool Date: Tue, 2 Jan 2024 15:03:26 -0800 Subject: [PATCH 04/18] Try nextest in our CI --- .github/workflows/tests.yml | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index ae45fe44..6946ab57 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -105,6 +105,7 @@ jobs: strategy: matrix: shard: [0, 1, 2, 3, 4, 5, 6, 7] + test_tool: [cargo, nextest] steps: - uses: actions/checkout@v3 - uses: dtolnay/rust-toolchain@master @@ -115,12 +116,17 @@ jobs: uses: actions/download-artifact@v3 with: name: cargo-mutants-linux + - uses: taiki-e/install-action@v2 + name: Install nextest using install-action + with: + tool: nextest - name: Install binary artifact run: | install cargo-mutants $HOME/.cargo/bin/ - name: Mutants - run: | + run: > cargo mutants --no-shuffle -vV --shard ${{ matrix.shard }}/8 + --test-tool ${{ matrix.test_tool }} - name: Archive mutants.out uses: actions/upload-artifact@v3 if: always() From 49c3e55f6390e4d63ceea760099fea2f823b9c56 Mon Sep 17 00:00:00 2001 From: Martin Pool Date: Tue, 2 Jan 2024 15:08:59 -0800 Subject: [PATCH 05/18] Also try nextest for pr-mutants --- .github/workflows/tests.yml | 28 ++++++++++++++++++---------- 1 file changed, 18 insertions(+), 10 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 6946ab57..366bfd71 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -70,6 +70,9 @@ jobs: runs-on: ubuntu-latest needs: [release-binary] if: github.event_name == 'pull_request' + strategy: + matrix: + test_tool: [cargo, nextest] steps: - uses: actions/checkout@v3 with: @@ -82,16 +85,21 @@ jobs: with: toolchain: beta - uses: Swatinem/rust-cache@v2 - - name: Download release binary + - uses: taiki-e/install-action@v2 + name: Install nextest using install-action + with: + tool: nextest + - name: Download cargo-mutants binary uses: actions/download-artifact@v3 with: name: cargo-mutants-linux - - name: Install binary artifact + - name: Install cargo-mutants binary run: | install cargo-mutants $HOME/.cargo/bin/ - - name: Mutants - run: | - cargo mutants --no-shuffle -vV --in-diff git.diff + - name: Mutants in-diff + run: > + cargo mutants --no-shuffle -vV --in-diff git.diff --test-tool + ${{matrix.test_tool}} - name: Archive mutants.out uses: actions/upload-artifact@v3 if: always() @@ -112,15 +120,15 @@ jobs: with: toolchain: beta - uses: Swatinem/rust-cache@v2 - - name: Download release binary - uses: actions/download-artifact@v3 - with: - name: cargo-mutants-linux - uses: taiki-e/install-action@v2 name: Install nextest using install-action with: tool: nextest - - name: Install binary artifact + - name: Download cargo-mutants binary + uses: actions/download-artifact@v3 + with: + name: cargo-mutants-linux + - name: Install cargo-mutants binary run: | install cargo-mutants $HOME/.cargo/bin/ - name: Mutants From b71fe6f5a06b9c53f0b5b41f635deddf918a1bd9 Mon Sep 17 00:00:00 2001 From: Martin Pool Date: Tue, 2 Jan 2024 15:25:38 -0800 Subject: [PATCH 06/18] A note on test performance --- DESIGN.md | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/DESIGN.md b/DESIGN.md index 09273e7b..448dc01f 100644 --- a/DESIGN.md +++ b/DESIGN.md @@ -194,6 +194,18 @@ Cargo-mutants is primarily tested on its public interface, which is the command `cargo-mutants` runs as a subprocess of the test process so that we get the most realistic view of its behavior. In some cases it is run via the `cargo` command to test that this level of indirection works properly. +### Test performance + +Since cargo-mutants itself runs the test suite of the program under test many times there is a risk that the test suite can get slow. Aside from slowing down developers and CI, this has a multiplicative effect on the time to run `cargo mutants` on itself. + +To manage test time: + +* Although key behaviour should be tested through integration tests that run the CLI, it's OK to handle additional cases with unit tests that run much faster. + +* Whenever reasonable, CLI tests can only list mutants with `--list` rather than actually testing all of them: we have just a small set of tests that check that the mutants that are listed are actually run. + +* Use relatively small testdata trees that are sufficient to test the right behavior. + ### `testdata` trees The primary means of testing is Rust source trees under `testdata`: you can copy an existing tree and modify it to show the new behavior that you want to test. From f3a7ff2d6d8dc3fb3c0a915431762378b00ef14c Mon Sep 17 00:00:00 2001 From: Martin Pool Date: Tue, 2 Jan 2024 15:50:58 -0800 Subject: [PATCH 07/18] Better package description --- Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Cargo.toml b/Cargo.toml index 974708e8..7a4ef3a2 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -4,7 +4,7 @@ version = "23.12.2" edition = "2021" authors = ["Martin Pool"] license = "MIT" -description = "Find inadequately-tested code that can be removed without any tests failing." +description = "Inject bugs and see if your tests catch them" repository = "https://github.com/sourcefrog/cargo-mutants" homepage = "https://mutants.rs/" categories = ["development-tools::testing"] From 74d535cf0ee02d242d8550741d059802ff09338e Mon Sep 17 00:00:00 2001 From: Martin Pool Date: Tue, 2 Jan 2024 16:15:32 -0800 Subject: [PATCH 08/18] Add an integration test for nextest --- .github/workflows/tests.yml | 4 ++++ Cargo.toml | 1 - DESIGN.md | 4 ++++ tests/cli/main.rs | 1 + tests/cli/nextest.rs | 20 ++++++++++++++++++++ 5 files changed, 29 insertions(+), 1 deletion(-) create mode 100644 tests/cli/nextest.rs diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 366bfd71..4acad525 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -42,6 +42,10 @@ jobs: - uses: Swatinem/rust-cache@v2 - name: rustfmt run: cargo fmt --all -- --check + - uses: taiki-e/install-action@v2 + name: Install nextest using install-action + with: + tool: nextest - name: Build run: cargo build --all-targets - name: Test diff --git a/Cargo.toml b/Cargo.toml index 7a4ef3a2..8ebb5863 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -73,7 +73,6 @@ whoami = "1.2" [dependencies.nutmeg] version = "0.1.4" # git = "https://github.com/sourcefrog/nutmeg.git" -# branch = "const-new" [dependencies.proc-macro2] features = ["span-locations"] diff --git a/DESIGN.md b/DESIGN.md index 448dc01f..c7174de2 100644 --- a/DESIGN.md +++ b/DESIGN.md @@ -228,6 +228,10 @@ Many features can be tested adequately by only looking at the list of mutants pr Although we primarily want to test the public interface (which is the command line), unit tests can be added in a `mod test {}` within the source tree for any behavior that is inconvenient to exercise from the command line. +### Nextest tests + +cargo-mutants tests require `nextest` to be installed. + ## UI Style Always print paths with forward slashes, even on Windows. Use `path_slash`. diff --git a/tests/cli/main.rs b/tests/cli/main.rs index f94f4971..7b1a5cca 100644 --- a/tests/cli/main.rs +++ b/tests/cli/main.rs @@ -26,6 +26,7 @@ mod config; mod error_value; mod in_diff; mod jobs; +mod nextest; mod shard; mod trace; #[cfg(windows)] diff --git a/tests/cli/nextest.rs b/tests/cli/nextest.rs new file mode 100644 index 00000000..5ef14c94 --- /dev/null +++ b/tests/cli/nextest.rs @@ -0,0 +1,20 @@ +// Copyright 2024 Martin Pool + +//! Integration tests for cargo mutants calling nextest. + +use super::{copy_of_testdata, run}; + +#[test] +fn test_with_nextest_on_small_tree() { + let tmp_src_dir = copy_of_testdata("small_well_tested"); + let assert = run() + .args(["mutants", "--test-tool", "nextest", "-vV", "--no-shuffle"]) + .arg("-d") + .arg(tmp_src_dir.path()) + .assert() + .success(); + println!( + "stdout:\n{}", + String::from_utf8_lossy(&assert.get_output().stdout) + ); +} From 61fa6db377d0d628cda41a69f80fc83c9409024e Mon Sep 17 00:00:00 2001 From: Martin Pool Date: Tue, 2 Jan 2024 16:36:28 -0800 Subject: [PATCH 09/18] Book content about Nextest --- book/src/SUMMARY.md | 1 + book/src/nextest.md | 27 +++++++++++++++++++++++++++ 2 files changed, 28 insertions(+) create mode 100644 book/src/nextest.md diff --git a/book/src/SUMMARY.md b/book/src/SUMMARY.md index dffedcc8..da5a6848 100644 --- a/book/src/SUMMARY.md +++ b/book/src/SUMMARY.md @@ -16,6 +16,7 @@ - [Workspaces and packages](workspaces.md) - [Passing options to Cargo](cargo-args.md) - [Build directories](build-dirs.md) + - [Using nextest](nextest.md) - [Generating mutants](mutants.md) - [Error values](error-values.md) - [Improving performance](performance.md) diff --git a/book/src/nextest.md b/book/src/nextest.md new file mode 100644 index 00000000..b8b78a2f --- /dev/null +++ b/book/src/nextest.md @@ -0,0 +1,27 @@ +# cargo-mutants with nextest + +nextest is a tool for running Rust tests, as a replacement for `cargo test`. + +You can use nextest to run your tests with cargo-mutants, instead of `cargo test`, by either passing the `--test-tool=nextest` option, or setting `test_tool = "nextest"` in `.cargo/mutants.toml`. + +## How nextest works + +In the context of cargo-mutants the most important difference between cargo-test and nextest is that nextest runs each test in a separate process, and it can run tests from multiple test targets in parallel. (Nextest also has some nice UI improvements and other features, but they're not relevant here.) + +This means that nextest can stop faster if a single test fails, whereas cargo test will continue running all the tests within the test binary. + +This is beneficial for cargo-mutants, because it only needs to know whether at least one test caught the mutation, and so exiting as soon as there's a failure is better. + +However, running each test individually also means there is more per-test startup cost, and so on some trees nextest may be slower. + +In general, nextest will do relatively poorly on trees that have tests that are individually very fast, or on trees that establish shared or cached state across tests. + +So there are at least two reasons why you might want to use nextest: + +1. Some trees only support testing under nextest, and their tests fail under `cargo test`: in that case, you have to use this option! In particular, nextest's behavior of running each test in a separate process gives better isolation between tests. + +2. Some trees might be faster under nextest than under `cargo test`, because they have a lot of tests that fail quickly, and the startup time is a small fraction of the time for the average test. This may or may not be true for your tree, so you can try it and see. + +## nextest and doctests + +**Caution:** [nextest currently does not run doctests](https://github.com/nextest-rs/nextest/issues/16), so behaviors that are only caught by doctests will show as missed when using nextest. (cargo-mutants could separately run the doctests, but currently does not.) From 9c19f25570ab31962682cd4e4c06e6aa1eb12b15 Mon Sep 17 00:00:00 2001 From: Martin Pool Date: Sat, 6 Jan 2024 20:24:43 -0800 Subject: [PATCH 10/18] Update readme, including mentioning nextest --- README.md | 24 +++++++++++++++--------- 1 file changed, 15 insertions(+), 9 deletions(-) diff --git a/README.md b/README.md index d666a3f5..2fb9702f 100644 --- a/README.md +++ b/README.md @@ -22,12 +22,18 @@ the tests might be insufficient. **The main documentation is the user guide at .** +## Prerequisites + +cargo-mutants can help on trees with non-flaky tests that run under `cargo test` or [`cargo nextest run`](https://nexte.st/). + ## Install ```sh cargo install --locked cargo-mutants ``` +You can also install using [cargo-binstall](https://github.com/cargo-bins/cargo-binstall) or from binaries attached to GitHub releases. + ## Quick start From within a Rust source directory, just run @@ -42,8 +48,6 @@ To generate mutants in only one file: cargo mutants -f src/something.rs ``` -**For more, see the user guide is at .** - ## Help advance cargo-mutants If you use cargo-mutants or just like the idea you can help it get better: @@ -53,18 +57,20 @@ If you use cargo-mutants or just like the idea you can help it get better: ## Project status -As of October 2023 this is an actively-maintained spare time project. I expect to make [releases](https://github.com/sourcefrog/cargo-mutants/releases) about every one or two months. +As of January 2024 this is an actively-maintained spare time project. I expect to make [releases](https://github.com/sourcefrog/cargo-mutants/releases) about every one or two months. It's very usable at it is and there's room for lots more future improvement, especially in adding new types of mutation. +This software is provided as-is with no warranty of any kind. + ## Further reading See also: -- [cargo-mutants manual](https://mutants.rs/) -- [How cargo-mutants compares to other techniques and tools](https://github.com/sourcefrog/cargo-mutants/wiki/Compared). -- [Design notes](DESIGN.md) -- [Contributing](CONTRIBUTING.md) -- [Release notes](NEWS.md) -- [Discussions](https://github.com/sourcefrog/cargo-mutants/discussions) +* [cargo-mutants manual](https://mutants.rs/) +* [How cargo-mutants compares to other techniques and tools](https://github.com/sourcefrog/cargo-mutants/wiki/Compared). +* [Design notes](DESIGN.md) +* [Contributing](CONTRIBUTING.md) +* [Release notes](NEWS.md) +* [Discussions](https://github.com/sourcefrog/cargo-mutants/discussions) From 0c023dceae8a65d1f583be078c7a824eabea94a9 Mon Sep 17 00:00:00 2001 From: Martin Pool Date: Sun, 7 Jan 2024 08:59:58 -0800 Subject: [PATCH 11/18] More nextest docs --- CONTRIBUTING.md | 8 +++++++- DESIGN.md | 2 +- book/src/nextest.md | 2 +- 3 files changed, 9 insertions(+), 3 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index dd01b50a..9c4ad4ee 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -33,7 +33,13 @@ Please run `cargo fmt` and `cargo clippy`. These are checked in CI. ## Testing -Of course, please add tests for new features or bug fixes, and see the _Testing_ section of [the design doc](DESIGN.md). +Of course, please add tests for new features or bug fixes. See also the _Testing_ section of [the design doc](DESIGN.md). + +### Running the tests + +cargo-mutants tests require [`cargo-nextest`](https://nexte.st/) to be installed, so that they can exercise `--test-tool=nextest`. + +cargo-mutants tests can be run under either `cargo test` or `cargo nextest run`. ### Test naming diff --git a/DESIGN.md b/DESIGN.md index c7174de2..7a9ae4b1 100644 --- a/DESIGN.md +++ b/DESIGN.md @@ -226,7 +226,7 @@ Many features can be tested adequately by only looking at the list of mutants pr ### Unit tests -Although we primarily want to test the public interface (which is the command line), unit tests can be added in a `mod test {}` within the source tree for any behavior that is inconvenient to exercise from the command line. +Although we primarily want to test the public interface (which is the command line), unit tests can be added in a `mod test {}` within the source tree for any behavior that is inconvenient or overly slow to exercise from the command line. ### Nextest tests diff --git a/book/src/nextest.md b/book/src/nextest.md index b8b78a2f..1a34b410 100644 --- a/book/src/nextest.md +++ b/book/src/nextest.md @@ -1,6 +1,6 @@ # cargo-mutants with nextest -nextest is a tool for running Rust tests, as a replacement for `cargo test`. +[nextest](https://nexte.st) is a tool for running Rust tests, as a replacement for `cargo test`. You can use nextest to run your tests with cargo-mutants, instead of `cargo test`, by either passing the `--test-tool=nextest` option, or setting `test_tool = "nextest"` in `.cargo/mutants.toml`. From 09c152993c54b6c3c2818c6ed8a567411805ce3f Mon Sep 17 00:00:00 2001 From: Martin Pool Date: Wed, 10 Jan 2024 21:03:13 -0800 Subject: [PATCH 12/18] Some more docs, including about nextest --- book/src/cargo-args.md | 4 +++- book/src/getting-started.md | 11 +++++++++++ book/src/installation.md | 2 ++ book/src/nextest.md | 8 +++++--- 4 files changed, 21 insertions(+), 4 deletions(-) diff --git a/book/src/cargo-args.md b/book/src/cargo-args.md index 0d393856..cfa04746 100644 --- a/book/src/cargo-args.md +++ b/book/src/cargo-args.md @@ -27,7 +27,9 @@ additional_cargo_args = ["--all-features"] ## Arguments to `cargo test` Command-line options following a `--` delimiter are passed through to -`cargo test`. For example, this can be used to pass `--all-targets` which (unobviously) +`cargo test` (or to [nextest](nextest.md), if you're using that). + +For example, this can be used to pass `--all-targets` which (unobviously) excludes doctests. (If the doctests are numerous and slow, and not relied upon to catch bugs, this can improve performance.) ```shell diff --git a/book/src/getting-started.md b/book/src/getting-started.md index a6e9dfa1..2c5dc973 100644 --- a/book/src/getting-started.md +++ b/book/src/getting-started.md @@ -3,6 +3,17 @@ Just run `cargo mutants` in a Rust source directory, and it will point out functions that may be inadequately tested. +## Prerequisites + +For cargo-mutants to give useful results, your tree must already + +1. Be built with `cargo build`, and +2. Have reliable non-flaky tests that run under either `cargo test` or `cargo nextest`. + +If the tests are flaky, meaning that they can pass or fail depending on factors other than the source tree, then the cargo-mutants results will be meaningless. + +Cross-compilation is not currently supported, so the tree must be buildable for the host platform. + ## Example ```none diff --git a/book/src/installation.md b/book/src/installation.md index f4e1847b..ff8354e4 100644 --- a/book/src/installation.md +++ b/book/src/installation.md @@ -6,6 +6,8 @@ Install cargo-mutants from source: cargo install --locked cargo-mutants ``` +You can also use `cargo binstall` from [cargo-binstall](https://github.com/cargo-bins/cargo-binstall), or install binaries from GitHub releases. + ## Supported Rust versions Building cargo-mutants requires a reasonably recent stable (or nightly or beta) Rust toolchain. diff --git a/book/src/nextest.md b/book/src/nextest.md index 1a34b410..17cc335e 100644 --- a/book/src/nextest.md +++ b/book/src/nextest.md @@ -16,11 +16,13 @@ However, running each test individually also means there is more per-test startu In general, nextest will do relatively poorly on trees that have tests that are individually very fast, or on trees that establish shared or cached state across tests. -So there are at least two reasons why you might want to use nextest: +## When to use nextest -1. Some trees only support testing under nextest, and their tests fail under `cargo test`: in that case, you have to use this option! In particular, nextest's behavior of running each test in a separate process gives better isolation between tests. +There are at least two reasons why you might want to use nextest: -2. Some trees might be faster under nextest than under `cargo test`, because they have a lot of tests that fail quickly, and the startup time is a small fraction of the time for the average test. This may or may not be true for your tree, so you can try it and see. +1. Some Rust source trees only support testing under nextest, and their tests fail under `cargo test`: in that case, you have to use this option! In particular, nextest's behavior of running each test in a separate process gives better isolation between tests. + +2. Some trees might be faster under nextest than under `cargo test`, because they have a lot of tests that fail quickly, and the startup time is a small fraction of the time for the average test. This may or may not be true for your tree, so you can try it and see. Some trees, including cargo-mutants itself, are slower under nextest. ## nextest and doctests From f9cf56b6d4d88bb737e0619b8d923cbc4da12abc Mon Sep 17 00:00:00 2001 From: Martin Pool Date: Wed, 10 Jan 2024 21:07:52 -0800 Subject: [PATCH 13/18] FIx nextest link Hooray for mdbook-linkcheck --- NEWS.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/NEWS.md b/NEWS.md index a5977eaf..38d656a8 100644 --- a/NEWS.md +++ b/NEWS.md @@ -2,7 +2,7 @@ ## Unreleased -- New! `cargo mutants --test-tool nextest`, or `test_tool = "nextest"` in `.cargo/mutants.toml` runs tests under [Nextest](https://nextest.rs/). Some trees have tests that only work under Nextest, and this allows them to be tested. In other cases Nextest may be significantly faster, because it will exit soon after the first test failure. +- New! `cargo mutants --test-tool nextest`, or `test_tool = "nextest"` in `.cargo/mutants.toml` runs tests under [Nextest](https://nexte.st/). Some trees have tests that only work under Nextest, and this allows them to be tested. In other cases Nextest may be significantly faster, because it will exit soon after the first test failure. - Fixed: Fixed spurious "Patch input contains repeated filenames" error when `--in-diff` is given a patch that deletes multiple files. From e8f0ccc3499761e573226b3f76aba053a3bd4f6e Mon Sep 17 00:00:00 2001 From: Martin Pool Date: Wed, 10 Jan 2024 22:08:08 -0800 Subject: [PATCH 14/18] Move book build to book workflow Run only on changes to the book --- .../workflows/{deploy-book.yml => book.yml} | 40 ++++++++++++++++--- .github/workflows/tests.yml | 22 ---------- 2 files changed, 34 insertions(+), 28 deletions(-) rename .github/workflows/{deploy-book.yml => book.yml} (52%) diff --git a/.github/workflows/deploy-book.yml b/.github/workflows/book.yml similarity index 52% rename from .github/workflows/deploy-book.yml rename to .github/workflows/book.yml index 5175980d..83435940 100644 --- a/.github/workflows/deploy-book.yml +++ b/.github/workflows/book.yml @@ -5,6 +5,12 @@ on: - main # Allows you to run this workflow manually from the Actions tab workflow_dispatch: + pull_request: + branches: + - main + paths: + - .github/workflows/book.yml + - book/** # Sets permissions of the GITHUB_TOKEN to allow deployment to GitHub Pages permissions: @@ -12,17 +18,39 @@ permissions: pages: write id-token: write -# Allow one concurrent deployment -concurrency: - group: "pages" - cancel-in-progress: true - jobs: + build: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + - uses: dtolnay/rust-toolchain@master + with: + toolchain: beta + - uses: Swatinem/rust-cache@v2 + - name: Install mdbook and mdbook-linkcheck + uses: taiki-e/install-action@v2 + with: + tool: mdbook, mdbook-linkcheck + - name: Build mdbook + run: | + mdbook build book + - name: Archive book output + uses: actions/upload-artifact@v3 + if: always() + with: + name: book + path: book/book + deploy: - if: github.repository == 'sourcefrog/cargo-mutants' + if: + github.repository == 'sourcefrog/cargo-mutants' && github.ref_name == + 'main' environment: name: github-pages url: ${{ steps.deployment.outputs.page_url }} + concurrency: # Allow one concurrent deployment + group: "pages" + cancel-in-progress: true runs-on: ubuntu-latest steps: - uses: actions/checkout@v3 diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 112c33f8..4acad525 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -145,25 +145,3 @@ jobs: with: name: mutants.out path: mutants.out - - book: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v3 - - uses: dtolnay/rust-toolchain@master - with: - toolchain: beta - - uses: Swatinem/rust-cache@v2 - - name: Install mdbook and mdbook-linkcheck - uses: taiki-e/install-action@v2 - with: - tool: mdbook, mdbook-linkcheck - - name: Build mdbook - run: | - mdbook build book - - name: Archive book output - uses: actions/upload-artifact@v3 - if: always() - with: - name: book - path: book/book From 4c9152e256055fda134c7be69f09dd57feee0484 Mon Sep 17 00:00:00 2001 From: Martin Pool Date: Wed, 10 Jan 2024 22:08:50 -0800 Subject: [PATCH 15/18] Run tests only on relevant changes The mutants tests are getting time consuming and there's no need to run them on every change to the book --- .github/workflows/tests.yml | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 4acad525..080732f0 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -8,6 +8,14 @@ on: branches: - main pull_request: + paths: + - ".cargo/**" + - ".github/workflows/tests.yml" + - "Cargo.*" + - "mutants_attrs/**" + - "src/**" + - "testdata/**" + - "tests/**" # see https://matklad.github.io/2021/09/04/fast-rust-builds.html env: From 4052bbc2e4ea8c739dcd1f74cc87f0238e49d93d Mon Sep 17 00:00:00 2001 From: Martin Pool Date: Wed, 10 Jan 2024 22:10:00 -0800 Subject: [PATCH 16/18] Rename book workflow --- .github/workflows/book.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/book.yml b/.github/workflows/book.yml index 83435940..ad50ae11 100644 --- a/.github/workflows/book.yml +++ b/.github/workflows/book.yml @@ -1,4 +1,4 @@ -name: Deploy book +name: Book on: push: branches: From ac5ffb44c7db8be833afd8cc0066fb124e87ba97 Mon Sep 17 00:00:00 2001 From: Martin Pool Date: Wed, 10 Jan 2024 22:13:48 -0800 Subject: [PATCH 17/18] chore: Release --- Cargo.lock | 2 +- Cargo.toml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 976d0b96..248583b3 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -158,7 +158,7 @@ dependencies = [ [[package]] name = "cargo-mutants" -version = "23.12.2" +version = "24.1.0" dependencies = [ "anyhow", "assert_cmd", diff --git a/Cargo.toml b/Cargo.toml index 5b521ece..a8767a71 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "cargo-mutants" -version = "23.12.2" +version = "24.1.0" edition = "2021" authors = ["Martin Pool"] license = "MIT" From 35ac5981d9949bf6de738c98168ed49ff05bfac2 Mon Sep 17 00:00:00 2001 From: Martin Pool Date: Wed, 10 Jan 2024 22:14:58 -0800 Subject: [PATCH 18/18] Fix NEWS for new release Signed-off-by: Martin Pool --- NEWS.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/NEWS.md b/NEWS.md index 38d656a8..3b2090ab 100644 --- a/NEWS.md +++ b/NEWS.md @@ -2,6 +2,8 @@ ## Unreleased +## 24.1.0 + - New! `cargo mutants --test-tool nextest`, or `test_tool = "nextest"` in `.cargo/mutants.toml` runs tests under [Nextest](https://nexte.st/). Some trees have tests that only work under Nextest, and this allows them to be tested. In other cases Nextest may be significantly faster, because it will exit soon after the first test failure. - Fixed: Fixed spurious "Patch input contains repeated filenames" error when `--in-diff` is given a patch that deletes multiple files.