feat: Add a --profile option to choose a cargo profile (#398)

Fixes #338

.cargo/mutants.toml

@@ -2,3 +2,4 @@
 
 error_values = ["::anyhow::anyhow!(\"mutated!\")"]
 exclude_globs = ["src/console.rs"]
+profile = "mutants"                                # Build without debug symbols

Cargo.toml

@@ -42,7 +42,7 @@ indoc = "2.0.0"
 itertools = "0.12"
 jobserver = "0.1"
 mutants = "0.0.3"
-num_cpus ="1.16"
+num_cpus = "1.16"
 patch = "0.7"
 path-slash = "0.2"
 quote = "1.0.35"

NEWS.md

@@ -4,6 +4,8 @@
 
 - Fixed: Avoid generating empty string elements in `ENCODED_RUSTFLAGS` when `--cap-lints` is set. In some situations these could cause a compiler error complaining about the empty argument.
 
+- New: `--profile` option allows selecting a Cargo profile. In particular, it's recommended that you can use `--profile=mutants` and configure a custom profile in your `Cargo.toml` to optimize the build for mutants, by turning off debug symbols.
+
 - New: `--iterate` option skips mutants that were previously caught or unviable.
 
 - New: cargo-mutants starts a GNU jobserver, shared across all children, so that running multiple `--jobs` does not spawn an excessive number of compiler processes. The jobserver is on by default and can be turned off with `--jobserver false`.

book/src/performance.md

@@ -16,13 +16,33 @@ test binary. If you're using doctests only as testable documentation and not to
 assert correctness of the code, you can skip them with `cargo mutants --
 --all-targets`.
 
-## Optimized builds
+## Choosing a cargo profile
 
-On _some but not all_ projects, cargo-mutants can be faster if you use `-C --release`, which will make the build slower but may make the tests faster. Typically this will help on projects with very long CPU-intensive test suites.
+[Cargo profiles](https://doc.rust-lang.org/cargo/reference/profiles.html) provide a way to configure compiler settings including several that influence build and runtime performance.
 
-cargo-mutants now shows the breakdown of build versus test time which may help you work out if this will help: if the tests are much slower than the build it's worth trying more more compiler optimizations.
+By default, cargo-mutants will use the default profile selected for `cargo test`, which is also called `test`. This includes debug symbols but disables optimization.
+
+You can select a different profile using the `--profile` option or the `profile` configuration key.
+
+You may wish to define a `mutants` profile in `Cargo.toml`, such as:
+
+```toml
+[profile.mutants]
+inherits = "test"
+debug = "none"
+```
+
+and then configure this as the default in `.cargo/mutants.toml`:
 
-On projects like this you might also choose just to turn up optimization for all debug builds in [`.cargo/config.toml`](https://doc.rust-lang.org/cargo/reference/config.html).
+```toml
+profile = "mutants"
+```
+
+Turning off debug symbols will make the builds faster, at the expense of possibly giving less useful output when a test fails. In general, since mutants are expected to cause tests to fail, debug symbols may not be worth cost.
+
+If your project's tests take a long time to run then it may be worth experimenting with increasing the `opt` level or other optimization parameters in the profile, to trade off longer builds for faster test runs.
+
+cargo-mutants now shows the breakdown of build versus test time which may help you work out if this will help: if the tests are much slower than the build it's worth trying more more compiler optimizations.
 
 ## Ramdisks
 

src/cargo.rs

@@ -115,6 +115,16 @@ fn cargo_argv(
             cargo_args.push("--tests".to_string());
         }
     }
+    if let Some(profile) = &options.profile {
+        match options.test_tool {
+            TestTool::Cargo => {
+                cargo_args.push(format!("--profile={profile}"));
+            }
+            TestTool::Nextest => {
+                cargo_args.push(format!("--cargo-profile={profile}"));
+            }
+        }
+    }
     cargo_args.push("--verbose".to_string());
     if let Some([package]) = packages {
         // Use the unambiguous form for this case; it works better when the same
@@ -355,6 +365,44 @@ mod test {
         );
     }
 
+    #[test]
+    fn profile_arg_passed_to_cargo() {
+        let args = Args::try_parse_from(["mutants", "--profile", "mutants"].as_slice()).unwrap();
+        let options = Options::from_args(&args).unwrap();
+        let build_dir = Utf8Path::new("/tmp/buildXYZ");
+        assert_eq!(
+            cargo_argv(build_dir, None, Phase::Check, &options)[1..],
+            [
+                "check",
+                "--tests",
+                "--profile=mutants",
+                "--verbose",
+                "--workspace",
+            ]
+        );
+    }
+
+    #[test]
+    fn nextest_gets_special_cargo_profile_option() {
+        let args = Args::try_parse_from(
+            ["mutants", "--test-tool=nextest", "--profile", "mutants"].as_slice(),
+        )
+        .unwrap();
+        let options = Options::from_args(&args).unwrap();
+        let build_dir = Utf8Path::new("/tmp/buildXYZ");
+        assert_eq!(
+            cargo_argv(build_dir, None, Phase::Build, &options)[1..],
+            [
+                "nextest",
+                "run",
+                "--no-run",
+                "--cargo-profile=mutants",
+                "--verbose",
+                "--workspace",
+            ]
+        );
+    }
+
     rusty_fork_test! {
         #[test]
         fn rustflags_without_cap_lints_and_no_environment_variables() {

src/config.rs

@@ -45,6 +45,8 @@ pub struct Config {
     pub additional_cargo_test_args: Vec<String>,
     /// Minimum test timeout, in seconds, as a floor on the autoset value.
     pub minimum_test_timeout: Option<f64>,
+    /// Cargo profile.
+    pub profile: Option<String>,
     /// Choice of test tool: cargo or nextest.
     pub test_tool: Option<TestTool>,
     /// Timeout multiplier, relative to the baseline 'cargo test'.

src/main.rs

@@ -296,6 +296,10 @@ pub struct Args {
     #[arg(long, help_heading = "Execution")]
     no_shuffle: bool,
 
+    /// Build with this cargo profile.
+    #[arg(long, help_heading = "Build")]
+    profile: Option<String>,
+
     /// Run only one shard of all generated mutants: specify as e.g. 1/4.
     #[arg(long, help_heading = "Execution")]
     shard: Option<Shard>,

src/options.rs

@@ -85,6 +85,9 @@ pub struct Options {
     /// interesting results.
     pub shuffle: bool,
 
+    /// Cargo profile.
+    pub profile: Option<String>,
+
     /// Additional arguments for every cargo invocation.
     pub additional_cargo_args: Vec<String>,
 
@@ -205,6 +208,10 @@ impl Options {
                 &config.additional_cargo_test_args,
             ),
             baseline: args.baseline,
+            build_timeout: args.build_timeout.map(Duration::from_secs_f64),
+            build_timeout_multiplier: args
+                .build_timeout_multiplier
+                .or(config.build_timeout_multiplier),
             cap_lints: args.cap_lints.unwrap_or(config.cap_lints),
             check_only: args.check,
             colors: args.colors,
@@ -228,16 +235,13 @@ impl Options {
             output_in_dir: args.output.clone(),
             print_caught: args.caught,
             print_unviable: args.unviable,
+            profile: args.profile.as_ref().or(config.profile.as_ref()).cloned(),
             shuffle: !args.no_shuffle,
             show_line_col: args.line_col,
             show_times: !args.no_times,
             show_all_logs: args.all_logs,
             test_timeout: args.timeout.map(Duration::from_secs_f64),
             test_timeout_multiplier: args.timeout_multiplier.or(config.timeout_multiplier),
-            build_timeout: args.build_timeout.map(Duration::from_secs_f64),
-            build_timeout_multiplier: args
-                .build_timeout_multiplier
-                .or(config.build_timeout_multiplier),
             test_tool: args.test_tool.or(config.test_tool).unwrap_or_default(),
         };
         options.error_values.iter().for_each(|e| {
@@ -520,5 +524,28 @@ mod test {
             let options = Options::new(&args, &Config::default()).unwrap();
             assert_eq!(options.colors.forced_value(), None);
         }
+
+        #[test]
+        fn profile_option_from_args() {
+            let args = Args::parse_from(["mutants", "--profile=mutants"]);
+            let options = Options::new(&args, &Config::default()).unwrap();
+            assert_eq!(options.profile.unwrap(), "mutants");
+        }
+
+
+        #[test]
+        fn profile_from_config() {
+            let args = Args::parse_from(["mutants", "-j3"]);
+            let config = indoc! { r#"
+                profile = "mutants"
+                timeout_multiplier = 1.0
+                build_timeout_multiplier = 2.0
+            "#};
+            let mut config_file = NamedTempFile::new().unwrap();
+            config_file.write_all(config.as_bytes()).unwrap();
+            let config = Config::read_file(config_file.path()).unwrap();
+            let options = Options::new(&args, &config).unwrap();
+            assert_eq!(options.profile.unwrap(), "mutants");
+        }
     }
 }