Guard tests with features rather than ignore

trailofbits · May 11, 2024 · 53b3bf8 · 53b3bf8
1 parent ab7e2b6
commit 53b3bf8
Show file tree

Hide file tree

Showing 4 changed files with 35 additions and 60 deletions.
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
@@ -155,15 +155,21 @@ jobs:
 
       - name: Setup
         run: |
+          TEST_THIRD_PARTY_FULL=
+          if [[ ${{ github.event_name }} = 'schedule' ]] ||
+            git diff --name-only ${{ github.event.pull_request.base.sha }} | grep -w 'patches\|third_party' >/dev/null
+          then
+            TEST_THIRD_PARTY_FULL='--features=test-third-party-full'
+          fi
           SERDE_FORMAT='--features=test-fuzz/serde_${{ matrix.serde_format }}'
           SHUFFLE=
           if [[ ${{ matrix.toolchain }} = nightly ]]; then
             SHUFFLE='-Z unstable-options --shuffle --test-threads=1'
           fi
           SELF_TY_IN_MOD_NAME="$(${{ matrix.self_ty_in_mod_name }} && echo '--features test-fuzz/self_ty_in_mod_name')" || true
           CONFIG_GROUP_RUNNER="--config target.'cfg(all())'.runner='group-runner'"
-          BUILD_CMD="cargo build $SERDE_FORMAT $SELF_TY_IN_MOD_NAME --all-targets"
-          TEST_CMD="cargo test $SERDE_FORMAT $SELF_TY_IN_MOD_NAME $CONFIG_GROUP_RUNNER -- --nocapture $SHUFFLE"
+          BUILD_CMD="cargo build --features=test-install $TEST_THIRD_PARTY_FULL $SERDE_FORMAT $SELF_TY_IN_MOD_NAME --all-targets"
+          TEST_CMD="cargo test --features=test-install $TEST_THIRD_PARTY_FULL $SERDE_FORMAT $SELF_TY_IN_MOD_NAME $CONFIG_GROUP_RUNNER -- --nocapture $SHUFFLE"
           echo "BUILD_CMD=$BUILD_CMD" >> "$GITHUB_ENV"
           echo "TEST_CMD=$TEST_CMD" >> "$GITHUB_ENV"
 
@@ -173,11 +179,6 @@ jobs:
       - name: Test
         run: |
           $TEST_CMD
-          if [[ ${{ github.event_name }} = 'schedule' ]] ||
-            git diff --name-only ${{ github.event.pull_request.base.sha }} | grep -w 'patches\|third_party' >/dev/null
-          then
-            $TEST_CMD --ignored
-          fi
         env:
           RUST_BACKTRACE: 1
           RUST_LOG: warn

diff --git a/cargo-test-fuzz/Cargo.toml b/cargo-test-fuzz/Cargo.toml
@@ -16,6 +16,10 @@ path = "src/bin/cargo_test_fuzz/main.rs"
 [lib]
 doctest = false
 
+[[test]]
+name = "install"
+required-features = ["test-install"]
+
 [dependencies]
 anyhow = { version = "1.0", features = ["backtrace"] }
 bitflags = "2.5"
@@ -26,7 +30,6 @@ heck = "0.5"
 log = "0.4"
 mio = { version = "0.8", features = ["os-ext", "os-poll"] }
 num_cpus = "1.16"
-paste = "1.0"
 remain = "0.2"
 semver = "1.0"
 serde = { version = "1.0", features = ["derive"] }
@@ -52,8 +55,14 @@ xshell = "0.2"
 
 testing = { path = "../testing", package = "test-fuzz-testing" }
 
+[features]
+test-install = []
+test-third-party-full = []
+
 [lints]
 workspace = true
 
+# smoelius: `xshell` is used only by the `install` tests, which are guarded by the `test-install`
+# feature.
 [package.metadata.cargo-udeps.ignore]
-normal = ["paste"]
+normal = ["xshell"]
diff --git a/cargo-test-fuzz/tests/third_party.rs b/cargo-test-fuzz/tests/third_party.rs
@@ -3,7 +3,6 @@ use cargo_metadata::MetadataCommand;
 use once_cell::sync::Lazy;
 use option_set::option_set;
 use predicates::prelude::*;
-use regex::Regex;
 use rustc_version::{version_meta, Channel};
 use serde::Deserialize;
 use std::{
@@ -41,55 +40,27 @@ static TESTS: Lazy<Vec<Test>> = Lazy::new(|| {
     serde_json::from_str(&content).unwrap()
 });
 
-// smoelius: This should match `scripts/update_patches.sh`.
-const LINES_OF_CONTEXT: u32 = 2;
-
-mod cheap_tests {
-    use super::*;
-
-    #[cfg_attr(dylint_lib = "supplementary", allow(commented_code))]
-    #[test]
-    fn test() {
-        let version_meta = version_meta().unwrap();
-        for test in TESTS.iter() {
-            run_test(
-                module_path!(),
-                test,
-                test.flags.contains(Flags::EXPENSIVE)
+#[cfg_attr(dylint_lib = "supplementary", allow(commented_code))]
+#[test]
+fn test() {
+    let version_meta = version_meta().unwrap();
+    for test in TESTS.iter() {
+        run_test(
+            test,
+            (!cfg!(feature = "test-third-party-full") && test.flags.contains(Flags::EXPENSIVE))
                     // || test.flags.contains(Flags::SKIP)
                     || (test.flags.contains(Flags::SKIP_NIGHTLY)
                         && version_meta.channel == Channel::Nightly),
-            );
-        }
-    }
-}
-
-mod all_tests {
-    use super::*;
-    #[test]
-    #[ignore]
-    fn test() {
-        let version_meta = version_meta().unwrap();
-        for test in TESTS.iter() {
-            run_test(
-                module_path!(),
-                test,
-                // test.flags.contains(Flags::SKIP) ||
-                test.flags.contains(Flags::SKIP_NIGHTLY)
-                    && version_meta.channel == Channel::Nightly,
-            );
-        }
+        );
     }
 }
 
 #[allow(clippy::too_many_lines)]
-fn run_test(module_path: &str, test: &Test, no_run: bool) {
-    let (_, module) = module_path.split_once("::").unwrap();
+fn run_test(test: &Test, no_run: bool) {
     #[allow(clippy::explicit_write)]
     writeln!(
         stderr(),
-        "{}: {}{}",
-        module,
+        "{}{}",
         test.url,
         if no_run { " (no-run)" } else { "" }
     )
@@ -227,10 +198,13 @@ where
     assert
 }
 
+#[cfg(feature = "test-third-party-full")]
 #[test]
-#[ignore]
 fn patches_are_current() {
-    let re = Regex::new(r"^index [[:xdigit:]]{7}\.\.[[:xdigit:]]{7} [0-7]{6}$").unwrap();
+    // smoelius: This should match `scripts/update_patches.sh`.
+    const LINES_OF_CONTEXT: u32 = 2;
+
+    let re = regex::Regex::new(r"^index [[:xdigit:]]{7}\.\.[[:xdigit:]]{7} [0-7]{6}$").unwrap();
 
     for test in TESTS.iter() {
         let tempdir = tempdir_in(env!("CARGO_MANIFEST_DIR")).unwrap();

diff --git a/test-fuzz/tests/supply_chain.json b/test-fuzz/tests/supply_chain.json
@@ -1059,15 +1059,6 @@
         "name": "Michal 'vorner' Vaner"
       }
     ],
-    "paste": [
-      {
-        "avatar": "https://avatars.githubusercontent.com/u/1940490?v=4",
-        "id": 3618,
-        "kind": "user",
-        "login": "dtolnay",
-        "name": "David Tolnay"
-      }
-    ],
     "ppv-lite86": [
       {
         "avatar": "https://avatars.githubusercontent.com/u/1047859?v=4",