From f40ef997ac99e6b202fbd6d25840672fc09bfbe7 Mon Sep 17 00:00:00 2001 From: "Celina G. Val" Date: Thu, 31 Aug 2023 15:11:26 -0700 Subject: [PATCH] Create tooling for end-to-end testing Create two different tools: - `test-drive`: A rustc_driver that compiles a crate and run a few sanity checks on StableMIR. - `compiletest`: A wrapper to run compiler tests using the `test-drive` tool. I am also adding a script to run a few rustc tests and a nightly workflow. The files diff is not quite working yet so most tests that fail compilation don't succeed yet. --- .github/scripts/run_rustc_tests.sh | 35 ++++++++ Cargo.toml | 12 +++ rust-toolchain.toml | 3 + tools/compiletest/Cargo.toml | 14 +++ tools/compiletest/build.rs | 16 ++++ tools/compiletest/src/args.rs | 40 +++++++++ tools/compiletest/src/main.rs | 12 +++ tools/test-drive/Cargo.toml | 12 +++ tools/test-drive/build.rs | 15 ++++ tools/test-drive/src/main.rs | 88 +++++++++++++++++++ tools/test-drive/src/sanity_checks.rs | 121 ++++++++++++++++++++++++++ 11 files changed, 368 insertions(+) create mode 100755 .github/scripts/run_rustc_tests.sh create mode 100644 Cargo.toml create mode 100644 rust-toolchain.toml create mode 100644 tools/compiletest/Cargo.toml create mode 100644 tools/compiletest/build.rs create mode 100644 tools/compiletest/src/args.rs create mode 100644 tools/compiletest/src/main.rs create mode 100644 tools/test-drive/Cargo.toml create mode 100644 tools/test-drive/build.rs create mode 100644 tools/test-drive/src/main.rs create mode 100644 tools/test-drive/src/sanity_checks.rs diff --git a/.github/scripts/run_rustc_tests.sh b/.github/scripts/run_rustc_tests.sh new file mode 100755 index 0000000..a6938b7 --- /dev/null +++ b/.github/scripts/run_rustc_tests.sh @@ -0,0 +1,35 @@ +#!/usr/bin/env bash + +set -e +set -u + +# Location of a rust repository. Clone one if path doesn't exist. +RUST_REPO="${RUST_REPO:?Missing path to rust repository. Set RUST_REPO}" +# Where we will store the SMIR tools (Optional). +TOOLS_BIN="${TOOLS_BIN:-"/tmp/smir/bin"}" +# Assume we are inside SMIR repository +SMIR_PATH=$(git rev-parse --show-toplevel) +export RUST_BACKTRACE=1 + +pushd "${SMIR_PATH}" +cargo +smir build -Z unstable-options --out-dir "${TOOLS_BIN}" +export PATH="${TOOLS_BIN}":"${PATH}" + +if [[ ! -e "${RUST_REPO}" ]]; then + mkdir -p "$(dirname ${RUST_REPO})" + git clone --depth 1 https://github.com/rust-lang/rust.git "${RUST_REPO}" +fi + +pushd "${RUST_REPO}" +SUITES=( + # Match https://github.com/rust-lang/rust/blob/master/src/bootstrap/test.rs for now + "tests/ui/cfg ui" +) +for suite_cfg in "${SUITES[@]}"; do + # Hack to work on older bash like the ones on MacOS. + suite_pair=($suite_cfg) + suite=${suite_pair[0]} + mode=${suite_pair[1]} + echo "${suite_cfg} pair: $suite_pair mode: $mode" + compiletest --driver-path="${TOOLS_BIN}/test-drive" --mode=${mode} --src-base="${suite}" --output-path "${RUST_REPO}/build" +done diff --git a/Cargo.toml b/Cargo.toml new file mode 100644 index 0000000..e90ec9c --- /dev/null +++ b/Cargo.toml @@ -0,0 +1,12 @@ +# Cargo workspace for utility tools used to check stable-mir in CI +[workspace] +resolver = "2" +members = [ + "tools/compiletest", + "tools/test-drive", +] + +exclude = [ + "build", + "target", +] diff --git a/rust-toolchain.toml b/rust-toolchain.toml new file mode 100644 index 0000000..ee9dc17 --- /dev/null +++ b/rust-toolchain.toml @@ -0,0 +1,3 @@ +[toolchain] +channel = "nightly" +components = ["llvm-tools-preview", "rustc-dev", "rust-src", "rustfmt"] diff --git a/tools/compiletest/Cargo.toml b/tools/compiletest/Cargo.toml new file mode 100644 index 0000000..94fea23 --- /dev/null +++ b/tools/compiletest/Cargo.toml @@ -0,0 +1,14 @@ +[package] +name = "compiletest" +description = "Run tests using compiletest-rs" +version = "0.0.0" +edition = "2021" + +[dependencies] +compiletest_rs = { version = "0.10.0", features = [ "rustc" ] } +clap = { version = "4.1.3", features = ["derive"] } + +[package.metadata.rust-analyzer] +# This crate uses #[feature(rustc_private)]. +# See https://github.com/rust-analyzer/rust-analyzer/pull/7891 +rustc_private = true diff --git a/tools/compiletest/build.rs b/tools/compiletest/build.rs new file mode 100644 index 0000000..63fcfb5 --- /dev/null +++ b/tools/compiletest/build.rs @@ -0,0 +1,16 @@ +use std::env; +use std::path::PathBuf; + +pub fn main() { + // Add rustup to the rpath in order to properly link with the correct rustc version. + let rustup_home = env::var("RUSTUP_HOME").unwrap(); + let toolchain = env::var("RUSTUP_TOOLCHAIN").unwrap(); + let rustc_lib: PathBuf = [&rustup_home, "toolchains", &toolchain, "lib"] + .iter() + .collect(); + println!( + "cargo:rustc-link-arg-bin=compiletest=-Wl,-rpath,{}", + rustc_lib.display() + ); + println!("cargo:rustc-env=RUSTC_LIB_PATH={}", rustc_lib.display()); +} diff --git a/tools/compiletest/src/args.rs b/tools/compiletest/src/args.rs new file mode 100644 index 0000000..72a5f83 --- /dev/null +++ b/tools/compiletest/src/args.rs @@ -0,0 +1,40 @@ +use compiletest_rs::Config; +use std::fmt::Debug; +use std::path::PathBuf; + +#[derive(Debug, clap::Parser)] +#[command(version, name = "compiletest")] +pub struct Args { + /// The path where all tests are + #[arg(long)] + src_base: PathBuf, + + /// The mode according to compiletest modes. + #[arg(long)] + mode: String, + + /// Path for the stable-mir driver. + #[arg(long)] + driver_path: PathBuf, + + /// Path for where the output should be stored. + #[arg(long)] + output_path: PathBuf, + + #[arg(long)] + verbose: bool, +} + +impl From for Config { + fn from(args: Args) -> Config { + let mut config = Config::default(); + config.mode = args.mode.parse().expect("Invalid mode"); + config.src_base = args.src_base; + config.rustc_path = args.driver_path; + config.build_base = args.output_path; + config.verbose = args.verbose; + config.run_lib_path = PathBuf::from(env!("RUSTC_LIB_PATH")); + config.link_deps(); + config + } +} diff --git a/tools/compiletest/src/main.rs b/tools/compiletest/src/main.rs new file mode 100644 index 0000000..b27f8ee --- /dev/null +++ b/tools/compiletest/src/main.rs @@ -0,0 +1,12 @@ +//! Run compiletest on a given folder. + +mod args; +use clap::Parser; +use compiletest_rs::Config; + +fn main() { + let args = args::Args::parse(); + println!("args: ${args:?}"); + let cfg = Config::from(args); + compiletest_rs::run_tests(&cfg); +} diff --git a/tools/test-drive/Cargo.toml b/tools/test-drive/Cargo.toml new file mode 100644 index 0000000..12b88aa --- /dev/null +++ b/tools/test-drive/Cargo.toml @@ -0,0 +1,12 @@ +[package] +name = "test-drive" +description = "A rustc wrapper that can be used to test stable-mir on a crate" +version = "0.0.0" +edition = "2021" + +[dependencies] + +[package.metadata.rust-analyzer] +# This crate uses #[feature(rustc_private)]. +# See https://github.com/rust-analyzer/rust-analyzer/pull/7891 +rustc_private = true diff --git a/tools/test-drive/build.rs b/tools/test-drive/build.rs new file mode 100644 index 0000000..44c9940 --- /dev/null +++ b/tools/test-drive/build.rs @@ -0,0 +1,15 @@ +use std::env; +use std::path::PathBuf; + +pub fn main() { + // Add rustup to the rpath in order to properly link with the correct rustc version. + let rustup_home = env::var("RUSTUP_HOME").unwrap(); + let toolchain = env::var("RUSTUP_TOOLCHAIN").unwrap(); + let rustc_lib: PathBuf = [&rustup_home, "toolchains", &toolchain, "lib"] + .iter() + .collect(); + println!( + "cargo:rustc-link-arg-bin=test-drive=-Wl,-rpath,{}", + rustc_lib.display() + ); +} diff --git a/tools/test-drive/src/main.rs b/tools/test-drive/src/main.rs new file mode 100644 index 0000000..d8f20d7 --- /dev/null +++ b/tools/test-drive/src/main.rs @@ -0,0 +1,88 @@ +//! Test that users are able to inspec the MIR body of functions and types + +#![feature(rustc_private)] +#![feature(assert_matches)] +#![feature(result_option_inspect)] + +mod sanity_checks; + +extern crate rustc_middle; +extern crate rustc_smir; + +use rustc_middle::ty::TyCtxt; +use rustc_smir::{rustc_internal, stable_mir}; +use std::panic::{catch_unwind, AssertUnwindSafe}; +use std::process::ExitCode; + +const CHECK_ARG: &str = "--check-smir"; + +type TestResult = Result<(), String>; + +/// This is a wrapper that can be used to replace rustc. +/// +/// Besides all supported rustc arguments, use `--check-smir` to run all the stable-mir checks. +/// This allows us to use this tool in cargo projects to analyze the target crate only by running +/// `cargo rustc --check-smir`. +fn main() -> ExitCode { + let mut check_smir = false; + let args: Vec<_> = std::env::args() + .filter(|arg| { + let is_check_arg = arg == CHECK_ARG; + check_smir |= is_check_arg; + !is_check_arg + }) + .collect(); + + + let callback = if check_smir { test_stable_mir } else { |_: TyCtxt| ExitCode::SUCCESS }; + let result = rustc_internal::StableMir::new(args, callback).continue_compilation().run(); + if let Ok(test_result) = result { + test_result + } else { + ExitCode::FAILURE + } +} + +macro_rules! run_tests { + ($( $test:path ),+) => { + [$({ + run_test(stringify!($test), || { $test() }) + },)+] + }; +} + +/// This function invoke other tests and process their results. +/// Tests should avoid panic, +fn test_stable_mir(tcx: TyCtxt<'_>) -> ExitCode { + let results = run_tests![ + sanity_checks::test_entry_fn, + sanity_checks::test_all_fns, + sanity_checks::test_traits, + sanity_checks::test_crates + ]; + let (success, failure): (Vec<_>, Vec<_>) = results.iter().partition(|r| r.is_ok()); + println!( + "Ran {} tests. {} succeeded. {} failed", + results.len(), + success.len(), + failure.len() + ); + if failure.is_empty() { + ExitCode::SUCCESS + } else { + ExitCode::FAILURE + } +} + +fn run_test TestResult>(name: &str, f: F) -> TestResult { + let result = match catch_unwind(AssertUnwindSafe(f)) { + Err(_) => Err("Panic: {}".to_string()), + Ok(result) => result, + }; + println!( + "Test {}: {}", + name, + result.as_ref().err().unwrap_or(&"Ok".to_string()) + ); + result +} diff --git a/tools/test-drive/src/sanity_checks.rs b/tools/test-drive/src/sanity_checks.rs new file mode 100644 index 0000000..8267db7 --- /dev/null +++ b/tools/test-drive/src/sanity_checks.rs @@ -0,0 +1,121 @@ +//! Module that contains sanity checks that Stable MIR APIs don't crash and that +//! their result is coherent. +//! +//! These checks should only depend on StableMIR APIs. See other modules for tests that compare +//! the result between StableMIR and internal APIs. +use crate::TestResult; +use rustc_middle::ty::TyCtxt; +use rustc_smir::stable_mir; +use std::collections::HashSet; +use std::fmt::Debug; +use std::hint::black_box; + +fn check_equal(val: T, expected: T, msg: &str) -> TestResult +where + T: Debug + PartialEq, +{ + if val != expected { + Err(format!( + "{}: \n Expected: {:?}\n Found: {:?}", + msg, expected, val + )) + } else { + Ok(()) + } +} + +pub fn check(val: bool, msg: String) -> TestResult { + if !val { + Err(msg) + } else { + Ok(()) + } +} + +// Test that if there is an entry point, the function is part of `all_local_items`. +pub fn test_entry_fn() -> TestResult { + let entry_fn = stable_mir::entry_fn(); + entry_fn.map_or(Ok(()), |entry_fn| { + let all_items = stable_mir::all_local_items(); + check( + all_items.contains(&entry_fn), + format!("Failed to find entry_fn `{:?}`", entry_fn), + ) + }) +} + +/// Check that the crate isn't empty and iterate over function bodies. +pub fn test_all_fns() -> TestResult { + let all_items = stable_mir::all_local_items(); + check( + !all_items.is_empty(), + "Failed to find any local item".to_string(), + )?; + + for item in all_items { + // Get body and iterate over items + let body = item.body(); + check_body(body); + } + Ok(()) +} + +/// FIXME: Create to track improvements to TraitDecls / ImplTraitDecls. +/// Using these structures will always follow calls to get more details about those structures. +/// Unless user is trying to find a specific type, this will get repetitive. +pub fn test_traits() -> TestResult { + let all_traits = stable_mir::all_trait_decls(); + for trait_decl in all_traits.iter().map(stable_mir::trait_decl) { + // Can't compare trait_decl, so just compare a field for now. + check_equal( + stable_mir::trait_decl(&trait_decl.def_id).specialization_kind, + trait_decl.specialization_kind, + "external crate mismatch", + )?; + } + + for trait_impl in stable_mir::all_trait_impls() + .iter() + .map(stable_mir::trait_impl) + { + check( + all_traits.contains(&trait_impl.value.def_id), + format!("Failed to find trait definition {trait_impl:?}"), + )?; + } + Ok(()) +} + +pub fn test_crates() -> TestResult { + for krate in stable_mir::external_crates() { + check_equal( + stable_mir::find_crate(&krate.name.as_str()), + Some(krate), + "external crate mismatch", + )?; + } + + let local = stable_mir::local_crate(); + check_equal( + stable_mir::find_crate(&local.name.as_str()), + Some(local), + "local crate mismatch", + ) +} + +/// Visit all local types, statements and terminator to ensure nothing crashes. +fn check_body(body: stable_mir::mir::Body) { + for bb in body.blocks { + for stmt in bb.statements { + black_box(matches!(stmt, stable_mir::mir::Statement::Assign(..))); + } + black_box(matches!( + bb.terminator, + stable_mir::mir::Terminator::Goto { .. } + )); + } + + for local in body.locals { + black_box(matches!(local.kind(), stable_mir::ty::TyKind::Alias(..))); + } +}