Create tooling for end-to-end testing

Create two different tools:

  - `test-drive`: A rustc_driver that compiles a crate and run a few
  sanity checks on StableMIR.
  - `compiletest`: A wrapper to run compiler tests using the
    `test-drive` tool.

I am also adding a script to run a few rustc tests and a nightly
workflow. The files diff is not quite working yet so most tests
that fail compilation don't succeed yet.

.github/scripts/run_rustc_tests.sh

@@ -0,0 +1,35 @@
+#!/usr/bin/env bash
+
+set -e
+set -u
+
+# Location of a rust repository. Clone one if path doesn't exist.
+RUST_REPO="${RUST_REPO:?Missing path to rust repository. Set RUST_REPO}"
+# Where we will store the SMIR tools (Optional).
+TOOLS_BIN="${TOOLS_BIN:-"/tmp/smir/bin"}"
+# Assume we are inside SMIR repository
+SMIR_PATH=$(git rev-parse --show-toplevel)
+export RUST_BACKTRACE=1
+
+pushd "${SMIR_PATH}"
+cargo +smir build -Z unstable-options --out-dir "${TOOLS_BIN}"
+export PATH="${TOOLS_BIN}":"${PATH}"
+
+if [[ ! -e "${RUST_REPO}" ]]; then
+ mkdir -p "$(dirname ${RUST_REPO})"
+ git clone --depth 1 https://github.com/rust-lang/rust.git "${RUST_REPO}"
+fi
+
+pushd "${RUST_REPO}"
+SUITES=(
+ # Match https://github.com/rust-lang/rust/blob/master/src/bootstrap/test.rs for now
+ "tests/ui/cfg ui"
+)
+for suite_cfg in "${SUITES[@]}"; do
+ # Hack to work on older bash like the ones on MacOS.
+ suite_pair=($suite_cfg)
+ suite=${suite_pair[0]}
+ mode=${suite_pair[1]}
+ echo "${suite_cfg} pair: $suite_pair mode: $mode"
+ compiletest --driver-path="${TOOLS_BIN}/test-drive" --mode=${mode} --src-base="${suite}" --output-path "${RUST_REPO}/build"
+done

Cargo.toml

@@ -0,0 +1,12 @@
+# Cargo workspace for utility tools used to check stable-mir in CI
+[workspace]
+resolver = "2"
+members = [
+ "tools/compiletest",
+ "tools/test-drive",
+]
+
+exclude = [
+ "build",
+ "target",
+]

rust-toolchain.toml

@@ -0,0 +1,3 @@
+[toolchain]
+channel = "nightly"
+components = ["llvm-tools-preview", "rustc-dev", "rust-src", "rustfmt"]

tools/compiletest/Cargo.toml

@@ -0,0 +1,14 @@
+[package]
+name = "compiletest"
+description = "Run tests using compiletest-rs"
+version = "0.0.0"
+edition = "2021"
+
+[dependencies]
+compiletest_rs = { version = "0.10.0", features = [ "rustc" ] }
+clap = { version = "4.1.3", features = ["derive"] }
+
+[package.metadata.rust-analyzer]
+# This crate uses #[feature(rustc_private)].
+# See https://github.com/rust-analyzer/rust-analyzer/pull/7891
+rustc_private = true

tools/compiletest/build.rs

@@ -0,0 +1,16 @@
+use std::env;
+use std::path::PathBuf;
+
+pub fn main() {
+ // Add rustup to the rpath in order to properly link with the correct rustc version.
+ let rustup_home = env::var("RUSTUP_HOME").unwrap();
+ let toolchain = env::var("RUSTUP_TOOLCHAIN").unwrap();
+ let rustc_lib: PathBuf = [&rustup_home, "toolchains", &toolchain, "lib"]
+ .iter()
+ .collect();
+ println!(
+ "cargo:rustc-link-arg-bin=compiletest=-Wl,-rpath,{}",
+ rustc_lib.display()
+ );
+ println!("cargo:rustc-env=RUSTC_LIB_PATH={}", rustc_lib.display());
+}

tools/compiletest/src/args.rs

@@ -0,0 +1,40 @@
+use compiletest_rs::Config;
+use std::fmt::Debug;
+use std::path::PathBuf;
+
+#[derive(Debug, clap::Parser)]
+#[command(version, name = "compiletest")]
+pub struct Args {
+ /// The path where all tests are
+ #[arg(long)]
+ src_base: PathBuf,
+
+ /// The mode according to compiletest modes.
+ #[arg(long)]
+ mode: String,
+
+ /// Path for the stable-mir driver.
+ #[arg(long)]
+ driver_path: PathBuf,
+
+ /// Path for where the output should be stored.
+ #[arg(long)]
+ output_path: PathBuf,
+
+ #[arg(long)]
+ verbose: bool,
+}
+
+impl From<Args> for Config {
+ fn from(args: Args) -> Config {
+ let mut config = Config::default();
+ config.mode = args.mode.parse().expect("Invalid mode");
+ config.src_base = args.src_base;
+ config.rustc_path = args.driver_path;
+ config.build_base = args.output_path;
+ config.verbose = args.verbose;
+ config.run_lib_path = PathBuf::from(env!("RUSTC_LIB_PATH"));
+ config.link_deps();
+ config
+ }
+}

tools/compiletest/src/main.rs

@@ -0,0 +1,12 @@
+//! Run compiletest on a given folder.
+
+mod args;
+use clap::Parser;
+use compiletest_rs::Config;
+
+fn main() {
+ let args = args::Args::parse();
+ println!("args: ${args:?}");
+ let cfg = Config::from(args);
+ compiletest_rs::run_tests(&cfg);
+}

tools/test-drive/Cargo.toml

@@ -0,0 +1,12 @@
+[package]
+name = "test-drive"
+description = "A rustc wrapper that can be used to test stable-mir on a crate"
+version = "0.0.0"
+edition = "2021"
+
+[dependencies]
+
+[package.metadata.rust-analyzer]
+# This crate uses #[feature(rustc_private)].
+# See https://github.com/rust-analyzer/rust-analyzer/pull/7891
+rustc_private = true

tools/test-drive/build.rs

@@ -0,0 +1,15 @@
+use std::env;
+use std::path::PathBuf;
+
+pub fn main() {
+ // Add rustup to the rpath in order to properly link with the correct rustc version.
+ let rustup_home = env::var("RUSTUP_HOME").unwrap();
+ let toolchain = env::var("RUSTUP_TOOLCHAIN").unwrap();
+ let rustc_lib: PathBuf = [&rustup_home, "toolchains", &toolchain, "lib"]
+ .iter()
+ .collect();
+ println!(
+ "cargo:rustc-link-arg-bin=test-drive=-Wl,-rpath,{}",
+ rustc_lib.display()
+ );
+}

tools/test-drive/src/main.rs

@@ -0,0 +1,88 @@
+//! Test that users are able to inspec the MIR body of functions and types
+
+#![feature(rustc_private)]
+#![feature(assert_matches)]
+#![feature(result_option_inspect)]
+
+mod sanity_checks;
+
+extern crate rustc_middle;
+extern crate rustc_smir;
+
+use rustc_middle::ty::TyCtxt;
+use rustc_smir::{rustc_internal, stable_mir};
+use std::panic::{catch_unwind, AssertUnwindSafe};
+use std::process::ExitCode;
+
+const CHECK_ARG: &str = "--check-smir";
+
+type TestResult = Result<(), String>;
+
+/// This is a wrapper that can be used to replace rustc.
+///
+/// Besides all supported rustc arguments, use `--check-smir` to run all the stable-mir checks.
+/// This allows us to use this tool in cargo projects to analyze the target crate only by running
+/// `cargo rustc --check-smir`.
+fn main() -> ExitCode {
+ let mut check_smir = false;
+ let args: Vec<_> = std::env::args()
+ .filter(|arg| {
+ let is_check_arg = arg == CHECK_ARG;
+ check_smir |= is_check_arg;
+ !is_check_arg
+ })
+ .collect();
+
+
+ let callback = if check_smir { test_stable_mir } else { |_: TyCtxt| ExitCode::SUCCESS };
+ let result = rustc_internal::StableMir::new(args, callback).continue_compilation().run();
+ if let Ok(test_result) = result {
+ test_result
+ } else {
+ ExitCode::FAILURE
+ }
+}
+
+macro_rules! run_tests {
+ ($( $test:path ),+) => {
+ [$({
+ run_test(stringify!($test), || { $test() })
+ },)+]
+ };
+}
+
+/// This function invoke other tests and process their results.
+/// Tests should avoid panic,
+fn test_stable_mir(tcx: TyCtxt<'_>) -> ExitCode {
+ let results = run_tests![
+ sanity_checks::test_entry_fn,
+ sanity_checks::test_all_fns,
+ sanity_checks::test_traits,
+ sanity_checks::test_crates
+ ];
+ let (success, failure): (Vec<_>, Vec<_>) = results.iter().partition(|r| r.is_ok());
+ println!(
+ "Ran {} tests. {} succeeded. {} failed",
+ results.len(),
+ success.len(),
+ failure.len()
+ );
+ if failure.is_empty() {
+ ExitCode::SUCCESS
+ } else {
+ ExitCode::FAILURE
+ }
+}
+
+fn run_test<F: FnOnce() -> TestResult>(name: &str, f: F) -> TestResult {
+ let result = match catch_unwind(AssertUnwindSafe(f)) {
+ Err(_) => Err("Panic: {}".to_string()),
+ Ok(result) => result,
+ };
+ println!(
+ "Test {}: {}",
+ name,
+ result.as_ref().err().unwrap_or(&"Ok".to_string())
+ );
+ result
+}

tools/test-drive/src/sanity_checks.rs

@@ -0,0 +1,121 @@
+//! Module that contains sanity checks that Stable MIR APIs don't crash and that
+//! their result is coherent.
+//!
+//! These checks should only depend on StableMIR APIs. See other modules for tests that compare
+//! the result between StableMIR and internal APIs.
+use crate::TestResult;
+use rustc_middle::ty::TyCtxt;
+use rustc_smir::stable_mir;
+use std::collections::HashSet;
+use std::fmt::Debug;
+use std::hint::black_box;
+
+fn check_equal<T>(val: T, expected: T, msg: &str) -> TestResult
+where
+ T: Debug + PartialEq,
+{
+ if val != expected {
+ Err(format!(
+ "{}: \n Expected: {:?}\n Found: {:?}",
+ msg, expected, val
+ ))
+ } else {
+ Ok(())
+ }
+}
+
+pub fn check(val: bool, msg: String) -> TestResult {
+ if !val {
+ Err(msg)
+ } else {
+ Ok(())
+ }
+}
+
+// Test that if there is an entry point, the function is part of `all_local_items`.
+pub fn test_entry_fn() -> TestResult {
+ let entry_fn = stable_mir::entry_fn();
+ entry_fn.map_or(Ok(()), |entry_fn| {
+ let all_items = stable_mir::all_local_items();
+ check(
+ all_items.contains(&entry_fn),
+ format!("Failed to find entry_fn `{:?}`", entry_fn),
+ )
+ })
+}
+
+/// Check that the crate isn't empty and iterate over function bodies.
+pub fn test_all_fns() -> TestResult {
+ let all_items = stable_mir::all_local_items();
+ check(
+ !all_items.is_empty(),
+ "Failed to find any local item".to_string(),
+ )?;
+
+ for item in all_items {
+ // Get body and iterate over items
+ let body = item.body();
+ check_body(body);
+ }
+ Ok(())
+}
+
+/// FIXME: Create <issue> to track improvements to TraitDecls / ImplTraitDecls.
+/// Using these structures will always follow calls to get more details about those structures.
+/// Unless user is trying to find a specific type, this will get repetitive.
+pub fn test_traits() -> TestResult {
+ let all_traits = stable_mir::all_trait_decls();
+ for trait_decl in all_traits.iter().map(stable_mir::trait_decl) {
+ // Can't compare trait_decl, so just compare a field for now.
+ check_equal(
+ stable_mir::trait_decl(&trait_decl.def_id).specialization_kind,
+ trait_decl.specialization_kind,
+ "external crate mismatch",
+ )?;
+ }
+
+ for trait_impl in stable_mir::all_trait_impls()
+ .iter()
+ .map(stable_mir::trait_impl)
+ {
+ check(
+ all_traits.contains(&trait_impl.value.def_id),
+ format!("Failed to find trait definition {trait_impl:?}"),
+ )?;
+ }
+ Ok(())
+}
+
+pub fn test_crates() -> TestResult {
+ for krate in stable_mir::external_crates() {
+ check_equal(
+ stable_mir::find_crate(&krate.name.as_str()),
+ Some(krate),
+ "external crate mismatch",
+ )?;
+ }
+
+ let local = stable_mir::local_crate();
+ check_equal(
+ stable_mir::find_crate(&local.name.as_str()),
+ Some(local),
+ "local crate mismatch",
+ )
+}
+
+/// Visit all local types, statements and terminator to ensure nothing crashes.
+fn check_body(body: stable_mir::mir::Body) {
+ for bb in body.blocks {
+ for stmt in bb.statements {
+ black_box(matches!(stmt, stable_mir::mir::Statement::Assign(..)));
+ }
+ black_box(matches!(
+ bb.terminator,
+ stable_mir::mir::Terminator::Goto { .. }
+ ));
+ }
+
+ for local in body.locals {
+ black_box(matches!(local.kind(), stable_mir::ty::TyKind::Alias(..)));
+ }
+}