Add a developer documentation to our book

.github/scripts/run_own_tests.sh

@@ -1,18 +1,22 @@
 #!/usr/bin/env bash
 # Execute our own set of tests using a local `compiletest` tool based on `ui_test`.
+# This will run cargo under the repository path and, by default, will run the toolchain
+# specified in the `rust-toolchain.toml` file.
 set -e
 set -u
 
 # Where we will store the SMIR tools (Optional).
 TOOLS_BIN="${TOOLS_BIN:-"/tmp/smir/bin"}"
 # Assume we are inside SMIR repository
 SMIR_PATH=$(git rev-parse --show-toplevel)
+REPO_TOOLCHAIN=$(rustup show active-toolchain | (read toolchain _; echo $toolchain))
+TOOLCHAIN="${TOOLCHAIN:-${REPO_TOOLCHAIN}}"
 export RUST_BACKTRACE=1
 
 # Build stable_mir tools
 function build_smir_tools() {
- pushd "${SMIR_PATH}"
- cargo +nightly build -Z unstable-options --out-dir "${TOOLS_BIN}"
+ echo "#### Build tools. Toolchain: ${TOOLCHAIN}"
+ cargo +${TOOLCHAIN} build -Z unstable-options --out-dir "${TOOLS_BIN}"
  export PATH="${TOOLS_BIN}":"${PATH}"
 }
 
@@ -37,5 +41,6 @@ function run_tests() {
  done
 }
 
+pushd "${SMIR_PATH}" > /dev/null
 build_smir_tools
 run_tests

.github/scripts/run_rustc_tests.sh

@@ -19,10 +19,14 @@ TOOLS_BIN="${TOOLS_BIN:-"/tmp/smir/bin"}"
 # Assume we are inside SMIR repository
 SMIR_PATH=$(git rev-parse --show-toplevel)
 
+# Set the toolchain to be used in this script
+REPO_TOOLCHAIN=$(rustup show active-toolchain | (read toolchain _; echo $toolchain))
+TOOLCHAIN="${TOOLCHAIN:-${REPO_TOOLCHAIN}}"
+
 # Build stable_mir tools
 function build_smir_tools() {
  pushd "${SMIR_PATH}"
- cargo +nightly build -Z unstable-options --out-dir "${TOOLS_BIN}"
+ cargo +${TOOLCHAIN} build -Z unstable-options --out-dir "${TOOLS_BIN}"
  export PATH="${TOOLS_BIN}":"${PATH}"
 }
 
@@ -32,7 +36,7 @@ function setup_rustc_repo() {
  mkdir -p "$(dirname ${RUST_REPO})"
  git clone -b master https://github.com/rust-lang/rust.git "${RUST_REPO}"
  pushd "${RUST_REPO}"
- commit="$(rustc +nightly -vV | awk '/^commit-hash/ { print $2 }')"
+ commit="$(rustc +${TOOLCHAIN} -vV | awk '/^commit-hash/ { print $2 }')"
  git checkout ${commit}
  git submodule init -- "${RUST_REPO}/library/stdarch"
  git submodule update
@@ -56,9 +60,9 @@ function run_tests() {
  #"pretty pretty" -- 2 failing tests
  )
 
- SYSROOT=$(rustc +nightly --print sysroot)
+ SYSROOT=$(rustc +${TOOLCHAIN} --print sysroot)
  PY_PATH=$(type -P python3)
- HOST=$(rustc +nightly -vV | awk '/^host/ { print $2 }')
+ HOST=$(rustc +${TOOLCHAIN} -vV | awk '/^host/ { print $2 }')
  FILE_CHECK="$(which FileCheck-12 || which FileCheck-13 || which FileCheck-14)"
 
  for suite_cfg in "${SUITES[@]}"; do
@@ -68,7 +72,7 @@ function run_tests() {
  mode=${suite_pair[1]}
 
  echo "#### Running suite: ${suite} mode: ${mode}"
- cargo +nightly run -p compiletest -- \
+ cargo +${TOOLCHAIN} run -p compiletest -- \
  --compile-lib-path="${SYSROOT}/lib" \
  --run-lib-path="${SYSROOT}/lib"\
  --python="${PY_PATH}" \

.gitignore

@@ -1 +1,7 @@
 /book
+**/target
+
+.idea
+*.swp
+*.swo
+.vscode

SUMMARY.md

@@ -4,3 +4,4 @@
 - [Charter](./CHARTER.md)
 - [Meetings](./meetings/README.md)
 - [Draft RFCs](./draft-rfcs/README.md)
+- [Developer Documentation](./dev/README.md)

dev/README.md

@@ -0,0 +1,121 @@
+# Developer documentation
+
+This folder contains some documentation useful for the development of Stable MIR
+
+## Tools
+
+We currently have two different tools that is used to help with testing StableMIR
+
+### TestDrive
+
+This is a small driver that we build on the top of the rust compiler.
+By default, this driver behaves exactly like `rustc`, and it can be used to build a crate
+or multiple packages using cargo.
+
+This driver also contains multiple checks that will inspect the stable MIR of the crate that was compiled.
+In order to run those tests, you need to pass an extra argument `--smir-check`.
+
+Let's say we have a crate `input.rs`, in order to run all checks, one should invoke:
+
+```shell
+cargo run -p test-drive -- --smir-check test.rs
+# or run the test-drive directly where ${BIN_PATH} is the binary location
+${BIN_PATH}/test-drive --smir-check test.rs
+```
+
+In order to run this as part of a cargo build, you can run from a workspace:
+
+```shell
+# Only check SMIR for the target crate
+RUSTC=${BIN_PATH} cargo rustc -p ${TARGET_PACKAGE} -- --smir-check
+
+# Check SMIR for all crates being compiled, including dependencies
+RUSTC=${BIN_PATH} RUSTFLAGS=--smir-check cargo build
+```
+
+This driver accepts a few other options that are all preceded with `--smir` prefix[^outdated]. For example
+ - `--smir-fixme`: Will run checks that currently trigger a known bug in StableMIR.
+ - `--smir-verbose`: Print status and failure messages.
+
+[^outdated]: Since these are test tools, this documentation may be outdated.
+
+### Compiletest
+
+This is a little utility built on the top of `ui_test` crate.
+It scans our test folders and run tests according to the selected mode.
+For more details on how to run this utility, check its help option:
+```shell
+cargo run -p compiletest -- --help
+```
+
+## Test Suites
+
+We have a few different test suites for Stable MIR:
+ - **Rust compiler [`ui-fulldeps/stable-mir`](https://github. com/rust-lang/rust/tree/master/tests/ui-fulldeps/stable-mir):**
+ These are very simple sanity checks that live inside the main rust repository.
+ These tests should cover very basic functionality related to the translation of internal structures to stable ones.
+ - **Rust compiler suites:** We are enabling the execution of rust compiler test suites.
+ These suites are run with the rust respository compiletest. 
+ To run them, I recommend using our script `scripts/run_rustc_tests.sh` since there are a lot of arguments.
+ - **Local suites:** These are suites hosted in this repository inside `tests/`.
+ These are run using our local `compilest` and
+ `scripts/run_own_tests.sh` is a script to run all local suites
+ - `fixme`: Single file crates that are currently failing.
+ These tests are run using `--smir-fixme`.
+ - `sanity-checks`: Small examples that exercise some specific construct.
+ These tests succeed if compilation and all smir checks succeed.
+
+## Toolchain versions
+
+Our CI currently run a nightly job that build and run our test suites against the latest nightly toolchain.
+This repository also contains a .toolchain file that specifies to use `nightly` channel.
+
+However, if you already have the nightly toolchain installed, this will not update the toolchain to
+the latest nightly.
+You need to explicitly do that.
+
+If you see some errors while compiling our tools, please ensure you have the latest nightly.
+You might also want to check [our nightly runs](https://github. com/rust-lang/project-stable-mir/actions/workflows/nightly.yml?query=branch%2Amain) to ensure they are not currently broken.
+If so, you can check what was the last successful nightly run, and use its nightly version.
+
+### Custom toolchain
+
+In order to run the tools and test suites using a local copy of `rustc`, do the following:
+ 1. Build stage 2 of the compiler.
+ See [`rustc` build guide](https://rustc-dev-guide.rust-lang.org/building/how-to-build-and-run.html) for more details. E.g.:
+```shell
+git clone https://github.com/rust-lang/rust.git
+cd rust
+git checkout ${COMMIT_ID:?"Missing rustc commit id"}
+./configure --enable-extended --tools=src,rustfmt,cargo --enable-debug --set=llvm.download-ci-llvm=true
+./x.py build -i --stage 2
+```
+
+ 2. Create a custom toolchain:
+```shell
+# This will create a toolchain named "local"
+# Set the TARGET variable, e.g.: x86_64-unknown-linux-gnu
+rustup toolchain link local build/${TARGET}/stage2
+cp build/${TARGET}/stage2-tools-bin/* build/${TARGET}/stage2/bin/
+```
+ 3. Override the current toolchain in your `project-stable-mir` repository.
+```shell
+cd ${SMIR_FOLDER}
+rustup override set local
+cargo clean
+cargo build
+```
+
+By default, the build scripts will use the active toolchain for the project folder.
+If you run step 3, the scripts should already pick up the local toolchain.
+Additionally, you can also set the rust toolchain by setting the TOOLCHAIN environment variable.
+E.g.:
+```shell
+# Clean old build
+cargo clean
+
+# Run test with local toolchain
+TOOLCHAIN=local ./.github/scripts/run_own_tests.sh
+```
+
+Note: To remove the override created on step 3, run `rustup override unset` in the same folder.