CI: Code Coverage (#35)

* coverage generation script

* generate all coverage

* get rid of chuffed stuff in git status

* deploy to pages

* fix typO

* fix

* print summaries

* also add things to latest

* only run latest on main

* update gen script

* fix output

* better table

* fix typo

* fix typo

* rename workflow

* rename workflow

* rename again

.github/workflows/coverage.yml

@@ -0,0 +1,101 @@
+name: 'Info: Code Coverage'
+on:
+ push:
+ pull_request:
+ workflow_dispatch:
+
+# see https://github.com/JamesIves/github-pages-deploy-action/tree/dev
+permissions:
+ contents: write
+
+jobs:
+ coverage:
+ name: "Coverage Report"
+ # only do coverage for ready PRs
+ if: ${{ github.event != 'pull_request' || ( github.event == 'pull_request' && (! github.event.pull_request.draft)) }}
+ runs-on: ubuntu-latest
+ steps:
+ - name: Checkout the repository
+ uses: actions/checkout@v2
+
+ - name: Get submodule hashes for cache invalidation
+ id: cache-hashes
+ run: |
+ echo "shas=$(git rev-parse HEAD:solvers/minion/vendor)-$(git rev-parse HEAD:solvers/chuffed/vendor)"
+ echo "shas=$(git rev-parse HEAD:solvers/minion/vendor)-$(git rev-parse HEAD:solvers/chuffed/vendor)" >> "$GITHUB_OUTPUT"
+
+ - name: Set up cache
+ uses: actions/cache@v3
+ with:
+ path: |
+ ~/.cargo/bin/
+ ~/.cargo/registry/index/
+ ~/.cargo/registry/cache/
+ ~/.cargo/git/db/
+ target/ 
+ solvers/minion/vendor
+ solvers/chuffed/vendor
+ key: nightly-${{ runner.os }}-solvers-${{ steps.cache-hashes.outputs.shas }}-cargo-${{ hashFiles('**/Cargo.lock') }}
+ restore-keys: nightly-${{ runner.os }}-solvers-${{ steps.cache-hashes.outputs.shas }}-cargo-
+
+ - name: Install rust nightly
+ working-directory: ./solvers/minion
+ run: rustup update nightly && rustup default nightly
+
+ - name: Generate coverage reports
+ working-directory: .
+ run: |
+ ./etc/scripts/gen_coverage_all.sh
+
+ - name: Move all html to correct folders for deployment
+ run: |
+ # put things both in the sha directory, and in latest/
+ mkdir -p deploy/conjure-oxide
+ mkdir -p deploy/minion
+ mkdir -p deploy/chuffed
+
+ cp -r coverage/html/* deploy/conjure-oxide
+ cp -r solvers/minion/coverage/html/* deploy/minion
+ cp -r solvers/chuffed/coverage/html/* deploy/chuffed
+
+ - name: Deploy to Github Pages 
+ uses: JamesIves/github-pages-deploy-action@v4
+ with:
+ folder: ./deploy
+ target-folder: "coverage/${{ github.sha }}"
+ branch: gh-pages
+ commit-message: "Actions: Code Coverage for ${{ github.sha }}"
+
+ - name: Update latest code coverage if on main.
+ if: github.ref == 'refs/heads/main'
+ uses: JamesIves/github-pages-deploy-action@v4
+ with:
+ folder: ./deploy
+ target-folder: "coverage/latest"
+ branch: gh-pages
+ commit-message: "Actions: Update latest code coverage for main (${{ github.sha }})"
+
+ - name: Format summary
+ run : |
+ CONJURE_OXIDE_URL="https://conjure-cp.github.io/conjure-oxide/coverage/${{ github.sha }}/conjure-oxide"
+ MINION_URL="https://conjure-cp.github.io/conjure-oxide/coverage/${{ github.sha }}/minion"
+ CHUFFED_URL="https://conjure-cp.github.io/conjure-oxide/coverage/${{ github.sha }}/chuffed"
+
+ CONJURE_OXIDE_LATEST="https://conjure-cp.github.io/conjure-oxide/coverage/latest/conjure-oxide"
+ MINION_LATEST="https://conjure-cp.github.io/conjure-oxide/coverage/latest/minion"
+ CHUFFED_LATEST="https://conjure-cp.github.io/conjure-oxide/coverage/latest/chuffed"
+ echo '# Code Coverage' >> $GITHUB_STEP_SUMMARY
+ echo "" >> $GITHUB_STEP_SUMMARY
+ echo "## This commit" >> $GITHUB_STEP_SUMMARY
+ echo "| Crate | | | " >> $GITHUB_STEP_SUMMARY
+ echo "| ----- | ----- | ----- |" >> $GITHUB_STEP_SUMMARY
+ echo "| Conjure-Oxide | ![](${CONJURE_OXIDE_URL}/badges/flat.svg) | [Full Report](${CONJURE_OXIDE_URL}) | " >> $GITHUB_STEP_SUMMARY
+ echo "| Minion | ![](${MINION_URL}/badges/flat.svg) | [Full Report](${MINION_URL}) | " >> $GITHUB_STEP_SUMMARY
+ echo "| Chuffed | ![](${CHUFFED_URL}/badges/flat.svg) | [Full Report](${CHUFFED_URL}) | " >> $GITHUB_STEP_SUMMARY
+ echo "" >> $GITHUB_STEP_SUMMARY
+ echo "## Main" >> $GITHUB_STEP_SUMMARY
+ echo "| Crate | | | " >> $GITHUB_STEP_SUMMARY
+ echo "| ----- | ----- | ----- |" >> $GITHUB_STEP_SUMMARY
+ echo "| Conjure-Oxide | ![](${CONJURE_OXIDE_LATEST}/badges/flat.svg) | [Full Report](${CONJURE_OXIDE_LATEST}) | " >> $GITHUB_STEP_SUMMARY
+ echo "| Minion | ![](${MINION_LATEST}/badges/flat.svg) | [Full Report](${MINION_LATEST}) | " >> $GITHUB_STEP_SUMMARY
+ echo "| Chuffed | ![](${CHUFFED_LATEST}/badges/flat.svg) | [Full Report](${CHUFFED_LATEST}) | " >> $GITHUB_STEP_SUMMARY

.gitignore

@@ -1,3 +1,3 @@
 target
-
 solvers/**/vendor/build
+coverage

.gitmodules

@@ -7,3 +7,4 @@
 [submodule "solvers/chuffed/vendor"]
  path = solvers/chuffed/vendor
  url = https://github.com/Kieranoski702/chuffed
+ ignore = dirty

etc/scripts/gen_coverage.sh

@@ -0,0 +1,68 @@
+#!/bin/bash
+
+# gen_coverage.sh
+echo_err () {
+ echo "$@" 1>&2
+}
+
+usage_and_fail () { 
+ echo_err 'gen_coverage.sh'
+ echo_err ''
+ echo_err 'Generate coverage reports for the current rust project.'
+ echo_err 'These will be output into <project_root>/coverage'
+ exit 1
+}
+
+if ! command -v rustup &> /dev/null 
+then
+ echo_err "rustup is not found!"
+ exit 1
+fi
+
+if ! command -v cargo &> /dev/null 
+then
+ echo_err "cargo is not found!"
+ exit 1
+fi
+
+if ! command -v jq &> /dev/null 
+then
+ echo_err "jq is not found!"
+ exit 1
+fi
+
+# Setup - enter rust project
+cargo locate-project &>/dev/null || { echo_err "Cannot find a rust project"; usage_and_fail; }
+
+PROJECT_ROOT=$(dirname $(cargo locate-project | jq -r .root 2> /dev/null))
+TARGET_DIR=$(cargo metadata 2> /dev/null | jq -r .target_directory 2>/dev/null)
+
+cd "$PROJECT_ROOT"
+
+rm -rf coverage
+mkdir -p coverage || { echo_err "Cannot create coverage directory"; exit 1; }
+
+
+
+# Install required tools
+echo_err "info: installing nightly rust:"
+rustup install nightly
+
+echo_err "info: installing llvm-tools-preview:"
+rustup component add llvm-tools-preview
+
+echo_err "info: installing grcov:"
+rustup run nightly cargo install grcov
+
+
+# Run tests, profile, generate coverage
+# See https://blog.rng0.io/how-to-do-code-coverage-in-rust
+# and https://doc.rust-lang.org/beta/unstable-book/language-features/profiler-runtime.html
+echo_err "info: running tests with nightly profiler:"
+CARGO_INCREMENTAL=0 RUSTFLAGS='-Cinstrument-coverage' LLVM_PROFILE_FILE='coverage/cargo-test-%p-%m.profraw' rustup run nightly cargo test
+
+cd coverage
+echo_err "info: generating coverage reports:"
+grcov . -s .. --binary-path "$TARGET_DIR/debug/deps" -t html --branch --ignore-not-existing --keep-only src/** --ignore **/main.rs -o html
+grcov . -s .. --binary-path "$TARGET_DIR/debug/deps" -t lcov --branch --ignore-not-existing --keep-only src/** --ignore **/main.rs -o lcov.info
+

etc/scripts/gen_coverage_all.sh

@@ -0,0 +1,41 @@
+# !/bin/bash
+# generate coverage for all crates in the workspace
+
+PATH_TO_GEN_COV="/etc/scripts/gen_coverage.sh"
+echo_err () {
+ echo "$@" 1>&2
+}
+
+if ! command -v cargo &> /dev/null 
+then
+ echo_err "cargo is not found!"
+ exit 1
+fi
+
+if ! command -v jq &> /dev/null 
+then
+ echo_err "jq is not found!"
+ exit 1
+fi
+
+# Setup - enter rust workspace
+cargo locate-project --workspace &>/dev/null || { echo_err "Cannot find a rust workspace"; usage_and_fail; }
+WORKSPACE_ROOT=$(dirname $(cargo locate-project --workspace | jq -r .root 2> /dev/null))
+
+cd "$WORKSPACE_ROOT"
+
+# conjure-oxide coverage
+echo_err "GENERATING COVERAGE FOR CONJURE-OXIDE"
+rm -rf coverage
+bash "$WORKSPACE_ROOT/$PATH_TO_GEN_COV"
+
+# solver coverage
+for dir in "$WORKSPACE_ROOT/solvers"/*/
+do 
+ echo_err "GENERATING COVERAGE FOR $dir"
+ cd "$dir"
+ rm -rf coverage
+ bash "$WORKSPACE_ROOT/$PATH_TO_GEN_COV"
+done
+
+