Merge pull request #37 from niklasdewally/pr/ci-fixes

CI Tweaks, Fix #38, and Single Cache
conjure-cp · Nov 4, 2023 · 6ae77bc · 6ae77bc
2 parents 6c2313e + bb2a780
commit 6ae77bc
Show file tree

Hide file tree

Showing 13 changed files with 1,483 additions and 239 deletions.
diff --git a/.github/workflows/chuffed-tests.yml → .github/workflows/chuffed.yml b/.github/workflows/chuffed-tests.yml → .github/workflows/chuffed.yml
@@ -2,6 +2,7 @@
 # https://ectobit.com/blog/speed-up-github-actions-rust-pipelines/
 name: "solvers/chuffed"
 on:
+ workflow_dispatch:
  push:
  paths:
  - "solvers/chuffed/**"
@@ -10,22 +11,20 @@ on:
  paths:
  - "solvers/chuffed/**"
  - "Cargo.*"
- workflow_dispatch:
 
 
 jobs:
  ubuntu:
- name: "solvers/chuffed: Ubuntu Build"
+ name: "Chuffed: Ubuntu Build"
  runs-on: ubuntu-latest
  steps:
  - uses: actions/checkout@v2
 
- # https://stackoverflow.com/questions/32327108/get-the-current-commit-id-of-specified-submodule
- # https://docs.github.com/en/actions/using-workflows/workflow-commands-for-github-actions#setting-an-environment-variable
- - name: Get chuffed hash for cache invalidation
- id: chuffed-cache
+ - name: Generate caching variables
+ id: cache-vars
  run: |
- echo "chuffed_sha=$(git rev-parse HEAD:solvers/chuffed/vendor)" >> "$GITHUB_OUTPUT"
+ echo -e "submodule_sha=$(./etc/ci/get_submodules_hash.sh)" >> "$GITHUB_OUTPUT"
+ echo -e "submodule_sha=$(./etc/ci/get_submodules_hash.sh)"
 
  - name: Set up cache
  uses: actions/cache@v3
@@ -37,9 +36,9 @@ jobs:
  ~/.cargo/git/db/
  target/ 
  solvers/chuffed/vendor
- key: ${{ runner.os }}-chuffed-${{ steps.chuffed-cache.outputs.chuffed_sha }}-cargo-${{ hashFiles('**/Cargo.lock') }}
- # match chuffed exactly, cargo inexactly
- restore-keys: ${{ runner.os }}-chuffed-${{ steps.chuffed-cache.outputs.chuffed_sha}}-cargo-
+
+ key: stable-${{ runner.os }}-gitmodules-${{ steps.cache-vars.outputs.submodule_sha }}-cargo-${{ hashFiles('**/Cargo.lock') }}
+ restore-keys: stable-${{ runner.os }}-gitmodules-
 
  - working-directory: ./solvers/chuffed
  run: rustup update stable && rustup default stable
@@ -48,15 +47,16 @@ jobs:
  run: cargo build -vv
 
  mac:
- name: "solvers/chuffed: Mac Build"
+ name: "Chuffed: Mac Build"
  runs-on: macos-latest
  steps:
  - uses: actions/checkout@v2
 
- - name: Get chuffed hash for cache invalidation
- id: chuffed-cache
+ - name: Generate caching variables
+ id: cache-vars
  run: |
- echo "chuffed_sha=$(git rev-parse HEAD:solvers/chuffed/vendor)" >> "$GITHUB_OUTPUT"
+ echo -e "submodule_sha=$(./etc/ci/get_submodules_hash.sh)" >> "$GITHUB_OUTPUT"
+ echo -e "submodule_sha=$(./etc/ci/get_submodules_hash.sh)"
 
  - name: Set up cache
  uses: actions/cache@v3
@@ -68,9 +68,9 @@ jobs:
  ~/.cargo/git/db/
  target/ 
  solvers/chuffed/vendor
- key: ${{ runner.os }}-chuffed-${{ steps.chuffed-cache.outputs.chuffed_sha }}-cargo-${{ hashFiles('**/Cargo.lock') }}
- # match chuffed exactly, cargo inexactly
- restore-keys: ${{ runner.os }}-chuffed-${{ steps.chuffed-cache.outputs.chuffed_sha}}-cargo-
+
+ key: stable-${{ runner.os }}-gitmodules-${{ steps.cache-vars.outputs.submodule_sha }}-cargo-${{ hashFiles('**/Cargo.lock') }}
+ restore-keys: stable-${{ runner.os }}-gitmodules-
 
  - working-directory: ./solvers/chuffed
  run: rustup update stable && rustup default stable
@@ -82,17 +82,19 @@ jobs:
  run: cargo build -vv
 
  tests:
- name: "solvers/chuffed: Tests"
+ name: "Chuffed: Tests"
  runs-on: ubuntu-latest
  steps:
  - uses: actions/checkout@v2
 
- # https://stackoverflow.com/questions/32327108/get-the-current-commit-id-of-specified-submodule
- # https://docs.github.com/en/actions/using-workflows/workflow-commands-for-github-actions#setting-an-environment-variable
- - name: Get chuffed hash for cache invalidation
- id: chuffed-cache
+ - name: Generate caching variables
+ id: cache-vars
  run: |
- echo "chuffed_sha=$(git rev-parse HEAD:solvers/chuffed/vendor)" >> "$GITHUB_OUTPUT"
+ echo -e "submodule_sha=$(./etc/ci/get_submodules_hash.sh)" >> "$GITHUB_OUTPUT"
+ echo -e "submodule_sha=$(./etc/ci/get_submodules_hash.sh)"
+
+ echo "cache_paths=$(./etc/ci/get_caching_paths.sh)" >> "$GITHUB_OUTPUT"
+ echo "cache_paths=$(./etc/ci/get_caching_paths.sh)"
 
  - name: Set up cache
  uses: actions/cache@v3
@@ -104,9 +106,9 @@ jobs:
  ~/.cargo/git/db/
  target/ 
  solvers/chuffed/vendor
- key: ${{ runner.os }}-chuffed-${{ steps.chuffed-cache.outputs.chuffed_sha }}-cargo-${{ hashFiles('**/Cargo.lock') }}
- # match chuffed exactly, cargo inexactly
- restore-keys: ${{ runner.os }}-chuffed-${{ steps.chuffed-cache.outputs.chuffed_sha}}-cargo-
+
+ key: stable-${{ runner.os }}-gitmodules-${{ steps.cache-vars.outputs.submodule_sha }}-cargo-${{ hashFiles('**/Cargo.lock') }}
+ restore-keys: stable-${{ runner.os }}-gitmodules-
 
  - working-directory: ./solvers/chuffed
  run: rustup update stable && rustup default stable

diff --git a/.github/workflows/coverage.yml → .github/workflows/code-coverage-deploy.yml b/.github/workflows/coverage.yml → .github/workflows/code-coverage-deploy.yml
@@ -1,85 +1,51 @@
-name: 'Info: Code Coverage'
+name: "Code Coverage (Deploy)"
 on:
- push:
- pull_request:
- workflow_dispatch:
+ workflow_run:
+ workflows: [code-coverage]
+ types:
+ - completed
+
 
 # see https://github.com/JamesIves/github-pages-deploy-action/tree/dev
 permissions:
  contents: write
 
 jobs:
- coverage:
- name: "Coverage Report"
- # only do coverage for ready PRs
- if: ${{ github.event != 'pull_request' || ( github.event == 'pull_request' && (! github.event.pull_request.draft)) }}
+ deploy-coverage: 
+ name: "Info: Code Coverage"
  runs-on: ubuntu-latest
  steps:
- - name: Checkout the repository
- uses: actions/checkout@v2
+ - uses: actions/checkout@v2
 
- - name: Get submodule hashes for cache invalidation
- id: cache-hashes
- run: |
- echo "shas=$(git rev-parse HEAD:solvers/minion/vendor)-$(git rev-parse HEAD:solvers/chuffed/vendor)"
- echo "shas=$(git rev-parse HEAD:solvers/minion/vendor)-$(git rev-parse HEAD:solvers/chuffed/vendor)" >> "$GITHUB_OUTPUT"
-
- - name: Set up cache
- uses: actions/cache@v3
+ - name: Download artifact
+ uses: dawidd6/action-download-artifact@v2
  with:
- path: |
- ~/.cargo/bin/
- ~/.cargo/registry/index/
- ~/.cargo/registry/cache/
- ~/.cargo/git/db/
- target/ 
- solvers/minion/vendor
- solvers/chuffed/vendor
- key: nightly-${{ runner.os }}-solvers-${{ steps.cache-hashes.outputs.shas }}-cargo-${{ hashFiles('**/Cargo.lock') }}
- restore-keys: nightly-${{ runner.os }}-solvers-${{ steps.cache-hashes.outputs.shas }}-cargo-
-
- - name: Install rust nightly
- working-directory: ./solvers/minion
- run: rustup update nightly && rustup default nightly
-
- - name: Generate coverage reports
- working-directory: .
- run: |
- ./etc/scripts/gen_coverage_all.sh
-
- - name: Move all html to correct folders for deployment
- run: |
- # put things both in the sha directory, and in latest/
- mkdir -p deploy/conjure-oxide
- mkdir -p deploy/minion
- mkdir -p deploy/chuffed
-
- cp -r coverage/html/* deploy/conjure-oxide
- cp -r solvers/minion/coverage/html/* deploy/minion
- cp -r solvers/chuffed/coverage/html/* deploy/chuffed
+ name: code-coverage-${{ github.event.workflow_run.head_sha }}
+ workflow: code-coverage.yml
+ path: ./deploy
 
  - name: Deploy to Github Pages 
  uses: JamesIves/github-pages-deploy-action@v4
  with:
  folder: ./deploy
- target-folder: "coverage/${{ github.sha }}"
+ target-folder: "coverage/${{ github.event.workflow_run.head_sha }}"
  branch: gh-pages
- commit-message: "Actions: Code Coverage for ${{ github.sha }}"
+ commit-message: "Actions: Code Coverage for ${{ github.event.workflow_run.head_sha }}"
 
- - name: Update latest code coverage if on main.
+ - name: If on main branch, copy coverage report to /latest.
  if: github.ref == 'refs/heads/main'
  uses: JamesIves/github-pages-deploy-action@v4
  with:
  folder: ./deploy
  target-folder: "coverage/latest"
  branch: gh-pages
- commit-message: "Actions: Update latest code coverage for main (${{ github.sha }})"
+ commit-message: "Actions: Update latest code coverage for main (${{ github.event.workflow_run.head_sha }})"
 
  - name: Format summary
  run : |
- CONJURE_OXIDE_URL="https://conjure-cp.github.io/conjure-oxide/coverage/${{ github.sha }}/conjure-oxide"
- MINION_URL="https://conjure-cp.github.io/conjure-oxide/coverage/${{ github.sha }}/minion"
- CHUFFED_URL="https://conjure-cp.github.io/conjure-oxide/coverage/${{ github.sha }}/chuffed"
+ CONJURE_OXIDE_URL="https://conjure-cp.github.io/conjure-oxide/coverage/${{ github.event.workflow_run.head_sha }}/conjure-oxide"
+ MINION_URL="https://conjure-cp.github.io/conjure-oxide/coverage/${{ github.event.workflow_run.head_sha }}/minion"
+ CHUFFED_URL="https://conjure-cp.github.io/conjure-oxide/coverage/${{ github.event.workflow_run.head_sha }}/chuffed"
 
  CONJURE_OXIDE_LATEST="https://conjure-cp.github.io/conjure-oxide/coverage/latest/conjure-oxide"
  MINION_LATEST="https://conjure-cp.github.io/conjure-oxide/coverage/latest/minion"
@@ -99,3 +65,30 @@ jobs:
  echo "| Conjure-Oxide | ![](${CONJURE_OXIDE_LATEST}/badges/flat.svg) | [Full Report](${CONJURE_OXIDE_LATEST}) | " >> $GITHUB_STEP_SUMMARY
  echo "| Minion | ![](${MINION_LATEST}/badges/flat.svg) | [Full Report](${MINION_LATEST}) | " >> $GITHUB_STEP_SUMMARY
  echo "| Chuffed | ![](${CHUFFED_LATEST}/badges/flat.svg) | [Full Report](${CHUFFED_LATEST}) | " >> $GITHUB_STEP_SUMMARY
+
+ indexes:
+ needs: deploy-coverage
+ name: "Regenerate indexes for coverage"
+ runs-on: ubuntu-latest
+ steps:
+ - name: Checkout the repository
+ uses: actions/checkout@v2
+
+ - name: Checkout the pages repository
+ uses: actions/checkout@v2
+ with:
+ ref: "gh-pages"
+ path: "pages"
+
+ - name: "Generate indexes"
+ run: |
+ ./etc/ci/genindex.py pages/coverage/${{ github.event.workflow_run.head_sha }}
+ ./etc/ci/genindex.py pages/coverage/latest
+ ./etc/ci/genindex.py pages/coverage/
+
+ - uses: JamesIves/github-pages-deploy-action@v4
+ with:
+ folder: ./pages/coverage/
+ target-folder: ./coverage/
+ branch: gh-pages
+ commit-message: "Actions: Update coverage indexes"
diff --git a/.github/workflows/code-coverage.yml b/.github/workflows/code-coverage.yml
@@ -0,0 +1,64 @@
+name: "Code Coverage (Generate)"
+on:
+ push:
+ pull_request:
+
+
+jobs:
+ coverage:
+ name: "Generate Code Coverage Reports"
+ # only do coverage for ready PRs
+ if: ${{ github.event != 'pull_request' || ( github.event == 'pull_request' && (! github.event.pull_request.draft)) }}
+ runs-on: ubuntu-latest
+ steps:
+ - name: Checkout the repository
+ uses: actions/checkout@v2
+
+ - name: Generate caching variables
+ id: cache-vars
+ run: |
+ echo -e "submodule_sha=$(./etc/ci/get_submodules_hash.sh)" >> "$GITHUB_OUTPUT"
+ echo -e "submodule_sha=$(./etc/ci/get_submodules_hash.sh)"
+
+ - name: Set up cache
+ uses: actions/cache@v3
+ with:
+ path: |
+ ~/.cargo/bin/
+ ~/.cargo/registry/index
+ ~/.cargo/git/
+ target/ 
+ solvers/minion/vendor
+ solvers/chuffed/vendor
+
+ key: nightly-${{ runner.os }}-gitmodules-${{ steps.cache-vars.outputs.submodule_sha }}-cargo-${{ hashFiles('**/Cargo.lock') }}
+ restore-keys: nightly-${{ runner.os }}-gitmodules-
+
+ - name: Install rust nightly
+ working-directory: ./solvers/minion
+ run: rustup update nightly && rustup default nightly
+
+ - name: Generate coverage reports
+ working-directory: .
+ run: |
+ ./etc/scripts/gen_coverage_all.sh
+ 
+ - name: Move all html to correct folders for deployment
+ run: |
+ # put things both in the sha directory, and in latest/
+ mkdir -p deploy/conjure-oxide
+ mkdir -p deploy/minion
+ mkdir -p deploy/chuffed
+
+ cp -r coverage/html/* deploy/conjure-oxide
+ cp -r solvers/minion/coverage/html/* deploy/minion
+ cp -r solvers/chuffed/coverage/html/* deploy/chuffed
+
+ - name: Archive code coverage results
+ uses: actions/upload-artifact@v3
+ with:
+ name: code-coverage-${{ github.sha }}
+ path: deploy/**
+
+
+