From 4799c42b4e649b68058e9deb880e67293af7ac31 Mon Sep 17 00:00:00 2001 From: Niklas Dewally Date: Fri, 3 Nov 2023 12:00:53 +0000 Subject: [PATCH 01/10] CI: utility scripts --- etc/ci/get_caching_paths.sh | 13 +++++++++++++ etc/ci/get_submodule_paths.sh | 13 +++++++++++++ etc/ci/get_submodules_hash.sh | 22 ++++++++++++++++++++++ 3 files changed, 48 insertions(+) create mode 100755 etc/ci/get_caching_paths.sh create mode 100755 etc/ci/get_submodule_paths.sh create mode 100755 etc/ci/get_submodules_hash.sh diff --git a/etc/ci/get_caching_paths.sh b/etc/ci/get_caching_paths.sh new file mode 100755 index 000000000..8a35e8fee --- /dev/null +++ b/etc/ci/get_caching_paths.sh @@ -0,0 +1,13 @@ +#!/usr/bin/env sh + +# get_caching_paths.sh +# +# get the paths to be cached by Github Actions. +# paths are seperated by new lines + +sh get_submodule_paths.sh +echo '~/.cargo/bin' +echo '~/.cargo/registry/index' +echo '~/.cargo/registry/cache' +echo '~/.cargo/git/db' +echo 'target/' diff --git a/etc/ci/get_submodule_paths.sh b/etc/ci/get_submodule_paths.sh new file mode 100755 index 000000000..b796b1748 --- /dev/null +++ b/etc/ci/get_submodule_paths.sh @@ -0,0 +1,13 @@ +#!/usr/bin/env sh + +# get_submodule_paths +# +# returns the paths of each submodule in this repository, seperated by a new line +# these paths are relative to the git repository. + +# go to top level of git repo +cd $(git rev-parse --show-toplevel) + +# https://stackoverflow.com/questions/12641469/list-submodules-in-a-git-repository +git config --file .gitmodules --get-regexp path | awk '{print $2}' + diff --git a/etc/ci/get_submodules_hash.sh b/etc/ci/get_submodules_hash.sh new file mode 100755 index 000000000..337949535 --- /dev/null +++ b/etc/ci/get_submodules_hash.sh @@ -0,0 +1,22 @@ +#!/usr/bin/env sh + +# get_submodules_hash +# +# compute a hash that represents the current state of all git submodule +# dependencies. Primarily used for cache invalidation in GitHub Actions. + + +CI_SCRIPTS_DIR=$(realpath $(dirname "$0")) + +# go to top level of git repo +cd $(git rev-parse --show-toplevel) + +git submodule update --init --recursive + +{ + for module in $(sh "$CI_SCRIPTS_DIR/get_submodule_paths.sh") + do + git rev-parse "HEAD:$module" >> $SHAS + done; +} 2>/dev/null | sha256sum | head -c 40 # note: sha256sum print a - at the end - this is removed here. +echo '' From c54c6bba1b8e5d8228748c2eb9adef0d9f0b59c8 Mon Sep 17 00:00:00 2001 From: Niklas Dewally Date: Fri, 3 Nov 2023 12:08:19 +0000 Subject: [PATCH 02/10] CI: use single hash and rename things Qualify job names so they are easier to read in PRs. --- .../{chuffed-tests.yml => chuffed.yml} | 54 ++++---- .../{coverage.yml => code-coverage.yml} | 22 ++-- .github/workflows/doc-coverage.yml | 49 +++++--- .github/workflows/{rustfmt.yml => format.yml} | 12 +- .../{kissat-tests.yml => kissat.yml} | 42 +++++-- .github/workflows/minion-tests.yml | 119 ------------------ .github/workflows/minion.yml | 115 +++++++++++++++++ etc/ci/get_caching_paths.sh | 13 -- etc/ci/get_submodules_hash.sh | 3 +- 9 files changed, 219 insertions(+), 210 deletions(-) rename .github/workflows/{chuffed-tests.yml => chuffed.yml} (52%) rename .github/workflows/{coverage.yml => code-coverage.yml} (84%) rename .github/workflows/{rustfmt.yml => format.yml} (66%) rename .github/workflows/{kissat-tests.yml => kissat.yml} (57%) delete mode 100644 .github/workflows/minion-tests.yml create mode 100644 .github/workflows/minion.yml delete mode 100755 etc/ci/get_caching_paths.sh diff --git a/.github/workflows/chuffed-tests.yml b/.github/workflows/chuffed.yml similarity index 52% rename from .github/workflows/chuffed-tests.yml rename to .github/workflows/chuffed.yml index e55c5b777..3313643d8 100644 --- a/.github/workflows/chuffed-tests.yml +++ b/.github/workflows/chuffed.yml @@ -2,6 +2,7 @@ # https://ectobit.com/blog/speed-up-github-actions-rust-pipelines/ name: "solvers/chuffed" on: + workflow_dispatch: push: paths: - "solvers/chuffed/**" @@ -10,22 +11,20 @@ on: paths: - "solvers/chuffed/**" - "Cargo.*" - workflow_dispatch: jobs: ubuntu: - name: "solvers/chuffed: Ubuntu Build" + name: "Chuffed: Ubuntu Build" runs-on: ubuntu-latest steps: - uses: actions/checkout@v2 - # https://stackoverflow.com/questions/32327108/get-the-current-commit-id-of-specified-submodule - # https://docs.github.com/en/actions/using-workflows/workflow-commands-for-github-actions#setting-an-environment-variable - - name: Get chuffed hash for cache invalidation - id: chuffed-cache + - name: Generate caching variables + id: cache-vars run: | - echo "chuffed_sha=$(git rev-parse HEAD:solvers/chuffed/vendor)" >> "$GITHUB_OUTPUT" + echo -e "submodule_sha=$(./etc/ci/get_submodules_hash.sh)" >> "$GITHUB_OUTPUT" + echo -e "submodule_sha=$(./etc/ci/get_submodules_hash.sh)" - name: Set up cache uses: actions/cache@v3 @@ -37,9 +36,9 @@ jobs: ~/.cargo/git/db/ target/ solvers/chuffed/vendor - key: ${{ runner.os }}-chuffed-${{ steps.chuffed-cache.outputs.chuffed_sha }}-cargo-${{ hashFiles('**/Cargo.lock') }} - # match chuffed exactly, cargo inexactly - restore-keys: ${{ runner.os }}-chuffed-${{ steps.chuffed-cache.outputs.chuffed_sha}}-cargo- + + key: stable-${{ runner.os }}-gitmodules-${{ steps.cache-vars.outputs.submodule_sha }}-cargo-${{ hashFiles('**/Cargo.lock') }} + restore-keys: stable-${{ runner.os }}-gitmodules- - working-directory: ./solvers/chuffed run: rustup update stable && rustup default stable @@ -48,15 +47,16 @@ jobs: run: cargo build -vv mac: - name: "solvers/chuffed: Mac Build" + name: "Chuffed: Mac Build" runs-on: macos-latest steps: - uses: actions/checkout@v2 - - name: Get chuffed hash for cache invalidation - id: chuffed-cache + - name: Generate caching variables + id: cache-vars run: | - echo "chuffed_sha=$(git rev-parse HEAD:solvers/chuffed/vendor)" >> "$GITHUB_OUTPUT" + echo -e "submodule_sha=$(./etc/ci/get_submodules_hash.sh)" >> "$GITHUB_OUTPUT" + echo -e "submodule_sha=$(./etc/ci/get_submodules_hash.sh)" - name: Set up cache uses: actions/cache@v3 @@ -68,9 +68,9 @@ jobs: ~/.cargo/git/db/ target/ solvers/chuffed/vendor - key: ${{ runner.os }}-chuffed-${{ steps.chuffed-cache.outputs.chuffed_sha }}-cargo-${{ hashFiles('**/Cargo.lock') }} - # match chuffed exactly, cargo inexactly - restore-keys: ${{ runner.os }}-chuffed-${{ steps.chuffed-cache.outputs.chuffed_sha}}-cargo- + + key: stable-${{ runner.os }}-gitmodules-${{ steps.cache-vars.outputs.submodule_sha }}-cargo-${{ hashFiles('**/Cargo.lock') }} + restore-keys: stable-${{ runner.os }}-gitmodules- - working-directory: ./solvers/chuffed run: rustup update stable && rustup default stable @@ -82,17 +82,19 @@ jobs: run: cargo build -vv tests: - name: "solvers/chuffed: Tests" + name: "Chuffed: Tests" runs-on: ubuntu-latest steps: - uses: actions/checkout@v2 - # https://stackoverflow.com/questions/32327108/get-the-current-commit-id-of-specified-submodule - # https://docs.github.com/en/actions/using-workflows/workflow-commands-for-github-actions#setting-an-environment-variable - - name: Get chuffed hash for cache invalidation - id: chuffed-cache + - name: Generate caching variables + id: cache-vars run: | - echo "chuffed_sha=$(git rev-parse HEAD:solvers/chuffed/vendor)" >> "$GITHUB_OUTPUT" + echo -e "submodule_sha=$(./etc/ci/get_submodules_hash.sh)" >> "$GITHUB_OUTPUT" + echo -e "submodule_sha=$(./etc/ci/get_submodules_hash.sh)" + + echo "cache_paths=$(./etc/ci/get_caching_paths.sh)" >> "$GITHUB_OUTPUT" + echo "cache_paths=$(./etc/ci/get_caching_paths.sh)" - name: Set up cache uses: actions/cache@v3 @@ -104,9 +106,9 @@ jobs: ~/.cargo/git/db/ target/ solvers/chuffed/vendor - key: ${{ runner.os }}-chuffed-${{ steps.chuffed-cache.outputs.chuffed_sha }}-cargo-${{ hashFiles('**/Cargo.lock') }} - # match chuffed exactly, cargo inexactly - restore-keys: ${{ runner.os }}-chuffed-${{ steps.chuffed-cache.outputs.chuffed_sha}}-cargo- + + key: stable-${{ runner.os }}-gitmodules-${{ steps.cache-vars.outputs.submodule_sha }}-cargo-${{ hashFiles('**/Cargo.lock') }} + restore-keys: stable-${{ runner.os }}-gitmodules- - working-directory: ./solvers/chuffed run: rustup update stable && rustup default stable diff --git a/.github/workflows/coverage.yml b/.github/workflows/code-coverage.yml similarity index 84% rename from .github/workflows/coverage.yml rename to .github/workflows/code-coverage.yml index 0f6176a8c..c099eaf25 100644 --- a/.github/workflows/coverage.yml +++ b/.github/workflows/code-coverage.yml @@ -1,4 +1,4 @@ -name: 'Info: Code Coverage' +name: 'Code Coverage' on: push: pull_request: @@ -10,7 +10,7 @@ permissions: jobs: coverage: - name: "Coverage Report" + name: "Info: Code Coverage Reports" # only do coverage for ready PRs if: ${{ github.event != 'pull_request' || ( github.event == 'pull_request' && (! github.event.pull_request.draft)) }} runs-on: ubuntu-latest @@ -18,25 +18,25 @@ jobs: - name: Checkout the repository uses: actions/checkout@v2 - - name: Get submodule hashes for cache invalidation - id: cache-hashes + - name: Generate caching variables + id: cache-vars run: | - echo "shas=$(git rev-parse HEAD:solvers/minion/vendor)-$(git rev-parse HEAD:solvers/chuffed/vendor)" - echo "shas=$(git rev-parse HEAD:solvers/minion/vendor)-$(git rev-parse HEAD:solvers/chuffed/vendor)" >> "$GITHUB_OUTPUT" + echo -e "submodule_sha=$(./etc/ci/get_submodules_hash.sh)" >> "$GITHUB_OUTPUT" + echo -e "submodule_sha=$(./etc/ci/get_submodules_hash.sh)" - name: Set up cache uses: actions/cache@v3 with: - path: | + path: ~/.cargo/bin/ ~/.cargo/registry/index/ ~/.cargo/registry/cache/ ~/.cargo/git/db/ target/ solvers/minion/vendor - solvers/chuffed/vendor - key: nightly-${{ runner.os }}-solvers-${{ steps.cache-hashes.outputs.shas }}-cargo-${{ hashFiles('**/Cargo.lock') }} - restore-keys: nightly-${{ runner.os }}-solvers-${{ steps.cache-hashes.outputs.shas }}-cargo- + + key: nightly-${{ runner.os }}-gitmodules-${{ steps.cache-vars.outputs.submodule_sha }}-cargo-${{ hashFiles('**/Cargo.lock') }} + restore-keys: nightly-${{ runner.os }}-gitmodules- - name: Install rust nightly working-directory: ./solvers/minion @@ -66,7 +66,7 @@ jobs: branch: gh-pages commit-message: "Actions: Code Coverage for ${{ github.sha }}" - - name: Update latest code coverage if on main. + - name: If on main branch, copy coverage report to /latest. if: github.ref == 'refs/heads/main' uses: JamesIves/github-pages-deploy-action@v4 with: diff --git a/.github/workflows/doc-coverage.yml b/.github/workflows/doc-coverage.yml index 31010e4c2..92eade5d7 100644 --- a/.github/workflows/doc-coverage.yml +++ b/.github/workflows/doc-coverage.yml @@ -1,4 +1,4 @@ -name: 'Info: Documentation Coverage' +name: "Documentation Coverage" on: push: pull_request: @@ -6,16 +6,16 @@ on: jobs: minion: - name: 'Minion' + name: 'Info: Minion Docs Coverage Report' runs-on: ubuntu-latest steps: - uses: actions/checkout@v2 - - - name: Get minion hash for cache invalidation - id: minion-cache + - name: Generate caching variables + id: cache-vars run: | - echo "minion_sha=$(git rev-parse HEAD:solvers/minion/vendor)" >> "$GITHUB_OUTPUT" + echo -e "submodule_sha=$(./etc/ci/get_submodules_hash.sh)" >> "$GITHUB_OUTPUT" + echo -e "submodule_sha=$(./etc/ci/get_submodules_hash.sh)" - name: Set up cache uses: actions/cache@v3 @@ -27,8 +27,9 @@ jobs: ~/.cargo/git/db/ target/ solvers/minion/vendor - key: nightly-${{ runner.os }}-minion-${{ steps.minion-cache.outputs.minion_sha }}-cargo-${{ hashFiles('**/Cargo.lock') }} - restore-keys: nightly-${{ runner.os }}-minion-${{ steps.minion-cache.outputs.minion_sha}}-cargo- + + key: nightly-${{ runner.os }}-gitmodules-${{ steps.cache-vars.outputs.submodule_sha }}-cargo-${{ hashFiles('**/Cargo.lock') }} + restore-keys: nightly-${{ runner.os }}-gitmodules- - name: Use nightly run: rustup update nightly && rustup default nightly @@ -41,15 +42,16 @@ jobs: echo '```' >> $GITHUB_STEP_SUMMARY chuffed: - name: 'Chuffed' + name: 'Info: Chuffed Docs Coverage Report' runs-on: ubuntu-latest steps: - uses: actions/checkout@v2 - - name: Get chuffed hash for cache invalidation - id: chuffed-cache + - name: Generate caching variables + id: cache-vars run: | - echo "chuffed_sha=$(git rev-parse HEAD:solvers/chuffed/vendor)" >> "$GITHUB_OUTPUT" + echo -e "submodule_sha=$(./etc/ci/get_submodules_hash.sh)" >> "$GITHUB_OUTPUT" + echo -e "submodule_sha=$(./etc/ci/get_submodules_hash.sh)" - name: Set up cache uses: actions/cache@v3 @@ -61,9 +63,9 @@ jobs: ~/.cargo/git/db/ target/ solvers/chuffed/vendor - key: nightly-${{ runner.os }}-chuffed-${{ steps.chuffed-cache.outputs.chuffed_sha }}-cargo-${{ hashFiles('**/Cargo.lock') }} - # match chuffed exactly, cargo inexactly - restore-keys: nightly-${{ runner.os }}-chuffed-${{ steps.chuffed-cache.outputs.chuffed_sha}}-cargo- + + key: nightly-${{ runner.os }}-gitmodules-${{ steps.cache-vars.outputs.submodule_sha }}-cargo-${{ hashFiles('**/Cargo.lock') }} + restore-keys: nightly-${{ runner.os }}-gitmodules- - name: Use nightly run: rustup update nightly && rustup default nightly @@ -76,11 +78,17 @@ jobs: echo '```' >> $GITHUB_STEP_SUMMARY conjure-oxide: - name: 'Conjure Oxide' + name: 'Info: Conjure Oxide Docs Coverage Report' runs-on: ubuntu-latest steps: - uses: actions/checkout@v2 + - name: Generate caching variables + id: cache-vars + run: | + echo -e "submodule_sha=$(./etc/ci/get_submodules_hash.sh)" >> "$GITHUB_OUTPUT" + echo -e "submodule_sha=$(./etc/ci/get_submodules_hash.sh)" + - name: Set up cache uses: actions/cache@v3 with: @@ -89,11 +97,12 @@ jobs: ~/.cargo/registry/index/ ~/.cargo/registry/cache/ ~/.cargo/git/db/ - ~/solvers/minion/vendor - ~/solvers/chuffed/vendor target/ - key: nightly-conjureoxide-${{ runner.os }}-cargo-${{ hashFiles('**/Cargo.lock') }} - restore-keys: nightly-conjureoxide-${{ runner.os }}-cargo + solvers/chuffed/vendor + solvers/minion/vendor + + key: nightly-${{ runner.os }}-gitmodules-${{ steps.cache-vars.outputs.submodule_sha }}-cargo-${{ hashFiles('**/Cargo.lock') }} + restore-keys: nightly-${{ runner.os }}-gitmodules- - name: Use nightly run: rustup update nightly && rustup default nightly diff --git a/.github/workflows/rustfmt.yml b/.github/workflows/format.yml similarity index 66% rename from .github/workflows/rustfmt.yml rename to .github/workflows/format.yml index 8c5712c53..2b9bfd0ae 100644 --- a/.github/workflows/rustfmt.yml +++ b/.github/workflows/format.yml @@ -1,20 +1,18 @@ # see: https://github.com/marketplace/actions/rust-rustfmt-check -name: rustfmt +name: "Formatting" on: pull_request: - -permissions: - pull-requests: write + push: jobs: - format: + rust: + name: "Check Rust Formatting" runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 - uses: dtolnay/rust-toolchain@stable with: components: rustfmt - - name: Check rust formatting - run: cargo fmt --check + - run: cargo fmt --check diff --git a/.github/workflows/kissat-tests.yml b/.github/workflows/kissat.yml similarity index 57% rename from .github/workflows/kissat-tests.yml rename to .github/workflows/kissat.yml index c493f6b63..5cc1ffd90 100644 --- a/.github/workflows/kissat-tests.yml +++ b/.github/workflows/kissat.yml @@ -3,9 +3,6 @@ name: 'solvers/kissat' on: push: - paths: - - 'solvers/kissat/**' - - "Cargo.*" pull_request: paths: - 'solvers/kissat/**' @@ -14,11 +11,17 @@ on: jobs: ubuntu: - name: 'solvers/kissat: Ubuntu Build' + name: 'Kissat: Ubuntu Build' runs-on: ubuntu-latest steps: - uses: actions/checkout@v2 + - name: Generate caching variables + id: cache-vars + run: | + echo -e "submodule_sha=$(./etc/ci/get_submodules_hash.sh)" >> "$GITHUB_OUTPUT" + echo -e "submodule_sha=$(./etc/ci/get_submodules_hash.sh)" + - name: Set up cache uses: actions/cache@v3 with: @@ -28,8 +31,9 @@ jobs: ~/.cargo/registry/cache/ ~/.cargo/git/db/ target/ - key: ${{ runner.os }}-cargo-${{ hashFiles('**/Cargo.lock') }} - restore-keys: ${{ runner.os }}-cargo- + + key: stable-${{ runner.os }}-gitmodules-${{ steps.cache-vars.outputs.submodule_sha }}-cargo-${{ hashFiles('**/Cargo.lock') }} + restore-keys: stable-${{ runner.os }}-gitmodules- - working-directory: ./solvers/kissat run: rustup update stable && rustup default stable @@ -38,11 +42,17 @@ jobs: run: cargo build -vv mac: - name: 'solvers/kissat: Mac Build' + name: "Kissat: Mac Build" runs-on: macos-latest steps: - uses: actions/checkout@v2 + - name: Generate caching variables + id: cache-vars + run: | + echo -e "submodule_sha=$(./etc/ci/get_submodules_hash.sh)" >> "$GITHUB_OUTPUT" + echo -e "submodule_sha=$(./etc/ci/get_submodules_hash.sh)" + - name: Set up cache uses: actions/cache@v3 with: @@ -52,8 +62,9 @@ jobs: ~/.cargo/registry/cache/ ~/.cargo/git/db/ target/ - key: ${{ runner.os }}-cargo-${{ hashFiles('**/Cargo.lock') }} - restore-keys: ${{ runner.os }}-cargo- + + key: stable-${{ runner.os }}-gitmodules-${{ steps.cache-vars.outputs.submodule_sha }}-cargo-${{ hashFiles('**/Cargo.lock') }} + restore-keys: stable-${{ runner.os }}-gitmodules- - working-directory: ./solvers/kissat run: rustup update stable && rustup default stable @@ -65,11 +76,17 @@ jobs: run: cargo build -vv tests: - name: 'solvers/kissat: Tests' + name: "Kissat: Tests" runs-on: ubuntu-latest steps: - uses: actions/checkout@v2 + - name: Generate caching variables + id: cache-vars + run: | + echo -e "submodule_sha=$(./etc/ci/get_submodules_hash.sh)" >> "$GITHUB_OUTPUT" + echo -e "submodule_sha=$(./etc/ci/get_submodules_hash.sh)" + - name: Set up cache uses: actions/cache@v3 with: @@ -79,8 +96,9 @@ jobs: ~/.cargo/registry/cache/ ~/.cargo/git/db/ target/ - key: ${{ runner.os }}-cargo-${{ hashFiles('**/Cargo.lock') }} - restore-keys: ${{ runner.os }}-cargo- + + key: stable-${{ runner.os }}-gitmodules-${{ steps.cache-vars.outputs.submodule_sha }}-cargo-${{ hashFiles('**/Cargo.lock') }} + restore-keys: stable-${{ runner.os }}-gitmodules- - working-directory: ./solvers/kissat run: rustup update stable && rustup default stable diff --git a/.github/workflows/minion-tests.yml b/.github/workflows/minion-tests.yml deleted file mode 100644 index c08b7252b..000000000 --- a/.github/workflows/minion-tests.yml +++ /dev/null @@ -1,119 +0,0 @@ -# https://doc.rust-lang.org/cargo/guide/continuous-integration.html -# https://ectobit.com/blog/speed-up-github-actions-rust-pipelines/ -name: 'solvers/minion' -on: - push: - paths: - - 'solvers/minion/**' - - "Cargo.*" - pull_request: - paths: - - 'solvers/minion/**' - - "Cargo.*" - workflow_dispatch: - -jobs: - ubuntu: - name: 'solvers/minion: Ubuntu Build' - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v2 - - # https://stackoverflow.com/questions/32327108/get-the-current-commit-id-of-specified-submodule - # https://docs.github.com/en/actions/using-workflows/workflow-commands-for-github-actions#setting-an-environment-variable - - name: Get minion hash for cache invalidation - id: minion-cache - run: | - echo "minion_sha=$(git rev-parse HEAD:solvers/minion/vendor)" >> "$GITHUB_OUTPUT" - - - name: Set up cache - uses: actions/cache@v3 - with: - path: | - ~/.cargo/bin/ - ~/.cargo/registry/index/ - ~/.cargo/registry/cache/ - ~/.cargo/git/db/ - target/ - solvers/minion/vendor - key: ${{ runner.os }}-minion-${{ steps.minion-cache.outputs.minion_sha }}-cargo-${{ hashFiles('**/Cargo.lock') }} - # match minion exactly, cargo inexactly - restore-keys: ${{ runner.os }}-minion-${{ steps.minion-cache.outputs.minion_sha}}-cargo- - - - working-directory: ./solvers/minion - run: rustup update stable && rustup default stable - - - working-directory: ./solvers/minion - run: cargo build -vv - - mac: - name: 'solvers/minion: Mac Build' - runs-on: macos-latest - steps: - - uses: actions/checkout@v2 - - - name: Get minion hash for cache invalidation - id: minion-cache - run: | - echo "minion_sha=$(git rev-parse HEAD:solvers/minion/vendor)" >> "$GITHUB_OUTPUT" - - - name: Set up cache - uses: actions/cache@v3 - with: - path: | - ~/.cargo/bin/ - ~/.cargo/registry/index/ - ~/.cargo/registry/cache/ - ~/.cargo/git/db/ - target/ - solvers/minion/vendor - key: ${{ runner.os }}-minion-${{ steps.minion-cache.outputs.minion_sha }}-cargo-${{ hashFiles('**/Cargo.lock') }} - # match minion exactly, cargo inexactly - restore-keys: ${{ runner.os }}-minion-${{ steps.minion-cache.outputs.minion_sha}}-cargo- - - - working-directory: ./solvers/minion - run: rustup update stable && rustup default stable - - - working-directory: ./solvers/minion - run: rustup target add aarch64-apple-darwin - - - working-directory: ./solvers/minion - run: cargo build -vv - - tests: - name: 'solvers/minion: Tests' - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v2 - - # https://stackoverflow.com/questions/32327108/get-the-current-commit-id-of-specified-submodule - # https://docs.github.com/en/actions/using-workflows/workflow-commands-for-github-actions#setting-an-environment-variable - - name: Get minion hash for cache invalidation - id: minion-cache - run: | - echo "minion_sha=$(git rev-parse HEAD:solvers/minion/vendor)" >> "$GITHUB_OUTPUT" - - - name: Set up cache - uses: actions/cache@v3 - with: - path: | - ~/.cargo/bin/ - ~/.cargo/registry/index/ - ~/.cargo/registry/cache/ - ~/.cargo/git/db/ - target/ - solvers/minion/vendor - key: ${{ runner.os }}-minion-${{ steps.minion-cache.outputs.minion_sha }}-cargo-${{ hashFiles('**/Cargo.lock') }} - # match minion exactly, cargo inexactly - restore-keys: ${{ runner.os }}-minion-${{ steps.minion-cache.outputs.minion_sha}}-cargo- - - - working-directory: ./solvers/minion - run: rustup update stable && rustup default stable - - - working-directory: ./solvers/minion - run: cargo test -- --test-threads=1 - - - - - diff --git a/.github/workflows/minion.yml b/.github/workflows/minion.yml new file mode 100644 index 000000000..b84894d95 --- /dev/null +++ b/.github/workflows/minion.yml @@ -0,0 +1,115 @@ +name: "solvers/minion" +on: + push: + paths: + - 'solvers/minion/**' + - "Cargo.*" + pull_request: + paths: + - 'solvers/minion/**' + - "Cargo.*" + workflow_dispatch: + +jobs: + ubuntu: + name: "Minion: Ubuntu Build" + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + + - name: Generate caching variables + id: cache-vars + run: | + echo -e "submodule_sha=$(./etc/ci/get_submodules_hash.sh)" >> "$GITHUB_OUTPUT" + echo -e "submodule_sha=$(./etc/ci/get_submodules_hash.sh)" + + - name: Set up cache + uses: actions/cache@v3 + with: + path: + ~/.cargo/bin/ + ~/.cargo/registry/index/ + ~/.cargo/registry/cache/ + ~/.cargo/git/db/ + target/ + solvers/minion/vendor + + key: stable-${{ runner.os }}-gitmodules-${{ steps.cache-vars.outputs.submodule_sha }}-cargo-${{ hashFiles('**/Cargo.lock') }} + restore-keys: stable-${{ runner.os }}-gitmodules- + + - working-directory: ./solvers/minion + run: rustup update stable && rustup default stable + + - working-directory: ./solvers/minion + run: cargo build -vv + + mac: + name: "Minion: Mac Build" + runs-on: macos-latest + steps: + - uses: actions/checkout@v2 + + - name: Generate caching variables + id: cache-vars + run: | + echo -e "submodule_sha=$(./etc/ci/get_submodules_hash.sh)" >> "$GITHUB_OUTPUT" + echo -e "submodule_sha=$(./etc/ci/get_submodules_hash.sh)" + + - name: Set up cache + uses: actions/cache@v3 + with: + path: + ~/.cargo/bin/ + ~/.cargo/registry/index/ + ~/.cargo/registry/cache/ + ~/.cargo/git/db/ + target/ + solvers/minion/vendor + + key: stable-${{ runner.os }}-gitmodules-${{ steps.cache-vars.outputs.submodule_sha }}-cargo-${{ hashFiles('**/Cargo.lock') }} + restore-keys: stable-${{ runner.os }}-gitmodules- + + - working-directory: ./solvers/minion + run: rustup update stable && rustup default stable + + - working-directory: ./solvers/minion + run: rustup target add aarch64-apple-darwin + + - working-directory: ./solvers/minion + run: cargo build -vv + + tests: + name: "Minion: Tests" + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + + - name: Generate caching variables + id: cache-vars + run: | + echo -e "submodule_sha=$(./etc/ci/get_submodules_hash.sh)" >> "$GITHUB_OUTPUT" + echo -e "submodule_sha=$(./etc/ci/get_submodules_hash.sh)" + + - name: Set up cache + uses: actions/cache@v3 + with: + path: + ~/.cargo/bin/ + ~/.cargo/registry/index/ + ~/.cargo/registry/cache/ + ~/.cargo/git/db/ + target/ + solvers/minion/vendor + key: stable-${{ runner.os }}-gitmodules-${{ steps.cache-vars.outputs.submodule_sha }}-cargo-${{ hashFiles('**/Cargo.lock') }} + restore-keys: stable-${{ runner.os }}-gitmodules- + + - working-directory: ./solvers/minion + run: rustup update stable && rustup default stable + + - working-directory: ./solvers/minion + run: cargo test -- --test-threads=1 + + + + + diff --git a/etc/ci/get_caching_paths.sh b/etc/ci/get_caching_paths.sh deleted file mode 100755 index 8a35e8fee..000000000 --- a/etc/ci/get_caching_paths.sh +++ /dev/null @@ -1,13 +0,0 @@ -#!/usr/bin/env sh - -# get_caching_paths.sh -# -# get the paths to be cached by Github Actions. -# paths are seperated by new lines - -sh get_submodule_paths.sh -echo '~/.cargo/bin' -echo '~/.cargo/registry/index' -echo '~/.cargo/registry/cache' -echo '~/.cargo/git/db' -echo 'target/' diff --git a/etc/ci/get_submodules_hash.sh b/etc/ci/get_submodules_hash.sh index 337949535..fa5aed852 100755 --- a/etc/ci/get_submodules_hash.sh +++ b/etc/ci/get_submodules_hash.sh @@ -11,7 +11,7 @@ CI_SCRIPTS_DIR=$(realpath $(dirname "$0")) # go to top level of git repo cd $(git rev-parse --show-toplevel) -git submodule update --init --recursive +git submodule update --init --recursive 1>&2 2>/dev/null { for module in $(sh "$CI_SCRIPTS_DIR/get_submodule_paths.sh") @@ -19,4 +19,3 @@ git submodule update --init --recursive git rev-parse "HEAD:$module" >> $SHAS done; } 2>/dev/null | sha256sum | head -c 40 # note: sha256sum print a - at the end - this is removed here. -echo '' From 691cbf8ca71f162bed01428502922f98f9ddeefe Mon Sep 17 00:00:00 2001 From: Niklas Dewally Date: Fri, 3 Nov 2023 18:35:08 +0000 Subject: [PATCH 03/10] CI: build and test for Conjure Oxide --- .github/workflows/minion.yml | 2 +- .github/workflows/oxide.yml | 111 +++++++++++++++++++++++++++++++++++ 2 files changed, 112 insertions(+), 1 deletion(-) create mode 100644 .github/workflows/oxide.yml diff --git a/.github/workflows/minion.yml b/.github/workflows/minion.yml index b84894d95..0fa56f635 100644 --- a/.github/workflows/minion.yml +++ b/.github/workflows/minion.yml @@ -107,7 +107,7 @@ jobs: run: rustup update stable && rustup default stable - working-directory: ./solvers/minion - run: cargo test -- --test-threads=1 + run: cargo test diff --git a/.github/workflows/oxide.yml b/.github/workflows/oxide.yml new file mode 100644 index 000000000..b8e4aa9fa --- /dev/null +++ b/.github/workflows/oxide.yml @@ -0,0 +1,111 @@ +name: "conjure-oxide" +on: + push: + paths: + - 'src/**' + - "Cargo.*" + pull_request: + paths: + - 'src/**' + - "Cargo.*" + workflow_dispatch: + +jobs: + ubuntu: + name: "Conjure Oxide: Ubuntu Build" + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + + - name: Generate caching variables + id: cache-vars + run: | + echo -e "submodule_sha=$(./etc/ci/get_submodules_hash.sh)" >> "$GITHUB_OUTPUT" + echo -e "submodule_sha=$(./etc/ci/get_submodules_hash.sh)" + + - name: Set up cache + uses: actions/cache@v3 + with: + path: + ~/.cargo/bin/ + ~/.cargo/registry/index/ + ~/.cargo/registry/cache/ + ~/.cargo/git/db/ + target/ + solvers/minion/vendor + solvers/chuffed/vendor + + key: stable-${{ runner.os }}-gitmodules-${{ steps.cache-vars.outputs.submodule_sha }}-cargo-${{ hashFiles('**/Cargo.lock') }} + restore-keys: stable-${{ runner.os }}-gitmodules- + + - run: rustup update stable && rustup default stable + + - run: cargo build -vv + + mac: + name: "Conjure Oxide: Mac Build" + runs-on: macos-latest + steps: + - uses: actions/checkout@v2 + + - name: Generate caching variables + id: cache-vars + run: | + echo -e "submodule_sha=$(./etc/ci/get_submodules_hash.sh)" >> "$GITHUB_OUTPUT" + echo -e "submodule_sha=$(./etc/ci/get_submodules_hash.sh)" + + - name: Set up cache + uses: actions/cache@v3 + with: + path: + ~/.cargo/bin/ + ~/.cargo/registry/index/ + ~/.cargo/registry/cache/ + ~/.cargo/git/db/ + target/ + solvers/minion/vendor + solvers/chuffed/vendor + + key: stable-${{ runner.os }}-gitmodules-${{ steps.cache-vars.outputs.submodule_sha }}-cargo-${{ hashFiles('**/Cargo.lock') }} + restore-keys: stable-${{ runner.os }}-gitmodules- + + - run: rustup update stable && rustup default stable + + - run: rustup target add aarch64-apple-darwin + + - run: cargo build -vv + + tests: + name: "Conjure Oxide: Tests" + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + + - name: Generate caching variables + id: cache-vars + run: | + echo -e "submodule_sha=$(./etc/ci/get_submodules_hash.sh)" >> "$GITHUB_OUTPUT" + echo -e "submodule_sha=$(./etc/ci/get_submodules_hash.sh)" + + - name: Set up cache + uses: actions/cache@v3 + with: + path: + ~/.cargo/bin/ + ~/.cargo/registry/index/ + ~/.cargo/registry/cache/ + ~/.cargo/git/db/ + target/ + solvers/minion/vendor + solvers/chuffed/vendor + key: stable-${{ runner.os }}-gitmodules-${{ steps.cache-vars.outputs.submodule_sha }}-cargo-${{ hashFiles('**/Cargo.lock') }} + restore-keys: stable-${{ runner.os }}-gitmodules- + + - run: rustup update stable && rustup default stable + + - run: cargo test + + + + + From 6dc659a974f8c6d20ba083e6bd13000306968d45 Mon Sep 17 00:00:00 2001 From: Niklas Dewally Date: Fri, 3 Nov 2023 19:21:35 +0000 Subject: [PATCH 04/10] CI: add directory indexes to coverage pages. --- .github/workflows/code-coverage.yml | 40 +- etc/ci/genindex.py | 955 ++++++++++++++++++++++++++++ 2 files changed, 991 insertions(+), 4 deletions(-) create mode 100755 etc/ci/genindex.py diff --git a/.github/workflows/code-coverage.yml b/.github/workflows/code-coverage.yml index c099eaf25..2569c26e8 100644 --- a/.github/workflows/code-coverage.yml +++ b/.github/workflows/code-coverage.yml @@ -27,13 +27,13 @@ jobs: - name: Set up cache uses: actions/cache@v3 with: - path: + path: | ~/.cargo/bin/ - ~/.cargo/registry/index/ - ~/.cargo/registry/cache/ - ~/.cargo/git/db/ + ~/.cargo/registry/index + ~/.cargo/git/ target/ solvers/minion/vendor + solvers/chuffed/vendor key: nightly-${{ runner.os }}-gitmodules-${{ steps.cache-vars.outputs.submodule_sha }}-cargo-${{ hashFiles('**/Cargo.lock') }} restore-keys: nightly-${{ runner.os }}-gitmodules- @@ -99,3 +99,35 @@ jobs: echo "| Conjure-Oxide | ![](${CONJURE_OXIDE_LATEST}/badges/flat.svg) | [Full Report](${CONJURE_OXIDE_LATEST}) | " >> $GITHUB_STEP_SUMMARY echo "| Minion | ![](${MINION_LATEST}/badges/flat.svg) | [Full Report](${MINION_LATEST}) | " >> $GITHUB_STEP_SUMMARY echo "| Chuffed | ![](${CHUFFED_LATEST}/badges/flat.svg) | [Full Report](${CHUFFED_LATEST}) | " >> $GITHUB_STEP_SUMMARY + + indexes: + needs: coverage + name: "Regenerate indexes for coverage" + runs-on: ubuntu-latest + steps: + - name: Checkout the repository + uses: actions/checkout@v2 + + - name: Checkout the pages repository + uses: actions/checkout@v2 + with: + ref: "gh-pages" + path: "pages" + + - name: "Generate indexes" + run: | + ./etc/ci/genindex.py pages/coverage/${{ github.sha }} + ./etc/ci/genindex.py pages/coverage/latest + ./etc/ci/genindex.py pages/coverage/ + + - uses: JamesIves/github-pages-deploy-action@v4 + with: + folder: ./pages/coverage/ + target-folder: ./coverage/ + branch: gh-pages + commit-message: "Actions: Update coverage indexes" + + + + + diff --git a/etc/ci/genindex.py b/etc/ci/genindex.py new file mode 100755 index 000000000..12c591dee --- /dev/null +++ b/etc/ci/genindex.py @@ -0,0 +1,955 @@ +#!/usr/bin/env python3 +# --- +# Copyright 2023 glowinthedark +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# +# You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# +# See the License for the specific language governing permissions and limitations under the License. +# --- +# +# - Generate index.html in a directory tree. +# - handle symlinked files and folders: displayed with custom icons +# - by default only the current folder is processed unless `-r` (`--recursive`) is specified +# - hidden files (starting with a dot) are skipped; use `--include-hidden` to force inclusion +# - skip specific files by regex, e.g.: `--exclude-regex "(build|node_modules|target|__pycache__)"` + +import argparse +import datetime +import os +import re +import sys +from pathlib import Path +from urllib.parse import quote + +DEFAULT_OUTPUT_FILE = 'index.html' + +EXTENSION_TYPES = { + 'id_rsa': 'cert', + 'LICENSE': 'license', + 'README': 'license', + '.jpg': 'image', + '.jpeg': 'image', + '.png': 'image', + '.gif': 'image', + '.webp': 'image', + '.tiff': 'image', + '.bmp': 'image', + '.heif': 'image', + '.heic': 'image', + '.svg': 'image', + '.mp4': 'video', + '.mov': 'video', + '.mpeg': 'video', + '.avi': 'video', + '.ogv': 'video', + '.webm': 'video', + '.mkv': 'video', + '.vob': 'video', + '.gifv': 'video', + '.3gp': 'video', + '.mp3': 'audio', + '.m4a': 'audio', + '.aac': 'audio', + '.ogg': 'audio', + '.flac': 'audio', + '.wav': 'audio', + '.wma': 'audio', + '.midi': 'audio', + '.cda': 'audio', + '.aiff': 'audio', + '.aif': 'audio', + '.caf': 'audio', + '.pdf': 'pdf', + '.csv': 'csv', + '.txt': 'doc', + '.doc': 'doc', + '.docx': 'doc', + '.odt': 'doc', + '.fodt': 'doc', + '.rtf': 'doc', + '.abw': 'doc', + '.pages': 'doc', + '.xls': 'sheet', + '.xlsx': 'sheet', + '.ods': 'sheet', + '.fods': 'sheet', + '.numbers': 'sheet', + '.ppt': 'ppt', + '.pptx': 'ppt', + '.odp': 'ppt', + '.fodp': 'ppt', + '.zip': 'archive', + '.gz': 'archive', + '.xz': 'archive', + '.tar': 'archive', + '.7z': 'archive', + '.rar': 'archive', + '.zst': 'archive', + '.bz2': 'archive', + '.bzip': 'archive', + '.arj': 'archive', + '.z': 'archive', + '.deb': 'deb', + '.dpkg': 'deb', + '.rpm': 'dist', + '.exe': 'dist', + '.flatpak': 'dist', + '.appimage': 'dist', + '.jar': 'dist', + '.msi': 'dist', + '.apk': 'dist', + '.ps1': 'ps1', + '.py': 'py', + '.pyc': 'py', + '.pyo': 'py', + '.egg': 'py', + '.sh': 'sh', + '.bash': 'sh', + '.com': 'sh', + '.bat': 'sh', + '.dll': 'sh', + '.so': 'sh', + '.dmg': 'dmg', + '.iso': 'iso', + '.img': 'iso', + '.md': 'md', + '.mdown': 'md', + '.markdown': 'md', + '.ttf': 'font', + '.ttc': 'font', + '.otf': 'font', + '.woff': 'font', + '.woff2': 'font', + '.eof': 'font', + '.apf': 'font', + '.go': 'go', + '.html': 'html', + '.htm': 'html', + '.php': 'html', + '.php3': 'html', + '.asp': 'html', + '.aspx': 'html', + '.css': 'css', + '.scss': 'css', + '.less': 'css', + '.json': 'json', + '.json5': 'json', + '.jsonc': 'json', + '.ts': 'ts', + '.sql': 'sql', + '.db': 'db', + '.sqlite': 'db', + '.mdb': 'db', + '.odb': 'db', + '.eml': 'email', + '.email': 'email', + '.mailbox': 'email', + '.mbox': 'email', + '.msg': 'email', + '.crt': 'cert', + '.pem': 'cert', + '.x509': 'cert', + '.cer': 'cert', + '.der': 'cert', + '.ca-bundle': 'cert', + '.key': 'keystore', + '.keystore': 'keystore', + '.jks': 'keystore', + '.p12': 'keystore', + '.pfx': 'keystore', + '.pub': 'keystore', + 'symlink': 'symlink', + 'generic': 'generic' +} + + +def process_dir(top_dir, opts): + glob_patt = opts.filter or '*' + + path_top_dir = Path(top_dir) + + index_path = Path(path_top_dir, opts.output_file) + + if opts.verbose: + print(f'Traversing dir {path_top_dir.absolute()}') + + try: + index_file = open(index_path, 'w') + except Exception as e: + print('cannot create file %s %s' % (index_path, e)) + return + + index_file.write(""" + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+

""" + f'{path_top_dir.name}' + """

+
+
+
+ + + + + + + + + + + + + + + + + + + """) + + # sort dirs first + sorted_entries = sorted(path_top_dir.glob(glob_patt), key=lambda p: (not p.is_dir(), p.name)) + + entry: Path + for entry in sorted_entries: + + # - don't include index.html in the file listing + # - skip .hidden dot files unless explicitly requested + # - skip by regex if defined + if (entry.name.lower() == opts.output_file.lower()) \ + or (not opts.include_hidden and entry.name.startswith('.')) \ + or (opts.exclude_regex and opts.exclude_regex.search(entry.name)): + continue + + if entry.is_dir() and opts.recursive: + process_dir(entry, opts) + + # From Python 3.6, os.access() accepts path-like objects + if (not entry.is_symlink()) and not os.access(str(entry), os.R_OK): + print(f"*** WARNING *** entry {entry.absolute()} is not readable! SKIPPING!") + continue + if opts.verbose: + print(f'{entry.absolute()}') + + size_bytes = -1 # is a folder + size_pretty = '—' + last_modified = '-' + last_modified_human_readable = '-' + last_modified_iso = '' + try: + if entry.is_file(): + size_bytes = entry.stat().st_size + size_pretty = pretty_size(size_bytes) + + if entry.is_dir() or entry.is_file(): + last_modified = datetime.datetime.fromtimestamp(entry.stat().st_mtime).replace(microsecond=0) + last_modified_iso = last_modified.isoformat() + last_modified_human_readable = last_modified.strftime("%c") + + except Exception as e: + print('ERROR accessing file name:', e, entry) + continue + + entry_path = str(entry.name) + + icon_xlink = None + css_class_svg = "" + + if entry.is_dir() and not entry.is_symlink(): + icon_xlink = 'folder' + css_class_svg = "folder_filled" + + if os.name not in ('nt',): + # append trailing slash to dirs, unless it's windows + entry_path = os.path.join(entry.name, '') + + elif entry.is_dir() and entry.is_symlink(): + icon_xlink = 'folder-symlink' + + print('dir-symlink', entry.absolute()) + + elif entry.is_file() and entry.is_symlink(): + icon_xlink = 'symlink' + + print('file-symlink', entry.absolute()) + + elif entry.is_file(): + + if '.' in entry.name: + icon_xlink = EXTENSION_TYPES.get(entry.suffix.lower()) + else: + icon_xlink = EXTENSION_TYPES.get(entry.name) + + if icon_xlink is None: + icon_xlink = 'generic' + + index_file.write(f""" + + + + + + + +""") + + index_file.write(""" + +
NameSize + Modified +
+ + + + + ..——
+ + + + {entry.name} + + {size_pretty}
+
+
+ +""") + if index_file: + index_file.close() + + +# bytes pretty-printing +UNITS_MAPPING = [ + (1024 ** 5, ' PB'), + (1024 ** 4, ' TB'), + (1024 ** 3, ' GB'), + (1024 ** 2, ' MB'), + (1024 ** 1, ' KB'), + (1024 ** 0, (' byte', ' bytes')), +] + + +def pretty_size(bytes, units=UNITS_MAPPING): + """Human-readable file sizes. + + ripped from https://pypi.python.org/pypi/hurry.filesize/ + """ + for factor, suffix in units: + if bytes >= factor: + break + amount = int(bytes / factor) + + if isinstance(suffix, tuple): + singular, multiple = suffix + if amount == 1: + suffix = singular + else: + suffix = multiple + return str(amount) + suffix + + +def type_regex(s): + if not s: + return None + try: + return re.compile(s) + except re.error as e: + raise argparse.ArgumentTypeError(f"Invalid regular expression: {e}") + + +if __name__ == "__main__": + parser = argparse.ArgumentParser(description='''DESCRIPTION: + Generate directory index files (recursive is OFF by default). + Start from current dir or from folder passed as first positional argument. + Optionally filter by file types with --filter "*.py". ''') + + parser.add_argument('top_dir', + nargs='?', + action='store', + help='top folder from which to start generating indexes, ' + 'use current folder if not specified', + default=os.getcwd()) + + parser.add_argument('--filter', '-f', + help='only include files matching glob', + required=False) + + parser.add_argument('--output-file', '-o', + metavar='filename', + default=DEFAULT_OUTPUT_FILE, + help=f'Custom output file, by default "{DEFAULT_OUTPUT_FILE}"') + + parser.add_argument('--recursive', '-r', + action='store_true', + help="recursively process nested dirs (FALSE by default)", + required=False) + + parser.add_argument('--include-hidden', '-i', + action='store_true', + help="include dot hidden files (FALSE by default)", + required=False) + + parser.add_argument('--exclude-regex', '-x', + type=type_regex, + help="exclude files matching regular expression", + required=False) + + parser.add_argument('--verbose', '-v', + action='store_true', + help='WARNING: can take longer time with complex file tree structures on slow terminals -' + ' verbosely list every processed file', + required=False) + + config = parser.parse_args(sys.argv[1:]) + process_dir(config.top_dir, config) From 9bb6f80bdc4660684d0d0af002e26e00550a47d3 Mon Sep 17 00:00:00 2001 From: Niklas Dewally Date: Fri, 3 Nov 2023 23:39:37 +0000 Subject: [PATCH 05/10] CI: fix #38 --- .github/workflows/code-coverage-deploy.yml | 94 ++++++++++++++++++++++ .github/workflows/code-coverage.yml | 83 ++----------------- 2 files changed, 101 insertions(+), 76 deletions(-) create mode 100644 .github/workflows/code-coverage-deploy.yml diff --git a/.github/workflows/code-coverage-deploy.yml b/.github/workflows/code-coverage-deploy.yml new file mode 100644 index 000000000..291e7cc4c --- /dev/null +++ b/.github/workflows/code-coverage-deploy.yml @@ -0,0 +1,94 @@ +name: coverage-deploy +on: + workflow_run: + workflows: [code-coverage] + types: + - completed + + +# see https://github.com/JamesIves/github-pages-deploy-action/tree/dev +permissions: + contents: write + +jobs: + deploy-coverage: + name: "Info: Code Coverage" + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + + - name: Download artifact + uses: dawidd6/action-download-artifact@v2 + with: + name: code-coverage-${{ github.event.workflow_run.head_sha }} + workflow: code-coverage.yml + path: ./deploy + + - name: Deploy to Github Pages + uses: JamesIves/github-pages-deploy-action@v4 + with: + folder: ./deploy + target-folder: "coverage/${{ github.event.workflow_run.head_sha }}" + branch: gh-pages + commit-message: "Actions: Code Coverage for ${{ github.event.workflow_run.head_sha }}" + + - name: If on main branch, copy coverage report to /latest. + if: github.ref == 'refs/heads/main' + uses: JamesIves/github-pages-deploy-action@v4 + with: + folder: ./deploy + target-folder: "coverage/latest" + branch: gh-pages + commit-message: "Actions: Update latest code coverage for main (${{ github.event.workflow_run.head_sha }})" + + - name: Format summary + run : | + CONJURE_OXIDE_URL="https://conjure-cp.github.io/conjure-oxide/coverage/${{ github.event.workflow_run.head_sha }}/conjure-oxide" + MINION_URL="https://conjure-cp.github.io/conjure-oxide/coverage/${{ github.event.workflow_run.head_sha }}/minion" + CHUFFED_URL="https://conjure-cp.github.io/conjure-oxide/coverage/${{ github.event.workflow_run.head_sha }}/chuffed" + + CONJURE_OXIDE_LATEST="https://conjure-cp.github.io/conjure-oxide/coverage/latest/conjure-oxide" + MINION_LATEST="https://conjure-cp.github.io/conjure-oxide/coverage/latest/minion" + CHUFFED_LATEST="https://conjure-cp.github.io/conjure-oxide/coverage/latest/chuffed" + echo '# Code Coverage' >> $GITHUB_STEP_SUMMARY + echo "" >> $GITHUB_STEP_SUMMARY + echo "## This commit" >> $GITHUB_STEP_SUMMARY + echo "| Crate | | | " >> $GITHUB_STEP_SUMMARY + echo "| ----- | ----- | ----- |" >> $GITHUB_STEP_SUMMARY + echo "| Conjure-Oxide | ![](${CONJURE_OXIDE_URL}/badges/flat.svg) | [Full Report](${CONJURE_OXIDE_URL}) | " >> $GITHUB_STEP_SUMMARY + echo "| Minion | ![](${MINION_URL}/badges/flat.svg) | [Full Report](${MINION_URL}) | " >> $GITHUB_STEP_SUMMARY + echo "| Chuffed | ![](${CHUFFED_URL}/badges/flat.svg) | [Full Report](${CHUFFED_URL}) | " >> $GITHUB_STEP_SUMMARY + echo "" >> $GITHUB_STEP_SUMMARY + echo "## Main" >> $GITHUB_STEP_SUMMARY + echo "| Crate | | | " >> $GITHUB_STEP_SUMMARY + echo "| ----- | ----- | ----- |" >> $GITHUB_STEP_SUMMARY + echo "| Conjure-Oxide | ![](${CONJURE_OXIDE_LATEST}/badges/flat.svg) | [Full Report](${CONJURE_OXIDE_LATEST}) | " >> $GITHUB_STEP_SUMMARY + echo "| Minion | ![](${MINION_LATEST}/badges/flat.svg) | [Full Report](${MINION_LATEST}) | " >> $GITHUB_STEP_SUMMARY + echo "| Chuffed | ![](${CHUFFED_LATEST}/badges/flat.svg) | [Full Report](${CHUFFED_LATEST}) | " >> $GITHUB_STEP_SUMMARY + + indexes: + needs: deploy-coverage + name: "Regenerate indexes for coverage" + runs-on: ubuntu-latest + steps: + - name: Checkout the repository + uses: actions/checkout@v2 + + - name: Checkout the pages repository + uses: actions/checkout@v2 + with: + ref: "gh-pages" + path: "pages" + + - name: "Generate indexes" + run: | + ./etc/ci/genindex.py pages/coverage/${{ github.event.workflow_run.head_sha }} + ./etc/ci/genindex.py pages/coverage/latest + ./etc/ci/genindex.py pages/coverage/ + + - uses: JamesIves/github-pages-deploy-action@v4 + with: + folder: ./pages/coverage/ + target-folder: ./coverage/ + branch: gh-pages + commit-message: "Actions: Update coverage indexes" diff --git a/.github/workflows/code-coverage.yml b/.github/workflows/code-coverage.yml index 2569c26e8..3da4f8e8d 100644 --- a/.github/workflows/code-coverage.yml +++ b/.github/workflows/code-coverage.yml @@ -1,16 +1,12 @@ -name: 'Code Coverage' +name: code-coverage on: push: pull_request: - workflow_dispatch: -# see https://github.com/JamesIves/github-pages-deploy-action/tree/dev -permissions: - contents: write jobs: coverage: - name: "Info: Code Coverage Reports" + name: "Generate Code Coverage Reports" # only do coverage for ready PRs if: ${{ github.event != 'pull_request' || ( github.event == 'pull_request' && (! github.event.pull_request.draft)) }} runs-on: ubuntu-latest @@ -46,7 +42,7 @@ jobs: working-directory: . run: | ./etc/scripts/gen_coverage_all.sh - + - name: Move all html to correct folders for deployment run: | # put things both in the sha directory, and in latest/ @@ -58,76 +54,11 @@ jobs: cp -r solvers/minion/coverage/html/* deploy/minion cp -r solvers/chuffed/coverage/html/* deploy/chuffed - - name: Deploy to Github Pages - uses: JamesIves/github-pages-deploy-action@v4 - with: - folder: ./deploy - target-folder: "coverage/${{ github.sha }}" - branch: gh-pages - commit-message: "Actions: Code Coverage for ${{ github.sha }}" - - - name: If on main branch, copy coverage report to /latest. - if: github.ref == 'refs/heads/main' - uses: JamesIves/github-pages-deploy-action@v4 + - name: Archive code coverage results + uses: actions/upload-artifact@v3 with: - folder: ./deploy - target-folder: "coverage/latest" - branch: gh-pages - commit-message: "Actions: Update latest code coverage for main (${{ github.sha }})" - - - name: Format summary - run : | - CONJURE_OXIDE_URL="https://conjure-cp.github.io/conjure-oxide/coverage/${{ github.sha }}/conjure-oxide" - MINION_URL="https://conjure-cp.github.io/conjure-oxide/coverage/${{ github.sha }}/minion" - CHUFFED_URL="https://conjure-cp.github.io/conjure-oxide/coverage/${{ github.sha }}/chuffed" - - CONJURE_OXIDE_LATEST="https://conjure-cp.github.io/conjure-oxide/coverage/latest/conjure-oxide" - MINION_LATEST="https://conjure-cp.github.io/conjure-oxide/coverage/latest/minion" - CHUFFED_LATEST="https://conjure-cp.github.io/conjure-oxide/coverage/latest/chuffed" - echo '# Code Coverage' >> $GITHUB_STEP_SUMMARY - echo "" >> $GITHUB_STEP_SUMMARY - echo "## This commit" >> $GITHUB_STEP_SUMMARY - echo "| Crate | | | " >> $GITHUB_STEP_SUMMARY - echo "| ----- | ----- | ----- |" >> $GITHUB_STEP_SUMMARY - echo "| Conjure-Oxide | ![](${CONJURE_OXIDE_URL}/badges/flat.svg) | [Full Report](${CONJURE_OXIDE_URL}) | " >> $GITHUB_STEP_SUMMARY - echo "| Minion | ![](${MINION_URL}/badges/flat.svg) | [Full Report](${MINION_URL}) | " >> $GITHUB_STEP_SUMMARY - echo "| Chuffed | ![](${CHUFFED_URL}/badges/flat.svg) | [Full Report](${CHUFFED_URL}) | " >> $GITHUB_STEP_SUMMARY - echo "" >> $GITHUB_STEP_SUMMARY - echo "## Main" >> $GITHUB_STEP_SUMMARY - echo "| Crate | | | " >> $GITHUB_STEP_SUMMARY - echo "| ----- | ----- | ----- |" >> $GITHUB_STEP_SUMMARY - echo "| Conjure-Oxide | ![](${CONJURE_OXIDE_LATEST}/badges/flat.svg) | [Full Report](${CONJURE_OXIDE_LATEST}) | " >> $GITHUB_STEP_SUMMARY - echo "| Minion | ![](${MINION_LATEST}/badges/flat.svg) | [Full Report](${MINION_LATEST}) | " >> $GITHUB_STEP_SUMMARY - echo "| Chuffed | ![](${CHUFFED_LATEST}/badges/flat.svg) | [Full Report](${CHUFFED_LATEST}) | " >> $GITHUB_STEP_SUMMARY - - indexes: - needs: coverage - name: "Regenerate indexes for coverage" - runs-on: ubuntu-latest - steps: - - name: Checkout the repository - uses: actions/checkout@v2 - - - name: Checkout the pages repository - uses: actions/checkout@v2 - with: - ref: "gh-pages" - path: "pages" - - - name: "Generate indexes" - run: | - ./etc/ci/genindex.py pages/coverage/${{ github.sha }} - ./etc/ci/genindex.py pages/coverage/latest - ./etc/ci/genindex.py pages/coverage/ - - - uses: JamesIves/github-pages-deploy-action@v4 - with: - folder: ./pages/coverage/ - target-folder: ./coverage/ - branch: gh-pages - commit-message: "Actions: Update coverage indexes" - - + name: code-coverage-${{ github.sha }} + path: deploy/** From 9e61f6b0835a95b5edb0618069b97e9a271566df Mon Sep 17 00:00:00 2001 From: Niklas Dewally Date: Sat, 4 Nov 2023 09:50:23 +0000 Subject: [PATCH 06/10] CI: generate full cache on main --- .github/workflows/doc-coverage.yml | 2 +- .github/workflows/update-caches.yml | 60 +++++++++++++++++++++++++++++ etc/ci/genindex.py | 1 + 3 files changed, 62 insertions(+), 1 deletion(-) create mode 100644 .github/workflows/update-caches.yml diff --git a/.github/workflows/doc-coverage.yml b/.github/workflows/doc-coverage.yml index 92eade5d7..d12c54cd2 100644 --- a/.github/workflows/doc-coverage.yml +++ b/.github/workflows/doc-coverage.yml @@ -102,7 +102,7 @@ jobs: solvers/minion/vendor key: nightly-${{ runner.os }}-gitmodules-${{ steps.cache-vars.outputs.submodule_sha }}-cargo-${{ hashFiles('**/Cargo.lock') }} - restore-keys: nightly-${{ runner.os }}-gitmodules- + restore-keys: nightly-${{ runner.os }} - name: Use nightly run: rustup update nightly && rustup default nightly diff --git a/.github/workflows/update-caches.yml b/.github/workflows/update-caches.yml new file mode 100644 index 000000000..edf29d0a1 --- /dev/null +++ b/.github/workflows/update-caches.yml @@ -0,0 +1,60 @@ +name: Full Caches +on: + push: + branches: + - main + +jobs: + caches: + strategy: + matrix: + version: [nightly,stable] + os: [ubuntu-latest, macos-latest] + runs-on: ${{ matrix.os }} + name: "Regenerate Caches (${{ matrix.version }}, ${{ matrix.os }})" + steps: + - name: Checkout the repository + uses: actions/checkout@v2 + + - name: Generate caching variables + id: cache-vars + run: | + echo -e "submodule_sha=$(./etc/ci/get_submodules_hash.sh)" >> "$GITHUB_OUTPUT" + echo -e "submodule_sha=$(./etc/ci/get_submodules_hash.sh)" + + - name: Set up cache + uses: actions/cache/restore@v3 + with: + path: | + ~/.cargo/bin/ + ~/.cargo/registry/ + ~/.cargo/git/ + target/ + solvers/minion/vendor + solvers/chuffed/vendor + + key: ${{ matrix.version }}-${{ runner.os }}-gitmodules-${{ steps.cache-vars.outputs.submodule_sha }}-cargo-${{ hashFiles('**/Cargo.lock') }} + restore-keys: ${{ matrix.version }}-${{ runner.os }} + + - name: Install rust ${{ matrix.version }} + run: rustup update ${{ matrix.version }} && rustup default ${{ matrix.version }} + + - run: cargo build --workspace + + - name: Delete cache if it already exists, so we can overwrite it + run: | + gh cache delete "${{ matrix.version }}-${{ runner.os }}-gitmodules-${{ steps.cache-vars.outputs.submodule_sha }}-cargo-${{ hashFiles('**/Cargo.lock') }}" + # always succeed even if cache does not exist + true + + - name: Save cache + uses: actions/cache/save@v3 + with: + path: | + ~/.cargo/bin/ + ~/.cargo/registry/ + ~/.cargo/git/ + target/ + solvers/minion/vendor + solvers/chuffed/vendor + key: ${{ matrix.version }}-${{ runner.os }}-gitmodules-${{ steps.cache-vars.outputs.submodule_sha }}-cargo-${{ hashFiles('**/Cargo.lock') }} diff --git a/etc/ci/genindex.py b/etc/ci/genindex.py index 12c591dee..501929fe7 100755 --- a/etc/ci/genindex.py +++ b/etc/ci/genindex.py @@ -1,5 +1,6 @@ #!/usr/bin/env python3 # --- +# # Copyright 2023 glowinthedark # # Licensed under the Apache License, Version 2.0 (the "License"); From 2ed39fc15887dbe3af068bd0f707f27d8ad75303 Mon Sep 17 00:00:00 2001 From: Niklas Dewally Date: Sat, 4 Nov 2023 10:06:33 +0000 Subject: [PATCH 07/10] CI: add link to genindex.py --- etc/ci/genindex.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/etc/ci/genindex.py b/etc/ci/genindex.py index 501929fe7..0e7f70cac 100755 --- a/etc/ci/genindex.py +++ b/etc/ci/genindex.py @@ -1,6 +1,6 @@ #!/usr/bin/env python3 # --- -# +# https://github.com/glowinthedark/index-html-generator/ # Copyright 2023 glowinthedark # # Licensed under the Apache License, Version 2.0 (the "License"); From b1136393b441943e51a6150df41ee8e8db67f2b0 Mon Sep 17 00:00:00 2001 From: Niklas Dewally Date: Sat, 4 Nov 2023 10:16:37 +0000 Subject: [PATCH 08/10] CI: add CLI token to update-caches.yml --- .github/workflows/update-caches.yml | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/.github/workflows/update-caches.yml b/.github/workflows/update-caches.yml index edf29d0a1..1d6d46281 100644 --- a/.github/workflows/update-caches.yml +++ b/.github/workflows/update-caches.yml @@ -4,6 +4,9 @@ on: branches: - main +env: + GH_TOKEN: ${{ github.token }} + jobs: caches: strategy: @@ -16,6 +19,7 @@ jobs: - name: Checkout the repository uses: actions/checkout@v2 + - name: Generate caching variables id: cache-vars run: | @@ -43,9 +47,8 @@ jobs: - name: Delete cache if it already exists, so we can overwrite it run: | - gh cache delete "${{ matrix.version }}-${{ runner.os }}-gitmodules-${{ steps.cache-vars.outputs.submodule_sha }}-cargo-${{ hashFiles('**/Cargo.lock') }}" + gh cache delete "${{ matrix.version }}-${{ runner.os }}-gitmodules-${{ steps.cache-vars.outputs.submodule_sha }}-cargo-${{ hashFiles('**/Cargo.lock') }}" || true # always succeed even if cache does not exist - true - name: Save cache uses: actions/cache/save@v3 From 97ea56bd787cc4bb5f6811f72935a69f0cfb2bbc Mon Sep 17 00:00:00 2001 From: Niklas Dewally Date: Sat, 4 Nov 2023 11:11:02 +0000 Subject: [PATCH 09/10] CI: update genindex.py link --- etc/ci/genindex.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/etc/ci/genindex.py b/etc/ci/genindex.py index 0e7f70cac..a135d34d7 100755 --- a/etc/ci/genindex.py +++ b/etc/ci/genindex.py @@ -1,6 +1,6 @@ #!/usr/bin/env python3 # --- -# https://github.com/glowinthedark/index-html-generator/ +#https://github.com/glowinthedark/index-html-generator/tree/639db17b0a5856f1c4060826c83aebba2e6cc9dc # Copyright 2023 glowinthedark # # Licensed under the Apache License, Version 2.0 (the "License"); From ede2e28cf3caa200c050cf9e5a707d0f90f214ab Mon Sep 17 00:00:00 2001 From: Niklas Dewally Date: Sat, 4 Nov 2023 11:12:33 +0000 Subject: [PATCH 10/10] CI: update naming --- .github/workflows/code-coverage-deploy.yml | 2 +- .github/workflows/code-coverage.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/code-coverage-deploy.yml b/.github/workflows/code-coverage-deploy.yml index 291e7cc4c..498bfc024 100644 --- a/.github/workflows/code-coverage-deploy.yml +++ b/.github/workflows/code-coverage-deploy.yml @@ -1,4 +1,4 @@ -name: coverage-deploy +name: "Code Coverage (Deploy)" on: workflow_run: workflows: [code-coverage] diff --git a/.github/workflows/code-coverage.yml b/.github/workflows/code-coverage.yml index 3da4f8e8d..3f55df833 100644 --- a/.github/workflows/code-coverage.yml +++ b/.github/workflows/code-coverage.yml @@ -1,4 +1,4 @@ -name: code-coverage +name: "Code Coverage (Generate)" on: push: pull_request: