From a05dc47f5f658aa0801fa5b516a6e628f0e6be76 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20Gom=C3=A8s?= Date: Tue, 2 Apr 2024 16:14:14 +0200 Subject: [PATCH] feat: add a way of using GHA cache locally MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This is the first implementation of uploading the local cache as a single file to a remote cache for reuse in a future build. Right now it is only done for GHA as that was the intended scope¹, but one could adapt this system to other remote caches. Because of the immutability of GHACache, this commit only adds support for re-using the cache for the same version (as defined by the user through the `SCCACHE_GHA_VERSION` environment variable). A way of reusing incremental build within a given version or even across versions could be devised, but it falls outside the scope of this particular effort, and it's probably not trivial. [1] https://github.com/Mozilla-Actions/sccache-action/issues/81 --- .github/workflows/ci.yml | 376 ------------ .github/workflows/close-snap.yml | 19 - .github/workflows/integration-tests.yml | 739 +----------------------- .github/workflows/snap.yml | 26 - src/cache/cache.rs | 61 +- src/cache/gha.rs | 130 +++++ src/config.rs | 17 +- src/server.rs | 19 +- 8 files changed, 223 insertions(+), 1164 deletions(-) delete mode 100644 .github/workflows/ci.yml delete mode 100644 .github/workflows/close-snap.yml delete mode 100644 .github/workflows/snap.yml diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml deleted file mode 100644 index 6b8749cc4..000000000 --- a/.github/workflows/ci.yml +++ /dev/null @@ -1,376 +0,0 @@ -name: ci -on: [push, pull_request] - -jobs: - lint: - name: ${{ matrix.component }} ${{ matrix.os }} - runs-on: ${{ matrix.os }} - timeout-minutes: 15 - strategy: - fail-fast: false - matrix: - os: [ubuntu-latest, macOS-latest, windows-latest] - component: [clippy] - include: - - component: rustfmt - cargo_cmd: fmt -- --check - os: ubuntu-latest - - component: clippy - cargo_cmd: clippy --locked --all-targets -- -D warnings -A unknown-lints -A clippy::type_complexity -A clippy::new-without-default - steps: - - name: Clone repository - uses: actions/checkout@v4 - - - name: Install rust - uses: ./.github/actions/rust-toolchain - with: - components: ${{ matrix.component }} - # Oldest supported version, keep in sync with README.md - toolchain: "1.70.0" - - - name: clippy version - run: cargo clippy --version - if: ${{ matrix.component == 'clippy' }} - - - name: Check - run: cargo ${{ matrix.cargo_cmd }} - - check_features: - runs-on: ubuntu-latest - strategy: - matrix: - feature: [azure, gcs, gha, memcached, redis, s3, webdav] - steps: - - name: Clone repository - uses: actions/checkout@v4 - - - name: Check feature ${{ matrix.feature }} - run: cargo check --no-default-features --features ${{ matrix.feature }} - - toml_format: - runs-on: ubuntu-latest - steps: - - name: Clone repository - uses: actions/checkout@v4 - - - name: Check - run: npx --yes @taplo/cli fmt --check - - test: - name: test ${{ matrix.os }} rust ${{ matrix.rustc || 'stable' }} ${{ matrix.extra_desc }} - runs-on: ${{ matrix.os }} - continue-on-error: ${{ matrix.allow_failure || false }} - timeout-minutes: 30 - strategy: - fail-fast: false - matrix: - include: - - os: ubuntu-20.04 - # Oldest supported version, keep in sync with README.md - rustc: "1.70.0" - - os: ubuntu-22.04 - # Oldest supported version, keep in sync with README.md - rustc: "1.70.0" - extra_desc: dist-server - extra_args: --no-default-features --features=dist-tests test_dist_ -- --test-threads 1 - - os: ubuntu-20.04 - rustc: stable - - os: ubuntu-20.04 - rustc: beta - - os: ubuntu-20.04 - rustc: nightly - allow_failure: true - extra_args: --features=unstable - - os: ubuntu-20.04 - extra_desc: no-default-features - extra_args: --no-default-features - allow_failure: true - - os: ubuntu-22.04 - - os: macOS-11 - # M1 CPU - - os: macos-14 - - os: windows-2019 - # Oldest supported version, keep in sync with README.md - rustc: "1.70.0" - - os: windows-2019 - rustc: nightly - allow_failure: true - extra_args: --features=unstable - - os: windows-2019 - rustc: beta - env: - RUST_BACKTRACE: 1 - steps: - - name: Clone repository - uses: actions/checkout@v4 - - - name: Install rust - uses: ./.github/actions/rust-toolchain - with: - toolchain: ${{ matrix.rustc }} - - - name: Install gcc & clang for tests - run: sudo apt-get install -y clang gcc - if: ${{ matrix.os == 'ubuntu-20.04' || matrix.os == 'ubuntu-22.04' }} - - - name: Build tests - run: cargo test --no-run --locked --all-targets ${{ matrix.extra_args }} - - - name: Run tests - run: cargo test --locked --all-targets ${{ matrix.extra_args }} - - - name: Upload failure - if: failure() - uses: ./.github/actions/artifact_failure - with: - name: test-${{ matrix.os }}-${{ matrix.rustc || 'stable' }}-${{ matrix.extra_desc }} - - build: - name: build ${{ matrix.binary || 'sccache' }} ${{ matrix.target }} - runs-on: ${{ matrix.os }} - container: ${{ fromJson(matrix.container || '{"image":null}') }} - timeout-minutes: 30 - strategy: - fail-fast: false - matrix: - include: - - os: ubuntu-20.04 - target: x86_64-unknown-linux-musl - container: '{"image": "messense/rust-musl-cross:x86_64-musl"}' - - os: ubuntu-20.04 - binary: sccache-dist - extra_args: --no-default-features --features="dist-server" - target: x86_64-unknown-linux-musl - container: '{"image": "messense/rust-musl-cross:x86_64-musl"}' - - os: ubuntu-20.04 - target: aarch64-unknown-linux-musl - container: '{"image": "messense/rust-musl-cross:aarch64-musl"}' - - os: ubuntu-20.04 - target: armv7-unknown-linux-musleabi - container: '{"image": "messense/rust-musl-cross:armv7-musleabi"}' - - os: ubuntu-20.04 - target: i686-unknown-linux-musl - container: '{"image": "messense/rust-musl-cross:i686-musl"}' - - os: macOS-11 - target: x86_64-apple-darwin - macosx_deployment_target: 10.13 - developer_dir: /Applications/Xcode_11.7.app - sdkroot: /Applications/Xcode_11.7.app/Contents/Developer/Platforms/MacOSX.platform/Developer/SDKs/MacOSX10.15.sdk - - os: macos-14 - target: aarch64-apple-darwin - macosx_deployment_target: 11.0 - - os: windows-2019 - target: x86_64-pc-windows-msvc - rustflags: -Ctarget-feature=+crt-static - steps: - - name: Clone repository - uses: actions/checkout@v4 - - - name: Install rust - uses: ./.github/actions/rust-toolchain - with: - toolchain: ${{ matrix.target == 'aarch64-apple-darwin' && 'beta' || 'stable' }} - target: ${{ matrix.target }} - if: ${{ !matrix.container }} - - - name: Build - run: cargo build --locked --release --bin ${{ matrix.binary || 'sccache' }} --target ${{ matrix.target }} --features=openssl/vendored ${{ matrix.extra_args }} - env: - MACOSX_DEPLOYMENT_TARGET: ${{ matrix.macosx_deployment_target }} - DEVELOPER_DIR: ${{ matrix.developer_dir }} - SDKROOT: ${{ matrix.sdkroot }} - RUSTFLAGS: ${{ matrix.rustflags }} - - # Workaround for the lack of substring() function in github actions expressions. - - name: Id - id: id - shell: bash - run: echo "id=${ID#refs/tags/}" >> $GITHUB_OUTPUT - env: - ID: ${{ startsWith(github.ref, 'refs/tags/') && github.ref || github.sha }} - - - name: Upload artifacts - uses: actions/upload-artifact@v4 - with: - name: ${{ matrix.binary || 'sccache' }}-${{ steps.id.outputs.id }}-${{ matrix.target }} - path: target/${{ matrix.target }}/release/${{ matrix.binary || 'sccache' }}${{ endsWith(matrix.target, '-msvc') && '.exe' || '' }} - if-no-files-found: error - - coverage: - name: coverage ${{ matrix.os }} rust ${{ matrix.rustc || 'stable' }} ${{ matrix.extra_desc }} - runs-on: ${{ matrix.os }} - continue-on-error: ${{ matrix.allow_failure || false }} - timeout-minutes: 30 - strategy: - fail-fast: false - matrix: - include: - - os: ubuntu-20.04 - rustc: nightly - allow_failure: true - extra_args: --features=unstable - - os: macOS-11 - rustc: nightly - # Disable on Windows for now as it fails with: - # found invalid metadata files for crate `vte_generate_state_changes` - # - os: windows-2019 - # rustc: nightly - env: - RUST_BACKTRACE: 1 - steps: - - name: Clone repository - uses: actions/checkout@v4 - - - name: Install rust - uses: ./.github/actions/rust-toolchain - with: - toolchain: ${{ matrix.rustc }} - - - name: Install gcc & clang for tests - run: sudo apt-get install -y clang gcc - if: ${{ matrix.os == 'ubuntu-20.04' }} - - - name: "`grcov` ~ install" - run: cargo install grcov - - - name: Execute tests - run: cargo test --no-fail-fast --locked --all-targets ${{ matrix.extra_args }} - env: - CARGO_INCREMENTAL: "0" - RUSTC_WRAPPER: "" - RUSTFLAGS: "-Zprofile -Ccodegen-units=1 -Copt-level=0 -Clink-dead-code -Coverflow-checks=off" - - - name: Generate coverage data (via `grcov`) - id: coverage - shell: bash - run: | - ## Generate coverage data - COVERAGE_REPORT_DIR="target/debug" - COVERAGE_REPORT_FILE="${COVERAGE_REPORT_DIR}/lcov.info" - # GRCOV_IGNORE_OPTION='--ignore build.rs --ignore "/*" --ignore "[a-zA-Z]:/*"' ## `grcov` ignores these params when passed as an environment variable (why?) - # GRCOV_EXCLUDE_OPTION='--excl-br-line "^\s*((debug_)?assert(_eq|_ne)?!|#\[derive\()"' ## `grcov` ignores these params when passed as an environment variable (why?) - mkdir -p "${COVERAGE_REPORT_DIR}" - # display coverage files - grcov . --output-type files --ignore build.rs --ignore "/*" --ignore "[a-zA-Z]:/*" --excl-br-line "^\s*((debug_)?assert(_eq|_ne)?!|#\[derive\()" | sort --unique - # generate coverage report - grcov . --output-type lcov --output-path "${COVERAGE_REPORT_FILE}" --branch --ignore build.rs --ignore "/*" --ignore "[a-zA-Z]:/*" --excl-br-line "^\s*((debug_)?assert(_eq|_ne)?!|#\[derive\()" - echo "report=${COVERAGE_REPORT_FILE}" >> $GITHUB_OUTPUT - - - name: Upload coverage results (to Codecov.io) - uses: codecov/codecov-action@v4 - with: - file: ${{ steps.coverage.outputs.report }} - ## flags: IntegrationTests, UnitTests, ${{ steps.vars.outputs.CODECOV_FLAGS }} - flags: ${{ steps.vars.outputs.CODECOV_FLAGS }} - name: codecov-umbrella - fail_ci_if_error: false - - test_freebsd: - name: test freebsd-13.2 rust stable - runs-on: ${{ matrix.job.os }} - timeout-minutes: 70 - strategy: - fail-fast: false - matrix: - job: - - { os: ubuntu-22.04 } - steps: - - uses: actions/checkout@v4 - - name: Prepare, build and test - uses: vmactions/freebsd-vm@v1.0.7 - with: - mem: 8192 - usesh: true - sync: rsync - copyback: false - prepare: pkg install -y ca_root_nss curl gmake gtar pot sudo - run: | - ##################################################################################### - ### Prepare, build, and test - ##################################################################################### - ### based on ref: - ### and on ref: - ### * NOTE: All steps need to be run in this block, otherwise, we are operating back - ### on the mac host. - set -exo pipefail - # - ### Basic user setup ################################################################ - TEST_USER=tester - TEST_USER_HOME="/opt/$TEST_USER" - REPO_NAME=${GITHUB_WORKSPACE##*/} - WORKSPACE_PARENT="/home/runner/work/${REPO_NAME}" - WORKSPACE="${WORKSPACE_PARENT}/${REPO_NAME}" - export WORKSPACE - # - mkdir -p "$TEST_USER_HOME" - pw adduser -n "$TEST_USER" -d "$TEST_USER_HOME" -c "Tester" -h - - chown -R "$TEST_USER":"$TEST_USER" "$TEST_USER_HOME" - chown -R "$TEST_USER":"$TEST_USER" "/$WORKSPACE_PARENT"/ - cat > /usr/local/etc/sudoers.d/wheel< "$d.tar.gz.sha256" - if [[ $d =~ (sccache-)(.*)?(x86_64-pc-windows)(.*)? ]]; then - zip -r "$d.zip" "$d" - echo -n "$(shasum -ba 256 "$d.zip" | cut -d " " -f 1)" > "$d.zip.sha256" - fi - done - - - name: Create release - run: | - sudo apt-get update && sudo apt-get install -y hub - tag_name=${GITHUB_REF#refs/tags/} - for f in sccache-*.tar.gz* sccache-*.zip*; do - if [[ -f "$f" ]]; then - files="$files -a $f"; - fi - done - hub release create -m $tag_name $tag_name $files - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} diff --git a/.github/workflows/close-snap.yml b/.github/workflows/close-snap.yml deleted file mode 100644 index 3563cbe18..000000000 --- a/.github/workflows/close-snap.yml +++ /dev/null @@ -1,19 +0,0 @@ -name: Close Snaps - -on: - pull_request: - types: [closed] - -jobs: - close: - runs-on: ubuntu-latest - - timeout-minutes: 5 - - steps: - - name: Close obsolete channels - uses: canonical/actions/close-snap@release - continue-on-error: true - with: - channel: edge/pr${{ github.event.number }} - snapcraft-token: ${{ secrets.SNAPCRAFT_TOKEN }} diff --git a/.github/workflows/integration-tests.yml b/.github/workflows/integration-tests.yml index a3b85fce3..a167316a5 100644 --- a/.github/workflows/integration-tests.yml +++ b/.github/workflows/integration-tests.yml @@ -27,218 +27,6 @@ jobs: name: integration-tests path: ./target/debug/sccache - redis-deprecated: - runs-on: ubuntu-latest - needs: build - - services: - redis: - image: redis - ports: - - 6379:6379 - - env: - SCCACHE_REDIS: redis://127.0.0.1 - RUSTC_WRAPPER: /home/runner/.cargo/bin/sccache - - steps: - - name: Clone repository - uses: actions/checkout@v4 - - - name: Install rust - uses: ./.github/actions/rust-toolchain - with: - toolchain: "stable" - - - uses: actions/download-artifact@v4 - with: - name: integration-tests - path: /home/runner/.cargo/bin/ - - name: Chmod for binary - run: chmod +x ${SCCACHE_PATH} - - - name: Test - run: cargo clean && cargo build - - - name: Output - run: | - ${SCCACHE_PATH} --show-stats - - ${SCCACHE_PATH} --show-stats | grep redis - - - name: Test Twice for Cache Read - run: cargo clean && cargo build - - - name: Output - run: | - ${SCCACHE_PATH} --show-stats - - ${SCCACHE_PATH} --show-stats | grep -e "Cache hits\s*[1-9]" - - redis: - runs-on: ubuntu-latest - needs: build - - services: - redis: - image: redis - ports: - - 6379:6379 - - env: - SCCACHE_REDIS_ENDPOINT: tcp://127.0.0.1 - RUSTC_WRAPPER: /home/runner/.cargo/bin/sccache - - steps: - - name: Clone repository - uses: actions/checkout@v4 - - - name: Install rust - uses: ./.github/actions/rust-toolchain - with: - toolchain: "stable" - - - uses: actions/download-artifact@v4 - with: - name: integration-tests - path: /home/runner/.cargo/bin/ - - name: Chmod for binary - run: chmod +x ${SCCACHE_PATH} - - - name: Test - run: cargo clean && cargo build - - - name: Output - run: | - ${SCCACHE_PATH} --show-stats - - ${SCCACHE_PATH} --show-stats | grep redis - - - name: Test Twice for Cache Read - run: cargo clean && cargo build - - - name: Output - run: | - ${SCCACHE_PATH} --show-stats - - ${SCCACHE_PATH} --show-stats | grep -e "Cache hits\s*[1-9]" - - s3_minio: - runs-on: ubuntu-latest - needs: build - - # Setup minio server - services: - minio: - image: wktk/minio-server - ports: - - 9000:9000 - env: - MINIO_ACCESS_KEY: "minioadmin" - MINIO_SECRET_KEY: "minioadmin" - - env: - SCCACHE_BUCKET: test - SCCACHE_ENDPOINT: http://127.0.0.1:9000/ - SCCACHE_REGION: us-east-1 - AWS_ACCESS_KEY_ID: "minioadmin" - AWS_SECRET_ACCESS_KEY: "minioadmin" - AWS_EC2_METADATA_DISABLED: "true" - RUSTC_WRAPPER: /home/runner/.cargo/bin/sccache - - steps: - - name: Clone repository - uses: actions/checkout@v4 - - - name: Setup test bucket - run: aws --endpoint-url http://127.0.0.1:9000/ s3 mb s3://test - - - name: Install rust - uses: ./.github/actions/rust-toolchain - with: - toolchain: "stable" - - - uses: actions/download-artifact@v4 - with: - name: integration-tests - path: /home/runner/.cargo/bin/ - - name: Chmod for binary - run: chmod +x ${SCCACHE_PATH} - - - name: Test - run: cargo clean && cargo build - - - name: Output - run: | - ${SCCACHE_PATH} --show-stats - - ${SCCACHE_PATH} --show-stats | grep s3 - - - name: Test Twice for Cache Read - run: cargo clean && cargo build - - - name: Output - run: | - ${SCCACHE_PATH} --show-stats - - ${SCCACHE_PATH} --show-stats | grep -e "Cache hits\s*[1-9]" - - azblob_azurite: - runs-on: ubuntu-latest - needs: build - - # Setup azurite server - services: - azurite: - image: mcr.microsoft.com/azure-storage/azurite - ports: - - 10000:10000 - - env: - SCCACHE_AZURE_BLOB_CONTAINER: "test" - SCCACHE_AZURE_CONNECTION_STRING: "DefaultEndpointsProtocol=http;AccountName=devstoreaccount1;AccountKey=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==;BlobEndpoint=http://127.0.0.1:10000/devstoreaccount1;" - RUSTC_WRAPPER: /home/runner/.cargo/bin/sccache - - steps: - - name: Clone repository - uses: actions/checkout@v4 - - - name: Setup test bucket - run: | - az storage container create \ - --name test \ - --connection-string ${SCCACHE_AZURE_CONNECTION_STRING} - - - name: Install rust - uses: ./.github/actions/rust-toolchain - with: - toolchain: "stable" - - - uses: actions/download-artifact@v4 - with: - name: integration-tests - path: /home/runner/.cargo/bin/ - - name: Chmod for binary - run: chmod +x ${SCCACHE_PATH} - - - name: Test - run: cargo clean && cargo build - - - name: Output - run: | - ${SCCACHE_PATH} --show-stats - - ${SCCACHE_PATH} --show-stats | grep azblob - - - name: Test Twice for Cache Read - run: cargo clean && cargo build - - - name: Output - run: | - ${SCCACHE_PATH} --show-stats - - ${SCCACHE_PATH} --show-stats | grep -e "Cache hits\s*[1-9]" - gha: runs-on: ubuntu-latest needs: build @@ -288,137 +76,25 @@ jobs: ${SCCACHE_PATH} --show-stats | grep -e "Cache hits\s*[1-9]" - memcached-deprecated: + gha-as-local: runs-on: ubuntu-latest needs: build - # Setup memcached server - services: - memcached: - image: bitnami/memcached - env: - # memcache's max item size is 1MiB, But our tests - # will produce larger file. - # - # Specify the setting here to make our test happy. - MEMCACHED_MAX_ITEM_SIZE: 16777216 - ports: - - 11211:11211 - env: - SCCACHE_MEMCACHED: "tcp://127.0.0.1:11211" - RUSTC_WRAPPER: /home/runner/.cargo/bin/sccache - - steps: - - name: Clone repository - uses: actions/checkout@v4 - - - name: Install rust - uses: ./.github/actions/rust-toolchain - with: - toolchain: "stable" - - - uses: actions/download-artifact@v4 - with: - name: integration-tests - path: /home/runner/.cargo/bin/ - - name: Chmod for binary - run: chmod +x ${SCCACHE_PATH} - - - name: Test - run: cargo clean && cargo build - - - name: Output - run: | - ${SCCACHE_PATH} --show-stats - - ${SCCACHE_PATH} --show-stats | grep memcached - - - name: Test Twice for Cache Read - run: cargo clean && cargo build - - - name: Output - run: | - ${SCCACHE_PATH} --show-stats - - ${SCCACHE_PATH} --show-stats | grep -e "Cache hits\s*[1-9]" - - memcached: - runs-on: ubuntu-latest - needs: build - - # Setup memcached server - services: - memcached: - image: bitnami/memcached - env: - # memcache's max item size is 1MiB, But our tests - # will produce larger file. - # - # Specify the setting here to make our test happy. - MEMCACHED_MAX_ITEM_SIZE: 16777216 - ports: - - 11211:11211 - - env: - SCCACHE_MEMCACHED_ENDPOINT: "tcp://127.0.0.1:11211" + SCCACHE_GHA_ENABLED: "on" + SCCACHE_GHA_AS_LOCAL: "on" RUSTC_WRAPPER: /home/runner/.cargo/bin/sccache steps: - name: Clone repository uses: actions/checkout@v4 - - name: Install rust - uses: ./.github/actions/rust-toolchain - with: - toolchain: "stable" - - - uses: actions/download-artifact@v4 + - name: Configure Cache Env + uses: actions/github-script@v7 with: - name: integration-tests - path: /home/runner/.cargo/bin/ - - name: Chmod for binary - run: chmod +x ${SCCACHE_PATH} - - - name: Test - run: cargo clean && cargo build - - - name: Output - run: | - ${SCCACHE_PATH} --show-stats - - ${SCCACHE_PATH} --show-stats | grep memcached - - - name: Test Twice for Cache Read - run: cargo clean && cargo build - - - name: Output - run: | - ${SCCACHE_PATH} --show-stats - - ${SCCACHE_PATH} --show-stats | grep -e "Cache hits\s*[1-9]" - - webdav: - runs-on: ubuntu-latest - needs: build - - env: - SCCACHE_WEBDAV_ENDPOINT: "http://127.0.0.1:8080" - SCCACHE_WEBDAV_USERNAME: "bar" - SCCACHE_WEBDAV_PASSWORD: "baz" - RUSTC_WRAPPER: /home/runner/.cargo/bin/sccache - - steps: - - uses: actions/checkout@v4 - - - name: Start nginx - shell: bash - run: | - sudo apt install -y nginx-full - - mkdir /tmp/static - cp `pwd`/tests/htpasswd /tmp/htpasswd - nginx -c `pwd`/tests/nginx_http_cache.conf + script: | + core.exportVariable('ACTIONS_CACHE_URL', process.env.ACTIONS_CACHE_URL || ''); + core.exportVariable('ACTIONS_RUNTIME_TOKEN', process.env.ACTIONS_RUNTIME_TOKEN || ''); - name: Install rust uses: ./.github/actions/rust-toolchain @@ -439,7 +115,7 @@ jobs: run: | ${SCCACHE_PATH} --show-stats - ${SCCACHE_PATH} --show-stats | grep webdav + ${SCCACHE_PATH} --show-stats | grep gha - name: Test Twice for Cache Read run: cargo clean && cargo build @@ -449,400 +125,3 @@ jobs: ${SCCACHE_PATH} --show-stats ${SCCACHE_PATH} --show-stats | grep -e "Cache hits\s*[1-9]" - - test-mock-msvc: - runs-on: windows-2019 - env: - TARGET: x86_64-pc-windows-msvc - SCCACHE_EXE: ${{ github.workspace }}\\target\\x86_64-pc-windows-msvc\\debug\\sccache.exe - SCCACHE_LOG: "trace" - SCCACHE_ERROR_LOG: "${{ github.workspace }}\\server_log.txt" - - steps: - - uses: ilammy/msvc-dev-cmd@v1 - - - name: Clone repository - uses: actions/checkout@v4 - - - name: Install rust - uses: ./.github/actions/rust-toolchain - with: - toolchain: "stable" - target: $TARGET - - - name: Build - run: cargo build --bin sccache --target $env:TARGET --features=openssl/vendored - - - name: Compile MSVC (no cache) - shell: bash - working-directory: ./tests/msvc - run: | - cl "@args.rsp" - test -e ./foo.o || { echo "No compiler output found"; exit -1; } - - - name: Start Server - shell: bash - run: $SCCACHE_EXE --start-server - - - name: Compile - Cache Miss - shell: bash - working-directory: ./tests/msvc - run: | - rm ./foo.o || true - $SCCACHE_EXE "$(where cl.exe)" -c "@args.rsp" - $SCCACHE_EXE --show-stats - $SCCACHE_EXE --show-stats | grep -e "Cache misses\s*[1-9]" - test -e ./foo.o || { echo "No compiler output found"; exit -1; } - test -e ./foo.o.json || { echo "No dependency list found"; exit -1; } - - - name: Compile - Cache Hit - shell: bash - working-directory: ./tests/msvc - run: | - rm ./foo.o || true - $SCCACHE_EXE "$(where cl.exe)" -c "@args.rsp" - $SCCACHE_EXE --show-stats - $SCCACHE_EXE --show-stats | grep -e "Cache hits\s*[1-9]" - test -e ./foo.o || { echo "No compiler output found"; exit -1; } - test -e ./foo.o.json || { echo "No dependency list found"; exit -1; } - - - name: Compile - Preprocessing Compiler Bug - shell: bash - working-directory: ./tests/msvc-preprocessing - run: | - $SCCACHE_EXE "$(where cl.exe)" -c "@args.rsp" - $SCCACHE_EXE --show-stats - - - name: Stop Server - if: success() || failure() - shell: bash - run: $SCCACHE_EXE --stop-server - - - name: Show Server Log - if: success() || failure() - shell: bash - run: cat "$SCCACHE_ERROR_LOG" - - - clang: - runs-on: ubuntu-latest - needs: build - - env: - LLVM_VERSION: "16" - SCCACHE_GHA_ENABLED: "on" - - steps: - - uses: actions/checkout@v4 - - - name: Configure Cache Env - uses: actions/github-script@v7 - with: - script: | - core.exportVariable('ACTIONS_CACHE_URL', process.env.ACTIONS_CACHE_URL || ''); - core.exportVariable('ACTIONS_RUNTIME_TOKEN', process.env.ACTIONS_RUNTIME_TOKEN || '') - - - name: Install clang - run: | - wget https://apt.llvm.org/llvm.sh - chmod +x llvm.sh - sudo ./llvm.sh "${LLVM_VERSION}" - - - uses: actions/download-artifact@v4 - with: - name: integration-tests - path: /home/runner/.cargo/bin/ - - name: Chmod for binary - run: chmod +x ${SCCACHE_PATH} - - - name: Test - run: | - export CXX="${SCCACHE_PATH} clang++" - $CXX -c `pwd`/tests/test_clang_multicall.c - - - name: Output - run: | - ${SCCACHE_PATH} --show-stats - - - name: Test Twice for Cache Read - run: | - export CXX="${SCCACHE_PATH} clang++" - $CXX -c `pwd`/tests/test_clang_multicall.c - - - name: Output - run: | - ${SCCACHE_PATH} --show-stats - - ${SCCACHE_PATH} --show-stats | grep -e "Cache hits\s*[1-9]" - - hip: - # Probably wouldn't matter anyway since we run in a container, but staying - # close to the version is better than not. - runs-on: ubuntu-22.04 - needs: build - container: - image: rocm/dev-ubuntu-22.04:6.0 - - env: - # SCCACHE_GHA_ENABLED: "on" - ROCM_PATH: "/opt/rocm" - - steps: - - uses: actions/checkout@v4 - - # I don't want to break the cache during testing. Will turn on after I - # make sure it's working. - # - # - name: Configure Cache Env - # uses: actions/github-script@v7 - # with: - # script: | - # core.exportVariable('ACTIONS_CACHE_URL', process.env.ACTIONS_CACHE_URL || ''); - # core.exportVariable('ACTIONS_RUNTIME_TOKEN', process.env.ACTIONS_RUNTIME_TOKEN || '') - - # - name: Configure ROCm Env - # uses: actions/github-script@v7 - # with: - # script: | - # core.exportVariable('ROCM_PATH', process.env.ROCM_PATH || ''); - - - uses: actions/download-artifact@v4 - with: - name: integration-tests - path: /home/runner/.cargo/bin/ - - name: Chmod for binary - run: chmod +x ${SCCACHE_PATH} - - - name: Install dependencies - shell: bash - run: | - ## Install dependencies - sudo apt-get update - sudo apt-get install -y cmake - - # Ensure that HIPCC isn't already borken - - name: Sanity Check - run: | - hipcc -o vectoradd_hip --offload-arch=gfx900 tests/cmake-hip/vectoradd_hip.cpp - - - name: Test - run: | - cmake -B build -S tests/cmake-hip -DCMAKE_HIP_COMPILER_LAUNCHER=${SCCACHE_PATH} -DCMAKE_HIP_ARCHITECTURES=gfx900 - cmake --build build - - - name: Output - run: | - ${SCCACHE_PATH} --show-stats - - - name: Test Twice for Cache Read - run: | - rm -rf build - cmake -B build -S tests/cmake-hip -DCMAKE_HIP_COMPILER_LAUNCHER=${SCCACHE_PATH} -DCMAKE_HIP_ARCHITECTURES=gfx900 - cmake --build build - - - name: Output - run: | - ${SCCACHE_PATH} --show-stats - - ${SCCACHE_PATH} --show-stats | grep -e "Cache hits\s*[1-9]" - - - gcc: - runs-on: ubuntu-latest - needs: build - - env: - SCCACHE_GHA_ENABLED: "on" - - steps: - - uses: actions/checkout@v4 - - - name: Configure Cache Env - uses: actions/github-script@v7 - with: - script: | - core.exportVariable('ACTIONS_CACHE_URL', process.env.ACTIONS_CACHE_URL || ''); - core.exportVariable('ACTIONS_RUNTIME_TOKEN', process.env.ACTIONS_RUNTIME_TOKEN || '') - - - uses: actions/download-artifact@v4 - with: - name: integration-tests - path: /home/runner/.cargo/bin/ - - name: Chmod for binary - run: chmod +x ${SCCACHE_PATH} - - - name: Test - run: | - export CXX="${SCCACHE_PATH} g++" - $CXX -c `pwd`/tests/test_clang_multicall.c - - - name: Output - run: | - ${SCCACHE_PATH} --show-stats - - - name: Test Twice for Cache Read - run: | - export CXX="${SCCACHE_PATH} g++" - $CXX -c `pwd`/tests/test_clang_multicall.c - - - name: Output - run: | - ${SCCACHE_PATH} --show-stats - - ${SCCACHE_PATH} --show-stats | grep -e "Cache hits\s*[1-9]" - - - autotools: - runs-on: ubuntu-latest - needs: build - - env: - SCCACHE_GHA_ENABLED: "on" - - steps: - - uses: actions/checkout@v4 - - - name: Configure Cache Env - uses: actions/github-script@v7 - with: - script: | - core.exportVariable('ACTIONS_CACHE_URL', process.env.ACTIONS_CACHE_URL || ''); - core.exportVariable('ACTIONS_RUNTIME_TOKEN', process.env.ACTIONS_RUNTIME_TOKEN || '') - - - uses: actions/download-artifact@v4 - with: - name: integration-tests - path: /home/runner/.cargo/bin/ - - name: Chmod for binary - run: chmod +x ${SCCACHE_PATH} - - - name: Install dependencies - shell: bash - run: | - ## Install dependencies - sudo apt-get update - sudo apt-get install autoconf automake libtool - - - name: Test - run: | - cd `pwd`/tests/autotools/ - autoreconf||true - automake --add-missing - ./configure CXX="${SCCACHE_PATH} g++" - make - - - name: Output - run: | - ${SCCACHE_PATH} --show-stats - - - name: Test Twice for Cache Read - run: | - cd `pwd`/tests/autotools/ - make distclean - ./configure CXX="${SCCACHE_PATH} g++" - make - - - name: Output - run: | - ${SCCACHE_PATH} --show-stats - - ${SCCACHE_PATH} --show-stats | grep -e "Cache hits\s*[1-9]" - - cmake: - runs-on: ubuntu-latest - needs: build - - env: - SCCACHE_GHA_ENABLED: "on" - - steps: - - uses: actions/checkout@v4 - - - name: Configure Cache Env - uses: actions/github-script@v7 - with: - script: | - core.exportVariable('ACTIONS_CACHE_URL', process.env.ACTIONS_CACHE_URL || ''); - core.exportVariable('ACTIONS_RUNTIME_TOKEN', process.env.ACTIONS_RUNTIME_TOKEN || '') - - - uses: actions/download-artifact@v4 - with: - name: integration-tests - path: /home/runner/.cargo/bin/ - - name: Chmod for binary - run: chmod +x ${SCCACHE_PATH} - - - name: Install dependencies - shell: bash - run: | - ## Install dependencies - sudo apt-get update - sudo apt-get install cmake - - - name: Test - run: | - cd `pwd`/tests/cmake/ - mkdir build - cd build - cmake -DCMAKE_C_COMPILER_LAUNCHER=${SCCACHE_PATH} -DCMAKE_CXX_COMPILER_LAUNCHER=${SCCACHE_PATH} .. - make - - - name: Output - run: | - ${SCCACHE_PATH} --show-stats - - - name: Test Twice for Cache Read - run: | - cd `pwd`/tests/cmake/ - rm -rf build - mkdir build - cd build - cmake -DCMAKE_C_COMPILER_LAUNCHER=${SCCACHE_PATH} -DCMAKE_CXX_COMPILER_LAUNCHER=${SCCACHE_PATH} .. - make - - - name: Output - run: | - ${SCCACHE_PATH} --show-stats - - ${SCCACHE_PATH} --show-stats | grep -e "Cache hits\s*[1-9]" - - rust-test-coverage: - runs-on: ubuntu-latest - needs: build - - env: - RUSTC_WRAPPER: /home/runner/.cargo/bin/sccache - CARGO_INCREMENTAL: "0" - RUSTFLAGS: "-Zprofile -Ccodegen-units=1 -Copt-level=0 -Clink-dead-code -Coverflow-checks=off -Zpanic_abort_tests -Cpanic=abort" - RUSTDOCFLAGS: "-Cpanic=abort" - - steps: - - uses: actions/download-artifact@v4 - with: - name: integration-tests - path: /home/runner/.cargo/bin/ - - name: Chmod for binary - run: chmod +x ${SCCACHE_PATH} - - - name: Prepare - run: | - rustup toolchain install nightly - cargo new coverage-test - cd coverage-test - echo "serde = { version = \"1.0\", features = [\"derive\"] }" >> Cargo.toml - - - name: "Coverage test #1" - working-directory: ./coverage-test - run: cargo clean && cargo +nightly test - - - name: Output - run: ${SCCACHE_PATH} --show-stats - - - name: "Coverage test #2" - working-directory: ./coverage-test - run: cargo clean && cargo +nightly test - - - name: Output - run: | - ${SCCACHE_PATH} --show-stats - ${SCCACHE_PATH} --show-stats | grep -e "Cache hits\s*[1-9]" diff --git a/.github/workflows/snap.yml b/.github/workflows/snap.yml deleted file mode 100644 index d2d1c024b..000000000 --- a/.github/workflows/snap.yml +++ /dev/null @@ -1,26 +0,0 @@ -name: Snap - -on: - pull_request: - types: [opened, synchronize, reopened, ready_for_review] - -jobs: - Snap: - runs-on: ubuntu-latest - - timeout-minutes: 45 - - steps: - - name: Check out code - uses: actions/checkout@v4 - with: - fetch-depth: 0 # needed for version determination - - - name: Build and publish the snap - uses: canonical/actions/build-snap@release - with: - snapcraft-channel: edge - review-opts: --allow-classic - snapcraft-token: ${{ secrets.SNAPCRAFT_TOKEN }} - publish: ${{ github.event_name == 'pull_request' && github.repository == github.event.pull_request.head.repo.full_name }} - publish-channel: edge/pr${{ github.event.number }} diff --git a/src/cache/cache.rs b/src/cache/cache.rs index 9d61788c8..aa6cd5778 100644 --- a/src/cache/cache.rs +++ b/src/cache/cache.rs @@ -591,12 +591,24 @@ pub fn storage_from_config( return Ok(Arc::new(storage)); } #[cfg(feature = "gha")] - CacheType::GHA(config::GHACacheConfig { ref version, .. }) => { - debug!("Init gha cache with version {version}"); - - let storage = GHACache::build(version) - .map_err(|err| anyhow!("create gha cache failed: {err:?}"))?; - return Ok(Arc::new(storage)); + CacheType::GHA(config::GHACacheConfig { + ref version, + as_local, + .. + }) => { + if *as_local { + debug!("Init gha as local cache"); + let downloaded_path = pool + .block_on(GHACache::download_to_local(config, version)) + .map_err(|err| anyhow!("download gha cache as local failed: {err:?}"))?; + let storage = disk_cache_from_config(config, pool, downloaded_path)?; + return Ok(storage); + } else { + debug!("Init gha cache with version {version}"); + let storage = GHACache::build(version) + .map_err(|err| anyhow!("create gha cache failed: {err:?}"))?; + return Ok(Arc::new(storage)); + } } #[cfg(feature = "memcached")] CacheType::Memcached(config::MemcachedCacheConfig { @@ -724,7 +736,21 @@ pub fn storage_from_config( } } - let (dir, size) = (&config.fallback_cache.dir, config.fallback_cache.size); + disk_cache_from_config(config, pool, None) +} + +fn disk_cache_from_config( + config: &Config, + pool: &tokio::runtime::Handle, + root_override: Option, +) -> Result> { + let (mut dir, size) = ( + config.fallback_cache.dir.to_owned(), + config.fallback_cache.size, + ); + if let Some(new_root) = root_override { + dir = dir.join(new_root); + } let preprocessor_cache_mode_config = config.fallback_cache.preprocessor_cache_mode; let rw_mode = config.fallback_cache.rw_mode.into(); debug!("Init disk cache with dir {:?}, size {}", dir, size); @@ -737,6 +763,27 @@ pub fn storage_from_config( ))) } +#[cfg(feature = "gha")] +pub async fn upload_local_cache(config: &Config) -> Result<()> { + match &config.cache { + Some(CacheType::GHA(gha_config)) => { + if !gha_config.enabled { + debug!("GHA cache is disabled in config"); + return Ok(()); + } + if !gha_config.as_local { + debug!("GHA not configured `as_local`"); + return Ok(()); + } + GHACache::upload_local_cache(config).await + } + _ => { + debug!("Uploading the local cache is only possible when using GitHub Actions"); + Ok(()) + } + } +} + #[cfg(test)] mod test { use super::*; diff --git a/src/cache/gha.rs b/src/cache/gha.rs index 8d8373430..078f57b60 100644 --- a/src/cache/gha.rs +++ b/src/cache/gha.rs @@ -12,13 +12,18 @@ // See the License for the specific language governing permissions and // limitations under the License. +use std::path::PathBuf; + use opendal::layers::LoggingLayer; use opendal::services::Ghac; use opendal::Operator; +use crate::config::Config; use crate::errors::*; use crate::VERSION; +const FULL_GHA_CACHE_ROOT: &str = "sccache-full"; + /// A cache that stores entries in GHA Cache Services. pub struct GHACache; @@ -43,4 +48,129 @@ impl GHACache { .finish(); Ok(op) } + + /// Download a copy of the entire GHA cache from the given version + /// and return the path to the root folder on the local disk. + /// + /// It is the user's responsibility to split the caches according + /// to anything relevant like architecture, OS, etc. by using the `version`. + pub async fn download_to_local(config: &Config, version: &str) -> Result> { + let tarball_path = local_cache_tarball_path(config); + let mut builder = Ghac::default(); + + // TODO somehow loop over decreasingly "fresh" versions of the cache + // like in + // https://docs.github.com/en/actions/using-workflows/caching-dependencies-to-speed-up-workflows#matching-a-cache-key + // For now the behavior is to match the same version, which would + // speed up rebuilds in the same (Git) branch. + if version.is_empty() { + builder.version(&format!("sccache-v{VERSION}")); + } else { + builder.version(&format!("sccache-v{VERSION}-{version}")); + } + + let op = Operator::new(builder)? + .layer(LoggingLayer::default()) + .finish(); + + if !op.is_exist(FULL_GHA_CACHE_ROOT).await? { + info!("Remote full gha cache does not exist: nothing to do"); + return Ok(None); + } + debug!("Found full gha cache"); + + let mut reader = op.reader(FULL_GHA_CACHE_ROOT).await?; + std::fs::create_dir_all(tarball_path.parent().expect("root path"))?; + + let mut writer = tokio::fs::OpenOptions::new() + .write(true) + .create(true) + .open(&tarball_path) + .await + .context("opening the local tarball for writing")?; + + if let Err(error) = tokio::io::copy(&mut reader, &mut writer).await { + match error.kind() { + std::io::ErrorKind::NotFound => { + debug!("Remote full gha cache was deleted: nothing to do"); + // TOCTOU race with the above existence check and the cache + // being cleared. + return Ok(None); + } + _ => { + bail!(error) + } + } + }; + + let cache = local_cache_path(config); + let tarball = + std::fs::File::open(tarball_path).context("Failed to open the GHA cache tarball")?; + tar::Archive::new(tarball) + .unpack(&cache) + .context("Failed to extract the GHA cache tarball")?; + + Ok(Some(cache)) + } + + /// Upload a tarball of the local cache + pub async fn upload_local_cache(config: &Config) -> Result<()> { + let cache = local_cache_path(config); + if !cache.exists() { + info!("Local cache does not exist: nothing to do"); + return Ok(()); + } + debug!("Found local gha cache at {}", cache.display()); + + let op = Operator::new(Ghac::default())? + .layer(LoggingLayer::default()) + .finish(); + + // GHA cache is immutable, if the path has already been written within + // a given version, it cannot be changed again. + if op.is_exist(FULL_GHA_CACHE_ROOT).await? { + info!("Remote cache of this version already exists, cannot upload"); + return Ok(()); + } + + let mut tar_builder = tar::Builder::new(vec![]); + tar_builder + .append_dir_all(local_cache_path(config), ".") + .context("Failed to create GHA local cache tarball")?; + let source = local_cache_tarball_path(config); + std::fs::write(&source, tar_builder.into_inner()?) + .context("Failed to write the GHA local cache tarball to disk")?; + + let mut writer = op + .writer(FULL_GHA_CACHE_ROOT) + .await + .context("opening the remote tarball for writing")?; + + let mut reader = tokio::fs::File::open(&source) + .await + .context("opening the local tarball for reading")?; + + if let Err(error) = tokio::io::copy(&mut reader, &mut writer).await { + match error.kind() { + std::io::ErrorKind::AlreadyExists => { + debug!("Remote cache of this version raced us, cannot upload"); + // TOCTOU race with the above existence check and the cache + // being uploaded by another worker. + return Ok(()); + } + _ => bail!(error), + } + } + Ok(()) + } +} + +fn local_cache_tarball_path(config: &Config) -> PathBuf { + let mut path = config.fallback_cache.dir.join(FULL_GHA_CACHE_ROOT); + path.set_extension(".tar"); + path +} + +fn local_cache_path(config: &Config) -> PathBuf { + config.fallback_cache.dir.join(FULL_GHA_CACHE_ROOT) } diff --git a/src/config.rs b/src/config.rs index 4173bcaac..6ebd7f451 100644 --- a/src/config.rs +++ b/src/config.rs @@ -213,6 +213,11 @@ pub struct GHACacheConfig { /// Version for gha cache is a namespace. By setting different versions, /// we can avoid mixed caches. pub version: String, + /// Download the entire cache to be used like a local cache, then upload + /// it back if anything changed. + /// This is useful in CI contexts to reduce the number of requests, + /// hence avoiding rate limiting and improving overall cache speed. + pub as_local: bool, } /// Memcached's default value of expiration is 10800s (3 hours), which is too @@ -784,12 +789,13 @@ fn config_from_env() -> Result { }); // ======= GHA ======= - let gha = if let Ok(version) = env::var("SCCACHE_GHA_VERSION") { + let mut gha = if let Ok(version) = env::var("SCCACHE_GHA_VERSION") { // If SCCACHE_GHA_VERSION has been set, we don't need to check // SCCACHE_GHA_ENABLED's value anymore. Some(GHACacheConfig { enabled: true, version, + as_local: false, }) } else if bool_from_env_var("SCCACHE_GHA_ENABLED")?.unwrap_or(false) { // If only SCCACHE_GHA_ENABLED has been set to the true value, enable with @@ -797,11 +803,16 @@ fn config_from_env() -> Result { Some(GHACacheConfig { enabled: true, version: "".to_string(), + as_local: false, }) } else { None }; + if let Some(gha) = &mut gha { + gha.as_local = bool_from_env_var("SCCACHE_GHA_AS_LOCAL")?.unwrap_or(false); + } + // ======= Azure ======= let azure = if let (Ok(connection_string), Ok(container)) = ( env::var("SCCACHE_AZURE_CONNECTION_STRING"), @@ -1453,6 +1464,7 @@ service_account = "example_service_account" [cache.gha] enabled = true version = "sccache" +as_local = false [cache.memcached] # Deprecated alias for `endpoint` @@ -1519,7 +1531,8 @@ no_credentials = true }), gha: Some(GHACacheConfig { enabled: true, - version: "sccache".to_string() + version: "sccache".to_string(), + as_local: false, }), redis: Some(RedisCacheConfig { url: Some("redis://user:passwd@1.2.3.4:6379/?db=1".to_owned()), diff --git a/src/server.rs b/src/server.rs index 70dc4f859..d61d7f5e3 100644 --- a/src/server.rs +++ b/src/server.rs @@ -13,6 +13,8 @@ // limitations under the License.SCCACHE_MAX_FRAME_LENGTH use crate::cache::readonly::ReadOnlyStorage; +#[cfg(feature = "gha")] +use crate::cache::upload_local_cache; use crate::cache::{storage_from_config, CacheMode, Storage}; use crate::compiler::{ get_compiler_info, CacheControl, CompileResult, Compiler, CompilerArguments, CompilerHasher, @@ -466,7 +468,6 @@ pub fn start_server(config: &Config, port: u16) -> Result<()> { CacheMode::ReadOnly => Arc::new(ReadOnlyStorage(raw_storage)), _ => raw_storage, }; - let res = SccacheServer::::new(port, runtime, client, dist_client, storage); match res { @@ -474,7 +475,17 @@ pub fn start_server(config: &Config, port: u16) -> Result<()> { let port = srv.port(); info!("server started, listening on port {}", port); notify_server_startup(¬ify, ServerStartup::Ok { port })?; - srv.run(future::pending::<()>())?; + let runtime = srv.run(future::pending::<()>())?; + #[cfg(feature = "gha")] + if let Err(e) = runtime.block_on(upload_local_cache(config)) { + notify_server_startup( + ¬ify, + ServerStartup::Err { + reason: e.to_string(), + }, + )?; + } + Ok(()) } Err(e) => { @@ -589,7 +600,7 @@ impl SccacheServer { /// If the `shutdown` future resolves then the server will be shut down, /// otherwise the server may naturally shut down if it becomes idle for too /// long anyway. - pub fn run(self, shutdown: F) -> io::Result<()> + pub fn run(self, shutdown: F) -> io::Result where F: Future, C: Send, @@ -676,7 +687,7 @@ impl SccacheServer { info!("ok, fully shutting down now"); - Ok(()) + Ok(runtime) } }