From 38d71fa52a056a6035d3ba10d107173349bcd61a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20Gom=C3=A8s?= Date: Tue, 2 Apr 2024 16:14:14 +0200 Subject: [PATCH] feat: add a way of using GHA cache locally MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This is the first implementation of uploading the local cache as a single file to a remote cache for reuse in a future build. Right now it is only done for GHA as that was the intended scope¹, but one could adapt this system to other remote caches. Because of the immutability of GHACache, this commit only adds support for re-using the cache for the same version (as defined by the user through the `SCCACHE_GHA_VERSION` environment variable). A way of reusing incremental build within a given version or even across versions could be devised, but it falls outside the scope of this particular effort, and it's probably not trivial. [1] https://github.com/Mozilla-Actions/sccache-action/issues/81 --- .github/workflows/integration-tests.yml | 739 +----------------------- src/cache/cache.rs | 61 +- src/cache/gha.rs | 115 ++++ src/config.rs | 17 +- src/server.rs | 11 +- 5 files changed, 200 insertions(+), 743 deletions(-) diff --git a/.github/workflows/integration-tests.yml b/.github/workflows/integration-tests.yml index a3b85fce3..a167316a5 100644 --- a/.github/workflows/integration-tests.yml +++ b/.github/workflows/integration-tests.yml @@ -27,218 +27,6 @@ jobs: name: integration-tests path: ./target/debug/sccache - redis-deprecated: - runs-on: ubuntu-latest - needs: build - - services: - redis: - image: redis - ports: - - 6379:6379 - - env: - SCCACHE_REDIS: redis://127.0.0.1 - RUSTC_WRAPPER: /home/runner/.cargo/bin/sccache - - steps: - - name: Clone repository - uses: actions/checkout@v4 - - - name: Install rust - uses: ./.github/actions/rust-toolchain - with: - toolchain: "stable" - - - uses: actions/download-artifact@v4 - with: - name: integration-tests - path: /home/runner/.cargo/bin/ - - name: Chmod for binary - run: chmod +x ${SCCACHE_PATH} - - - name: Test - run: cargo clean && cargo build - - - name: Output - run: | - ${SCCACHE_PATH} --show-stats - - ${SCCACHE_PATH} --show-stats | grep redis - - - name: Test Twice for Cache Read - run: cargo clean && cargo build - - - name: Output - run: | - ${SCCACHE_PATH} --show-stats - - ${SCCACHE_PATH} --show-stats | grep -e "Cache hits\s*[1-9]" - - redis: - runs-on: ubuntu-latest - needs: build - - services: - redis: - image: redis - ports: - - 6379:6379 - - env: - SCCACHE_REDIS_ENDPOINT: tcp://127.0.0.1 - RUSTC_WRAPPER: /home/runner/.cargo/bin/sccache - - steps: - - name: Clone repository - uses: actions/checkout@v4 - - - name: Install rust - uses: ./.github/actions/rust-toolchain - with: - toolchain: "stable" - - - uses: actions/download-artifact@v4 - with: - name: integration-tests - path: /home/runner/.cargo/bin/ - - name: Chmod for binary - run: chmod +x ${SCCACHE_PATH} - - - name: Test - run: cargo clean && cargo build - - - name: Output - run: | - ${SCCACHE_PATH} --show-stats - - ${SCCACHE_PATH} --show-stats | grep redis - - - name: Test Twice for Cache Read - run: cargo clean && cargo build - - - name: Output - run: | - ${SCCACHE_PATH} --show-stats - - ${SCCACHE_PATH} --show-stats | grep -e "Cache hits\s*[1-9]" - - s3_minio: - runs-on: ubuntu-latest - needs: build - - # Setup minio server - services: - minio: - image: wktk/minio-server - ports: - - 9000:9000 - env: - MINIO_ACCESS_KEY: "minioadmin" - MINIO_SECRET_KEY: "minioadmin" - - env: - SCCACHE_BUCKET: test - SCCACHE_ENDPOINT: http://127.0.0.1:9000/ - SCCACHE_REGION: us-east-1 - AWS_ACCESS_KEY_ID: "minioadmin" - AWS_SECRET_ACCESS_KEY: "minioadmin" - AWS_EC2_METADATA_DISABLED: "true" - RUSTC_WRAPPER: /home/runner/.cargo/bin/sccache - - steps: - - name: Clone repository - uses: actions/checkout@v4 - - - name: Setup test bucket - run: aws --endpoint-url http://127.0.0.1:9000/ s3 mb s3://test - - - name: Install rust - uses: ./.github/actions/rust-toolchain - with: - toolchain: "stable" - - - uses: actions/download-artifact@v4 - with: - name: integration-tests - path: /home/runner/.cargo/bin/ - - name: Chmod for binary - run: chmod +x ${SCCACHE_PATH} - - - name: Test - run: cargo clean && cargo build - - - name: Output - run: | - ${SCCACHE_PATH} --show-stats - - ${SCCACHE_PATH} --show-stats | grep s3 - - - name: Test Twice for Cache Read - run: cargo clean && cargo build - - - name: Output - run: | - ${SCCACHE_PATH} --show-stats - - ${SCCACHE_PATH} --show-stats | grep -e "Cache hits\s*[1-9]" - - azblob_azurite: - runs-on: ubuntu-latest - needs: build - - # Setup azurite server - services: - azurite: - image: mcr.microsoft.com/azure-storage/azurite - ports: - - 10000:10000 - - env: - SCCACHE_AZURE_BLOB_CONTAINER: "test" - SCCACHE_AZURE_CONNECTION_STRING: "DefaultEndpointsProtocol=http;AccountName=devstoreaccount1;AccountKey=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==;BlobEndpoint=http://127.0.0.1:10000/devstoreaccount1;" - RUSTC_WRAPPER: /home/runner/.cargo/bin/sccache - - steps: - - name: Clone repository - uses: actions/checkout@v4 - - - name: Setup test bucket - run: | - az storage container create \ - --name test \ - --connection-string ${SCCACHE_AZURE_CONNECTION_STRING} - - - name: Install rust - uses: ./.github/actions/rust-toolchain - with: - toolchain: "stable" - - - uses: actions/download-artifact@v4 - with: - name: integration-tests - path: /home/runner/.cargo/bin/ - - name: Chmod for binary - run: chmod +x ${SCCACHE_PATH} - - - name: Test - run: cargo clean && cargo build - - - name: Output - run: | - ${SCCACHE_PATH} --show-stats - - ${SCCACHE_PATH} --show-stats | grep azblob - - - name: Test Twice for Cache Read - run: cargo clean && cargo build - - - name: Output - run: | - ${SCCACHE_PATH} --show-stats - - ${SCCACHE_PATH} --show-stats | grep -e "Cache hits\s*[1-9]" - gha: runs-on: ubuntu-latest needs: build @@ -288,137 +76,25 @@ jobs: ${SCCACHE_PATH} --show-stats | grep -e "Cache hits\s*[1-9]" - memcached-deprecated: + gha-as-local: runs-on: ubuntu-latest needs: build - # Setup memcached server - services: - memcached: - image: bitnami/memcached - env: - # memcache's max item size is 1MiB, But our tests - # will produce larger file. - # - # Specify the setting here to make our test happy. - MEMCACHED_MAX_ITEM_SIZE: 16777216 - ports: - - 11211:11211 - env: - SCCACHE_MEMCACHED: "tcp://127.0.0.1:11211" - RUSTC_WRAPPER: /home/runner/.cargo/bin/sccache - - steps: - - name: Clone repository - uses: actions/checkout@v4 - - - name: Install rust - uses: ./.github/actions/rust-toolchain - with: - toolchain: "stable" - - - uses: actions/download-artifact@v4 - with: - name: integration-tests - path: /home/runner/.cargo/bin/ - - name: Chmod for binary - run: chmod +x ${SCCACHE_PATH} - - - name: Test - run: cargo clean && cargo build - - - name: Output - run: | - ${SCCACHE_PATH} --show-stats - - ${SCCACHE_PATH} --show-stats | grep memcached - - - name: Test Twice for Cache Read - run: cargo clean && cargo build - - - name: Output - run: | - ${SCCACHE_PATH} --show-stats - - ${SCCACHE_PATH} --show-stats | grep -e "Cache hits\s*[1-9]" - - memcached: - runs-on: ubuntu-latest - needs: build - - # Setup memcached server - services: - memcached: - image: bitnami/memcached - env: - # memcache's max item size is 1MiB, But our tests - # will produce larger file. - # - # Specify the setting here to make our test happy. - MEMCACHED_MAX_ITEM_SIZE: 16777216 - ports: - - 11211:11211 - - env: - SCCACHE_MEMCACHED_ENDPOINT: "tcp://127.0.0.1:11211" + SCCACHE_GHA_ENABLED: "on" + SCCACHE_GHA_AS_LOCAL: "on" RUSTC_WRAPPER: /home/runner/.cargo/bin/sccache steps: - name: Clone repository uses: actions/checkout@v4 - - name: Install rust - uses: ./.github/actions/rust-toolchain - with: - toolchain: "stable" - - - uses: actions/download-artifact@v4 + - name: Configure Cache Env + uses: actions/github-script@v7 with: - name: integration-tests - path: /home/runner/.cargo/bin/ - - name: Chmod for binary - run: chmod +x ${SCCACHE_PATH} - - - name: Test - run: cargo clean && cargo build - - - name: Output - run: | - ${SCCACHE_PATH} --show-stats - - ${SCCACHE_PATH} --show-stats | grep memcached - - - name: Test Twice for Cache Read - run: cargo clean && cargo build - - - name: Output - run: | - ${SCCACHE_PATH} --show-stats - - ${SCCACHE_PATH} --show-stats | grep -e "Cache hits\s*[1-9]" - - webdav: - runs-on: ubuntu-latest - needs: build - - env: - SCCACHE_WEBDAV_ENDPOINT: "http://127.0.0.1:8080" - SCCACHE_WEBDAV_USERNAME: "bar" - SCCACHE_WEBDAV_PASSWORD: "baz" - RUSTC_WRAPPER: /home/runner/.cargo/bin/sccache - - steps: - - uses: actions/checkout@v4 - - - name: Start nginx - shell: bash - run: | - sudo apt install -y nginx-full - - mkdir /tmp/static - cp `pwd`/tests/htpasswd /tmp/htpasswd - nginx -c `pwd`/tests/nginx_http_cache.conf + script: | + core.exportVariable('ACTIONS_CACHE_URL', process.env.ACTIONS_CACHE_URL || ''); + core.exportVariable('ACTIONS_RUNTIME_TOKEN', process.env.ACTIONS_RUNTIME_TOKEN || ''); - name: Install rust uses: ./.github/actions/rust-toolchain @@ -439,7 +115,7 @@ jobs: run: | ${SCCACHE_PATH} --show-stats - ${SCCACHE_PATH} --show-stats | grep webdav + ${SCCACHE_PATH} --show-stats | grep gha - name: Test Twice for Cache Read run: cargo clean && cargo build @@ -449,400 +125,3 @@ jobs: ${SCCACHE_PATH} --show-stats ${SCCACHE_PATH} --show-stats | grep -e "Cache hits\s*[1-9]" - - test-mock-msvc: - runs-on: windows-2019 - env: - TARGET: x86_64-pc-windows-msvc - SCCACHE_EXE: ${{ github.workspace }}\\target\\x86_64-pc-windows-msvc\\debug\\sccache.exe - SCCACHE_LOG: "trace" - SCCACHE_ERROR_LOG: "${{ github.workspace }}\\server_log.txt" - - steps: - - uses: ilammy/msvc-dev-cmd@v1 - - - name: Clone repository - uses: actions/checkout@v4 - - - name: Install rust - uses: ./.github/actions/rust-toolchain - with: - toolchain: "stable" - target: $TARGET - - - name: Build - run: cargo build --bin sccache --target $env:TARGET --features=openssl/vendored - - - name: Compile MSVC (no cache) - shell: bash - working-directory: ./tests/msvc - run: | - cl "@args.rsp" - test -e ./foo.o || { echo "No compiler output found"; exit -1; } - - - name: Start Server - shell: bash - run: $SCCACHE_EXE --start-server - - - name: Compile - Cache Miss - shell: bash - working-directory: ./tests/msvc - run: | - rm ./foo.o || true - $SCCACHE_EXE "$(where cl.exe)" -c "@args.rsp" - $SCCACHE_EXE --show-stats - $SCCACHE_EXE --show-stats | grep -e "Cache misses\s*[1-9]" - test -e ./foo.o || { echo "No compiler output found"; exit -1; } - test -e ./foo.o.json || { echo "No dependency list found"; exit -1; } - - - name: Compile - Cache Hit - shell: bash - working-directory: ./tests/msvc - run: | - rm ./foo.o || true - $SCCACHE_EXE "$(where cl.exe)" -c "@args.rsp" - $SCCACHE_EXE --show-stats - $SCCACHE_EXE --show-stats | grep -e "Cache hits\s*[1-9]" - test -e ./foo.o || { echo "No compiler output found"; exit -1; } - test -e ./foo.o.json || { echo "No dependency list found"; exit -1; } - - - name: Compile - Preprocessing Compiler Bug - shell: bash - working-directory: ./tests/msvc-preprocessing - run: | - $SCCACHE_EXE "$(where cl.exe)" -c "@args.rsp" - $SCCACHE_EXE --show-stats - - - name: Stop Server - if: success() || failure() - shell: bash - run: $SCCACHE_EXE --stop-server - - - name: Show Server Log - if: success() || failure() - shell: bash - run: cat "$SCCACHE_ERROR_LOG" - - - clang: - runs-on: ubuntu-latest - needs: build - - env: - LLVM_VERSION: "16" - SCCACHE_GHA_ENABLED: "on" - - steps: - - uses: actions/checkout@v4 - - - name: Configure Cache Env - uses: actions/github-script@v7 - with: - script: | - core.exportVariable('ACTIONS_CACHE_URL', process.env.ACTIONS_CACHE_URL || ''); - core.exportVariable('ACTIONS_RUNTIME_TOKEN', process.env.ACTIONS_RUNTIME_TOKEN || '') - - - name: Install clang - run: | - wget https://apt.llvm.org/llvm.sh - chmod +x llvm.sh - sudo ./llvm.sh "${LLVM_VERSION}" - - - uses: actions/download-artifact@v4 - with: - name: integration-tests - path: /home/runner/.cargo/bin/ - - name: Chmod for binary - run: chmod +x ${SCCACHE_PATH} - - - name: Test - run: | - export CXX="${SCCACHE_PATH} clang++" - $CXX -c `pwd`/tests/test_clang_multicall.c - - - name: Output - run: | - ${SCCACHE_PATH} --show-stats - - - name: Test Twice for Cache Read - run: | - export CXX="${SCCACHE_PATH} clang++" - $CXX -c `pwd`/tests/test_clang_multicall.c - - - name: Output - run: | - ${SCCACHE_PATH} --show-stats - - ${SCCACHE_PATH} --show-stats | grep -e "Cache hits\s*[1-9]" - - hip: - # Probably wouldn't matter anyway since we run in a container, but staying - # close to the version is better than not. - runs-on: ubuntu-22.04 - needs: build - container: - image: rocm/dev-ubuntu-22.04:6.0 - - env: - # SCCACHE_GHA_ENABLED: "on" - ROCM_PATH: "/opt/rocm" - - steps: - - uses: actions/checkout@v4 - - # I don't want to break the cache during testing. Will turn on after I - # make sure it's working. - # - # - name: Configure Cache Env - # uses: actions/github-script@v7 - # with: - # script: | - # core.exportVariable('ACTIONS_CACHE_URL', process.env.ACTIONS_CACHE_URL || ''); - # core.exportVariable('ACTIONS_RUNTIME_TOKEN', process.env.ACTIONS_RUNTIME_TOKEN || '') - - # - name: Configure ROCm Env - # uses: actions/github-script@v7 - # with: - # script: | - # core.exportVariable('ROCM_PATH', process.env.ROCM_PATH || ''); - - - uses: actions/download-artifact@v4 - with: - name: integration-tests - path: /home/runner/.cargo/bin/ - - name: Chmod for binary - run: chmod +x ${SCCACHE_PATH} - - - name: Install dependencies - shell: bash - run: | - ## Install dependencies - sudo apt-get update - sudo apt-get install -y cmake - - # Ensure that HIPCC isn't already borken - - name: Sanity Check - run: | - hipcc -o vectoradd_hip --offload-arch=gfx900 tests/cmake-hip/vectoradd_hip.cpp - - - name: Test - run: | - cmake -B build -S tests/cmake-hip -DCMAKE_HIP_COMPILER_LAUNCHER=${SCCACHE_PATH} -DCMAKE_HIP_ARCHITECTURES=gfx900 - cmake --build build - - - name: Output - run: | - ${SCCACHE_PATH} --show-stats - - - name: Test Twice for Cache Read - run: | - rm -rf build - cmake -B build -S tests/cmake-hip -DCMAKE_HIP_COMPILER_LAUNCHER=${SCCACHE_PATH} -DCMAKE_HIP_ARCHITECTURES=gfx900 - cmake --build build - - - name: Output - run: | - ${SCCACHE_PATH} --show-stats - - ${SCCACHE_PATH} --show-stats | grep -e "Cache hits\s*[1-9]" - - - gcc: - runs-on: ubuntu-latest - needs: build - - env: - SCCACHE_GHA_ENABLED: "on" - - steps: - - uses: actions/checkout@v4 - - - name: Configure Cache Env - uses: actions/github-script@v7 - with: - script: | - core.exportVariable('ACTIONS_CACHE_URL', process.env.ACTIONS_CACHE_URL || ''); - core.exportVariable('ACTIONS_RUNTIME_TOKEN', process.env.ACTIONS_RUNTIME_TOKEN || '') - - - uses: actions/download-artifact@v4 - with: - name: integration-tests - path: /home/runner/.cargo/bin/ - - name: Chmod for binary - run: chmod +x ${SCCACHE_PATH} - - - name: Test - run: | - export CXX="${SCCACHE_PATH} g++" - $CXX -c `pwd`/tests/test_clang_multicall.c - - - name: Output - run: | - ${SCCACHE_PATH} --show-stats - - - name: Test Twice for Cache Read - run: | - export CXX="${SCCACHE_PATH} g++" - $CXX -c `pwd`/tests/test_clang_multicall.c - - - name: Output - run: | - ${SCCACHE_PATH} --show-stats - - ${SCCACHE_PATH} --show-stats | grep -e "Cache hits\s*[1-9]" - - - autotools: - runs-on: ubuntu-latest - needs: build - - env: - SCCACHE_GHA_ENABLED: "on" - - steps: - - uses: actions/checkout@v4 - - - name: Configure Cache Env - uses: actions/github-script@v7 - with: - script: | - core.exportVariable('ACTIONS_CACHE_URL', process.env.ACTIONS_CACHE_URL || ''); - core.exportVariable('ACTIONS_RUNTIME_TOKEN', process.env.ACTIONS_RUNTIME_TOKEN || '') - - - uses: actions/download-artifact@v4 - with: - name: integration-tests - path: /home/runner/.cargo/bin/ - - name: Chmod for binary - run: chmod +x ${SCCACHE_PATH} - - - name: Install dependencies - shell: bash - run: | - ## Install dependencies - sudo apt-get update - sudo apt-get install autoconf automake libtool - - - name: Test - run: | - cd `pwd`/tests/autotools/ - autoreconf||true - automake --add-missing - ./configure CXX="${SCCACHE_PATH} g++" - make - - - name: Output - run: | - ${SCCACHE_PATH} --show-stats - - - name: Test Twice for Cache Read - run: | - cd `pwd`/tests/autotools/ - make distclean - ./configure CXX="${SCCACHE_PATH} g++" - make - - - name: Output - run: | - ${SCCACHE_PATH} --show-stats - - ${SCCACHE_PATH} --show-stats | grep -e "Cache hits\s*[1-9]" - - cmake: - runs-on: ubuntu-latest - needs: build - - env: - SCCACHE_GHA_ENABLED: "on" - - steps: - - uses: actions/checkout@v4 - - - name: Configure Cache Env - uses: actions/github-script@v7 - with: - script: | - core.exportVariable('ACTIONS_CACHE_URL', process.env.ACTIONS_CACHE_URL || ''); - core.exportVariable('ACTIONS_RUNTIME_TOKEN', process.env.ACTIONS_RUNTIME_TOKEN || '') - - - uses: actions/download-artifact@v4 - with: - name: integration-tests - path: /home/runner/.cargo/bin/ - - name: Chmod for binary - run: chmod +x ${SCCACHE_PATH} - - - name: Install dependencies - shell: bash - run: | - ## Install dependencies - sudo apt-get update - sudo apt-get install cmake - - - name: Test - run: | - cd `pwd`/tests/cmake/ - mkdir build - cd build - cmake -DCMAKE_C_COMPILER_LAUNCHER=${SCCACHE_PATH} -DCMAKE_CXX_COMPILER_LAUNCHER=${SCCACHE_PATH} .. - make - - - name: Output - run: | - ${SCCACHE_PATH} --show-stats - - - name: Test Twice for Cache Read - run: | - cd `pwd`/tests/cmake/ - rm -rf build - mkdir build - cd build - cmake -DCMAKE_C_COMPILER_LAUNCHER=${SCCACHE_PATH} -DCMAKE_CXX_COMPILER_LAUNCHER=${SCCACHE_PATH} .. - make - - - name: Output - run: | - ${SCCACHE_PATH} --show-stats - - ${SCCACHE_PATH} --show-stats | grep -e "Cache hits\s*[1-9]" - - rust-test-coverage: - runs-on: ubuntu-latest - needs: build - - env: - RUSTC_WRAPPER: /home/runner/.cargo/bin/sccache - CARGO_INCREMENTAL: "0" - RUSTFLAGS: "-Zprofile -Ccodegen-units=1 -Copt-level=0 -Clink-dead-code -Coverflow-checks=off -Zpanic_abort_tests -Cpanic=abort" - RUSTDOCFLAGS: "-Cpanic=abort" - - steps: - - uses: actions/download-artifact@v4 - with: - name: integration-tests - path: /home/runner/.cargo/bin/ - - name: Chmod for binary - run: chmod +x ${SCCACHE_PATH} - - - name: Prepare - run: | - rustup toolchain install nightly - cargo new coverage-test - cd coverage-test - echo "serde = { version = \"1.0\", features = [\"derive\"] }" >> Cargo.toml - - - name: "Coverage test #1" - working-directory: ./coverage-test - run: cargo clean && cargo +nightly test - - - name: Output - run: ${SCCACHE_PATH} --show-stats - - - name: "Coverage test #2" - working-directory: ./coverage-test - run: cargo clean && cargo +nightly test - - - name: Output - run: | - ${SCCACHE_PATH} --show-stats - ${SCCACHE_PATH} --show-stats | grep -e "Cache hits\s*[1-9]" diff --git a/src/cache/cache.rs b/src/cache/cache.rs index 9d61788c8..aa6cd5778 100644 --- a/src/cache/cache.rs +++ b/src/cache/cache.rs @@ -591,12 +591,24 @@ pub fn storage_from_config( return Ok(Arc::new(storage)); } #[cfg(feature = "gha")] - CacheType::GHA(config::GHACacheConfig { ref version, .. }) => { - debug!("Init gha cache with version {version}"); - - let storage = GHACache::build(version) - .map_err(|err| anyhow!("create gha cache failed: {err:?}"))?; - return Ok(Arc::new(storage)); + CacheType::GHA(config::GHACacheConfig { + ref version, + as_local, + .. + }) => { + if *as_local { + debug!("Init gha as local cache"); + let downloaded_path = pool + .block_on(GHACache::download_to_local(config, version)) + .map_err(|err| anyhow!("download gha cache as local failed: {err:?}"))?; + let storage = disk_cache_from_config(config, pool, downloaded_path)?; + return Ok(storage); + } else { + debug!("Init gha cache with version {version}"); + let storage = GHACache::build(version) + .map_err(|err| anyhow!("create gha cache failed: {err:?}"))?; + return Ok(Arc::new(storage)); + } } #[cfg(feature = "memcached")] CacheType::Memcached(config::MemcachedCacheConfig { @@ -724,7 +736,21 @@ pub fn storage_from_config( } } - let (dir, size) = (&config.fallback_cache.dir, config.fallback_cache.size); + disk_cache_from_config(config, pool, None) +} + +fn disk_cache_from_config( + config: &Config, + pool: &tokio::runtime::Handle, + root_override: Option, +) -> Result> { + let (mut dir, size) = ( + config.fallback_cache.dir.to_owned(), + config.fallback_cache.size, + ); + if let Some(new_root) = root_override { + dir = dir.join(new_root); + } let preprocessor_cache_mode_config = config.fallback_cache.preprocessor_cache_mode; let rw_mode = config.fallback_cache.rw_mode.into(); debug!("Init disk cache with dir {:?}, size {}", dir, size); @@ -737,6 +763,27 @@ pub fn storage_from_config( ))) } +#[cfg(feature = "gha")] +pub async fn upload_local_cache(config: &Config) -> Result<()> { + match &config.cache { + Some(CacheType::GHA(gha_config)) => { + if !gha_config.enabled { + debug!("GHA cache is disabled in config"); + return Ok(()); + } + if !gha_config.as_local { + debug!("GHA not configured `as_local`"); + return Ok(()); + } + GHACache::upload_local_cache(config).await + } + _ => { + debug!("Uploading the local cache is only possible when using GitHub Actions"); + Ok(()) + } + } +} + #[cfg(test)] mod test { use super::*; diff --git a/src/cache/gha.rs b/src/cache/gha.rs index 8d8373430..81c9e52ae 100644 --- a/src/cache/gha.rs +++ b/src/cache/gha.rs @@ -12,13 +12,18 @@ // See the License for the specific language governing permissions and // limitations under the License. +use std::path::PathBuf; + use opendal::layers::LoggingLayer; use opendal::services::Ghac; use opendal::Operator; +use crate::config::Config; use crate::errors::*; use crate::VERSION; +const FULL_GHA_CACHE_ROOT: &str = "sccache-full"; + /// A cache that stores entries in GHA Cache Services. pub struct GHACache; @@ -43,4 +48,114 @@ impl GHACache { .finish(); Ok(op) } + + /// Download a copy of the entire GHA cache from the given version + /// and return the path to the root folder on the local disk. + /// + /// It is the user's responsibility to split the caches according + /// to anything relevant like architecture, OS, etc. by using the `version`. + pub async fn download_to_local(config: &Config, version: &str) -> Result> { + let tarball_path = local_cache_tarball_path(config); + let mut builder = Ghac::default(); + + // TODO somehow loop over decreasingly "fresh" versions of the cache + // like in + // https://docs.github.com/en/actions/using-workflows/caching-dependencies-to-speed-up-workflows#matching-a-cache-key + // For now the behavior is to match the same version, which would + // speed up rebuilds in the same (Git) branch. + if version.is_empty() { + builder.version(&format!("sccache-v{VERSION}")); + } else { + builder.version(&format!("sccache-v{VERSION}-{version}")); + } + + debug!("creating the operator"); + let op = Operator::new(builder)? + .layer(LoggingLayer::default()) + .finish(); + + debug!("created the operator"); + match op.reader(FULL_GHA_CACHE_ROOT).await { + Ok(mut reader) => { + debug!("Found full gha cache"); + let mut writer = tokio::fs::OpenOptions::new() + .write(true) + .open(&tarball_path) + .await + .context("opening the local tarball for writing")?; + tokio::io::copy(&mut reader, &mut writer) + .await + .context("writing to the local tarball")?; + } + Err(error) => match error.kind() { + opendal::ErrorKind::NotFound => { + debug!("Full gha cache not found"); + return Ok(None); + } + _ => { + debug!("{:?}", error); + bail!(error) + } + }, + }; + + let cache = local_cache_path(config); + let tarball = + std::fs::File::open(tarball_path).context("Failed to open the GHA cache tarball")?; + tar::Archive::new(tarball) + .unpack(&cache) + .context("Failed to extract the GHA cache tarball")?; + + Ok(Some(cache)) + } + + /// Upload a tarball of the local cache + pub async fn upload_local_cache(config: &Config) -> Result<()> { + let cache = local_cache_path(config); + if !cache.exists() { + info!("Local cache does not exist: nothing to do"); + return Ok(()) + } + debug!("Found local gha cache at {}", cache.display()); + + let op = Operator::new(Ghac::default())? + .layer(LoggingLayer::default()) + .finish(); + + // GHA cache is immutable, if the path has already been written within + // a given version, it cannot be changed again. + if op.is_exist(FULL_GHA_CACHE_ROOT).await? { + info!("remote cache of this version already exists, cannot upload"); + return Ok(()) + } + + let mut tar_builder = tar::Builder::new(vec![]); + tar_builder + .append_dir_all(local_cache_path(config), ".") + .context("Failed to create GHA local cache tarball")?; + let source = local_cache_tarball_path(config); + std::fs::write(&source, tar_builder.into_inner()?) + .context("Failed to write the GHA local cache tarball to disk")?; + + match op.writer(FULL_GHA_CACHE_ROOT).await { + Ok(mut writer) => { + let mut reader = tokio::fs::File::open(&source).await?; + tokio::io::copy(&mut reader, &mut writer).await?; + Ok(()) + } + // TODO handle error gracefully in case of TOCTOU from the + // check at the start of the function + Err(error) => bail!(error), + } + } +} + +fn local_cache_tarball_path(config: &Config) -> PathBuf { + let mut path = config.fallback_cache.dir.join(FULL_GHA_CACHE_ROOT); + path.set_extension(".tar"); + path +} + +fn local_cache_path(config: &Config) -> PathBuf { + config.fallback_cache.dir.join(FULL_GHA_CACHE_ROOT) } diff --git a/src/config.rs b/src/config.rs index 4173bcaac..6ebd7f451 100644 --- a/src/config.rs +++ b/src/config.rs @@ -213,6 +213,11 @@ pub struct GHACacheConfig { /// Version for gha cache is a namespace. By setting different versions, /// we can avoid mixed caches. pub version: String, + /// Download the entire cache to be used like a local cache, then upload + /// it back if anything changed. + /// This is useful in CI contexts to reduce the number of requests, + /// hence avoiding rate limiting and improving overall cache speed. + pub as_local: bool, } /// Memcached's default value of expiration is 10800s (3 hours), which is too @@ -784,12 +789,13 @@ fn config_from_env() -> Result { }); // ======= GHA ======= - let gha = if let Ok(version) = env::var("SCCACHE_GHA_VERSION") { + let mut gha = if let Ok(version) = env::var("SCCACHE_GHA_VERSION") { // If SCCACHE_GHA_VERSION has been set, we don't need to check // SCCACHE_GHA_ENABLED's value anymore. Some(GHACacheConfig { enabled: true, version, + as_local: false, }) } else if bool_from_env_var("SCCACHE_GHA_ENABLED")?.unwrap_or(false) { // If only SCCACHE_GHA_ENABLED has been set to the true value, enable with @@ -797,11 +803,16 @@ fn config_from_env() -> Result { Some(GHACacheConfig { enabled: true, version: "".to_string(), + as_local: false, }) } else { None }; + if let Some(gha) = &mut gha { + gha.as_local = bool_from_env_var("SCCACHE_GHA_AS_LOCAL")?.unwrap_or(false); + } + // ======= Azure ======= let azure = if let (Ok(connection_string), Ok(container)) = ( env::var("SCCACHE_AZURE_CONNECTION_STRING"), @@ -1453,6 +1464,7 @@ service_account = "example_service_account" [cache.gha] enabled = true version = "sccache" +as_local = false [cache.memcached] # Deprecated alias for `endpoint` @@ -1519,7 +1531,8 @@ no_credentials = true }), gha: Some(GHACacheConfig { enabled: true, - version: "sccache".to_string() + version: "sccache".to_string(), + as_local: false, }), redis: Some(RedisCacheConfig { url: Some("redis://user:passwd@1.2.3.4:6379/?db=1".to_owned()), diff --git a/src/server.rs b/src/server.rs index 70dc4f859..3a03f9a7d 100644 --- a/src/server.rs +++ b/src/server.rs @@ -13,6 +13,8 @@ // limitations under the License.SCCACHE_MAX_FRAME_LENGTH use crate::cache::readonly::ReadOnlyStorage; +#[cfg(feature = "gha")] +use crate::cache::upload_local_cache; use crate::cache::{storage_from_config, CacheMode, Storage}; use crate::compiler::{ get_compiler_info, CacheControl, CompileResult, Compiler, CompilerArguments, CompilerHasher, @@ -466,7 +468,6 @@ pub fn start_server(config: &Config, port: u16) -> Result<()> { CacheMode::ReadOnly => Arc::new(ReadOnlyStorage(raw_storage)), _ => raw_storage, }; - let res = SccacheServer::::new(port, runtime, client, dist_client, storage); match res { @@ -474,7 +475,9 @@ pub fn start_server(config: &Config, port: u16) -> Result<()> { let port = srv.port(); info!("server started, listening on port {}", port); notify_server_startup(¬ify, ServerStartup::Ok { port })?; - srv.run(future::pending::<()>())?; + let runtime = srv.run(future::pending::<()>())?; + #[cfg(feature = "gha")] + runtime.block_on(upload_local_cache(config))?; Ok(()) } Err(e) => { @@ -589,7 +592,7 @@ impl SccacheServer { /// If the `shutdown` future resolves then the server will be shut down, /// otherwise the server may naturally shut down if it becomes idle for too /// long anyway. - pub fn run(self, shutdown: F) -> io::Result<()> + pub fn run(self, shutdown: F) -> io::Result where F: Future, C: Send, @@ -676,7 +679,7 @@ impl SccacheServer { info!("ok, fully shutting down now"); - Ok(()) + Ok(runtime) } }