feat: use lynx cache

web-infra-dev · Dec 3, 2024 · e922772 · e922772
1 parent dea40a2
commit e922772
Show file tree

Hide file tree

Showing 8 changed files with 110 additions and 6 deletions.
diff --git a/.github/actions/pnpm-cache/action.yml b/.github/actions/pnpm-cache/action.yml
@@ -3,6 +3,14 @@ name: pnpm cache
 description: Install Node.js with pnpm global cache
 
 inputs:
+  type:
+    description: "Use github or local or lynx cache"
+    required: true
+    type: string
+    choices:
+      - github
+      - local
+      - lynx
   node-version:
     default: "20"
     required: false
@@ -46,15 +54,29 @@ runs:
         echo "STORE_PATH is $(pnpm store path)"
         echo "STORE_PATH=$(pnpm store path)" >> $GITHUB_OUTPUT
 
-    - name: Restore pnpm cache
-      id: restore
-      if: ${{ startsWith(runner.name, 'GitHub Actions') }}
-      uses: actions/cache/restore@v4
+    - name: Restore pnpm cache from Github
+      id: restore-from-github
+      if: ${{ inputs.type == 'github' }}
+      uses: actions/cache@v4
+      with:
+        path: ${{ steps.pnpm-cache.outputs.STORE_PATH }}
+        key: node-cache-${{ runner.os }}-pnpm-${{ hashFiles('pnpm-lock.yaml') }}
+        restore-keys: |
+          node-cache-${{ runner.os }}-pnpm-
+
+    - name: Restore pnpm cache from lynx
+      if: ${{ inputs.type == 'lynx' }}
+      uses: lynx-infra/cache@main
       with:
         path: ${{ steps.pnpm-cache.outputs.STORE_PATH }}
         key: node-cache-${{ runner.os }}-pnpm-${{ hashFiles('pnpm-lock.yaml') }}
         restore-keys: |
           node-cache-${{ runner.os }}-pnpm-
+      env:
+        ACCESS_KEY: ${{ secrets.TOS_ACCESS_KEY }}
+        SECRET_KEY: ${{ secrets.TOS_SECRET_KEY }}
+        BUCKET_NAME: ${{ vars.TOS_BUCKET_NAME }}
+        REGION: ${{ vars.TOS_REGION }}
 
     - name: Install dependencies
       shell: bash
@@ -67,7 +89,7 @@ runs:
 
     - name: Save pnpm cache
       uses: actions/cache/save@v4
-      if: ${{ startsWith(runner.name, 'GitHub Actions') && inputs.save-if == 'true' && steps.restore.outputs.cache-hit != 'true' }}
+      if: ${{ inputs.type == 'github' && inputs.save-if == 'true' && steps.restore-from-github.outputs.cache-hit != 'true' }}
       with:
         path: ${{ steps.pnpm-cache.outputs.STORE_PATH }}
         key: node-cache-${{ runner.os }}-pnpm-${{ hashFiles('pnpm-lock.yaml') }}
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
@@ -73,6 +73,7 @@ jobs:
       profile: "debug"
       runner: ${{ needs.get-runner-labels.outputs.LINUX_RUNNER_LABELS }}
       runner-type: ${{ needs.get-runner-labels.outputs.LINUX_RUNNER_TYPE }}
+      cache-type: ${{ needs.get-runner-labels.outputs.LINUX_CACHE_TYPE }}
       skipable: ${{ needs.check-changed.outputs.changed != 'true' }}
 
   test-release-linux:
@@ -92,6 +93,7 @@ jobs:
       target: ${{ matrix.array.target }}
       runner: ${{ needs.get-runner-labels.outputs.LINUX_RUNNER_LABELS }}
       runner-type: ${{ needs.get-runner-labels.outputs.LINUX_RUNNER_TYPE }}
+      cache-type: ${{ needs.get-runner-labels.outputs.LINUX_CACHE_TYPE }}
 
   test-windows:
     name: Test Windows
@@ -102,6 +104,7 @@ jobs:
       profile: "debug"
       runner: ${{ needs.get-runner-labels.outputs.WINDOWS_RUNNER_LABELS }}
       runner-type: ${{ needs.get-runner-labels.outputs.WINDOWS_RUNNER_TYPE }}
+      cache-type: ${{ needs.get-runner-labels.outputs.WINDOWS_CACHE_TYPE }}
       skipable: ${{ needs.check-changed.outputs.changed != 'true' }}
 
   test-release-windows:
@@ -120,6 +123,7 @@ jobs:
       target: ${{ matrix.array.target }}
       runner: ${{ needs.get-runner-labels.outputs.WINDOWS_RUNNER_LABELS }}
       runner-type: ${{ needs.get-runner-labels.outputs.WINDOWS_RUNNER_TYPE }}
+      cache-type: ${{ needs.get-runner-labels.outputs.WINDOWS_CACHE_TYPE }}
 
   test-mac:
     name: Test Mac
@@ -131,6 +135,7 @@ jobs:
       profile: "debug"
       runner: ${{ needs.get-runner-labels.outputs.MACOS_RUNNER_LABELS }}
       runner-type: ${{ needs.get-runner-labels.outputs.MACOS_RUNNER_TYPE }}
+      cache-type: ${{ needs.get-runner-labels.outputs.MACOS_CACHE_TYPE }}
       skipable: ${{ needs.check-changed.outputs.changed  != 'true' }}
 
   test-release-mac:
@@ -148,6 +153,7 @@ jobs:
       target: ${{ matrix.array.target }}
       runner: ${{ needs.get-runner-labels.outputs.MACOS_RUNNER_LABELS }}
       runner-type: ${{ needs.get-runner-labels.outputs.MACOS_RUNNER_TYPE }}
+      cache-type: ${{ needs.get-runner-labels.outputs.MACOS_CACHE_TYPE }}
 
   cargo-deny:
     name: Check license of dependencies
@@ -196,6 +202,8 @@ jobs:
 
       - name: Pnpm Cache
         uses: ./.github/actions/pnpm-cache
+        with:
+          type: github
 
       - name: Lint js
         if: steps.changes.outputs.src == 'true'
@@ -218,6 +226,8 @@ jobs:
 
       - name: Pnpm Cache
         uses: ./.github/actions/pnpm-cache
+        with:
+          type: github
 
       - name: Run
         run: |
@@ -256,6 +266,7 @@ jobs:
         uses: ./.github/actions/pnpm-cache
         with:
           frozen-lockfile: true
+          type: github
 
   rust_check:
     name: Rust check
@@ -274,6 +285,8 @@ jobs:
 
       - name: Pnpm Cache # Required by some tests
         uses: ./.github/actions/pnpm-cache
+        with:
+          type: ${{ needs.get-runner-labels.outputs.LINUX_CACHE_TYPE }}
 
       - name: Run Cargo Check
         run: cargo check --workspace --all-targets --locked # Not using --release because it uses too much cache, and is also slow.
@@ -317,6 +330,8 @@ jobs:
 
       - name: Pnpm Cache # Required by some tests
         uses: ./.github/actions/pnpm-cache
+        with:
+          type: ${{ needs.get-runner-labels.outputs.LINUX_CACHE_TYPE }}
 
       - name: Install Rust Toolchain
         uses: ./.github/actions/rustup
@@ -345,6 +360,8 @@ jobs:
 
       - name: Pnpm Cache # Required by some tests
         uses: ./.github/actions/pnpm-cache
+        with:
+          type: ${{ needs.get-runner-labels.outputs.LINUX_CACHE_TYPE }}
 
       - name: Install Rust Toolchain
         uses: ./.github/actions/rustup
@@ -374,6 +391,8 @@ jobs:
       - uses: actions/checkout@v4
       - name: Pnpm Cache # Required by some tests
         uses: ./.github/actions/pnpm-cache
+        with:
+          type: github
 
       - name: Install Rust Toolchain
         uses: ./.github/actions/rustup

diff --git a/.github/workflows/get-runner-labels.yml b/.github/workflows/get-runner-labels.yml
@@ -26,12 +26,15 @@ jobs:
     outputs:
       LINUX_RUNNER_LABELS: ${{ steps.run.outputs.LINUX_RUNNER_LABELS }}
       LINUX_RUNNER_TYPE: ${{ steps.run.outputs.LINUX_RUNNER_TYPE }}
+      LINUX_CACHE_TYPE: ${{ steps.run.outputs.LINUX_CACHE_TYPE }}
 
       MACOS_RUNNER_LABELS: ${{ steps.run.outputs.MACOS_RUNNER_LABELS }}
       MACOS_RUNNER_TYPE: ${{ steps.run.outputs.MACOS_RUNNER_TYPE }}
+      MACOS_CACHE_TYPE: ${{ steps.run.outputs.MACOS_CACHE_TYPE }}
 
       WINDOWS_RUNNER_LABELS: ${{ steps.run.outputs.WINDOWS_RUNNER_LABELS }}
       WINDOWS_RUNNER_TYPE: ${{ steps.run.outputs.WINDOWS_RUNNER_TYPE }}
+      WINDOWS_CACHE_TYPE: ${{ steps.run.outputs.WINDOWS_CACHE_TYPE }}
     steps:
       - id: run
         shell: bash
@@ -41,6 +44,7 @@ jobs:
             MACOS_RUNNER_LABELS='${{ vars.MACOS_RUNNER_LABELS }}';
             WINDOWS_RUNNER_LABELS='${{ vars.WINDOWS_RUNNER_LABELS }}';
           fi
+
           # set default value
           if [[ -z "$LINUX_RUNNER_LABELS" ]]; then
             LINUX_RUNNER_LABELS='"ubuntu-latest"';
@@ -54,21 +58,30 @@ jobs:
 
           if [[ "$LINUX_RUNNER_LABELS" == rspack-* ]]; then
             LINUX_RUNNER_TYPE="hosted"
+            LINUX_CACHE_TYPE="hosted"
           else
             LINUX_RUNNER_TYPE="github"
+            LINUX_CACHE_TYPE="github"
           fi
-          if [[ "$LINUX_RUNNER_LABELS" == rspack-* ]]; then
+
+          if [[ "$MACOS_RUNNER_LABELS" == rspack-* ]]; then
             MACOS_RUNNER_EPHEMERAL="hosted"
+            MACOS_CACHE_TYPE="hosted"
           else
             MACOS_RUNNER_EPHEMERAL="github"
+            MACOS_CACHE_TYPE="github"
           fi
 
           # set output
           echo "LINUX_RUNNER_LABELS=$LINUX_RUNNER_LABELS" >> "$GITHUB_OUTPUT"
           echo "LINUX_RUNNER_TYPE=$LINUX_RUNNER_TYPE" >> "$GITHUB_OUTPUT"
+          echo "LINUX_CACHE_TYPE=$LINUX_CACHE_TYPE" >> "$GITHUB_OUTPUT"
 
           echo "MACOS_RUNNER_LABELS=$MACOS_RUNNER_LABELS" >> "$GITHUB_OUTPUT"
           echo "MACOS_RUNNER_EPHEMERAL=$MACOS_RUNNER_EPHEMERAL" >> "$GITHUB_OUTPUT"
+          echo "MACOS_CACHE_TYPE=$MACOS_CACHE_TYPE" >> "$GITHUB_OUTPUT"
 
+          # TODO: test
           echo "WINDOWS_RUNNER_LABELS='"volc-windows-2022-medium"'" >> "$GITHUB_OUTPUT"
           echo "WINDOWS_RUNNER_TYPE='k8s'" >> "$GITHUB_OUTPUT"
+          echo "WINDOWS_CACHE_TYPE='lynx'" >> "$GITHUB_OUTPUT"
diff --git a/.github/workflows/preview-commit.yml b/.github/workflows/preview-commit.yml
@@ -37,35 +37,45 @@ jobs:
           - target: x86_64-unknown-linux-gnu
             runner: ${{ needs.get-runner-labels.outputs.LINUX_RUNNER_LABELS }}
             runner-type: ${{ needs.get-runner-labels.outputs.LINUX_RUNNER_TYPE }}
+            cache-type: ${{ needs.get-runner-labels.outputs.LINUX_CACHE_TYPE }}
           - target: aarch64-unknown-linux-gnu
             runner: "'ubuntu-latest'"
             runner-type: ${{ needs.get-runner-labels.outputs.LINUX_RUNNER_TYPE }}
+            cache-type: ${{ needs.get-runner-labels.outputs.LINUX_CACHE_TYPE }}
           - target: x86_64-unknown-linux-musl
             runner: ${{ needs.get-runner-labels.outputs.LINUX_RUNNER_LABELS }}
             runner-type: ${{ needs.get-runner-labels.outputs.LINUX_RUNNER_TYPE }}
+            cache-type: ${{ needs.get-runner-labels.outputs.LINUX_CACHE_TYPE }}
           - target: aarch64-unknown-linux-musl
             runner: ${{ needs.get-runner-labels.outputs.LINUX_RUNNER_LABELS }}
             runner-type: ${{ needs.get-runner-labels.outputs.LINUX_RUNNER_TYPE }}
+            cache-type: ${{ needs.get-runner-labels.outputs.LINUX_CACHE_TYPE }}
           - target: i686-pc-windows-msvc
             runner: ${{ needs.get-runner-labels.outputs.WINDOWS_RUNNER_LABELS }}
             runner-type: ${{ needs.get-runner-labels.outputs.WINDOWS_RUNNER_TYPE }}
+            cache-type: ${{ needs.get-runner-labels.outputs.WINDOWS_CACHE_TYPE }}
           - target: x86_64-pc-windows-msvc
             runner: ${{ needs.get-runner-labels.outputs.WINDOWS_RUNNER_LABELS }}
             runner-type: ${{ needs.get-runner-labels.outputs.WINDOWS_RUNNER_TYPE }}
+            cache-type: ${{ needs.get-runner-labels.outputs.WINDOWS_CACHE_TYPE }}
           - target: aarch64-pc-windows-msvc
             runner: ${{ needs.get-runner-labels.outputs.WINDOWS_RUNNER_LABELS }}
             runner-type: ${{ needs.get-runner-labels.outputs.WINDOWS_RUNNER_TYPE }}
+            cache-type: ${{ needs.get-runner-labels.outputs.WINDOWS_CACHE_TYPE }}
           - target: x86_64-apple-darwin
             runner: ${{ needs.get-runner-labels.outputs.MACOS_RUNNER_LABELS }}
             runner-type: ${{ needs.get-runner-labels.outputs.MACOS_RUNNER_TYPE }}
+            cache-type: ${{ needs.get-runner-labels.outputs.MACOS_CACHE_TYPE }}
           - target: aarch64-apple-darwin
             runner: ${{ needs.get-runner-labels.outputs.MACOS_RUNNER_LABELS }}
             runner-type: ${{ needs.get-runner-labels.outputs.MACOS_RUNNER_TYPE }}
+            cache-type: ${{ needs.get-runner-labels.outputs.MACOS_CACHE_TYPE }}
     uses: ./.github/workflows/reusable-build.yml
     with:
       target: ${{ matrix.array.target }}
       runner: ${{ matrix.array.runner }}
       runner-type: ${{ matrix.array.runner-type }}
+      cache-type: ${{ needs.get-runner-labels.outputs.cache-type }}
       profile: "release-prod"
       test: false
 
@@ -80,6 +90,8 @@ jobs:
 
       - name: Pnpm Cache
         uses: ./.github/actions/pnpm-cache
+        with:
+          type: github
 
       - name: Download artifacts
         uses: actions/download-artifact@v4.1.7

diff --git a/.github/workflows/release-canary.yml b/.github/workflows/release-canary.yml
@@ -28,28 +28,38 @@ jobs:
         array:
           - target: x86_64-unknown-linux-gnu
             runner: ${{ needs.get-runner-labels.outputs.LINUX_RUNNER_LABELS }}
+            cache-type: ${{ needs.get-runner-labels.outputs.LINUX_CACHE_TYPE }}
           - target: aarch64-unknown-linux-gnu
             runner: ${{ needs.get-runner-labels.outputs.LINUX_RUNNER_LABELS }}
+            cache-type: ${{ needs.get-runner-labels.outputs.LINUX_CACHE_TYPE }}
           - target: x86_64-unknown-linux-musl
             runner: ${{ needs.get-runner-labels.outputs.LINUX_RUNNER_LABELS }}
+            cache-type: ${{ needs.get-runner-labels.outputs.LINUX_CACHE_TYPE }}
           - target: aarch64-unknown-linux-musl
             runner: ${{ needs.get-runner-labels.outputs.LINUX_RUNNER_LABELS }}
+            cache-type: ${{ needs.get-runner-labels.outputs.LINUX_CACHE_TYPE }}
           - target: i686-pc-windows-msvc
             runner: ${{ needs.get-runner-labels.outputs.WINDOWS_RUNNER_LABELS }}
+            cache-type: ${{ needs.get-runner-labels.outputs.WINDOWS_CACHE_TYPE }}
           - target: x86_64-pc-windows-msvc
             runner: ${{ needs.get-runner-labels.outputs.WINDOWS_RUNNER_LABELS }}
+            cache-type: ${{ needs.get-runner-labels.outputs.WINDOWS_CACHE_TYPE }}
           - target: aarch64-pc-windows-msvc
             runner: ${{ needs.get-runner-labels.outputs.WINDOWS_RUNNER_LABELS }}
+            cache-type: ${{ needs.get-runner-labels.outputs.WINDOWS_CACHE_TYPE }}
           - target: x86_64-apple-darwin
             runner: ${{ needs.get-runner-labels.outputs.MACOS_RUNNER_LABELS }}
+            cache-type: ${{ needs.get-runner-labels.outputs.MACOS_CACHE_TYPE }}
           - target: aarch64-apple-darwin
             runner: ${{ needs.get-runner-labels.outputs.MACOS_RUNNER_LABELS }}
+            cache-type: ${{ needs.get-runner-labels.outputs.MACOS_CACHE_TYPE }}
     uses: ./.github/workflows/reusable-build.yml
     with:
       ref: ${{inputs.commit}}
       target: ${{ matrix.array.target }}
       runner: ${{ matrix.array.runner }}
       test: false
+      cache-type: ${{ matrix.array.cache-type }}
 
   release:
     name: Release Canary
@@ -64,6 +74,8 @@ jobs:
 
       - name: Pnpm Cache
         uses: ./.github/actions/pnpm-cache
+        with:
+          type: github
 
       - name: Download artifacts
         uses: actions/download-artifact@v4.1.7

diff --git a/.github/workflows/release-nightly.yml b/.github/workflows/release-nightly.yml
@@ -29,26 +29,36 @@ jobs:
         array:
           - target: x86_64-unknown-linux-gnu
             runner: ${{ needs.get-runner-labels.outputs.LINUX_RUNNER_LABELS }}
+            cache-type: ${{ needs.get-runner-labels.outputs.LINUX_CACHE_TYPE }}
           - target: aarch64-unknown-linux-gnu
             runner: ${{ needs.get-runner-labels.outputs.LINUX_RUNNER_LABELS }}
+            cache-type: ${{ needs.get-runner-labels.outputs.LINUX_CACHE_TYPE }}
           - target: x86_64-unknown-linux-musl
             runner: ${{ needs.get-runner-labels.outputs.LINUX_RUNNER_LABELS }}
+            cache-type: ${{ needs.get-runner-labels.outputs.LINUX_CACHE_TYPE }}
           - target: aarch64-unknown-linux-musl
             runner: ${{ needs.get-runner-labels.outputs.LINUX_RUNNER_LABELS }}
+            cache-type: ${{ needs.get-runner-labels.outputs.LINUX_CACHE_TYPE }}
           - target: i686-pc-windows-msvc
             runner: ${{ needs.get-runner-labels.outputs.WINDOWS_RUNNER_LABELS }}
+            cache-type: ${{ needs.get-runner-labels.outputs.WINDOWS_CACHE_TYPE }}
           - target: x86_64-pc-windows-msvc
             runner: ${{ needs.get-runner-labels.outputs.WINDOWS_RUNNER_LABELS }}
+            cache-type: ${{ needs.get-runner-labels.outputs.WINDOWS_CACHE_TYPE }}
           - target: aarch64-pc-windows-msvc
             runner: ${{ needs.get-runner-labels.outputs.WINDOWS_RUNNER_LABELS }}
+            cache-type: ${{ needs.get-runner-labels.outputs.WINDOWS_CACHE_TYPE }}
           - target: x86_64-apple-darwin
             runner: ${{ needs.get-runner-labels.outputs.MACOS_RUNNER_LABELS }}
+            cache-type: ${{ needs.get-runner-labels.outputs.MACOS_CACHE_TYPE }}
           - target: aarch64-apple-darwin
             runner: ${{ needs.get-runner-labels.outputs.MACOS_RUNNER_LABELS }}
+            cache-type: ${{ needs.get-runner-labels.outputs.MACOS_CACHE_TYPE }}
     uses: ./.github/workflows/reusable-build.yml
     with:
       target: ${{ matrix.array.target }}
       runner: ${{ matrix.array.runner }}
+      cache-type: ${{ matrix.array.cache-type }}
 
   release:
     name: Release Nightly
@@ -61,6 +71,8 @@ jobs:
 
       - name: Pnpm Cache
         uses: ./.github/actions/pnpm-cache
+        with:
+          type: github
 
       - name: Download artifacts
         uses: actions/download-artifact@v4.1.7

diff --git a/.github/workflows/release-pull-request.yml b/.github/workflows/release-pull-request.yml
@@ -42,6 +42,8 @@ jobs:
 
       - name: Pnpm Cache
         uses: ./.github/actions/pnpm-cache
+        with:
+          type: github
 
       - name: Create Release Pull Request
         # https://github.com/rspack-contrib/rspack-action