Skip to content

Commit

Permalink
CI: Rewrite the workflow to update the cache once before all tests
Browse files Browse the repository at this point in the history
  • Loading branch information
@sigprof
sigprof committed Nov 5, 2023
1 parent 1d03efb commit 26f0b06
Showing 1 changed file with 153 additions and 64 deletions.
217 changes: 153 additions & 64 deletions .github/workflows/ci.yml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -11,20 +11,27 @@ on:
description: 'Branch (user[/repo]:branch)'
type: string
required: false
os:
description: 'Machine type'
type: string
required: false
schedule:
- cron: '15 3 * * *'

env:
guix_cached_paths: |
~/.cache/guix
~/.cache/guix-install-action
~/.cache/guix-root
/gnu
/var/guix/db/db.sqlite*
/var/guix/gcroots
/var/guix/profiles
jobs:
# The `setup` job determines the strategy for the real build job.
setup:
name: Setup
runs-on: ubuntu-latest
outputs:
strategy: ${{ steps.strategy.outputs.result }}
cache_name: ${{ steps.saved_cache_name.outputs.name }}
steps:
- id: strategy
name: Determine build strategy
Expand All @@ -36,18 +43,13 @@ jobs:
{ repo: "qmk/qmk_firmware", branch: "master" },
{ repo: "qmk/qmk_firmware", branch: "develop" },
];
const defaultOS = [
"ubuntu-latest",
];
// Read workflow inputs.
let inputSource = "";
let inputOS = "";
if (context.eventName == "workflow_dispatch") {
const payload = context.payload;
const inputs = payload && payload.inputs;
inputSource = inputs && inputs.source && inputs.source.trim() || "";
inputOS = inputs && inputs.os && inputs.os.trim() || "";
}
// Parse the `source` input.
Expand All @@ -65,18 +67,11 @@ jobs:
}
}
// Parse the `os` input.
let matrixOS = defaultOS;
if (inputOS != "") {
matrixOS = [ inputOS ];
}
// Determine build strategy.
const strategy = {
"fail-fast": false,
"matrix": {
"source": matrixSource,
"os": matrixOS,
},
};
Expand All @@ -88,6 +83,134 @@ jobs:
// Return the strategy as the step output in the JSON format.
return strategy;
- name: Format the cache name
id: cache_name
run: |
# Format the cache name
base="guix"
date="$(date --utc +%Y%m%d)"
sha="${{github.sha}}"
echo "ok=${base}-ok-${date}-${sha}" >> $GITHUB_OUTPUT
echo "ok_date_only_prefix=${base}-ok-${date}-" >> $GITHUB_OUTPUT
echo "ok_any_prefix=${base}-ok-" >> $GITHUB_OUTPUT
echo "fail=${base}-fail-${date}-${sha}-${{github.run_id}}-${{github.run_attempt}}" >> $GITHUB_OUTPUT
echo "fail_same_run_id_prefix=${base}-fail-${date}-${sha}-${{github.run_id}}-" >> $GITHUB_OUTPUT
echo "fail_date_only_prefix=${base}-fail-${date}-" >> $GITHUB_OUTPUT
echo "any_prefix=${base}-" >> $GITHUB_OUTPUT
- name: Check whether the Guix cache needs updating
id: check_cache
uses: actions/cache/restore@v3.3.2
with:
path: ${{ env.guix_cached_paths }}
key: ${{ steps.cache_name.outputs.ok }}
lookup-only: true

- name: Prepare the Guix store before restoring from cache
if: ${{ !steps.check_cache.outputs.cache-hit }}
run: |
# Make the Guix store writable for the current user
sudo mkdir -p /gnu/store /var/guix/db /var/guix/gcroots /var/guix/profiles
sudo chown -R ${USER}: /gnu /var/guix
sudo chmod -R u+w /gnu /var/guix
- name: Restore the Guix store from cache
id: restore_cache
if: ${{ !steps.check_cache.outputs.cache-hit }}
uses: actions/cache/restore@v3.3.2
with:
path: ${{ env.guix_cached_paths }}
key: ${{ steps.cache_name.outputs.ok }}
restore-keys: |
${{ steps.cache_name.outputs.fail_same_run_id_prefix }}
${{ steps.cache_name.outputs.ok_date_only_prefix }}
${{ steps.cache_name.outputs.ok_any_prefix }}
${{ steps.cache_name.outputs.any_prefix }}
- name: Fix up permissions for the Guix store
if: ${{ !steps.check_cache.outputs.cache-hit }}
run: |
# Fix up permissions for the Guix store when restoring from cache
sudo chown -R root:root /gnu /var/guix
sudo chmod 755 /gnu
sudo chmod -R u-w /gnu/store
sudo chmod 1775 /gnu/store
# Move the Guix cache for root to the proper location
if [ -d ~/.cache/guix-root ]; then
sudo mkdir -p ~root/.cache
sudo rm -rf ~root/.cache/guix
sudo mv ~/.cache/guix-root ~root/.cache/guix
sudo chown -R root:root ~root/.cache/guix
fi
- name: Install Guix
if: ${{ !steps.check_cache.outputs.cache-hit }}
uses: sigprof/guix-install-action@358f4a3f978111a172cbaab16ab90b530701c761
with:
useExistingGuix: ${{ steps.restore_cache.outputs.cache-matched-key != '' }}
pullAfterInstall: >-
${{
!startsWith(steps.restore_cache.outputs.cache-matched-key, steps.cache_name.outputs.ok_date_only_prefix)
&& !startsWith(steps.restore_cache.outputs.cache-matched-key, steps.cache_name.outputs.fail_date_only_prefix)
}}
- name: Ensure no locale warning
id: guix_setup_check
if: ${{ !steps.check_cache.outputs.cache-hit }}
run: test -z "$(guix --version 2>&1 >/dev/null)"

- name: Checkout the project source
if: ${{ !steps.check_cache.outputs.cache-hit }}
uses: actions/checkout@v4.1.1

- name: Build the Guix shell environment
if: ${{ !steps.check_cache.outputs.cache-hit }}
run: guix shell -m manifest.scm -- true && false

- name: Set cache name for a successful update
if: always() && !cancelled() && success()
run: echo 'final_cache_name=${{ steps.cache_name.outputs.ok}}' >> $GITHUB_ENV

- name: Set cache name for a failed update
if: always() && !cancelled() && !success()
run: echo 'final_cache_name=${{ steps.cache_name.outputs.fail}}' >> $GITHUB_ENV

- name: Determine the resulting cache name
id: saved_cache_name
if: always() && !cancelled()
run: echo "name=$final_cache_name" >> $GITHUB_OUTPUT

- name: Prepare to save the Guix store
id: guix_shutdown
if: >-
always() && !cancelled()
&& !steps.check_cache.outputs.cache-hit
&& (steps.guix_setup_check.outcome == 'success')
run: |
sudo systemctl stop guix-daemon.service
sudo systemctl stop gnu-store.mount
sudo mkdir -p ~root/.cache/guix
sudo mv ~root/.cache/guix ~/.cache/guix-root
sudo chown -R ${USER}: ~/.cache/guix-root
set -f -x
set -- ${guix_cached_paths}
set +f
du -sh --total ${*/#\~/$HOME}
set -x
- name: Save the Guix store in the cache
if: >-
always() && !cancelled()
&& !steps.check_cache.outputs.cache-hit
&& (steps.guix_shutdown.outcome == 'success')
uses: actions/cache/save@v3.3.2
with:
path: ${{ env.guix_cached_paths }}
key: ${{ steps.saved_cache_name.outputs.name }}


test:
needs: setup
strategy: ${{ fromJSON(needs.setup.outputs.strategy) }}
Expand All @@ -97,7 +220,7 @@ jobs:
shell: bash
working-directory: qmk_firmware

runs-on: ${{ matrix.os }}
runs-on: ubuntu-latest
steps:
- name: Prepare the Guix store before restoring from cache
working-directory: .
Expand All @@ -111,17 +234,9 @@ jobs:
id: restore_cache
uses: actions/cache/restore@v3.3.2
with:
path: |
~/.cache/guix
~/.cache/guix-install-action
/gnu
/var/guix/db/db.sqlite*
/var/guix/gcroots
/var/guix/profiles
key: guix-store-${{ github.sha }}-${{ github.run_id }}
restore-keys: |
guix-store-${{ github.sha }}-
guix-store-
path: ${{ env.guix_cached_paths }}
key: ${{ needs.setup.outputs.cache_name }}
fail-on-cache-miss: true

- name: Fix up permissions for the Guix store
working-directory: .
Expand All @@ -132,26 +247,21 @@ jobs:
sudo chmod -R u-w /gnu/store
sudo chmod 1775 /gnu/store
# Fix up old cache
svc_cache_path="$HOME/.cache/guix-install-action/systemd"
orig_svc_path="/var/guix/profiles/per-user/root/current-guix-1-link/lib/systemd/system"
mkdir -p "$svc_cache_path"
for svc in gnu-store.mount guix-daemon.service; do
if ! [ -e "$svc_cache_path/$svc" ]; then
if [ -e "$orig_svc_path/$svc" ]; then
cp -v "$orig_svc_path/$svc" "$svc_cache_path/$svc"
fi
fi
done
# Move the Guix cache for root to the proper location
if [ -d ~/.cache/guix-root ]; then
sudo mkdir -p ~root/.cache
sudo rm -rf ~root/.cache/guix
sudo mv ~/.cache/guix-root ~root/.cache/guix
sudo chown -R root:root ~root/.cache/guix
fi
- name: Install Guix
uses: sigprof/guix-install-action@358f4a3f978111a172cbaab16ab90b530701c761
with:
useExistingGuix: ${{ steps.restore_cache.outputs.cache-matched-key != '' }}
pullAfterInstall: ${{ steps.restore_cache.outputs.cache-matched-key == '' }}
useExistingGuix: true
pullAfterInstall: false

- name: Ensure no locale warning
id: guix_setup_check
working-directory: .
run: test -z "$(guix --version 2>&1 >/dev/null)"

Expand All @@ -175,7 +285,7 @@ jobs:
- name: Configure the 'upstream' remote
run: |
git remote add -t master -t develop upstream https://github.com/qmk/qmk_firmware
git fetch --no-recurse-submodules upstream
git fetch --depth=1 --no-tags --no-recurse-submodules upstream
- name: Configure the udev rules
if: ${{ runner.os == 'Linux' }}
Expand Down Expand Up @@ -216,27 +326,6 @@ jobs:
- name: Test 'qmk clean'
run: guix shell -m ../guix-qmk/manifest.scm -- qmk clean

- name: Prepare to save the Guix store
id: guix_shutdown
if: always() && !cancelled() && (steps.guix_setup_check.outcome == 'success')
working-directory: .
run: |
sudo systemctl stop guix-daemon.service
sudo systemctl stop gnu-store.mount
- name: Save the Guix store in the cache
if: always() && !cancelled() && (steps.guix_shutdown.outcome == 'success')
uses: actions/cache/save@v3.3.2
with:
path: |
~/.cache/guix
~/.cache/guix-install-action
/gnu
/var/guix/db/db.sqlite*
/var/guix/gcroots
/var/guix/profiles
key: guix-store-${{ github.sha }}-${{ github.run_id }}

finish:
needs:
- setup
Expand Down

0 comments on commit 26f0b06

Please sign in to comment.