Merge pull request #18514 from wolfi-dev/wolfictl-0b43ece1-e744-4258-… #18
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Build Wolfi OS | |
| on: | |
| push: | |
| branches: ['main'] | |
| paths-ignore: | |
| - '**.md' | |
| - '**.txt' | |
| workflow_dispatch: | |
| # Only run one build at a time to prevent out of sync signatures. | |
| concurrency: build | |
| jobs: | |
| build: | |
| name: Build packages | |
| if: github.repository == 'wolfi-dev/os' | |
| strategy: | |
| matrix: | |
| arch: [ "x86_64", "aarch64" ] | |
| fail-fast: false | |
| runs-on: | |
| group: wolfi-os-builder-${{ matrix.arch }} | |
| permissions: | |
| contents: read | |
| container: | |
| image: ghcr.io/wolfi-dev/sdk:latest@sha256:7729ea91fad59487bec2d4949b0e51cee40b107e068d2421597a6f40ce316bad | |
| # TODO: Deprivilege | |
| options: | | |
| --cap-add NET_ADMIN --cap-add SYS_ADMIN --device /dev/fuse --security-opt seccomp=unconfined --security-opt apparmor:unconfined | |
| steps: | |
| - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 | |
| - name: 'Trust the github workspace' | |
| run: | | |
| # This is to avoid fatal errors about "dubious ownership" because we are | |
| # running inside of a container action with the workspace mounted in. | |
| git config --global --add safe.directory "$(pwd)" | |
| # Build with a local key, we'll resign this with the real key later | |
| - name: 'Generate local signing key' | |
| run: | | |
| make local-melange.rsa | |
| # Touch it with the epoch date to convince `make` that we don't need to | |
| # rebuild the targets that depend on this (all) | |
| touch -d @0 local-melange.rsa | |
| - name: 'Prepare package repository' | |
| run: | | |
| # yay wolfi! | |
| apk add gcsfuse google-cloud-sdk | |
| # Set up a gcsfuse RO mount to the public bucket. This is a cheap and | |
| # cheerful way to recreate the make targets (class A HEADs) locally | |
| # without syncing the whole bucket (class A+B). | |
| mkdir -p /gcsfuse/wolfi-registry | |
| gcsfuse -o ro --implicit-dirs --only-dir os wolfi-production-registry-destination /gcsfuse/wolfi-registry | |
| mkdir -p ./packages/${{ matrix.arch }} | |
| # Symlink the gcsfuse mount to ./packages/ to workaround the Makefile CWD assumptions | |
| for f in /gcsfuse/wolfi-registry/${{ matrix.arch }}/*.apk; do | |
| ln -s "$f" ./packages/${{ matrix.arch }}/ | |
| done | |
| # Make a copy of the APKINDEX.* since we'll need to write to it on package builds | |
| cp /gcsfuse/wolfi-registry/${{ matrix.arch }}/APKINDEX.* ./packages/${{ matrix.arch }}/ | |
| # TODO: Remove the previous and next steps by folding the logic into wolfictl. | |
| - name: 'Build Wolfi' | |
| run: | | |
| wolfictl build \ | |
| --runner bubblewrap \ | |
| --keyring-append /gcsfuse/wolfi-registry/wolfi-signing.rsa.pub \ | |
| --repository-append ./packages \ | |
| --keyring-append local-melange.rsa.pub \ | |
| --repository-append https://packages.wolfi.dev/os \ | |
| --keyring-append https://packages.wolfi.dev/os/wolfi-signing.rsa.pub \ | |
| --signing-key local-melange.rsa \ | |
| --arch ${{ matrix.arch }} \ | |
| --namespace wolfi \ | |
| --pipeline-dir ./pipelines/ \ | |
| --destination-repository https://packages.wolfi.dev/os \ | |
| --trace ./packages/${{ matrix.arch }}/trace.json | |
| # Always run this step for https://github.com/wolfi-dev/os/issues/8698 | |
| - if: ${{ always() }} | |
| name: 'Create artifacts tarball' | |
| run: | | |
| set -x | |
| set -e | |
| set -o pipefail | |
| # Pick up any stragglers that didn't get uploaded in previous builds. | |
| cat ./packages/${{ matrix.arch }}/APKINDEX.tar.gz | tar -Oxz APKINDEX | awk -F':' '$1 == "P" {printf "%s-", $2} $1 == "V" {printf "%s.apk\n", $2}' | sort > indexed.txt | |
| # TODO: Figure out why ls through gcsfuse is so slow. | |
| gcloud storage ls gs://wolfi-production-registry-destination/os/${{ matrix.arch }} | grep ".apk$" | xargs -n1 basename | sort > uploaded.txt | |
| # Lines that are only in uploaded.txt and not indexed.txt. | |
| comm -13 indexed.txt uploaded.txt > missing.txt | |
| # Clean up the symlinks to keep only packages we built. | |
| find ./packages/${{ matrix.arch }} -type l -exec rm -f {} \; | |
| # Merge any missing APKs into our new index. | |
| for missed in $(cat missing.txt); do | |
| # We could do this in one command instead of a loop, but it takes things on argv, which is a bit annoying. | |
| melange index --merge \ | |
| --source ./packages/${{ matrix.arch }}/APKINDEX.tar.gz \ | |
| --output new.tar.gz \ | |
| /gcsfuse/wolfi-registry/${{ matrix.arch }}/${missed} | |
| # Overwrite what we're going to upload (and for the next loop). | |
| mv new.tar.gz ./packages/${{ matrix.arch }}/APKINDEX.tar.gz | |
| done | |
| diff \ | |
| <(cat /gcsfuse/wolfi-registry/${{ matrix.arch }}/APKINDEX.tar.gz | tar -Oxz APKINDEX) \ | |
| <(cat ./packages/${{ matrix.arch }}/APKINDEX.tar.gz | tar -Oxz APKINDEX) || true | |
| # Move logs so we can upload them separately. | |
| mv ./packages/${{ matrix.arch }}/buildlogs /tmp/buildlogs | |
| # Move trace so we can upload it separately. | |
| mv ./packages/${{ matrix.arch }}/trace.json /tmp/trace.json | |
| # Create an archive for uploading | |
| tar -cvzf /tmp/packages-${{ matrix.arch }}.tar.gz ./packages/${{ matrix.arch }} | |
| # Always run these steps for https://github.com/wolfi-dev/os/issues/8698 | |
| - if: ${{ always() }} | |
| name: 'Upload logs archive to GitHub Artifacts' | |
| uses: actions/upload-artifact@5d5d22a31266ced268874388b861e4b58bb5c2f3 # v4.3.1 | |
| with: | |
| name: logs-${{ matrix.arch }} | |
| path: /tmp/buildlogs/ | |
| if-no-files-found: warn | |
| - if: ${{ always() }} | |
| name: 'Upload trace to GitHub Artifacts' | |
| uses: actions/upload-artifact@5d5d22a31266ced268874388b861e4b58bb5c2f3 # v4.3.1 | |
| with: | |
| name: trace-${{ matrix.arch }} | |
| path: /tmp/trace.json | |
| if-no-files-found: warn | |
| - if: ${{ always() }} | |
| name: 'Upload built packages archive to GitHub Artifacts' | |
| uses: actions/upload-artifact@5d5d22a31266ced268874388b861e4b58bb5c2f3 # v4.3.1 | |
| with: | |
| name: packages-${{ matrix.arch }} | |
| path: /tmp/packages-${{ matrix.arch }}.tar.gz | |
| retention-days: 1 # Low ttl since this is just an intermediary used once | |
| if-no-files-found: warn | |
| upload-packages: | |
| runs-on: ubuntu-latest-16-cores | |
| needs: build | |
| # Always run this job for https://github.com/wolfi-dev/os/issues/8698 | |
| if: ${{ always() }} | |
| permissions: | |
| id-token: write | |
| contents: read | |
| container: | |
| # NOTE: This step only signs and uploads, so it doesn't need any privileges | |
| image: ghcr.io/wolfi-dev/sdk:latest@sha256:7729ea91fad59487bec2d4949b0e51cee40b107e068d2421597a6f40ce316bad | |
| steps: | |
| - name: Harden Runner | |
| uses: step-security/harden-runner@63c24ba6bd7ba022e95695ff85de572c04a18142 # v2.7.0 | |
| with: | |
| egress-policy: audit | |
| - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 | |
| - name: Free up runner disk space | |
| run: | | |
| set -x | |
| rm -rf /usr/share/dotnet | |
| rm -rf "$AGENT_TOOLSDIRECTORY" | |
| - name: 'Trust the github workspace' | |
| run: | | |
| # This is to avoid fatal errors about "dubious ownership" because we are | |
| # running inside of a container action with the workspace mounted in. | |
| git config --global --add safe.directory "$(pwd)" | |
| - name: 'Download x86_64 package archives' | |
| uses: actions/download-artifact@21e5c25de9cf2ee24742cd3e822327f3be6dd2a3 # v4.1.1 | |
| with: | |
| path: /tmp/artifacts/ | |
| name: packages-x86_64 | |
| - name: 'Download aarch64 package archives' | |
| uses: actions/download-artifact@21e5c25de9cf2ee24742cd3e822327f3be6dd2a3 # v4.1.1 | |
| with: | |
| path: /tmp/artifacts/ | |
| name: packages-aarch64 | |
| # This is managed here: https://github.com/chainguard-dev/secrets/blob/main/wolfi-dev.tf | |
| - uses: google-github-actions/auth@a6e2e39c0a0331da29f7fd2c2a20a427e8d3ad1f # v2.1.1 | |
| id: auth | |
| with: | |
| workload_identity_provider: "projects/12758742386/locations/global/workloadIdentityPools/github-pool/providers/github-provider" | |
| service_account: "wolfi-dev@chainguard-github-secrets.iam.gserviceaccount.com" | |
| - uses: google-github-actions/setup-gcloud@5a5f7b85fca43e76e53463acaa9d408a03c98d3a # v2.0.1 | |
| with: | |
| project_id: "chainguard-github-secrets" | |
| - uses: 'google-github-actions/get-secretmanager-secrets@ae0d4054c32840e2ced71207a9df55161ae3debc' # v2.0.0 | |
| id: secrets | |
| with: | |
| secrets: |- | |
| token:chainguard-github-secrets/wolfi-dev-signing-key | |
| - run: echo "${{ steps.secrets.outputs.token }}" > ./wolfi-signing.rsa | |
| - run: | | |
| mkdir -p /etc/apk/keys | |
| cp ./wolfi-signing.rsa.pub /etc/apk/keys/wolfi-signing.rsa.pub | |
| - name: 'Update the APKINDEX' | |
| run: | | |
| for arch in "aarch64" "x86_64"; do | |
| mkdir -p ./packages/${arch} | |
| # Consolidate with the built artifacts | |
| tar xvf /tmp/artifacts/packages-${arch}.tar.gz | |
| # Sign the APK index | |
| melange sign-index -f --signing-key ./wolfi-signing.rsa packages/${arch}/APKINDEX.tar.gz | |
| # Only attempt to sign when *.apk's exist. | |
| apks=$(ls ./packages/${arch}/*.apk 2>/dev/null || true) | |
| if [ -n "$apks" ]; then | |
| melange sign --signing-key ./wolfi-signing.rsa ./packages/${arch}/*.apk | |
| fi | |
| done | |
| # Clean up the signing key before uploading to storage out | |
| # of an abundance of caution. | |
| - run: rm ./wolfi-signing.rsa | |
| # We use a different GSA for our interaction with GCS. | |
| - uses: google-github-actions/auth@f6de81663f7788d05bd15bcce18f0e57f23f0846 # v2.0.1 | |
| with: | |
| workload_identity_provider: "projects/618116202522/locations/global/workloadIdentityPools/prod-shared-e350/providers/prod-shared-gha" | |
| service_account: "prod-images-ci@prod-images-c6e5.iam.gserviceaccount.com" | |
| - uses: google-github-actions/setup-gcloud@5a5f7b85fca43e76e53463acaa9d408a03c98d3a # v2.0.1 | |
| with: | |
| project_id: "prod-images-c6e5" | |
| - name: 'Upload packages to GCS' | |
| run: | | |
| for arch in "aarch64" "x86_64"; do | |
| # Only attempt to upload when *.apk's exist | |
| apks=$(ls ./packages/${arch}/*.apk 2>/dev/null || true) | |
| if [ -n "$apks" ]; then | |
| # apks will be cached in CDN for an hour by default. | |
| # Don't upload the object if it already exists. | |
| gcloud --quiet storage cp \ | |
| --no-clobber \ | |
| "./packages/${arch}/*.apk" "gs://wolfi-production-registry-destination/os/${arch}/" | |
| fi | |
| done | |
| - name: 'Create APKINDEX tarball' | |
| run: | | |
| # Tar up any 'APKINDEX.*' files {aarch64,x86_64} x {tar.gz,json} | |
| find ./packages/ -name 'APKINDEX.*' > to-include | |
| tar -cvzf /tmp/indexes.tar.gz --files-from to-include | |
| - name: 'Upload APKINDEX archive to GitHub Artifacts' | |
| uses: actions/upload-artifact@5d5d22a31266ced268874388b861e4b58bb5c2f3 # v4.3.1 | |
| with: | |
| name: indexes | |
| path: /tmp/indexes.tar.gz | |
| retention-days: 1 # Low ttl since this is just an intermediary used once | |
| if-no-files-found: warn | |
| upload-index: | |
| runs-on: ubuntu-latest-16-cores | |
| needs: upload-packages | |
| permissions: | |
| id-token: write | |
| contents: read | |
| container: | |
| # NOTE: This step only signs and uploads, so it doesn't need any privileges | |
| image: ghcr.io/wolfi-dev/sdk:latest@sha256:7729ea91fad59487bec2d4949b0e51cee40b107e068d2421597a6f40ce316bad | |
| steps: | |
| - name: Harden Runner | |
| uses: step-security/harden-runner@63c24ba6bd7ba022e95695ff85de572c04a18142 # v2.7.0 | |
| with: | |
| egress-policy: audit | |
| - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 | |
| - name: 'Trust the github workspace' | |
| run: | | |
| # This is to avoid fatal errors about "dubious ownership" because we are | |
| # running inside of a container action with the workspace mounted in. | |
| git config --global --add safe.directory "$(pwd)" | |
| - id: auth | |
| name: 'Authenticate to Google Cloud' | |
| uses: google-github-actions/auth@55bd3a7c6e2ae7cf1877fd1ccb9d54c0503c457c # v2.1.2 | |
| with: | |
| workload_identity_provider: "projects/618116202522/locations/global/workloadIdentityPools/prod-shared-e350/providers/prod-shared-gha" | |
| service_account: "prod-images-ci@prod-images-c6e5.iam.gserviceaccount.com" | |
| - uses: google-github-actions/setup-gcloud@98ddc00a17442e89a24bbf282954a3b65ce6d200 # v2.1.0 | |
| with: | |
| project_id: prod-images-c6e5 | |
| - name: 'Download index archive' | |
| uses: actions/download-artifact@21e5c25de9cf2ee24742cd3e822327f3be6dd2a3 # v4.1.1 | |
| with: | |
| path: /tmp/artifacts/ | |
| name: indexes | |
| - name: 'Upload indexes to GCS' | |
| run: | | |
| tar xvf /tmp/artifacts/indexes.tar.gz | |
| for arch in "aarch64" "x86_64"; do | |
| # Don't cache the APKINDEX. | |
| gcloud --quiet storage cp \ | |
| --cache-control=no-store \ | |
| "./packages/${arch}/APKINDEX.tar.gz" "gs://wolfi-production-registry-destination/os/${arch}/" | |
| gcloud --quiet storage cp \ | |
| --cache-control=no-store \ | |
| "./packages/${arch}/APKINDEX.json" "gs://wolfi-production-registry-destination/os/${arch}/" | |
| done | |
| postrun: | |
| name: Notify Slack | |
| runs-on: ubuntu-latest | |
| if: failure() | |
| needs: [build, upload-packages, upload-index] | |
| steps: | |
| - name: Harden Runner | |
| uses: step-security/harden-runner@63c24ba6bd7ba022e95695ff85de572c04a18142 # v2.7.0 | |
| with: | |
| egress-policy: audit | |
| - uses: rtCamp/action-slack-notify@f05987dc91a66984f1666f486497def2cf85183d # v2.2.1 | |
| env: | |
| SLACK_ICON: http://github.com/chainguard-dev.png?size=48 | |
| SLACK_USERNAME: guardian | |
| SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK_URL }} | |
| SLACK_CHANNEL: C047DK5BUNP | |
| SLACK_MSG_AUTHOR: wolfi-bot | |
| SLACK_COLOR: '#8E1600' | |
| MSG_MINIMAL: 'true' | |
| SLACK_TITLE: '[build-wolfi-os] failure: https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}' | |
| SLACK_MESSAGE: | | |
| https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }} |