.github/workflows/build.yaml

name: Build Wolfi OS

on:
  push:
    branches: ['main']
    paths-ignore:
      - '**.md'
      - '**.txt'

  workflow_dispatch:

# Only run one build at a time to prevent out of sync signatures.
concurrency: build

jobs:
  build:
    name: Build packages
    if: github.repository == 'wolfi-dev/os'

    strategy:
      matrix:
        arch: [ "x86_64", "aarch64" ]
      fail-fast: false

    runs-on:
      group: wolfi-os-builder-${{ matrix.arch }}

    permissions:
      contents: read

    container:
      image: ghcr.io/wolfi-dev/sdk:latest@sha256:420bc33eb98b6c9fb2d785d7d9dfd6ed2084fa7eb8768a5114cc4d6723e5f1da
      # TODO: Deprivilege
      options: |
        --cap-add NET_ADMIN --cap-add SYS_ADMIN --device /dev/fuse --security-opt seccomp=unconfined --security-opt apparmor:unconfined

    steps:
      - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1

      - name: 'Trust the github workspace'
        run: |
          # This is to avoid fatal errors about "dubious ownership" because we are
          # running inside of a container action with the workspace mounted in.
          git config --global --add safe.directory "$(pwd)"

      # Build with a local key, we'll resign this with the real key later
      - name: 'Generate local signing key'
        run: |
          make local-melange.rsa

          # Touch it with the epoch date to convince `make` that we don't need to
          # rebuild the targets that depend on this (all)
          touch -d @0 local-melange.rsa

      - name: 'Prepare package repository'
        run: |
          # yay wolfi!
          apk add gcsfuse google-cloud-sdk

          # Set up a gcsfuse RO mount to the public bucket. This is a cheap and
          # cheerful way to recreate the make targets (class A HEADs) locally
          # without syncing the whole bucket (class A+B).
          mkdir -p /gcsfuse/wolfi-registry
          gcsfuse -o ro --implicit-dirs --only-dir os wolfi-production-registry-destination /gcsfuse/wolfi-registry

          mkdir -p ./packages/${{ matrix.arch }}
          # Symlink the gcsfuse mount to ./packages/ to workaround the Makefile CWD assumptions
          for f in /gcsfuse/wolfi-registry/${{ matrix.arch }}/*.apk; do
            ln -s "$f" ./packages/${{ matrix.arch }}/
          done

          # Make a copy of the APKINDEX.* since we'll need to write to it on package builds
          cp /gcsfuse/wolfi-registry/${{ matrix.arch }}/APKINDEX.* ./packages/${{ matrix.arch }}/

      # TODO: Remove the previous and next steps by folding the logic into wolfictl.
      - name: 'Build Wolfi'
        run: |
          wolfictl build \
            --runner bubblewrap \
            --keyring-append /gcsfuse/wolfi-registry/wolfi-signing.rsa.pub \
            --repository-append ./packages \
            --keyring-append local-melange.rsa.pub \
            --repository-append https://packages.wolfi.dev/os \
            --keyring-append https://packages.wolfi.dev/os/wolfi-signing.rsa.pub \
            --signing-key local-melange.rsa \
            --arch ${{ matrix.arch }} \
            --namespace wolfi \
            --pipeline-dir ./pipelines/ \
            --destination-repository https://packages.wolfi.dev/os \
            --trace ./packages/${{ matrix.arch }}/trace.json

      # Always run this step for https://github.com/wolfi-dev/os/issues/8698
      - if: ${{ always() }}
        name: 'Create artifacts tarball'
        run: |
          set -x
          set -e
          set -o pipefail

          # Pick up any stragglers that didn't get uploaded in previous builds.
          cat ./packages/${{ matrix.arch }}/APKINDEX.tar.gz | tar -Oxz APKINDEX | awk -F':' '$1 == "P" {printf "%s-", $2} $1 == "V" {printf "%s.apk\n", $2}' | sort > indexed.txt
          # TODO: Figure out why ls through gcsfuse is so slow.
          gcloud storage ls gs://wolfi-production-registry-destination/os/${{ matrix.arch }} | grep ".apk$" | xargs -n1 basename | sort > uploaded.txt

          # Lines that are only in uploaded.txt and not indexed.txt.
          comm -13 indexed.txt uploaded.txt > missing.txt

          # Clean up the symlinks to keep only packages we built.
          find ./packages/${{ matrix.arch }} -type l -exec rm -f {} \;

          # Merge any missing APKs into our new index.
          for missed in $(cat missing.txt); do
            # We could do this in one command instead of a loop, but it takes things on argv, which is a bit annoying.
            melange index --merge \
              --source ./packages/${{ matrix.arch }}/APKINDEX.tar.gz \
              --output new.tar.gz \
              /gcsfuse/wolfi-registry/${{ matrix.arch }}/${missed}

            # Overwrite what we're going to upload (and for the next loop).
            mv new.tar.gz ./packages/${{ matrix.arch }}/APKINDEX.tar.gz
          done

          diff \
            <(cat /gcsfuse/wolfi-registry/${{ matrix.arch }}/APKINDEX.tar.gz | tar -Oxz APKINDEX) \
            <(cat ./packages/${{ matrix.arch }}/APKINDEX.tar.gz | tar -Oxz APKINDEX) || true

          # Move logs so we can upload them separately.
          mv ./packages/${{ matrix.arch }}/buildlogs /tmp/buildlogs

          # Move trace so we can upload it separately.
          mv ./packages/${{ matrix.arch }}/trace.json /tmp/trace.json

          # Create an archive for uploading
          tar -cvzf /tmp/packages-${{ matrix.arch }}.tar.gz ./packages/${{ matrix.arch }}

      # Always run these steps for https://github.com/wolfi-dev/os/issues/8698
      - if: ${{ always() }}
        name: 'Upload logs archive to GitHub Artifacts'
        uses: actions/upload-artifact@5d5d22a31266ced268874388b861e4b58bb5c2f3 # v4.3.1
        with:
          name: logs-${{ matrix.arch }}
          path: /tmp/buildlogs/
          if-no-files-found: warn
      - if: ${{ always() }}
        name: 'Upload trace to GitHub Artifacts'
        uses: actions/upload-artifact@5d5d22a31266ced268874388b861e4b58bb5c2f3 # v4.3.1
        with:
          name: trace-${{ matrix.arch }}
          path: /tmp/trace.json
          if-no-files-found: warn
      - if: ${{ always() }}
        name: 'Upload built packages archive to GitHub Artifacts'
        uses: actions/upload-artifact@5d5d22a31266ced268874388b861e4b58bb5c2f3 # v4.3.1
        with:
          name: packages-${{ matrix.arch }}
          path: /tmp/packages-${{ matrix.arch }}.tar.gz
          retention-days: 1 # Low ttl since this is just an intermediary used once
          if-no-files-found: warn

  upload-packages:
    runs-on: ubuntu-latest-16-cores
    needs: build

    # Always run this job for https://github.com/wolfi-dev/os/issues/8698
    if: ${{ always() }}

    permissions:
      id-token: write
      contents: read

    container:
      # NOTE: This step only signs and uploads, so it doesn't need any privileges
      image: ghcr.io/wolfi-dev/sdk:latest@sha256:420bc33eb98b6c9fb2d785d7d9dfd6ed2084fa7eb8768a5114cc4d6723e5f1da

    steps:
      - name: Harden Runner
        uses: step-security/harden-runner@63c24ba6bd7ba022e95695ff85de572c04a18142 # v2.7.0
        with:
          egress-policy: audit

      - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1

      - name: Free up runner disk space
        run: |
          set -x
          rm -rf /usr/share/dotnet
          rm -rf "$AGENT_TOOLSDIRECTORY"

      - name: 'Trust the github workspace'
        run: |
          # This is to avoid fatal errors about "dubious ownership" because we are
          # running inside of a container action with the workspace mounted in.
          git config --global --add safe.directory "$(pwd)"

      - name: 'Download x86_64 package archives'
        uses: actions/download-artifact@21e5c25de9cf2ee24742cd3e822327f3be6dd2a3 # v4.1.1
        with:
          path: /tmp/artifacts/
          name: packages-x86_64

      - name: 'Download aarch64 package archives'
        uses: actions/download-artifact@21e5c25de9cf2ee24742cd3e822327f3be6dd2a3 # v4.1.1
        with:
          path: /tmp/artifacts/
          name: packages-aarch64

      # This is managed here: https://github.com/chainguard-dev/secrets/blob/main/wolfi-dev.tf
      - uses: google-github-actions/auth@a6e2e39c0a0331da29f7fd2c2a20a427e8d3ad1f # v2.1.1
        id: auth
        with:
          workload_identity_provider: "projects/12758742386/locations/global/workloadIdentityPools/github-pool/providers/github-provider"
          service_account: "wolfi-dev@chainguard-github-secrets.iam.gserviceaccount.com"
      - uses: google-github-actions/setup-gcloud@5a5f7b85fca43e76e53463acaa9d408a03c98d3a # v2.0.1
        with:
          project_id: "chainguard-github-secrets"
      - uses: 'google-github-actions/get-secretmanager-secrets@ae0d4054c32840e2ced71207a9df55161ae3debc' # v2.0.0
        id: secrets
        with:
          secrets: |-
            token:chainguard-github-secrets/wolfi-dev-signing-key

      - run: echo "${{ steps.secrets.outputs.token }}" > ./wolfi-signing.rsa
      - run: |
          mkdir -p /etc/apk/keys
          cp ./wolfi-signing.rsa.pub /etc/apk/keys/wolfi-signing.rsa.pub

      - name: 'Update the APKINDEX'
        run: |
          for arch in "aarch64" "x86_64"; do
            mkdir -p ./packages/${arch}

            # Consolidate with the built artifacts
            tar xvf /tmp/artifacts/packages-${arch}.tar.gz

            # Sign the APK index
            melange sign-index -f --signing-key ./wolfi-signing.rsa packages/${arch}/APKINDEX.tar.gz

            # Only attempt to sign when *.apk's exist.
            apks=$(ls ./packages/${arch}/*.apk 2>/dev/null || true)
            if [ -n "$apks" ]; then
              melange sign --signing-key ./wolfi-signing.rsa ./packages/${arch}/*.apk
            fi
          done

      # Clean up the signing key before uploading to storage out
      # of an abundance of caution.
      - run: rm ./wolfi-signing.rsa

      # We use a different GSA for our interaction with GCS.
      - uses: google-github-actions/auth@f6de81663f7788d05bd15bcce18f0e57f23f0846 # v2.0.1
        with:
          workload_identity_provider: "projects/618116202522/locations/global/workloadIdentityPools/prod-shared-e350/providers/prod-shared-gha"
          service_account: "prod-images-ci@prod-images-c6e5.iam.gserviceaccount.com"
      - uses: google-github-actions/setup-gcloud@5a5f7b85fca43e76e53463acaa9d408a03c98d3a # v2.0.1
        with:
          project_id: "prod-images-c6e5"

      - name: 'Upload packages to GCS'
        run: |
          for arch in "aarch64" "x86_64"; do
            # Only attempt to upload when *.apk's exist
            apks=$(ls ./packages/${arch}/*.apk 2>/dev/null || true)
            if [ -n "$apks" ]; then
              # apks will be cached in CDN for an hour by default.
              # Don't upload the object if it already exists.
              gcloud --quiet storage cp \
                  --no-clobber \
                  "./packages/${arch}/*.apk" "gs://wolfi-production-registry-destination/os/${arch}/"
            fi
          done

      - name: 'Create APKINDEX tarball'
        run: |
          # Tar up any 'APKINDEX.*' files {aarch64,x86_64} x {tar.gz,json}
          find ./packages/ -name 'APKINDEX.*' > to-include
          tar -cvzf /tmp/indexes.tar.gz --files-from to-include

      - name: 'Upload APKINDEX archive to GitHub Artifacts'
        uses: actions/upload-artifact@5d5d22a31266ced268874388b861e4b58bb5c2f3 # v4.3.1
        with:
          name: indexes
          path: /tmp/indexes.tar.gz
          retention-days: 1 # Low ttl since this is just an intermediary used once
          if-no-files-found: warn

  upload-index:
    runs-on: ubuntu-latest-16-cores
    needs: upload-packages

    permissions:
      id-token: write
      contents: read

    container:
      # NOTE: This step only signs and uploads, so it doesn't need any privileges
      image: ghcr.io/wolfi-dev/sdk:latest@sha256:420bc33eb98b6c9fb2d785d7d9dfd6ed2084fa7eb8768a5114cc4d6723e5f1da

    steps:
      - name: Harden Runner
        uses: step-security/harden-runner@63c24ba6bd7ba022e95695ff85de572c04a18142 # v2.7.0
        with:
          egress-policy: audit

      - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1

      - name: 'Trust the github workspace'
        run: |
          # This is to avoid fatal errors about "dubious ownership" because we are
          # running inside of a container action with the workspace mounted in.
          git config --global --add safe.directory "$(pwd)"

      - id: auth
        name: 'Authenticate to Google Cloud'
        uses: google-github-actions/auth@55bd3a7c6e2ae7cf1877fd1ccb9d54c0503c457c # v2.1.2
        with:
          workload_identity_provider: "projects/618116202522/locations/global/workloadIdentityPools/prod-shared-e350/providers/prod-shared-gha"
          service_account: "prod-images-ci@prod-images-c6e5.iam.gserviceaccount.com"

      - uses: google-github-actions/setup-gcloud@98ddc00a17442e89a24bbf282954a3b65ce6d200 # v2.1.0
        with:
          project_id: prod-images-c6e5

      - name: 'Download index archive'
        uses: actions/download-artifact@21e5c25de9cf2ee24742cd3e822327f3be6dd2a3 # v4.1.1
        with:
          path: /tmp/artifacts/
          name: indexes

      - name: 'Upload indexes to GCS'
        run: |
          tar xvf /tmp/artifacts/indexes.tar.gz

          for arch in "aarch64" "x86_64"; do
            # Don't cache the APKINDEX.
            gcloud --quiet storage cp \
                --cache-control=no-store \
                "./packages/${arch}/APKINDEX.tar.gz" "gs://wolfi-production-registry-destination/os/${arch}/"

            gcloud --quiet storage cp \
                --cache-control=no-store \
                "./packages/${arch}/APKINDEX.json" "gs://wolfi-production-registry-destination/os/${arch}/"
          done

  postrun:
    name: Notify Slack
    runs-on: ubuntu-latest
    if: failure()
    needs: [build, upload-packages, upload-index]
    steps:
      - name: Harden Runner
        uses: step-security/harden-runner@63c24ba6bd7ba022e95695ff85de572c04a18142 # v2.7.0
        with:
          egress-policy: audit

      - uses: rtCamp/action-slack-notify@f05987dc91a66984f1666f486497def2cf85183d # v2.2.1
        env:
          SLACK_ICON: http://github.com/chainguard-dev.png?size=48
          SLACK_USERNAME: guardian
          SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK_URL }}
          SLACK_CHANNEL: C047DK5BUNP
          SLACK_MSG_AUTHOR: wolfi-bot
          SLACK_COLOR: '#8E1600'
          MSG_MINIMAL: 'true'
          SLACK_TITLE: '[build-wolfi-os] failure: https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}'
          SLACK_MESSAGE: |
            https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}