docker-build-and-test.yml

---
# Test built docker images by building simple projects inside them

name: dagger
on:
  pull_request:
    paths:
      - '.dagger-ci'
      - '.github/workflows/docker-build-and-test.yml'
      - 'docker/**'
      - 'tests/**'
  push:
    branches: ['main']
    paths:
      - '.dagger-ci'
      - '.github/workflows/docker-build-and-test.yml'
      - 'docker/**'
      - 'tests/**'
  release:
  schedule:
    # First day of the month at midnight
    - cron: '0 0 1 * 0'
  workflow_dispatch:

env:
  REGISTRY: ghcr.io

permissions:
  contents: read

jobs:
  #=============================
  # Dynamically generate matrix
  #=============================

  get-matrix:
    runs-on: ubuntu-latest
    steps:
      - name: Checkout
        uses: actions/checkout@v4
      - name: get-matrix
        id: get-matrix
        run: |
          # Disable SC2046: Quote this to prevent word splitting
          # I can't quote this, just look at it
          # shellcheck disable=SC2046
          echo matrix=$( yq --input-format yaml --output-format json '.services | keys[]' docker/compose.yaml | sed 's/"//g' | jq -Rs 'split("\n") | del(.[-1])' | jq -c ) >> "${GITHUB_OUTPUT}"
      - name: Check
        run: |
          jq . <<< '${{ steps.get-matrix.outputs.matrix }}'
    outputs:
      matrix: ${{ steps.get-matrix.outputs.matrix }}

  get-matrix-coreboot:
    runs-on: ubuntu-latest
    steps:
      - name: Checkout
        uses: actions/checkout@v4
      - name: get-matrix
        id: get-matrix
        run: |
          # shellcheck disable=SC2046
          echo matrix=$( yq --input-format yaml --output-format json '.services | keys[] | select(. | test("coreboot.*"))' docker/compose.yaml | sed 's/"//g' | jq -Rs 'split("\n") | del(.[-1])' | jq -c ) >> "${GITHUB_OUTPUT}"
      - name: Check
        run: |
          jq . <<< '${{ steps.get-matrix.outputs.matrix }}'
    outputs:
      matrix: ${{ steps.get-matrix.outputs.matrix }}

  #=====================
  # Coreboot toolchains
  #=====================

  build-coreboot-toolchains:
    timeout-minutes: 120
    needs:
      - get-matrix-coreboot
    strategy:
      fail-fast: false
      matrix:
        arch: ['amd64', 'arm64']
        dockerfile: ${{ fromJson(needs.get-matrix-coreboot.outputs.matrix) }}
    runs-on: ${{ matrix.arch == 'arm64' && 'ARM64' || 'ubuntu-latest' }}
    container:
      # At the time of writing (2024-10) we cannot use ubuntu:noble as it is broken
      image: ubuntu:jammy
    env:
      DEBIAN_FRONTEND: noninteractive
      # Use coreboot mirrors
      BUILDGCC_OPTIONS: -m
    steps:
      - name: Install dependencies for CI
        run: |
          apt-get update
          apt-get install -y --no-install-recommends \
            build-essential \
            ca-certificates \
            curl \
            git \
            jq \
            sudo \
            tzdata \
            upx-ucl \
            wget
          update-ca-certificates
      - name: Get yq
        # the --no-check-certificate is needed because GitHub
        run: |
          wget -q --no-check-certificate -O /usr/local/bin/yq https://github.com/mikefarah/yq/releases/latest/download/yq_linux_${{ matrix.arch }}
          chmod 755 /usr/local/bin/yq
      - name: Configure tzdata
        run: |
          dpkg-reconfigure --frontend noninteractive tzdata
      - name: Checkout
        uses: actions/checkout@v4
      - name: Get coreboot version
        id: version
        run: |
          yq -r '.services.["${{ matrix.dockerfile }}"].build.args[] | select(test("COREBOOT_VERSION=.*"))' docker/compose.yaml >> "${GITHUB_OUTPUT}"

      - name: Clone coreboot
        run: |
          git clone --depth 1 "https://review.coreboot.org/coreboot.git" -b "${{ steps.version.outputs.COREBOOT_VERSION }}"

      - name: Get coreboot commit hash
        id: coreboot-hash
        run: |
          cd coreboot
          COREBOOT_HASH="$( git rev-parse --short HEAD )"
          echo "${COREBOOT_HASH}"
          echo "COREBOOT_HASH=${COREBOOT_HASH}" >> "${GITHUB_OUTPUT}"
      - name: Artefact and cache key
        id: cache-key
        run: |
          CACHE_KEY="coreboot-${{ steps.version.outputs.COREBOOT_VERSION }}-${{ steps.coreboot-hash.outputs.COREBOOT_HASH }}-${{ matrix.arch }}"
          echo "${CACHE_KEY}"
          echo "CACHE_KEY=${CACHE_KEY}" >> "${GITHUB_OUTPUT}"

      - name: Restore cached toolchains
        id: cache-toolchains
        uses: actions/cache/restore@v4
        with:
          path: coreboot/util/crossgcc/xgcc
          key: ${{ steps.cache-key.outputs.CACHE_KEY }}-xgcc
      - name: Restore cached utils
        id: cache-utils
        uses: actions/cache/restore@v4
        with:
          path: /usr/local/bin
          key: ${{ steps.cache-key.outputs.CACHE_KEY }}-utils

      - name: Debug list crossgcc
        if: steps.cache-toolchains.outputs.cache-hit == 'true'
        run: |
          ls -a1lh coreboot/util/crossgcc/xgcc
      - name: Debug list utils
        if: steps.cache-utils.outputs.cache-hit == 'true'
        run: |
          ls -a1lh /usr/local/bin

      - name: Install dependencies if needed
        if: steps.cache-toolchains.outputs.cache-hit != 'true'
        run: |
          apt-get install -y --no-install-recommends \
            acpica-tools \
            bc \
            bison \
            bsdmainutils \
            build-essential \
            flex \
            gnat \
            imagemagick \
            libelf-dev \
            libncurses5-dev \
            libnss3-dev \
            libssl-dev \
            m4 \
            nasm \
            openssh-client \
            pkgconf \
            python-is-python3 \
            python3-pip \
            qemu-system-x86 \
            upx-ucl \
            uuid-dev \
            zlib1g-dev
      - name: Install dependencies if needed (amd64)
        if: matrix.arch == 'amd64' && steps.cache-toolchains.outputs.cache-hit != 'true'
        run: |
          apt-get install -y --no-install-recommends \
            iucode-tool

      - name: Build coreboot toolchains
        if: steps.cache-toolchains.outputs.cache-hit != 'true'
        run: |
          cd coreboot
          make crossgcc CPUS="$(nproc)"
      - name: Build coreboot utils
        if: steps.cache-utils.outputs.cache-hit != 'true'
        run: |
          cd coreboot
          make -C util/ifdtool install
          make -C util/cbfstool install
      - name: Compress toolchain binaries
        # This step should shrink the size of single toolchain from 1.5 GB down to around 700 MB
        # I think it is save to compress all binaries except libraries, hence the '-wholename'
        if: steps.cache-toolchains.outputs.cache-hit != 'true'
        run: |
          cd coreboot/util/crossgcc/xgcc
          # shellcheck disable=SC2016
          find . -type f -wholename '*/bin/*' -exec bash -c 'upx-ucl -9 "$1"' shell {} \; || true

      # Store toolchains and utils in cache
      - name: Cache toolchains
        uses: actions/cache/save@v4
        if: steps.cache-toolchains.outputs.cache-hit != 'true'
        with:
          path: coreboot/util/crossgcc/xgcc
          key: ${{ steps.cache-key.outputs.CACHE_KEY }}-xgcc
      - name: Cache utils
        uses: actions/cache/save@v4
        if: steps.cache-utils.outputs.cache-hit != 'true'
        with:
          path: /usr/local/bin
          key: ${{ steps.cache-key.outputs.CACHE_KEY }}-utils

      # Upload toolchains and utils as artifacts
      - name: Tar toolchain to prevent permission loss
        # Docs: https://github.com/actions/upload-artifact?tab=readme-ov-file#permission-loss
        run: |
          if [ ! -f "coreboot/util/crossgcc/xgcc-tar/${{ steps.cache-key.outputs.CACHE_KEY }}-xgcc.tar" ]; then
            mv coreboot/util/crossgcc/xgcc coreboot/util/crossgcc/${{ matrix.arch }}-xgcc
            tar -cf ${{ steps.cache-key.outputs.CACHE_KEY }}-xgcc.tar coreboot/util/crossgcc/${{ matrix.arch }}-xgcc
          fi
      - name: Upload toolchain
        uses: actions/upload-artifact@v4.4.3
        with:
          name: ${{ steps.cache-key.outputs.CACHE_KEY }}-xgcc
          path: ${{ steps.cache-key.outputs.CACHE_KEY }}-xgcc.tar
          retention-days: 30
          include-hidden-files: true
          compression-level: 9
      - name: Upload utils
        uses: actions/upload-artifact@v4.4.3
        with:
          name: ${{ steps.cache-key.outputs.CACHE_KEY }}-utils
          path: /usr/local/bin
          retention-days: 30
          include-hidden-files: true
          compression-level: 9

  #=========================
  # Build Docker containers
  #=========================

  build:
    name: build_test_publish
    runs-on: ubuntu-latest
    timeout-minutes: 60
    needs:
      - get-matrix
      - build-coreboot-toolchains
    strategy:
      fail-fast: false
      matrix:
        dockerfile: ${{ fromJson(needs.get-matrix.outputs.matrix) }}
    permissions:
      contents: read
      packages: write
    steps:
      # We have to use my own fork of actions/delete-package-versions at the moment
      #   to have access to 'dry-run' and 'ignore-versions-include-tags' features
      # We can switch to upstream whe following PRs get merged:
      #   - [dry-run](https://github.com/actions/delete-package-versions/pull/119/commits)
      #   - [tags](https://github.com/actions/delete-package-versions/pull/104
      - name: Delete old packages
        uses: AtomicFS/delete-package-versions@main
        continue-on-error:
          true
          # we have continue-on-error because when I make a fork of this repo to debug something,
          # the Docker containers would not build because this step fails to fetch existing containers
          # (in fresh fork there are none)
        with:
          package-name: firmware-action/${{ matrix.dockerfile }}
          package-type: container
          min-versions-to-keep: 5
          ignore-versions:
            '^(main|latest|v(\d+\.?)+)$'
            # ignore:
            # - main
            # - latest
            # - vX
            # - vX.X
            # - vX.X.X
          dry-run: false
          ignore-versions-include-tags: true

      - name: Setup python
        uses: actions/setup-python@v5
        with:
          python-version: '3.x'

      - name: Checkout
        uses: actions/checkout@v4

      - name: Get coreboot version
        id: version
        if: startsWith(matrix.dockerfile, 'coreboot')
        run: yq -r '.services.["${{ matrix.dockerfile }}"].build.args[] | select(test("COREBOOT_VERSION=.*"))' docker/compose.yaml >> "${GITHUB_OUTPUT}"
      - name: Clone coreboot
        if: startsWith(matrix.dockerfile, 'coreboot')
        run: |
          git clone --depth 1 "https://review.coreboot.org/coreboot.git" -b "${{ steps.version.outputs.COREBOOT_VERSION }}"
      - name: Get coreboot commit hash
        id: coreboot-hash
        if: startsWith(matrix.dockerfile, 'coreboot')
        run: |
          cd coreboot
          COREBOOT_HASH="$( git rev-parse --short HEAD )"
          echo "${COREBOOT_HASH}"
          echo "COREBOOT_HASH=${COREBOOT_HASH}" >> "${GITHUB_OUTPUT}"
      - name: Artefact and cache key
        id: cache-key
        if: startsWith(matrix.dockerfile, 'coreboot')
        run: |
          CACHE_KEY="coreboot-${{ steps.version.outputs.COREBOOT_VERSION }}-${{ steps.coreboot-hash.outputs.COREBOOT_HASH }}"
          echo "CACHE_KEY=${CACHE_KEY}"
          echo "CACHE_KEY=${CACHE_KEY}" >> "${GITHUB_OUTPUT}"

      #=================================
      # Download artifacts for coreboot
      #=================================

      - name: Download coreboot toolchains from current run (if possible)
        id: artifacts-toolchains-current
        if: startsWith(matrix.dockerfile, 'coreboot')
        uses: dawidd6/action-download-artifact@v6
        with:
          name: ${{ steps.cache-key.outputs.CACHE_KEY }}-.*
          name_is_regexp: true
          if_no_artifact_found: warn
          run_id: ${{ github.event.workflow_run.id }}
        # It is possible that current run did not produce any artifacts (to save bandwidth on self-hosted runners)
        # In which case we have to look for artifacts in older runs
      - name: Download coreboot toolchains from older run
        if: startsWith(matrix.dockerfile, 'coreboot') && steps.artifacts-toolchains-current.outputs.found_artifact != 'true'
        uses: dawidd6/action-download-artifact@v6
        with:
          name: ${{ steps.cache-key.outputs.CACHE_KEY }}-.*
          name_is_regexp: true
          search_artifacts: true

      - name: Prepare toolchains
        if: startsWith(matrix.dockerfile, 'coreboot')
        run: |
          mkdir -p docker/coreboot/coreboot-${{ steps.version.outputs.COREBOOT_VERSION }}
          for f in ${{ steps.cache-key.outputs.CACHE_KEY }}-*-xgcc/*.tar; do
            ARCH=$( basename "${f}" | sed -E 's/coreboot\-[0-9\.]+\-[a-z0-9]+\-([a-z0-9]+)\-.*/\1/g' )
            echo "extracting ${f} -> ${{ steps.version.outputs.COREBOOT_VERSION }} / ${ARCH}"
            mkdir -p "${f}.dir/"
            tar -xf "${f}" -C "${f}.dir/"
            mv "${f}.dir/coreboot/util/crossgcc/${ARCH}-xgcc" "docker/coreboot/coreboot-${{ steps.version.outputs.COREBOOT_VERSION }}/xgcc-${ARCH}"
          done
      - name: Prepare utils
        if: startsWith(matrix.dockerfile, 'coreboot')
        run: |
          for f in ${{ steps.cache-key.outputs.CACHE_KEY }}-*-utils; do
            ARCH=$( basename "${f}" | sed -E 's/coreboot\-[0-9\.]+\-[a-z0-9]+\-([a-z0-9]+)\-.*/\1/g' )
            echo "${f} -> ${{ steps.version.outputs.COREBOOT_VERSION }} / ${ARCH}"
            chmod +rx "${f}"/*
            mv "${f}" "docker/coreboot/coreboot-${{ steps.version.outputs.COREBOOT_VERSION }}/utils-${ARCH}"
          done

      - name: Debug list artifacts
        if: startsWith(matrix.dockerfile, 'coreboot')
        run: |
          ls -a1lh docker/coreboot/coreboot-${{ steps.version.outputs.COREBOOT_VERSION }}/
      - name: Debug list xgcc (amd64)
        if: startsWith(matrix.dockerfile, 'coreboot')
        run: |
          ls -a1lh docker/coreboot/coreboot-${{ steps.version.outputs.COREBOOT_VERSION }}/xgcc-*
      - name: Debug list xgcc/bin (amd64)
        if: startsWith(matrix.dockerfile, 'coreboot')
        run: |
          ls -a1lh docker/coreboot/coreboot-${{ steps.version.outputs.COREBOOT_VERSION }}/xgcc-*/bin
      - name: Debug list utils (amd64)
        if: startsWith(matrix.dockerfile, 'coreboot')
        run: |
          ls -a1lh docker/coreboot/coreboot-${{ steps.version.outputs.COREBOOT_VERSION }}/utils-*

      #============================
      # Build the docker container
      #============================

      - name: Setup docker-compose
        uses: KengoTODA/actions-setup-docker-compose@v1
        env:
          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

      - name: Validate compose file
        run: docker-compose -f docker/compose.yaml config

      - name: Install python dependencies
        run: pip install -r ./.dagger-ci/daggerci/requirements.txt

      - name: Run dagger pipeline
        timeout-minutes: 60
        run: |
          if [[ "${GITHUB_EVENT_NAME}" == 'release' ]] || [[ "${GITHUB_REF}" == *'main' ]] || [[ "${GITHUB_REF_TYPE}" == 'tag' ]]; then
            echo "Enable publishing"
            python .dagger-ci/daggerci/main.py -d ${{ matrix.dockerfile }} --publish
          else
            echo "Disable publishing"
            python .dagger-ci/daggerci/main.py -d ${{ matrix.dockerfile }}
          fi
        shell: bash
        env:
          GITHUB_REGISTRY: ${{ env.REGISTRY }}
          GITHUB_ACTOR: ${{ github.actor }}
          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}