landing-builds.yml

name: Landings

# Run on landings, or when this action itself is changed.
on:
  push:
    branches:
      - master
      - 'release/*'
  pull_request:
    paths:
      - .github/workflows/landing-builds.yml
      - 'utils/docker/Dockerfile.*'
      - 'utils/scripts/install-*.sh'
      - 'utils/scripts/helpers/*'
      - utils/ci/**
      - ci/**
      - requirements-build.txt
      - requirements-utest.txt

permissions: {}

jobs:

  # Build a base Docker image, and save it with a key based on the hash of the dependencies, and a
  # copy of the git hash.  Do not attempt to load the cache here, but rather start each build
  # cleanly.  As the file hash comes before the git hash then each master build gets to populate
  # a new hash key, PRs then fetch based only on the file hash so they get the most recent master
  # build to complete.
  # This workflow only runs on landings, so the cache will only be built/saved when this changes,
  # builds for specific PRs will attempt to read this cache only, but not save anything.
  # For that reason set concurrency to 1 here for maximum reliability.
  # Set max-parallel and cache concurrency to 1 to avoid https 429 errors on cache save.
  # Distros where we build on PR should be covered here as the cache is populated for PRs to read,
  # for distros where we only want to build on master and not PRs see the Build-branch job below.
  Prepare:
    name: Create Docker images
    runs-on: ubuntu-22.04
    strategy:
      fail-fast: false
      max-parallel: 1
      matrix:
        distro: [ubuntu, rocky, fedora, leap.15]
        include:
          - distro: ubuntu
            base: ubuntu
            with: ubuntu:mantic
          - distro: rocky
            base: el.9
            with: rockylinux/rockylinux:9
          - distro: fedora
            base: el.8
            with: fedora:38
          - distro: leap.15
            base: leap.15
            with: opensuse/leap:15.5
    env:
      DEPS_JOBS: 10
      BASE_DISTRO: ${{ matrix.with }}
      DOCKER_BASE: ${{ matrix.base }}
    steps:
      - name: Checkout code
        uses: actions/checkout@v4
        with:
          fetch-depth: 500
      - name: Setup git hash
        run: ./ci/gha_helper.py --single
        id: commit-hash
      - name: Setup docker cache
        uses: jpribyl/action-docker-layer-caching@v0.1.1
        with:
          key: ${{ steps.commit-hash.outputs.key }}
          restore-keys: ${{ steps.commit-hash.outputs.restore }}
          concurrency: 1
      - name: Prepare base image in Docker
        run: docker build . --file utils/docker/Dockerfile.${{ matrix.base }}
                            --build-arg DAOS_DEPS_BUILD=no
                            --build-arg DEPS_JOBS
                            --build-arg BASE_DISTRO
      - name: Build dependencies in Docker
        run: docker build . --file utils/docker/Dockerfile.${{ matrix.base }}
                            --build-arg DAOS_BUILD=no
                            --build-arg DEPS_JOBS
                            --build-arg BASE_DISTRO
      - name: Prune images not required for build.
        run: docker images --all --filter label=DAOS=true --quiet | xargs docker rmi --no-prune

  Build-and-test:
    name: Run DAOS/NLT tests
    needs: Prepare
    runs-on: ubuntu-22.04
    permissions:
      # https://github.com/EnricoMi/publish-unit-test-result-action#permissions
      checks: write
      pull-requests: write
    strategy:
      matrix:
        distro: [ubuntu]
        include:
          - distro: ubuntu
            base: ubuntu
            with: ubuntu:mantic
    env:
      DEPS_JOBS: 10
      BASE_DISTRO: ${{ matrix.with }}
      DOCKER_BASE: ${{ matrix.base }}
      COMPILER: clang
    steps:
      - name: Checkout code
        uses: actions/checkout@v4
        with:
          submodules: true
          fetch-depth: 500
      - name: Setup git hash
        run: ./ci/gha_helper.py --single
        id: commit-hash
      - uses: jpribyl/action-docker-layer-caching@v0.1.1
        continue-on-error: true
        with:
          key: ${{ steps.commit-hash.outputs.key }}
          restore-keys: |
            ${{ steps.commit-hash.outputs.restore }}
            ${{ steps.commit-hash.outputs.restore_prev }}
          skip-save: true
      - name: Update dependencies in image.
        run: docker build . --file utils/docker/Dockerfile.${{ matrix.base }}
                            --build-arg DAOS_BUILD=no
                            --build-arg DEPS_JOBS
                            --build-arg BASE_DISTRO
                            --build-arg DAOS_KEEP_SRC=yes
                            --tag build-image
      - name: Build and Test
        run: docker run --name build-post --mount type=tmpfs,destination=/mnt/daos_0,tmpfs-mode=1777
              --env COMPILER --env DEPS_JOBS --user root:root build-image
              ./daos/utils/ci/run_in_gha.sh
      - name: Fetch results
        if: always()
        run: docker cp build-post:/home/daos/daos/nlt-junit.xml ./
      - name: Publish NLT test results
        if: always()
        uses: EnricoMi/publish-unit-test-result-action@v1.17
        with:
          github_token: ${{ secrets.GITHUB_TOKEN }}
          files: nlt-junit.xml
          comment_mode: off  # yamllint disable-line rule:truthy
          fail_on: nothing

  Build:
    name: Build DAOS in image
    needs: Prepare
    runs-on: ubuntu-22.04
    strategy:
      fail-fast: false
      max-parallel: 2
      matrix:
        distro: [ubuntu, rocky, fedora, leap.15]
        compiler: [clang, gcc]
        include:
          - distro: ubuntu
            base: ubuntu
            with: ubuntu:mantic
          - distro: rocky
            base: el.9
            with: rockylinux/rockylinux:9
          - distro: fedora
            base: el.8
            with: fedora:38
          - distro: leap.15
            base: leap.15
            with: opensuse/leap:15.5
    env:
      DEPS_JOBS: 10
      BASE_DISTRO: ${{ matrix.with }}
      DOCKER_BASE: ${{ matrix.base }}
      COMPILER: ${{ matrix.compiler }}
    steps:
      - name: Checkout code
        uses: actions/checkout@v4
        with:
          submodules: true
          fetch-depth: 500
      - name: Setup git hash
        run: ./ci/gha_helper.py --single
        id: commit-hash
      - uses: jpribyl/action-docker-layer-caching@v0.1.1
        continue-on-error: true
        with:
          key: ${{ steps.commit-hash.outputs.key }}
          restore-keys: |
            ${{ steps.commit-hash.outputs.restore }}
            ${{ steps.commit-hash.outputs.restore_prev }}
          skip-save: true
      - name: Build in docker.
        run: docker build . --file utils/docker/Dockerfile.${{ matrix.base }}
                            --build-arg DEPS_JOBS
                            --build-arg BASE_DISTRO
                            --build-arg DAOS_JAVA_BUILD=no
                            --build-arg COMPILER
                            --build-arg DAOS_KEEP_SRC=yes
                            --tag build-image
      - name: Build Java in docker.
        run: docker build . --file utils/docker/Dockerfile.${{ matrix.base }}
                            --build-arg DEPS_JOBS
                            --build-arg BASE_DISTRO
                            --build-arg DAOS_JAVA_BUILD=yes
                            --build-arg COMPILER
      - name: Build debug in docker.
        run: docker build . --file utils/docker/Dockerfile.${{ matrix.base }}
                            --build-arg DEPS_JOBS
                            --build-arg BASE_DISTRO
                            --build-arg DAOS_JAVA_BUILD=no
                            --build-arg DAOS_BUILD_TYPE=debug
                            --build-arg COMPILER
      - name: Build devel in docker.
        run: docker build . --file utils/docker/Dockerfile.${{ matrix.base }}
                            --build-arg DEPS_JOBS
                            --build-arg BASE_DISTRO
                            --build-arg DAOS_JAVA_BUILD=no
                            --build-arg DAOS_BUILD_TYPE=dev
                            --build-arg COMPILER
        # Fails with Ubuntu still for the spdk issue.
        # - name: Run NLT
        # run: docker run --mount type=tmpfs,destination=/mnt/daos_0,tmpfs-mode=1777
        #        --user root:root build-image ./daos/utils/node_local_test.py --no-root
        #        --memcheck no --test cont_copy

  Build-branch:
    name: Build DAOS
    runs-on: ubuntu-22.04
    strategy:
      fail-fast: false
      matrix:
        distro: [alma.8, alma.9, rocky.8, ubuntu]
        include:
          - distro: alma.8
            base: el.8
            with: almalinux:8
          - distro: alma.9
            base: el.9
            with: almalinux:9
          - distro: rocky.8
            base: el.8
            with: rockylinux/rockylinux:8
          - distro: ubuntu
            base: ubuntu
            with: ubuntu:22.04
    env:
      DEPS_JOBS: 10
      BASE_DISTRO: ${{ matrix.with }}
    steps:
      - name: Checkout code
        uses: actions/checkout@v4
        with:
          submodules: true
      - name: Build dependencies in image.
        run: docker build . --file utils/docker/Dockerfile.${{ matrix.base }}
                            --build-arg DEPS_JOBS
                            --build-arg BASE_DISTRO
                            --build-arg DAOS_BUILD=no
      - name: Build in docker with clang
        run: docker build . --file utils/docker/Dockerfile.${{ matrix.base }}
                            --build-arg DEPS_JOBS
                            --build-arg BASE_DISTRO
                            --build-arg DAOS_JAVA_BUILD=no
                            --build-arg COMPILER=clang
                            --build-arg DAOS_KEEP_SRC=yes
                            --tag build-image
      - name: Build Java in docker.
        run: docker build . --file utils/docker/Dockerfile.${{ matrix.base }}
                            --build-arg DEPS_JOBS
                            --build-arg BASE_DISTRO
                            --build-arg DAOS_JAVA_BUILD=yes
                            --build-arg COMPILER=clang
      - name: Build debug in docker with clang.
        run: docker build . --file utils/docker/Dockerfile.${{ matrix.base }}
                            --build-arg DEPS_JOBS
                            --build-arg BASE_DISTRO
                            --build-arg DAOS_JAVA_BUILD=no
                            --build-arg DAOS_BUILD_TYPE=debug
                            --build-arg COMPILER=clang
      - name: Build devel in docker with clang
        run: docker build . --file utils/docker/Dockerfile.${{ matrix.base }}
                            --build-arg DEPS_JOBS
                            --build-arg BASE_DISTRO
                            --build-arg DAOS_JAVA_BUILD=no
                            --build-arg DAOS_BUILD_TYPE=dev
                            --build-arg COMPILER=clang
      - name: Build in docker with gcc
        run: docker build . --file utils/docker/Dockerfile.${{ matrix.base }}
                            --build-arg DEPS_JOBS
                            --build-arg BASE_DISTRO
                            --build-arg DAOS_JAVA_BUILD=no
                            --build-arg COMPILER=gcc
      - name: Build debug in docker with gcc
        run: docker build . --file utils/docker/Dockerfile.${{ matrix.base }}
                            --build-arg DEPS_JOBS
                            --build-arg BASE_DISTRO
                            --build-arg DAOS_JAVA_BUILD=no
                            --build-arg DAOS_BUILD_TYPE=debug
                            --build-arg COMPILER=gcc
      - name: Build devel in docker with gcc
        run: docker build . --file utils/docker/Dockerfile.${{ matrix.base }}
                            --build-arg DEPS_JOBS
                            --build-arg BASE_DISTRO
                            --build-arg DAOS_JAVA_BUILD=no
                            --build-arg DAOS_BUILD_TYPE=dev
                            --build-arg COMPILER=gcc
      - name: Run NLT
        run: docker run --mount type=tmpfs,destination=/mnt/daos_0,tmpfs-mode=1777 --user root:root
                 build-image ./daos/utils/node_local_test.py --no-root
                 --memcheck no --test cont_copy --system-ram-reserved 1

  Build-branch-ARM:
    name: Build DAOS (ARM64)
    if: github.repository == 'daos-stack/daos'
    runs-on: [self-hosted, ARM64]
    strategy:
      fail-fast: false
      matrix:
        distro: [ubuntu, alma.8, leap.15]
        compiler: [clang]
        include:
          - distro: ubuntu
            base: ubuntu
            with: ubuntu:mantic
          - distro: alma.8
            base: el.8
            with: almalinux:8
          - distro: leap.15
            base: leap.15
            with: opensuse/leap:15.5
    env:
      DEPS_JOBS: 10
      BASE_DISTRO: ${{ matrix.with }}
      DOCKER_BASE: ${{ matrix.base }}
      COMPILER: ${{ matrix.compiler }}
    steps:
      - name: Checkout code
        uses: actions/checkout@v4
        with:
          submodules: true
      - name: Build dependencies in image.
        run: docker build . --file utils/docker/Dockerfile.${{ matrix.base }}
                            --build-arg DEPS_JOBS
                            --build-arg BASE_DISTRO
                            --build-arg DAOS_BUILD=no
      - name: Build in docker with clang
        run: docker build . --file utils/docker/Dockerfile.${{ matrix.base }}
                            --build-arg DEPS_JOBS
                            --build-arg BASE_DISTRO
                            --build-arg DAOS_JAVA_BUILD=no
                            --build-arg COMPILER=clang
                            --build-arg DAOS_KEEP_SRC=yes
                            --tag build-image
      - name: Build Java in docker.
        run: docker build . --file utils/docker/Dockerfile.${{ matrix.base }}
                            --build-arg DEPS_JOBS
                            --build-arg BASE_DISTRO
                            --build-arg DAOS_JAVA_BUILD=yes
                            --build-arg COMPILER=clang
      - name: Build debug in docker with clang.
        run: docker build . --file utils/docker/Dockerfile.${{ matrix.base }}
                            --build-arg DEPS_JOBS
                            --build-arg BASE_DISTRO
                            --build-arg DAOS_JAVA_BUILD=no
                            --build-arg DAOS_BUILD_TYPE=debug
                            --build-arg COMPILER=clang
      - name: Build devel in docker with clang
        run: docker build . --file utils/docker/Dockerfile.${{ matrix.base }}
                            --build-arg DEPS_JOBS
                            --build-arg BASE_DISTRO
                            --build-arg DAOS_JAVA_BUILD=no
                            --build-arg DAOS_BUILD_TYPE=dev
                            --build-arg COMPILER=clang
      - name: Build in docker with gcc
        run: docker build . --file utils/docker/Dockerfile.${{ matrix.base }}
                            --build-arg DEPS_JOBS
                            --build-arg BASE_DISTRO
                            --build-arg DAOS_JAVA_BUILD=no
                            --build-arg COMPILER=gcc
      - name: Build debug in docker with gcc
        run: docker build . --file utils/docker/Dockerfile.${{ matrix.base }}
                            --build-arg DEPS_JOBS
                            --build-arg BASE_DISTRO
                            --build-arg DAOS_JAVA_BUILD=no
                            --build-arg DAOS_BUILD_TYPE=debug
                            --build-arg COMPILER=gcc
      - name: Build devel in docker with gcc
        run: docker build . --file utils/docker/Dockerfile.${{ matrix.base }}
                            --build-arg DEPS_JOBS
                            --build-arg BASE_DISTRO
                            --build-arg DAOS_JAVA_BUILD=no
                            --build-arg DAOS_BUILD_TYPE=dev
                            --build-arg COMPILER=gcc
      - name: Run NLT
        run: docker run --mount type=tmpfs,destination=/mnt/daos_0,tmpfs-mode=1777 --user root:root
                 build-image ./daos/utils/node_local_test.py --no-root
                 --memcheck no --test cont_copy --system-ram-reserved 6