Merge pull request #3583 from apostasie/optimize-build-time

GHA cache image build dependencies
containerd · Nov 2, 2024 · b00f8cd · b00f8cd
2 parents cabd5bf + de193b3
commit b00f8cd
Show file tree

Hide file tree

Showing 2 changed files with 111 additions and 37 deletions.
diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
@@ -15,7 +15,47 @@ env:
   LONG_TIMEOUT: 60
 
 jobs:
+  # This job builds the dependency target of the test docker image for all supported architectures and cache it in GHA
+  build-dependencies:
+    timeout-minutes: 10
+    name: dependencies |  ${{ matrix.containerd }} | ${{ matrix.arch }}
+    runs-on: "${{ matrix.runner }}"
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+          - runner: ubuntu-24.04
+            containerd: v1.6.36
+            arch: amd64
+          - runner: ubuntu-24.04
+            containerd: v1.7.23
+            arch: amd64
+          - runner: ubuntu-24.04
+            containerd: v2.0.0-rc.5
+            arch: amd64
+          - runner: arm64-8core-32gb
+            containerd: v2.0.0-rc.5
+            arch: arm64
+    env:
+      CONTAINERD_VERSION: "${{ matrix.containerd }}"
+      ARCH: "${{ matrix.arch }}"
+    steps:
+      - uses: actions/checkout@v4.2.1
+        with:
+          fetch-depth: 1
+      - name: "Expose GitHub Runtime variables for gha"
+        uses: crazy-max/ghaction-github-runtime@v3
+      - name: "Build dependencies for the integration test environment image"
+        run: |
+          docker buildx create --name with-gha --use
+          docker buildx build \
+            --output=type=docker \
+            --cache-to type=gha,mode=max,scope=${ARCH}-${CONTAINERD_VERSION} \
+            --cache-from type=gha,scope=${ARCH}-${CONTAINERD_VERSION} \
+            --target build-dependencies --build-arg CONTAINERD_VERSION=${CONTAINERD_VERSION} .
+
   test-unit:
+    # FIXME:
     # Supposed to work: https://docs.github.com/en/actions/writing-workflows/choosing-what-your-workflow-does/evaluate-expressions-in-workflows-and-actions#example-returning-a-json-data-type
     # Apparently does not
     # timeout-minutes: ${{ fromJSON(env.SHORT_TIMEOUT) }}
@@ -56,7 +96,8 @@ jobs:
         run: make test-unit
 
   test-integration:
-    timeout-minutes: 60
+    needs: build-dependencies
+    timeout-minutes: 30
     name: rootful | ${{ matrix.containerd }} | ${{ matrix.runner }}
     runs-on: "${{ matrix.runner }}"
     strategy:
@@ -67,24 +108,36 @@ jobs:
           - ubuntu: 20.04
             containerd: v1.6.36
             runner: "ubuntu-20.04"
+            arch: amd64
           - ubuntu: 22.04
             containerd: v1.7.23
             runner: "ubuntu-22.04"
+            arch: amd64
           - ubuntu: 24.04
             containerd: v2.0.0-rc.5
             runner: "ubuntu-24.04"
+            arch: amd64
           - ubuntu: 24.04
             containerd: v2.0.0-rc.5
-            runner: github-arm64-2c-8gb
+            runner: arm64-8core-32gb
+            arch: arm64
     env:
-      UBUNTU_VERSION: "${{ matrix.ubuntu }}"
       CONTAINERD_VERSION: "${{ matrix.containerd }}"
+      ARCH: "${{ matrix.arch }}"
+      UBUNTU_VERSION: "${{ matrix.ubuntu }}"
     steps:
       - uses: actions/checkout@v4.2.2
         with:
           fetch-depth: 1
+      - name: "Expose GitHub Runtime variables for gha"
+        uses: crazy-max/ghaction-github-runtime@v3
       - name: "Prepare integration test environment"
-        run: docker build -t test-integration --target test-integration --build-arg UBUNTU_VERSION=${UBUNTU_VERSION} --build-arg CONTAINERD_VERSION=${CONTAINERD_VERSION} .
+        run: |
+          docker buildx create --name with-gha --use
+          docker buildx build \
+            --output=type=docker \
+            --cache-from type=gha,scope=${ARCH}-${CONTAINERD_VERSION} \
+            -t test-integration --target test-integration --build-arg UBUNTU_VERSION=${UBUNTU_VERSION} --build-arg CONTAINERD_VERSION=${CONTAINERD_VERSION} .
       - name: "Remove snap loopback devices (conflicts with our loopback devices in TestRunDevice)"
         run: |
           sudo systemctl disable --now snapd.service snapd.socket
@@ -106,19 +159,21 @@ jobs:
         run: docker run -t --rm --privileged test-integration ./hack/test-integration.sh -test.only-flaky=true
 
   test-integration-ipv6:
-    timeout-minutes: 60
+    needs: build-dependencies
+    timeout-minutes: 10
     name: ipv6 | ${{ matrix.containerd }} | ${{ matrix.ubuntu }}
     runs-on: "ubuntu-${{ matrix.ubuntu }}"
     strategy:
       fail-fast: false
       matrix:
-        # ubuntu-20.04: cgroup v1, ubuntu-22.04 and later: cgroup v2
         include:
           - ubuntu: 24.04
             containerd: v2.0.0-rc.5
+            arch: amd64
     env:
-      UBUNTU_VERSION: "${{ matrix.ubuntu }}"
       CONTAINERD_VERSION: "${{ matrix.containerd }}"
+      ARCH: "${{ matrix.arch }}"
+      UBUNTU_VERSION: "${{ matrix.ubuntu }}"
     steps:
       - uses: actions/checkout@v4.2.2
         with:
@@ -127,13 +182,20 @@ jobs:
         run: |
           sudo sysctl -w net.ipv6.conf.all.forwarding=1
           sudo sysctl -w net.ipv4.ip_forward=1
-      - name: Enable IPv6 for Docker
+      - name: "Expose GitHub Runtime variables for gha"
+        uses: crazy-max/ghaction-github-runtime@v3
+      - name: Enable IPv6 for Docker, and configure docker to use containerd for gha
         run: |
           sudo mkdir -p /etc/docker
           echo '{"ipv6": true, "fixed-cidr-v6": "2001:db8:1::/64", "experimental": true, "ip6tables": true}' | sudo tee /etc/docker/daemon.json
           sudo systemctl restart docker
       - name: "Prepare integration test environment"
-        run: docker build -t test-integration --target test-integration --build-arg UBUNTU_VERSION=${UBUNTU_VERSION} --build-arg CONTAINERD_VERSION=${CONTAINERD_VERSION} .
+        run: |
+          docker buildx create --name with-gha --use
+          docker buildx build \
+            --output=type=docker \
+            --cache-from type=gha,scope=${ARCH}-${CONTAINERD_VERSION} \
+            -t test-integration --target test-integration --build-arg UBUNTU_VERSION=${UBUNTU_VERSION} --build-arg CONTAINERD_VERSION=${CONTAINERD_VERSION} .
       - name: "Remove snap loopback devices (conflicts with our loopback devices in TestRunDevice)"
         run: |
           sudo systemctl disable --now snapd.service snapd.socket
@@ -158,7 +220,8 @@ jobs:
         run: docker run --network host -t --rm --privileged test-integration ./hack/test-integration.sh -test.only-ipv6
 
   test-integration-rootless:
-    timeout-minutes: 60
+    needs: build-dependencies
+    timeout-minutes: 30
     name: "${{ matrix.target }} | ${{ matrix.containerd }} | ${{ matrix.rootlesskit }} | ${{ matrix.ubuntu }}"
     runs-on: "ubuntu-${{ matrix.ubuntu }}"
     strategy:
@@ -170,21 +233,26 @@ jobs:
             containerd: v1.6.36
             rootlesskit: v1.1.1  # Deprecated
             target: rootless
+            arch: amd64
           - ubuntu: 22.04
             containerd: v1.7.23
             rootlesskit: v2.3.1
             target: rootless
+            arch: amd64
           - ubuntu: 24.04
             containerd: v2.0.0-rc.5
             rootlesskit: v2.3.1
             target: rootless
+            arch: amd64
           - ubuntu: 24.04
             containerd: v1.7.23
             rootlesskit: v2.3.1
             target: rootless-port-slirp4netns
+            arch: amd64
     env:
-      UBUNTU_VERSION: "${{ matrix.ubuntu }}"
       CONTAINERD_VERSION: "${{ matrix.containerd }}"
+      ARCH: "${{ matrix.arch }}"
+      UBUNTU_VERSION: "${{ matrix.ubuntu }}"
       ROOTLESSKIT_VERSION: "${{ matrix.rootlesskit }}"
       TEST_TARGET: "test-integration-${{ matrix.target }}"
     steps:
@@ -215,8 +283,15 @@ jobs:
           docker run --privileged --rm tonistiigi/binfmt --install linux/amd64
           docker run --privileged --rm tonistiigi/binfmt --install linux/arm64
           docker run --privileged --rm tonistiigi/binfmt --install linux/arm/v7
+      - name: "Expose GitHub Runtime variables for gha"
+        uses: crazy-max/ghaction-github-runtime@v3
       - name: "Prepare (network driver=slirp4netns, port driver=builtin)"
-        run: docker build -t ${TEST_TARGET} --target ${TEST_TARGET} --build-arg UBUNTU_VERSION=${UBUNTU_VERSION} --build-arg CONTAINERD_VERSION=${CONTAINERD_VERSION} --build-arg ROOTLESSKIT_VERSION=${ROOTLESSKIT_VERSION} .
+        run: |
+          docker buildx create --name with-gha --use
+          docker buildx build \
+            --output=type=docker \
+            --cache-from type=gha,scope=${ARCH}-${CONTAINERD_VERSION} \
+            -t ${TEST_TARGET} --target ${TEST_TARGET} --build-arg UBUNTU_VERSION=${UBUNTU_VERSION} --build-arg CONTAINERD_VERSION=${CONTAINERD_VERSION} --build-arg ROOTLESSKIT_VERSION=${ROOTLESSKIT_VERSION} .
       - name: "Disable BuildKit for RootlessKit v1 (workaround for issue #622)"
         run: |
           # https://github.com/containerd/nerdctl/issues/622
@@ -250,7 +325,7 @@ jobs:
         run: GO_VERSION="$(echo ${{ matrix.go-version }} | sed -e s/.x//)" make binaries
 
   test-integration-docker-compatibility:
-    timeout-minutes: 60
+    timeout-minutes: 30
     name: docker
     runs-on: ubuntu-24.04
     steps:
@@ -262,11 +337,6 @@ jobs:
           go-version: ${{ env.GO_VERSION }}
           cache: true
           check-latest: true
-      - name: "Print docker info"
-        run: |
-          set -eux -o pipefail
-          docker info
-          docker version
       - name: "Register QEMU (tonistiigi/binfmt)"
         run: |
           # `--install all` will only install emulation for architectures that cannot be natively executed
@@ -324,13 +394,14 @@ jobs:
         run: ./hack/test-integration.sh -test.only-flaky=true
 
   test-integration-freebsd:
-    timeout-minutes: 60
+    timeout-minutes: 30
     name: FreeBSD
     # ubuntu-24.04 lacks the vagrant package
     runs-on: ubuntu-22.04
-
     steps:
       - uses: actions/checkout@v4.2.2
+        with:
+          fetch-depth: 1
       - uses: actions/cache@v4
         with:
           path: /root/.vagrant.d

diff --git a/Dockerfile b/Dockerfile
@@ -69,6 +69,7 @@ RUN xx-apt-get update -qq && xx-apt-get install -qq --no-install-recommends \
     libbtrfs-dev \
     libseccomp-dev \
     pkg-config
+RUN git config --global advice.detachedHead false
 
 FROM build-base-debian AS build-containerd
 ARG TARGETARCH
@@ -117,25 +118,19 @@ RUN xx-go --wrap && \
 
 FROM --platform=$BUILDPLATFORM golang:${GO_VERSION}-alpine AS build-base
 RUN apk add --no-cache make git curl
-COPY . /go/src/github.com/containerd/nerdctl
-WORKDIR /go/src/github.com/containerd/nerdctl
+RUN git config --global advice.detachedHead false
 
 FROM build-base AS build-minimal
 RUN BINDIR=/out/bin make binaries install
 # We do not set CMD to `go test` here, because it requires systemd
 
-FROM build-base AS build-full
+FROM build-base AS build-dependencies
 ARG TARGETARCH
 ENV GOARCH=${TARGETARCH}
-RUN BINDIR=/out/bin make binaries install
-WORKDIR /nowhere
 COPY ./Dockerfile.d/SHA256SUMS.d/ /SHA256SUMS.d
-COPY README.md /out/share/doc/nerdctl/
-COPY docs /out/share/doc/nerdctl/docs
+WORKDIR /nowhere
 RUN echo "${TARGETARCH:-amd64}" | sed -e s/amd64/x86_64/ -e s/arm64/aarch64/ | tee /target_uname_m
-RUN mkdir -p /out/share/doc/nerdctl-full && \
-  echo "# nerdctl (full distribution)" > /out/share/doc/nerdctl-full/README.md && \
-  echo "- nerdctl: $(cd /go/src/github.com/containerd/nerdctl && git describe --tags)" >> /out/share/doc/nerdctl-full/README.md
+RUN mkdir -p /out/share/doc/nerdctl-full && touch /out/share/doc/nerdctl-full/README.md
 ARG CONTAINERD_VERSION
 COPY --from=build-containerd /out/${TARGETARCH:-amd64}/* /out/bin/
 COPY --from=build-containerd /out/containerd.service /out/lib/systemd/system/containerd.service
@@ -181,13 +176,6 @@ RUN git clone https://github.com/containerd/imgcrypt.git /go/src/github.com/cont
   git checkout "${IMGCRYPT_VERSION}" && \
   CGO_ENABLED=0 make && DESTDIR=/out make install && \
   echo "- imgcrypt: ${IMGCRYPT_VERSION}" >> /out/share/doc/nerdctl-full/README.md
-ARG ROOTLESSKIT_VERSION
-RUN fname="rootlesskit-$(cat /target_uname_m).tar.gz" && \
-  curl -o "${fname}" -fsSL --proto '=https' --tlsv1.2 "https://github.com/rootless-containers/rootlesskit/releases/download/${ROOTLESSKIT_VERSION}/${fname}" && \
-  grep "${fname}" "/SHA256SUMS.d/rootlesskit-${ROOTLESSKIT_VERSION}" | sha256sum -c && \
-  tar xzf "${fname}" -C /out/bin && \
-  rm -f "${fname}" /out/bin/rootlesskit-docker-proxy && \
-  echo "- RootlessKit: ${ROOTLESSKIT_VERSION}" >> /out/share/doc/nerdctl-full/README.md
 ARG SLIRP4NETNS_VERSION
 RUN fname="slirp4netns-$(cat /target_uname_m)" && \
   curl -o "${fname}" -fsSL --proto '=https' --tlsv1.2 "https://github.com/rootless-containers/slirp4netns/releases/download/${SLIRP4NETNS_VERSION}/${fname}" && \
@@ -225,6 +213,13 @@ RUN fname="buildg-${BUILDG_VERSION}-${TARGETOS:-linux}-${TARGETARCH:-amd64}.tar.
   tar xzf "${fname}" -C /out/bin && \
   rm -f "${fname}" && \
   echo "- buildg: ${BUILDG_VERSION}" >> /out/share/doc/nerdctl-full/README.md
+ARG ROOTLESSKIT_VERSION
+RUN fname="rootlesskit-$(cat /target_uname_m).tar.gz" && \
+  curl -o "${fname}" -fsSL --proto '=https' --tlsv1.2 "https://github.com/rootless-containers/rootlesskit/releases/download/${ROOTLESSKIT_VERSION}/${fname}" && \
+  grep "${fname}" "/SHA256SUMS.d/rootlesskit-${ROOTLESSKIT_VERSION}" | sha256sum -c && \
+  tar xzf "${fname}" -C /out/bin && \
+  rm -f "${fname}" /out/bin/rootlesskit-docker-proxy && \
+  echo "- RootlessKit: ${ROOTLESSKIT_VERSION}" >> /out/share/doc/nerdctl-full/README.md
 
 RUN echo "" >> /out/share/doc/nerdctl-full/README.md && \
   echo "## License" >> /out/share/doc/nerdctl-full/README.md && \
@@ -237,6 +232,14 @@ RUN echo "" >> /out/share/doc/nerdctl-full/README.md && \
   mv /tmp/SHA256SUMS /out/share/doc/nerdctl-full/SHA256SUMS && \
   chown -R 0:0 /out
 
+FROM build-dependencies AS build-full
+COPY . /go/src/github.com/containerd/nerdctl
+RUN { echo "# nerdctl (full distribution)"; echo "- nerdctl: $(cd /go/src/github.com/containerd/nerdctl && git describe --tags)"; cat /out/share/doc/nerdctl-full/README.md; } > /out/share/doc/nerdctl-full/README.md.new; mv /out/share/doc/nerdctl-full/README.md.new /out/share/doc/nerdctl-full/README.md
+WORKDIR /go/src/github.com/containerd/nerdctl
+RUN BINDIR=/out/bin make binaries install
+COPY README.md /out/share/doc/nerdctl/
+COPY docs /out/share/doc/nerdctl/docs
+
 FROM scratch AS out-full
 COPY --from=build-full /out /