From 33245e5f870aaa001a675369beaeace9cee2819a Mon Sep 17 00:00:00 2001 From: Florent Poinsard <35779988+frouioui@users.noreply.github.com> Date: Wed, 18 Oct 2023 10:30:05 -0500 Subject: [PATCH] Build and push Docker `vitess/base` and component images to DockerHub from GitHub Actions (#14271) --- .github/workflows/docker_build_base.yml | 261 ++++++++++++++++++++++++ .github/workflows/docker_build_lite.yml | 2 +- doc/internal/release/how-to-release.md | 5 +- docker/release.sh | 73 ------- tools/back_to_dev_mode.sh | 1 - tools/create_release.sh | 1 - tools/release_utils.sh | 5 - 7 files changed, 263 insertions(+), 85 deletions(-) create mode 100644 .github/workflows/docker_build_base.yml delete mode 100755 docker/release.sh diff --git a/.github/workflows/docker_build_base.yml b/.github/workflows/docker_build_base.yml new file mode 100644 index 00000000000..af459215099 --- /dev/null +++ b/.github/workflows/docker_build_base.yml @@ -0,0 +1,261 @@ +name: Docker Build Base +on: + push: + branches: + - main + tags: + - '*' + +concurrency: + group: format('{0}-{1}', ${{ github.ref }}, 'Docker Build Base') + cancel-in-progress: true + +permissions: read-all + +jobs: + build_and_push_base: + name: Build and push vitess/base Docker images + runs-on: gh-hosted-runners-16cores-1 + if: github.repository == 'vitessio/vitess' + + strategy: + fail-fast: true + matrix: + branch: [ latest, mysql57, percona57, percona80 ] + + steps: + - name: Check out code + uses: actions/checkout@v3 + + - name: Login to Docker Hub + uses: docker/login-action@v3 + with: + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_TOKEN }} + + - name: Set Dockerfile path + run: | + if [[ "${{ matrix.branch }}" == "latest" ]]; then + echo "DOCKERFILE=./docker/base/Dockerfile" >> $GITHUB_ENV + else + echo "DOCKERFILE=./docker/base/Dockerfile.${{ matrix.branch }}" >> $GITHUB_ENV + fi + + - name: Build and push on main + if: github.ref == 'refs/heads/main' + uses: docker/build-push-action@v5 + with: + context: . + file: ${{ env.DOCKERFILE }} + push: true + tags: vitess/base:${{ matrix.branch }} + + ###### + # All code below only applies to new tags + ###### + + - name: Get the Git tag + if: startsWith(github.ref, 'refs/tags/') + run: echo "TAG_NAME=${GITHUB_REF#refs/tags/}" >> $GITHUB_ENV + + - name: Set Docker tag name + if: startsWith(github.ref, 'refs/tags/') && matrix.branch == 'latest' + run: | + if [[ "${{ matrix.branch }}" == "latest" ]]; then + echo "DOCKER_TAG=vitess/base:${TAG_NAME}" >> $GITHUB_ENV + fi + + - name: Build and push on tags + if: startsWith(github.ref, 'refs/tags/') && matrix.branch == 'latest' + uses: docker/build-push-action@v5 + with: + context: . + file: ${{ env.DOCKERFILE }} + push: true + tags: ${{ env.DOCKER_TAG }} + + build_and_push_k8s: + needs: build_and_push_base + name: Build and push vitess/k8s image + runs-on: gh-hosted-runners-16cores-1 + if: github.repository == 'vitessio/vitess' + + strategy: + fail-fast: true + matrix: + debian: [ bullseye, bookworm ] + + steps: + - name: Check out code + uses: actions/checkout@v3 + + - name: Login to Docker Hub + uses: docker/login-action@v3 + with: + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_TOKEN }} + + - name: Set Docker context path + run: | + echo "DOCKER_CTX=./docker/k8s" >> $GITHUB_ENV + + - name: Build and push on main latest tag + if: github.ref == 'refs/heads/main' && matrix.debian == 'bookworm' + uses: docker/build-push-action@v5 + with: + context: ${{ env.DOCKER_CTX }} + push: true + tags: vitess/k8s:latest + build-args: | + VT_BASE_VER=latest + DEBIAN_VER=${{ matrix.debian }}-slim + + - name: Build and push on main debian specific tag + if: github.ref == 'refs/heads/main' + uses: docker/build-push-action@v5 + with: + context: ${{ env.DOCKER_CTX }} + push: true + tags: vitess/k8s:latest-${{ matrix.debian }} + build-args: | + VT_BASE_VER=latest + DEBIAN_VER=${{ matrix.debian }}-slim + + ###### + # All code below only applies to new tags + ###### + + - name: Get the Git tag + if: startsWith(github.ref, 'refs/tags/') + run: echo "TAG_NAME=${GITHUB_REF#refs/tags/}" >> $GITHUB_ENV + + # We push git-tag-based k8s image to three tags, i.e. for 'v19.0.0' we push to: + # + # vitess/k8s:v19.0.0 (DOCKER_TAG_DEFAULT_DEBIAN) + # vitess/k8s:v19.0.0-bookworm (DOCKER_TAG) + # vitess/k8s:v19.0.0-bullseye (DOCKER_TAG) + # + - name: Set Docker tag name + if: startsWith(github.ref, 'refs/tags/') + run: | + echo "DOCKER_TAG_DEFAULT_DEBIAN=vitess/k8s:${TAG_NAME}" >> $GITHUB_ENV + echo "DOCKER_TAG=vitess/k8s:${TAG_NAME}-${{ matrix.debian }}" >> $GITHUB_ENV + + # Build and Push component image to DOCKER_TAG, applies to both debian version + - name: Build and push on tags using Debian extension + if: startsWith(github.ref, 'refs/tags/') + uses: docker/build-push-action@v5 + with: + context: ${{ env.DOCKER_CTX }} + push: true + tags: ${{ env.DOCKER_TAG }} + build-args: | + VT_BASE_VER=${{ env.TAG_NAME }} + DEBIAN_VER=${{ matrix.debian }}-slim + + # Build and Push component image to DOCKER_TAG_DEFAULT_DEBIAN, only applies when building the default Debian version (bookworm) + # It is fine to build a second time here when "matrix.debian == 'bookworm'" as we have cached the first build already + - name: Build and push on tags without Debian extension + if: startsWith(github.ref, 'refs/tags/') && matrix.debian == 'bookworm' + uses: docker/build-push-action@v5 + with: + context: ${{ env.DOCKER_CTX }} + push: true + tags: ${{ env.DOCKER_TAG_DEFAULT_DEBIAN }} + build-args: | + VT_BASE_VER=${{ env.TAG_NAME }} + DEBIAN_VER=${{ matrix.debian }}-slim + + + build_and_push_components: + needs: build_and_push_k8s + name: Build and push vitess components Docker images + runs-on: gh-hosted-runners-16cores-1 + if: github.repository == 'vitessio/vitess' + + strategy: + fail-fast: true + matrix: + debian: [ bullseye, bookworm ] + component: [ vtadmin, vtorc, vtgate, vttablet, mysqlctld, mysqlctl, vtctl, vtctlclient, vtctld, logrotate, logtail ] + + steps: + - name: Check out code + uses: actions/checkout@v3 + + - name: Login to Docker Hub + uses: docker/login-action@v3 + with: + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_TOKEN }} + + - name: Set Docker context path + run: | + echo "DOCKER_CTX=./docker/k8s/${{ matrix.component }}" >> $GITHUB_ENV + + - name: Build and push on main latest tag + if: github.ref == 'refs/heads/main' && matrix.debian == 'bookworm' + uses: docker/build-push-action@v5 + with: + context: ${{ env.DOCKER_CTX }} + push: true + tags: vitess/${{ matrix.component }}:latest + build-args: | + VT_BASE_VER=latest + DEBIAN_VER=${{ matrix.debian }}-slim + + - name: Build and push on main debian specific tag + if: github.ref == 'refs/heads/main' + uses: docker/build-push-action@v5 + with: + context: ${{ env.DOCKER_CTX }} + push: true + tags: vitess/${{ matrix.component }}:latest-${{ matrix.debian }} + build-args: | + VT_BASE_VER=latest + DEBIAN_VER=${{ matrix.debian }}-slim + + ###### + # All code below only applies to new tags + ###### + + - name: Get the Git tag + if: startsWith(github.ref, 'refs/tags/') + run: echo "TAG_NAME=${GITHUB_REF#refs/tags/}" >> $GITHUB_ENV + + # We push git-tag-based images to three tags, i.e. for 'v19.0.0' we push to: + # + # vitess/${{ matrix.component }}:v19.0.0 (DOCKER_TAG_DEFAULT_DEBIAN) + # vitess/${{ matrix.component }}:v19.0.0-bookworm (DOCKER_TAG) + # vitess/${{ matrix.component }}:v19.0.0-bullseye (DOCKER_TAG) + # + - name: Set Docker tag name + if: startsWith(github.ref, 'refs/tags/') + run: | + echo "DOCKER_TAG_DEFAULT_DEBIAN=vitess/${{ matrix.component }}:${TAG_NAME}" >> $GITHUB_ENV + echo "DOCKER_TAG=vitess/${{ matrix.component }}:${TAG_NAME}-${{ matrix.debian }}" >> $GITHUB_ENV + + # Build and Push component image to DOCKER_TAG, applies to both debian version + - name: Build and push on tags using Debian extension + if: startsWith(github.ref, 'refs/tags/') + uses: docker/build-push-action@v5 + with: + context: ${{ env.DOCKER_CTX }} + push: true + tags: ${{ env.DOCKER_TAG }} + build-args: | + VT_BASE_VER=${{ env.TAG_NAME }} + DEBIAN_VER=${{ matrix.debian }}-slim + + # Build and Push component image to DOCKER_TAG_DEFAULT_DEBIAN, only applies when building the default Debian version (bookworm) + # It is fine to build a second time here when "matrix.debian == 'bookworm'" as we have cached the first build already + - name: Build and push on tags without Debian extension + if: startsWith(github.ref, 'refs/tags/') && matrix.debian == 'bookworm' + uses: docker/build-push-action@v5 + with: + context: ${{ env.DOCKER_CTX }} + push: true + tags: ${{ env.DOCKER_TAG_DEFAULT_DEBIAN }} + build-args: | + VT_BASE_VER=${{ env.TAG_NAME }} + DEBIAN_VER=${{ matrix.debian }}-slim diff --git a/.github/workflows/docker_build_lite.yml b/.github/workflows/docker_build_lite.yml index 54c2bcb2419..7f355ddfd32 100644 --- a/.github/workflows/docker_build_lite.yml +++ b/.github/workflows/docker_build_lite.yml @@ -63,7 +63,7 @@ jobs: echo "DOCKER_TAG=vitess/lite:${TAG_NAME}-${{ matrix.branch }}" >> $GITHUB_ENV fi - - name: Build and push on main + - name: Build and push on tags if: startsWith(github.ref, 'refs/tags/') uses: docker/build-push-action@v5 with: diff --git a/doc/internal/release/how-to-release.md b/doc/internal/release/how-to-release.md index 1d463c088da..fd5caa81e03 100644 --- a/doc/internal/release/how-to-release.md +++ b/doc/internal/release/how-to-release.md @@ -127,10 +127,7 @@ On the release day, there are several things to do: > - The benchmarks need to complete before announcing the blog posts or before they get cross-posted. - **Go back to dev mode on the release branch.** > - The version constants across the codebase must be updated to `SNAPSHOT`. -- **Build k8s Docker images and publish them.** - > - The docker image for `base`, `lite`, etc are built automatically by DockerHub. The k8s images however are dependent on these images and are required to be built manually. - > - These images should be built after the `base` image has been built and available on DockerHub. - > - To build and publish these images, checkout the new release tag that was just created and run `./release.sh` from the directory `./docker`. +- **Ensure the k8s images are available on DockerHub.** - **Close the current GitHub Milestone** > - Once we are done releasing the current version, we must close its corresponding GitHub Milestone as the development cycle for it is over. > - **This does not apply if we are releasing an RC release.** For instance, if we are releasing `v18.0.0-rc1` we want to keep the `v18.0.0` milestone opened as development is not fully done for `v18.0.0`. diff --git a/docker/release.sh b/docker/release.sh deleted file mode 100755 index 938cf91f1a4..00000000000 --- a/docker/release.sh +++ /dev/null @@ -1,73 +0,0 @@ -#!/bin/bash -set -ex - -vt_base_version='v18.0.0-SNAPSHOT' -debian_versions='bullseye bookworm' -default_debian_version='bookworm' - -docker pull --platform linux/amd64 vitess/base:$vt_base_version - -for debian_version in $debian_versions -do - echo "####### Building vitess/vt:$debian_version" - - docker build --platform linux/amd64 --build-arg VT_BASE_VER=$vt_base_version --build-arg DEBIAN_VER=$debian_version-slim -t vitess/k8s:$vt_base_version-$debian_version k8s - docker tag vitess/k8s:$vt_base_version-$debian_version vitess/k8s:$vt_base_version - docker push vitess/k8s:$vt_base_version-$debian_version - if [[ $debian_version == $default_debian_version ]]; then docker push vitess/k8s:$vt_base_version; fi - - docker build --platform linux/amd64 --build-arg VT_BASE_VER=$vt_base_version --build-arg DEBIAN_VER=$debian_version-slim -t vitess/vtadmin:$vt_base_version-$debian_version k8s/vtadmin - docker tag vitess/vtadmin:$vt_base_version-$debian_version vitess/vtadmin:$vt_base_version - docker push vitess/vtadmin:$vt_base_version-$debian_version - if [[ $debian_version == $default_debian_version ]]; then docker push vitess/vtadmin:$vt_base_version; fi - - docker build --platform linux/amd64 --build-arg VT_BASE_VER=$vt_base_version --build-arg DEBIAN_VER=$debian_version-slim -t vitess/vtorc:$vt_base_version-$debian_version k8s/vtorc - docker tag vitess/vtorc:$vt_base_version-$debian_version vitess/vtorc:$vt_base_version - docker push vitess/vtorc:$vt_base_version-$debian_version - if [[ $debian_version == $default_debian_version ]]; then docker push vitess/vtorc:$vt_base_version; fi - - docker build --platform linux/amd64 --build-arg VT_BASE_VER=$vt_base_version --build-arg DEBIAN_VER=$debian_version-slim -t vitess/vtgate:$vt_base_version-$debian_version k8s/vtgate - docker tag vitess/vtgate:$vt_base_version-$debian_version vitess/vtgate:$vt_base_version - docker push vitess/vtgate:$vt_base_version-$debian_version - if [[ $debian_version == $default_debian_version ]]; then docker push vitess/vtgate:$vt_base_version; fi - - docker build --platform linux/amd64 --build-arg VT_BASE_VER=$vt_base_version --build-arg DEBIAN_VER=$debian_version-slim -t vitess/vttablet:$vt_base_version-$debian_version k8s/vttablet - docker tag vitess/vttablet:$vt_base_version-$debian_version vitess/vttablet:$vt_base_version - docker push vitess/vttablet:$vt_base_version-$debian_version - if [[ $debian_version == $default_debian_version ]]; then docker push vitess/vttablet:$vt_base_version; fi - - docker build --platform linux/amd64 --build-arg VT_BASE_VER=$vt_base_version --build-arg DEBIAN_VER=$debian_version-slim -t vitess/mysqlctld:$vt_base_version-$debian_version k8s/mysqlctld - docker tag vitess/mysqlctld:$vt_base_version-$debian_version vitess/mysqlctld:$vt_base_version - docker push vitess/mysqlctld:$vt_base_version-$debian_version - if [[ $debian_version == $default_debian_version ]]; then docker push vitess/mysqlctld:$vt_base_version; fi - - docker build --platform linux/amd64 --build-arg VT_BASE_VER=$vt_base_version --build-arg DEBIAN_VER=$debian_version-slim -t vitess/mysqlctl:$vt_base_version-$debian_version k8s/mysqlctl - docker tag vitess/mysqlctl:$vt_base_version-$debian_version vitess/mysqlctl:$vt_base_version - docker push vitess/mysqlctl:$vt_base_version-$debian_version - if [[ $debian_version == $default_debian_version ]]; then docker push vitess/mysqlctl:$vt_base_version; fi - - docker build --platform linux/amd64 --build-arg VT_BASE_VER=$vt_base_version --build-arg DEBIAN_VER=$debian_version-slim -t vitess/vtctl:$vt_base_version-$debian_version k8s/vtctl - docker tag vitess/vtctl:$vt_base_version-$debian_version vitess/vtctl:$vt_base_version - docker push vitess/vtctl:$vt_base_version-$debian_version - if [[ $debian_version == $default_debian_version ]]; then docker push vitess/vtctl:$vt_base_version; fi - - docker build --platform linux/amd64 --build-arg VT_BASE_VER=$vt_base_version --build-arg DEBIAN_VER=$debian_version-slim -t vitess/vtctlclient:$vt_base_version-$debian_version k8s/vtctlclient - docker tag vitess/vtctlclient:$vt_base_version-$debian_version vitess/vtctlclient:$vt_base_version - docker push vitess/vtctlclient:$vt_base_version-$debian_version - if [[ $debian_version == $default_debian_version ]]; then docker push vitess/vtctlclient:$vt_base_version; fi - - docker build --platform linux/amd64 --build-arg VT_BASE_VER=$vt_base_version --build-arg DEBIAN_VER=$debian_version-slim -t vitess/vtctld:$vt_base_version-$debian_version k8s/vtctld - docker tag vitess/vtctld:$vt_base_version-$debian_version vitess/vtctld:$vt_base_version - docker push vitess/vtctld:$vt_base_version-$debian_version - if [[ $debian_version == $default_debian_version ]]; then docker push vitess/vtctld:$vt_base_version; fi - - docker build --platform linux/amd64 --build-arg VT_BASE_VER=$vt_base_version --build-arg DEBIAN_VER=$debian_version-slim -t vitess/logrotate:$vt_base_version-$debian_version k8s/logrotate - docker tag vitess/logrotate:$vt_base_version-$debian_version vitess/logrotate:$vt_base_version - docker push vitess/logrotate:$vt_base_version-$debian_version - if [[ $debian_version == $default_debian_version ]]; then docker push vitess/logrotate:$vt_base_version; fi - - docker build --platform linux/amd64 --build-arg VT_BASE_VER=$vt_base_version --build-arg DEBIAN_VER=$debian_version-slim -t vitess/logtail:$vt_base_version-$debian_version k8s/logtail - docker tag vitess/logtail:$vt_base_version-$debian_version vitess/logtail:$vt_base_version - docker push vitess/logtail:$vt_base_version-$debian_version - if [[ $debian_version == $default_debian_version ]]; then docker push vitess/logtail:$vt_base_version; fi -done diff --git a/tools/back_to_dev_mode.sh b/tools/back_to_dev_mode.sh index 9f4cc25b71f..a3b945fd87d 100755 --- a/tools/back_to_dev_mode.sh +++ b/tools/back_to_dev_mode.sh @@ -48,7 +48,6 @@ fi function doBackToDevMode () { # Preparing the "dev mode" commit updateJava $DEV_VERSION - updateDockerReleaseScript $DEV_VERSION updateVersionGo $DEV_VERSION git add --all diff --git a/tools/create_release.sh b/tools/create_release.sh index 77c9cb9d423..68e051f884e 100755 --- a/tools/create_release.sh +++ b/tools/create_release.sh @@ -61,7 +61,6 @@ function createRelease () { # Preparing the release commit updateVitessExamples $RELEASE_VERSION $VTOP_VERSION updateJava $RELEASE_VERSION - updateDockerReleaseScript $RELEASE_VERSION updateVersionGo $RELEASE_VERSION ## Create the commit for this release and tag it diff --git a/tools/release_utils.sh b/tools/release_utils.sh index ff7d348bcd3..d94d2fe7f31 100755 --- a/tools/release_utils.sh +++ b/tools/release_utils.sh @@ -24,11 +24,6 @@ function checkGitState() { fi } -function updateDockerReleaseScript () { - sed -i.bak -E "s/vt_base_version=.*/vt_base_version='v$1'/g" $ROOT/docker/release.sh - rm -f $ROOT/docker/release.sh.bak -} - function checkoutNewBranch () { branch_name=$1