diff --git a/.github/workflows/gha.yml b/.github/workflows/gha.yml index e8c98108e80..126a8caee35 100644 --- a/.github/workflows/gha.yml +++ b/.github/workflows/gha.yml @@ -47,9 +47,6 @@ jobs: - name: Run the build run: cd $WORKDIR && ${{ matrix.CONFIG }} ./build-CI.sh - - name: Push the image - run: cd $WORKDIR && source ./set-vars.sh && ${{ matrix.CONFIG }} /bin/bash -c "if [[ -f ${CI_FILE_PUSH_IMAGE_TO_REPO} ]]; then images/push-image.sh; fi" - windows: name: Windows runs-on: windows-2022 diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml index 4e08c8fac24..a58d45f87ec 100644 --- a/.github/workflows/nightly.yml +++ b/.github/workflows/nightly.yml @@ -46,10 +46,7 @@ jobs: fetch-depth: 50 - name: Pull or rebuild the image - run: cd $WORKDIR && ${{ matrix.CONFIG }} ./pull-or-rebuild-image.sh + run: cd $WORKDIR && ${{ matrix.CONFIG }} ./pull-or-rebuild-image.sh rebuild - name: Run the build run: cd $WORKDIR && ${{ matrix.CONFIG }} ./build-CI.sh - - - name: Push the image - run: cd $WORKDIR && source ./set-vars.sh && ${{ matrix.CONFIG }} /bin/bash -c "if [[ -f ${CI_FILE_PUSH_IMAGE_TO_REPO} ]]; then images/push-image.sh; fi" diff --git a/utils/docker/images/build-image.sh b/utils/docker/images/build-image.sh index f2a60a9e66f..37c5a6fdaac 100755 --- a/utils/docker/images/build-image.sh +++ b/utils/docker/images/build-image.sh @@ -1,6 +1,6 @@ #!/usr/bin/env bash # SPDX-License-Identifier: BSD-3-Clause -# Copyright 2016-2020, Intel Corporation +# Copyright 2016-2023, Intel Corporation # # build-image.sh - prepares a Docker image with -based @@ -20,8 +20,8 @@ function usage { echo "Usage:" echo " build-image.sh " echo "where:" - echo " - can be for example 'ubuntu-19.10' provided "\ - "a Dockerfile named 'Dockerfile.ubuntu-19.10' "\ + echo " - can be for example 'ubuntu-22.04' provided "\ + "a Dockerfile named 'Dockerfile.ubuntu-22.04' "\ "exists in the current directory and" echo " - is a CPU architecture, for example 'x86_64'" } @@ -40,8 +40,9 @@ if [[ ! -f "Dockerfile.$OS_VER" ]]; then exit 1 fi -if [[ -z "${DOCKER_REPO}" ]]; then - echo "Error: DOCKER_REPO environment variable is not set" +if [[ -z "${DOCKER_REPO}" || -z "${IMG_VER}" ]]; then + echo "Error: DOCKER_REPO (${DOCKER_REPO}) or IMG_VER (${IMG_VER}) " \ + "environment variables are not set." exit 1 fi diff --git a/utils/docker/images/push-image.sh b/utils/docker/images/push-image.sh index 59597c8a228..d0b7774afd2 100755 --- a/utils/docker/images/push-image.sh +++ b/utils/docker/images/push-image.sh @@ -1,6 +1,6 @@ #!/usr/bin/env bash # SPDX-License-Identifier: BSD-3-Clause -# Copyright 2016-2020, Intel Corporation +# Copyright 2016-2023, Intel Corporation # # push-image.sh - pushes the Docker image to $DOCKER_REPO. @@ -34,6 +34,11 @@ if [[ -z "${DOCKER_REPO}" ]]; then exit 1 fi +if [[ -z "$IMG_VER" ]]; then + echo "IMG_VER environment variable is not set" + exit 1 +fi + if [[ -z "${GH_CR_USER}" || -z "${GH_CR_PAT}" ]]; then echo "ERROR: variables GH_CR_USER=\"${GH_CR_USER}\" and GH_CR_PAT=\"${GH_CR_PAT}\"" \ "have to be set properly to allow login to the $DOCKER_REPO." @@ -53,5 +58,6 @@ fi # Log in to $DOCKER_REPO echo "${GH_CR_PAT}" | docker login "${DOCKER_REPO}" -u="${GH_CR_USER}" --password-stdin -# Push the image to $DOCKER_REPO +echo "Push the image '${DOCKER_REPO}:${TAG}' to the $DOCKER_REPO." docker push ${DOCKER_REPO}:${TAG} +echo "Image pushed." diff --git a/utils/docker/pull-or-rebuild-image.sh b/utils/docker/pull-or-rebuild-image.sh index 8e26af5bdd0..c21970411ae 100755 --- a/utils/docker/pull-or-rebuild-image.sh +++ b/utils/docker/pull-or-rebuild-image.sh @@ -1,21 +1,24 @@ #!/usr/bin/env bash # SPDX-License-Identifier: BSD-3-Clause -# Copyright 2016-2020, Intel Corporation +# Copyright 2016-2023, Intel Corporation # # pull-or-rebuild-image.sh - rebuilds the Docker image used in the -# current CI build if necessary. +# current build (if necessary) or pulls it from the $DOCKER_REPO. # -# The script rebuilds the Docker image if the Dockerfile for the current -# OS version (Dockerfile.${OS}-${OS_VER}) or any .sh script from the directory -# with Dockerfiles were modified and committed. +# Usage: pull-or-rebuild-image.sh [rebuild|pull] # -# If the CI build is not of the "pull_request" type (i.e. in case of -# merge after pull_request) and it succeed, the Docker image should be pushed -# to $DOCKER_REPO repository. An empty file is created to signal -# that to further scripts. +# If Docker was rebuilt and all requirements are fulfilled (more details in +# push_image function below) image will be pushed to the $DOCKER_REPO. # -# If the Docker image does not have to be rebuilt, it will be pulled from $DOCKER_REPO. +# The script rebuilds the Docker image if: +# 1. the Dockerfile for the current OS version (Dockerfile.${OS}-${OS_VER}) +# or any .sh script in the Dockerfiles directory were modified and committed, or +# 2. "rebuild" param was passed as a first argument to this script. +# +# The script pulls the Docker image if: +# 1. it does not have to be rebuilt (based on committed changes), or +# 2. "pull" param was passed as a first argument to this script. # set -e @@ -23,32 +26,69 @@ set -e source $(dirname $0)/set-ci-vars.sh source $(dirname $0)/set-vars.sh -if [[ "$CI_EVENT_TYPE" != "cron" && "$CI_BRANCH" != "coverity_scan" \ - && "$COVERITY" -eq 1 ]]; then - echo "INFO: Skip Coverity scan job if build is triggered neither by " \ - "'cron' nor by a push to 'coverity_scan' branch" - exit 0 -fi - -if [[ ( "$CI_EVENT_TYPE" == "cron" || "$CI_BRANCH" == "coverity_scan" )\ - && "$COVERITY" -ne 1 ]]; then - echo "INFO: Skip regular jobs if build is triggered either by 'cron'" \ - " or by a push to 'coverity_scan' branch" - exit 0 -fi +# Path to directory with Dockerfiles and image building scripts +images_dir_name=images +base_dir=utils/docker/$images_dir_name if [[ -z "$OS" || -z "$OS_VER" ]]; then echo "ERROR: The variables OS and OS_VER have to be set properly " \ - "(eg. OS=ubuntu, OS_VER=16.04)." + "(eg. OS=ubuntu, OS_VER=22.04)." exit 1 fi -if [[ -z "$HOST_WORKDIR" ]]; then - echo "ERROR: The variable HOST_WORKDIR has to contain a path to " \ - "the root of the PMDK project on the host machine" +if [[ -z "${DOCKER_REPO}" ]]; then + echo "ERROR: DOCKER_REPO environment variable is not set " \ + "(e.g. \"//\")." exit 1 fi +function build_image() { + echo "Building the Docker image for the Dockerfile.${OS}-${OS_VER}" + pushd $images_dir_name + ./build-image.sh ${OS}-${OS_VER} ${CI_CPU_ARCH} + popd +} + +function pull_image() { + echo "Pull the image from the DOCKER_REPO." + docker pull ${DOCKER_REPO}:${IMG_VER}-${OS}-${OS_VER}-${CI_CPU_ARCH} +} + +function push_image { + # Check if the image has to be pushed to the DOCKER_REPO: + # - only upstream (not forked) repository, + # - stable-*, devel-*, or master branch, + # - not a pull_request event, + # - and PUSH_IMAGE flag was set for current build. + if [[ "${CI_REPO_SLUG}" == "${GITHUB_REPO}" \ + && (${CI_BRANCH} == stable-* || ${CI_BRANCH} == devel-* || ${CI_BRANCH} == master) \ + && ${CI_EVENT_TYPE} != "pull_request" \ + && ${PUSH_IMAGE} == "1" ]] + then + echo "The image will be pushed to the Container Registry: ${DOCKER_REPO}" + pushd ${images_dir_name} + ./push-image.sh + popd + else + echo "Skip pushing the image to the ${DOCKER_REPO}." + fi +} + +# If "rebuild" or "pull" are passed to the script as param, force rebuild/pull. +if [[ "${1}" == "rebuild" ]]; then + build_image + push_image + exit 0 +elif [[ "${1}" == "pull" ]]; then + pull_image + exit 0 +fi + +# +# Determine if we need to rebuild the image or just pull it from +# the DOCKER_REPO, based on committed changes. +# + # Find all the commits for the current build if [ -n "$CI_COMMIT_RANGE" ]; then commits=$(git rev-list $CI_COMMIT_RANGE) @@ -59,42 +99,19 @@ fi echo "Commits in the commit range:" for commit in $commits; do echo $commit; done -# Get the list of files modified by the commits +echo "Files modified within the commit range:" files=$(for commit in $commits; do git diff-tree --no-commit-id --name-only \ -r $commit; done | sort -u) -echo "Files modified within the commit range:" for file in $files; do echo $file; done -# Path to directory with Dockerfiles and image building scripts -images_dir_name=images -base_dir=utils/docker/$images_dir_name - # Check if committed file modifications require the Docker image to be rebuilt for file in $files; do # Check if modified files are relevant to the current build if [[ $file =~ ^($base_dir)\/Dockerfile\.($OS)-($OS_VER)$ ]] \ || [[ $file =~ ^($base_dir)\/.*\.sh$ ]] then - # Rebuild Docker image for the current OS version - echo "Rebuilding the Docker image for the Dockerfile.$OS-$OS_VER" - pushd $images_dir_name - ./build-image.sh ${OS}-${OS_VER} ${CI_CPU_ARCH} - popd - - # Check if the image has to be pushed to $DOCKER_REPO - # (i.e. the build is triggered by commits to the $GITHUB_REPO - # repository's stable-*, devel-* or master branch, and the CI build is not - # of the "pull_request" type). In that case, create the empty file. - if [[ "$CI_REPO_SLUG" == "$GITHUB_REPO" \ - && ($CI_BRANCH == stable-* || $CI_BRANCH == devel-* || $CI_BRANCH == master) \ - && $CI_EVENT_TYPE != "pull_request" \ - && $PUSH_IMAGE == "1" ]] - then - echo "The image will be pushed to ${DOCKER_REPO}" - touch $CI_FILE_PUSH_IMAGE_TO_REPO - else - echo "Skip pushing the image to ${DOCKER_REPO}" - fi + build_image + push_image if [[ $PUSH_IMAGE == "1" ]] then @@ -105,6 +122,9 @@ for file in $files; do fi done -# Getting here means rebuilding the Docker image is not required. -# Pull the image from $DOCKER_REPO. -docker pull ${DOCKER_REPO}:${IMG_VER}-${OS}-${OS_VER}-${CI_CPU_ARCH} +# Getting here means rebuilding the Docker image isn't required (based on changed files). +# Pull the image from the $DOCKER_REPO or rebuild anyway, if pull fails. +if ! pull_image; then + build_image + push_image +fi diff --git a/utils/docker/set-vars.sh b/utils/docker/set-vars.sh index 9a531a84091..0fe2b01f150 100755 --- a/utils/docker/set-vars.sh +++ b/utils/docker/set-vars.sh @@ -1,6 +1,6 @@ #!/usr/bin/env bash # SPDX-License-Identifier: BSD-3-Clause -# Copyright 2019, Intel Corporation +# Copyright 2019-2023, Intel Corporation # # set-vars.sh - set required environment variables @@ -8,5 +8,4 @@ set -e -export CI_FILE_PUSH_IMAGE_TO_REPO=/tmp/push_image_to_repo_flag export CI_FILE_SKIP_BUILD_PKG_CHECK=/tmp/skip_build_package_check