Merge pull request #5565 from lukaszstolarczuk/update-CI

Update pull and rebuild script

.github/workflows/gha.yml

@@ -47,9 +47,6 @@ jobs:
        - name: Run the build
          run: cd $WORKDIR && ${{ matrix.CONFIG }} ./build-CI.sh
 
-       - name: Push the image
-         run: cd $WORKDIR && source ./set-vars.sh && ${{ matrix.CONFIG }} /bin/bash -c "if [[ -f ${CI_FILE_PUSH_IMAGE_TO_REPO} ]]; then images/push-image.sh; fi"
-
   windows:
     name: Windows
     runs-on: windows-2022

.github/workflows/nightly.yml

@@ -46,10 +46,7 @@ jobs:
             fetch-depth: 50
 
        - name: Pull or rebuild the image
-         run: cd $WORKDIR && ${{ matrix.CONFIG }} ./pull-or-rebuild-image.sh
+         run: cd $WORKDIR && ${{ matrix.CONFIG }} ./pull-or-rebuild-image.sh rebuild
 
        - name: Run the build
          run: cd $WORKDIR && ${{ matrix.CONFIG }} ./build-CI.sh
-
-       - name: Push the image
-         run: cd $WORKDIR && source ./set-vars.sh && ${{ matrix.CONFIG }} /bin/bash -c "if [[ -f ${CI_FILE_PUSH_IMAGE_TO_REPO} ]]; then images/push-image.sh; fi"

utils/docker/images/build-image.sh

@@ -1,6 +1,6 @@
 #!/usr/bin/env bash
 # SPDX-License-Identifier: BSD-3-Clause
-# Copyright 2016-2020, Intel Corporation
+# Copyright 2016-2023, Intel Corporation
 
 #
 # build-image.sh <OS-VER> <ARCH> - prepares a Docker image with <OS>-based
@@ -20,8 +20,8 @@ function usage {
 	echo "Usage:"
 	echo "    build-image.sh <OS-VER> <ARCH>"
 	echo "where:"
-	echo "  <OS-VER> - can be for example 'ubuntu-19.10' provided "\
-		"a Dockerfile named 'Dockerfile.ubuntu-19.10' "\
+	echo "  <OS-VER> - can be for example 'ubuntu-22.04' provided "\
+		"a Dockerfile named 'Dockerfile.ubuntu-22.04' "\
 		"exists in the current directory and"
 	echo "  <ARCH> - is a CPU architecture, for example 'x86_64'"
 }
@@ -40,8 +40,9 @@ if [[ ! -f "Dockerfile.$OS_VER" ]]; then
 	exit 1
 fi
 
-if [[ -z "${DOCKER_REPO}" ]]; then
-	echo "Error: DOCKER_REPO environment variable is not set"
+if [[ -z "${DOCKER_REPO}" || -z "${IMG_VER}" ]]; then
+	echo "Error: DOCKER_REPO (${DOCKER_REPO}) or IMG_VER (${IMG_VER}) " \
+		"environment variables are not set."
 	exit 1
 fi
 

utils/docker/images/push-image.sh

@@ -1,6 +1,6 @@
 #!/usr/bin/env bash
 # SPDX-License-Identifier: BSD-3-Clause
-# Copyright 2016-2020, Intel Corporation
+# Copyright 2016-2023, Intel Corporation
 
 #
 # push-image.sh - pushes the Docker image to $DOCKER_REPO.
@@ -34,6 +34,11 @@ if [[ -z "${DOCKER_REPO}" ]]; then
 	exit 1
 fi
 
+if [[ -z "$IMG_VER" ]]; then
+	echo "IMG_VER environment variable is not set"
+	exit 1
+fi
+
 if [[ -z "${GH_CR_USER}" || -z "${GH_CR_PAT}" ]]; then
 	echo "ERROR: variables GH_CR_USER=\"${GH_CR_USER}\" and GH_CR_PAT=\"${GH_CR_PAT}\"" \
 		"have to be set properly to allow login to the $DOCKER_REPO."
@@ -53,5 +58,6 @@ fi
 # Log in to $DOCKER_REPO
 echo "${GH_CR_PAT}" | docker login "${DOCKER_REPO}" -u="${GH_CR_USER}" --password-stdin
 
-# Push the image to $DOCKER_REPO
+echo "Push the image '${DOCKER_REPO}:${TAG}' to the $DOCKER_REPO."
 docker push ${DOCKER_REPO}:${TAG}
+echo "Image pushed."

utils/docker/pull-or-rebuild-image.sh

@@ -1,54 +1,94 @@
 #!/usr/bin/env bash
 # SPDX-License-Identifier: BSD-3-Clause
-# Copyright 2016-2020, Intel Corporation
+# Copyright 2016-2023, Intel Corporation
 
 #
 # pull-or-rebuild-image.sh - rebuilds the Docker image used in the
-#                            current CI build if necessary.
+#		current build (if necessary) or pulls it from the $DOCKER_REPO.
 #
-# The script rebuilds the Docker image if the Dockerfile for the current
-# OS version (Dockerfile.${OS}-${OS_VER}) or any .sh script from the directory
-# with Dockerfiles were modified and committed.
+# Usage: pull-or-rebuild-image.sh [rebuild|pull]
 #
-# If the CI build is not of the "pull_request" type (i.e. in case of
-# merge after pull_request) and it succeed, the Docker image should be pushed
-# to $DOCKER_REPO repository. An empty file is created to signal
-# that to further scripts.
+# If Docker was rebuilt and all requirements are fulfilled (more details in
+# push_image function below) image will be pushed to the $DOCKER_REPO.
 #
-# If the Docker image does not have to be rebuilt, it will be pulled from $DOCKER_REPO.
+# The script rebuilds the Docker image if:
+# 1. the Dockerfile for the current OS version (Dockerfile.${OS}-${OS_VER})
+#    or any .sh script in the Dockerfiles directory were modified and committed, or
+# 2. "rebuild" param was passed as a first argument to this script.
+#
+# The script pulls the Docker image if:
+# 1. it does not have to be rebuilt (based on committed changes), or
+# 2. "pull" param was passed as a first argument to this script.
 #
 
 set -e
 
 source $(dirname $0)/set-ci-vars.sh
 source $(dirname $0)/set-vars.sh
 
-if [[ "$CI_EVENT_TYPE" != "cron" && "$CI_BRANCH" != "coverity_scan" \
-	&& "$COVERITY" -eq 1 ]]; then
-	echo "INFO: Skip Coverity scan job if build is triggered neither by " \
-		"'cron' nor by a push to 'coverity_scan' branch"
-	exit 0
-fi
-
-if [[ ( "$CI_EVENT_TYPE" == "cron" || "$CI_BRANCH" == "coverity_scan" )\
-	&& "$COVERITY" -ne 1 ]]; then
-	echo "INFO: Skip regular jobs if build is triggered either by 'cron'" \
-		" or by a push to 'coverity_scan' branch"
-	exit 0
-fi
+# Path to directory with Dockerfiles and image building scripts
+images_dir_name=images
+base_dir=utils/docker/$images_dir_name
 
 if [[ -z "$OS" || -z "$OS_VER" ]]; then
 	echo "ERROR: The variables OS and OS_VER have to be set properly " \
-             "(eg. OS=ubuntu, OS_VER=16.04)."
+             "(eg. OS=ubuntu, OS_VER=22.04)."
 	exit 1
 fi
 
-if [[ -z "$HOST_WORKDIR" ]]; then
-	echo "ERROR: The variable HOST_WORKDIR has to contain a path to " \
-		"the root of the PMDK project on the host machine"
+if [[ -z "${DOCKER_REPO}" ]]; then
+	echo "ERROR: DOCKER_REPO environment variable is not set " \
+		"(e.g. \"<docker_repo_addr>/<org_name>/<package_name>\")."
 	exit 1
 fi
 
+function build_image() {
+	echo "Building the Docker image for the Dockerfile.${OS}-${OS_VER}"
+	pushd $images_dir_name
+	./build-image.sh ${OS}-${OS_VER} ${CI_CPU_ARCH}
+	popd
+}
+
+function pull_image() {
+	echo "Pull the image from the DOCKER_REPO."
+	docker pull ${DOCKER_REPO}:${IMG_VER}-${OS}-${OS_VER}-${CI_CPU_ARCH}
+}
+
+function push_image {
+	# Check if the image has to be pushed to the DOCKER_REPO:
+	# - only upstream (not forked) repository,
+	# - stable-*, devel-*, or master branch,
+	# - not a pull_request event,
+	# - and PUSH_IMAGE flag was set for current build.
+	if [[ "${CI_REPO_SLUG}" == "${GITHUB_REPO}" \
+		&& (${CI_BRANCH} == stable-* || ${CI_BRANCH} == devel-* || ${CI_BRANCH} == master) \
+		&& ${CI_EVENT_TYPE} != "pull_request" \
+		&& ${PUSH_IMAGE} == "1" ]]
+	then
+		echo "The image will be pushed to the Container Registry: ${DOCKER_REPO}"
+		pushd ${images_dir_name}
+		./push-image.sh
+		popd
+	else
+		echo "Skip pushing the image to the ${DOCKER_REPO}."
+	fi
+}
+
+# If "rebuild" or "pull" are passed to the script as param, force rebuild/pull.
+if [[ "${1}" == "rebuild" ]]; then
+	build_image
+	push_image
+	exit 0
+elif [[ "${1}" == "pull" ]]; then
+	pull_image
+	exit 0
+fi
+
+#
+# Determine if we need to rebuild the image or just pull it from
+# the DOCKER_REPO, based on committed changes.
+#
+
 # Find all the commits for the current build
 if [ -n "$CI_COMMIT_RANGE" ]; then
 	commits=$(git rev-list $CI_COMMIT_RANGE)
@@ -59,42 +99,19 @@ fi
 echo "Commits in the commit range:"
 for commit in $commits; do echo $commit; done
 
-# Get the list of files modified by the commits
+echo "Files modified within the commit range:"
 files=$(for commit in $commits; do git diff-tree --no-commit-id --name-only \
 	-r $commit; done | sort -u)
-echo "Files modified within the commit range:"
 for file in $files; do echo $file; done
 
-# Path to directory with Dockerfiles and image building scripts
-images_dir_name=images
-base_dir=utils/docker/$images_dir_name
-
 # Check if committed file modifications require the Docker image to be rebuilt
 for file in $files; do
 	# Check if modified files are relevant to the current build
 	if [[ $file =~ ^($base_dir)\/Dockerfile\.($OS)-($OS_VER)$ ]] \
 		|| [[ $file =~ ^($base_dir)\/.*\.sh$ ]]
 	then
-		# Rebuild Docker image for the current OS version
-		echo "Rebuilding the Docker image for the Dockerfile.$OS-$OS_VER"
-		pushd $images_dir_name
-		./build-image.sh ${OS}-${OS_VER} ${CI_CPU_ARCH}
-		popd
-
-		# Check if the image has to be pushed to $DOCKER_REPO
-		# (i.e. the build is triggered by commits to the $GITHUB_REPO
-		# repository's stable-*, devel-* or master branch, and the CI build is not
-		# of the "pull_request" type). In that case, create the empty file.
-		if [[ "$CI_REPO_SLUG" == "$GITHUB_REPO" \
-			&& ($CI_BRANCH == stable-* || $CI_BRANCH == devel-* || $CI_BRANCH == master) \
-			&& $CI_EVENT_TYPE != "pull_request" \
-			&& $PUSH_IMAGE == "1" ]]
-		then
-			echo "The image will be pushed to ${DOCKER_REPO}"
-			touch $CI_FILE_PUSH_IMAGE_TO_REPO
-		else
-			echo "Skip pushing the image to ${DOCKER_REPO}"
-		fi
+		build_image
+		push_image
 
 		if [[ $PUSH_IMAGE == "1" ]]
 		then
@@ -105,6 +122,9 @@ for file in $files; do
 	fi
 done
 
-# Getting here means rebuilding the Docker image is not required.
-# Pull the image from $DOCKER_REPO.
-docker pull ${DOCKER_REPO}:${IMG_VER}-${OS}-${OS_VER}-${CI_CPU_ARCH}
+# Getting here means rebuilding the Docker image isn't required (based on changed files).
+# Pull the image from the $DOCKER_REPO or rebuild anyway, if pull fails.
+if ! pull_image; then
+	build_image
+	push_image
+fi

utils/docker/set-vars.sh

@@ -1,12 +1,11 @@
 #!/usr/bin/env bash
 # SPDX-License-Identifier: BSD-3-Clause
-# Copyright 2019, Intel Corporation
+# Copyright 2019-2023, Intel Corporation
 
 #
 # set-vars.sh - set required environment variables
 #
 
 set -e
 
-export CI_FILE_PUSH_IMAGE_TO_REPO=/tmp/push_image_to_repo_flag
 export CI_FILE_SKIP_BUILD_PKG_CHECK=/tmp/skip_build_package_check