From a2374585c0432de89f951f3c854c0c62bed60ff9 Mon Sep 17 00:00:00 2001 From: Kris Jacque Date: Mon, 23 Sep 2024 16:18:21 -0600 Subject: [PATCH 1/8] DAOS-16621 build: Fix Go versions in rpm/deb packaging - Set Go minimum version to 1.21 in rpm and debian packaging spec files. - Remove scons Go version check, as it will be checked in Go build based on go.mod file. - Add a reminder in go.mod file so we remember the packaging files when bumping the minimum Go version in the future. Required-githooks: true Signed-off-by: Kris Jacque --- debian/changelog | 6 ++++++ debian/control | 4 ++-- site_scons/site_tools/go_builder.py | 6 ------ src/control/go.mod | 1 + utils/rpms/daos.spec | 9 ++++++--- 5 files changed, 15 insertions(+), 11 deletions(-) diff --git a/debian/changelog b/debian/changelog index bddf3f36d0b..6a63bcf0777 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,3 +1,9 @@ +daos (2.7.100-6) unstable; urgency=medium + [ Kris Jacque ] + * Bump minimum golang-go version to 1.21 + + -- Kris Jacque Mon, 23 Sep 2024 11:06:00 -0700 + daos (2.7.100-5) unstable; urgency=medium [ Michael MacDonald ] * Add libdaos_self_test.so to client package diff --git a/debian/control b/debian/control index bdc377e6d94..7c6da62b483 100644 --- a/debian/control +++ b/debian/control @@ -25,7 +25,7 @@ Build-Depends: debhelper (>= 10), dpdk-dev (>= 21.11.2), libisal-crypto-dev, libcunit1-dev, - golang-go (>= 1.18), + golang-go (>= 1.21), libboost-dev, libspdk-dev (>= 22.01.2), libipmctl-dev, @@ -117,7 +117,7 @@ Depends: python (>=3.8), python3, python-yaml, python3-yaml, ${shlibs:Depends}, ${misc:Depends}, daos-client (= ${binary:Version}), daos-admin (= ${binary:Version}), - golang-go (>=1.18), + golang-go (>=1.21), libcapstone-dev Description: The Distributed Asynchronous Object Storage (DAOS) is an open-source software-defined object store designed from the ground up for diff --git a/site_scons/site_tools/go_builder.py b/site_scons/site_tools/go_builder.py index 33455f16b68..5ba5f826da8 100644 --- a/site_scons/site_tools/go_builder.py +++ b/site_scons/site_tools/go_builder.py @@ -8,7 +8,6 @@ from SCons.Script import Configure, Exit, File, GetOption, Glob, Scanner GO_COMPILER = 'go' -MIN_GO_VERSION = '1.18.0' include_re = re.compile(r'\#include [<"](\S+[>"])', re.M) @@ -93,11 +92,6 @@ def _check_go_version(context): if go_version is None: context.Result(f'failed to get version from "{out}"') return 0 - if len([x for x, y in - zip(go_version.split('.'), MIN_GO_VERSION.split('.')) - if int(x) < int(y)]) > 0: - context.Result(f'{out} is too old (min supported: {MIN_GO_VERSION}) ') - return 0 context.Result(go_version) return 1 diff --git a/src/control/go.mod b/src/control/go.mod index 364179c1fb6..f3b507ccd1e 100644 --- a/src/control/go.mod +++ b/src/control/go.mod @@ -1,5 +1,6 @@ module github.com/daos-stack/daos/src/control +// NB: When updating minimum Go build version, don't forget to update rpm and debian packaging specs. go 1.21 toolchain go1.22.3 diff --git a/utils/rpms/daos.spec b/utils/rpms/daos.spec index f90e6d7010f..7aa6b5e81d5 100644 --- a/utils/rpms/daos.spec +++ b/utils/rpms/daos.spec @@ -81,7 +81,7 @@ BuildRequires: libyaml-devel BuildRequires: libcmocka-devel BuildRequires: valgrind-devel BuildRequires: systemd -BuildRequires: go >= 1.17 +BuildRequires: go >= 1.21 BuildRequires: pciutils-devel %if (0%{?rhel} >= 8) BuildRequires: numactl-devel @@ -218,7 +218,7 @@ Requires: dbench Requires: lbzip2 Requires: attr Requires: ior -Requires: go >= 1.18 +Requires: go >= 1.21 %if (0%{?suse_version} >= 1315) Requires: lua-lmod Requires: libcapstone-devel @@ -592,6 +592,9 @@ getent passwd daos_agent >/dev/null || useradd -s /sbin/nologin -r -g daos_agent # No files in a shim package %changelog +* Mon Sep 23 2024 Kris Jacque 2.3.103-6 +- Bump min supported go version to 1.21 + * Thu Aug 15 2024 Michael MacDonald 2.7.100-5 - Add libdaos_self_test.so to client RPM @@ -601,7 +604,7 @@ getent passwd daos_agent >/dev/null || useradd -s /sbin/nologin -r -g daos_agent * Thu Jul 11 2024 Dalton Bohning 2.7.100-3 - Add pciutils-devel build dep for client-tests package -* Thu Jun 24 2024 Tom Nabarro 2.7.100-2 +* Mon Jun 24 2024 Tom Nabarro 2.7.100-2 - Add pciutils runtime dep for daos_server lspci call - Add pciutils-devel build dep for pciutils CGO bindings From f17c1d5d341178cef8d8c5d9ed85611c7758bc70 Mon Sep 17 00:00:00 2001 From: Kris Jacque Date: Tue, 24 Sep 2024 17:46:08 -0600 Subject: [PATCH 2/8] Address review comments - Fix debian Go versions - In scons, get minimum Go version from go.mod - Add reminder comment in go.mod - Fix changelog version in rpm specfile - Use longsleep/go-backports repo for Go versions in Dockerfile Required-githooks: true Signed-off-by: Kris Jacque --- debian/control | 4 ++-- site_scons/site_tools/go_builder.py | 25 ++++++++++++++++++++++++- src/control/go.mod | 7 ++++--- utils/rpms/daos.spec | 2 +- utils/scripts/install-ubuntu.sh | 4 ++++ 5 files changed, 35 insertions(+), 7 deletions(-) diff --git a/debian/control b/debian/control index 7c6da62b483..53f84d7f9c1 100644 --- a/debian/control +++ b/debian/control @@ -25,7 +25,7 @@ Build-Depends: debhelper (>= 10), dpdk-dev (>= 21.11.2), libisal-crypto-dev, libcunit1-dev, - golang-go (>= 1.21), + golang-go (>= 2:1.21), libboost-dev, libspdk-dev (>= 22.01.2), libipmctl-dev, @@ -117,7 +117,7 @@ Depends: python (>=3.8), python3, python-yaml, python3-yaml, ${shlibs:Depends}, ${misc:Depends}, daos-client (= ${binary:Version}), daos-admin (= ${binary:Version}), - golang-go (>=1.21), + golang-go (>= 2:1.21), libcapstone-dev Description: The Distributed Asynchronous Object Storage (DAOS) is an open-source software-defined object store designed from the ground up for diff --git a/site_scons/site_tools/go_builder.py b/site_scons/site_tools/go_builder.py index 5ba5f826da8..42bb0bc6578 100644 --- a/site_scons/site_tools/go_builder.py +++ b/site_scons/site_tools/go_builder.py @@ -5,7 +5,7 @@ import re import subprocess # nosec B404 -from SCons.Script import Configure, Exit, File, GetOption, Glob, Scanner +from SCons.Script import Configure, Dir, Exit, File, GetOption, Glob, Scanner GO_COMPILER = 'go' include_re = re.compile(r'\#include [<"](\S+[>"])', re.M) @@ -48,6 +48,17 @@ def _scan_go_file(node, env, _path): return includes +def get_min_go_version(env): + """Get go minimum version from go.mod""" + go_mod_path = os.path.join(Dir('#').abspath, "src", "control", "go.mod") + with open(go_mod_path, 'r') as f: + for line in f: + if line.startswith('go '): # e.g. "go 1.21" + parts = line.split() + return parts[1] + ".0" # e.g. "1.21.0" + return None + + def get_go_version(output): """Capture only the version after 'go'""" ver_re = re.compile(r'go([0-9\.]+)') @@ -80,6 +91,13 @@ def _check_go_version(context): context.Result(0) return 0 + context.Display(f'Getting minimum {env.d_go_bin} version... ') + min_go_version = get_min_go_version(env) + if min_go_version is None: + context.Result('failed to extract minimum version from go.mod') + return 0 + context.Display(min_go_version + '\n') + context.Display(f'Checking {env.d_go_bin} version... ') cmd_rc = subprocess.run([env.d_go_bin, 'version'], check=True, stdout=subprocess.PIPE) out = cmd_rc.stdout.decode('utf-8').strip() @@ -92,6 +110,11 @@ def _check_go_version(context): if go_version is None: context.Result(f'failed to get version from "{out}"') return 0 + if len([x for x, y in zip(go_version.split('.'), min_go_version.split('.')) + if int(x) < int(y)]) > 0: + context.Result(f'{out} is too old (min supported: {min_go_version}) ') + return 0 + context.Result(go_version) return 1 diff --git a/src/control/go.mod b/src/control/go.mod index f3b507ccd1e..dde1a47b7da 100644 --- a/src/control/go.mod +++ b/src/control/go.mod @@ -1,10 +1,11 @@ module github.com/daos-stack/daos/src/control -// NB: When updating minimum Go build version, don't forget to update rpm and debian packaging specs. +// NB: When updating minimum Go build version, don't forget to update: +// - rpm packaging version checks: utils/rpms/daos.spec +// - debian packaging version checks: debian/control +// Scons uses this file to extract the minimum version. go 1.21 -toolchain go1.22.3 - require ( github.com/Jille/raft-grpc-transport v1.2.0 github.com/desertbit/grumble v1.1.3 diff --git a/utils/rpms/daos.spec b/utils/rpms/daos.spec index 7aa6b5e81d5..d9e0e0fee05 100644 --- a/utils/rpms/daos.spec +++ b/utils/rpms/daos.spec @@ -592,7 +592,7 @@ getent passwd daos_agent >/dev/null || useradd -s /sbin/nologin -r -g daos_agent # No files in a shim package %changelog -* Mon Sep 23 2024 Kris Jacque 2.3.103-6 +* Mon Sep 23 2024 Kris Jacque 2.7.100-6 - Bump min supported go version to 1.21 * Thu Aug 15 2024 Michael MacDonald 2.7.100-5 diff --git a/utils/scripts/install-ubuntu.sh b/utils/scripts/install-ubuntu.sh index ec21c92f474..d8e545db88d 100755 --- a/utils/scripts/install-ubuntu.sh +++ b/utils/scripts/install-ubuntu.sh @@ -14,6 +14,10 @@ set -e arch=$(uname -i) +# DAOS requires newer golang version than the one available in core ubuntu repo +add-apt-repository ppa:longsleep/golang-backports +apt update + apt-get install \ autoconf \ build-essential \ From 2942650faf59e470354a9f439dc7ee5a86686f75 Mon Sep 17 00:00:00 2001 From: Kris Jacque Date: Tue, 24 Sep 2024 17:53:28 -0600 Subject: [PATCH 3/8] Update Ubuntu install script Required-githooks: true Signed-off-by: Kris Jacque --- utils/scripts/install-ubuntu.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/utils/scripts/install-ubuntu.sh b/utils/scripts/install-ubuntu.sh index d8e545db88d..fef67d51808 100755 --- a/utils/scripts/install-ubuntu.sh +++ b/utils/scripts/install-ubuntu.sh @@ -15,6 +15,7 @@ set -e arch=$(uname -i) # DAOS requires newer golang version than the one available in core ubuntu repo +apt-get install software-properties-common add-apt-repository ppa:longsleep/golang-backports apt update From 408e2d4f871522ea3e9edcea7149395c49d9f839 Mon Sep 17 00:00:00 2001 From: Kris Jacque Date: Wed, 25 Sep 2024 09:34:47 -0600 Subject: [PATCH 4/8] Add gpg-agent to ubuntu install It's needed to install an external PPA. Required-githooks: true Signed-off-by: Kris Jacque --- utils/scripts/install-ubuntu.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/utils/scripts/install-ubuntu.sh b/utils/scripts/install-ubuntu.sh index fef67d51808..7d256bbba24 100755 --- a/utils/scripts/install-ubuntu.sh +++ b/utils/scripts/install-ubuntu.sh @@ -15,7 +15,7 @@ set -e arch=$(uname -i) # DAOS requires newer golang version than the one available in core ubuntu repo -apt-get install software-properties-common +apt-get install gpg-agent software-properties-common add-apt-repository ppa:longsleep/golang-backports apt update From 2b4106cf30603a8c29ae9dee416e7952619b8eb3 Mon Sep 17 00:00:00 2001 From: Kris Jacque Date: Wed, 25 Sep 2024 09:42:49 -0600 Subject: [PATCH 5/8] Fix pylint error Required-githooks: true Signed-off-by: Kris Jacque --- site_scons/site_tools/go_builder.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/site_scons/site_tools/go_builder.py b/site_scons/site_tools/go_builder.py index 42bb0bc6578..1f444d4c810 100644 --- a/site_scons/site_tools/go_builder.py +++ b/site_scons/site_tools/go_builder.py @@ -48,7 +48,7 @@ def _scan_go_file(node, env, _path): return includes -def get_min_go_version(env): +def get_min_go_version(): """Get go minimum version from go.mod""" go_mod_path = os.path.join(Dir('#').abspath, "src", "control", "go.mod") with open(go_mod_path, 'r') as f: @@ -92,7 +92,7 @@ def _check_go_version(context): return 0 context.Display(f'Getting minimum {env.d_go_bin} version... ') - min_go_version = get_min_go_version(env) + min_go_version = get_min_go_version() if min_go_version is None: context.Result('failed to extract minimum version from go.mod') return 0 From 95f6cd39b053035987c7a5493f425e473f2f2b2f Mon Sep 17 00:00:00 2001 From: Kris Jacque Date: Wed, 25 Sep 2024 18:50:34 -0600 Subject: [PATCH 6/8] Harden minimum Go version detection Required-githooks: true Signed-off-by: Kris Jacque --- site_scons/site_tools/go_builder.py | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/site_scons/site_tools/go_builder.py b/site_scons/site_tools/go_builder.py index 1f444d4c810..16de8deffd9 100644 --- a/site_scons/site_tools/go_builder.py +++ b/site_scons/site_tools/go_builder.py @@ -55,8 +55,8 @@ def get_min_go_version(): for line in f: if line.startswith('go '): # e.g. "go 1.21" parts = line.split() - return parts[1] + ".0" # e.g. "1.21.0" - return None + return get_go_version("go" + parts[1]) + raise Exception("no minimum go version found in go.mod") def get_go_version(output): @@ -93,9 +93,6 @@ def _check_go_version(context): context.Display(f'Getting minimum {env.d_go_bin} version... ') min_go_version = get_min_go_version() - if min_go_version is None: - context.Result('failed to extract minimum version from go.mod') - return 0 context.Display(min_go_version + '\n') context.Display(f'Checking {env.d_go_bin} version... ') From b1ee4cf7af2f4e0d4408b95ae5694a287da524a7 Mon Sep 17 00:00:00 2001 From: Kris Jacque Date: Thu, 26 Sep 2024 08:25:11 -0600 Subject: [PATCH 7/8] Address review comments Required-githooks: true Signed-off-by: Kris Jacque --- site_scons/site_tools/go_builder.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/site_scons/site_tools/go_builder.py b/site_scons/site_tools/go_builder.py index 16de8deffd9..8d5d86b9016 100644 --- a/site_scons/site_tools/go_builder.py +++ b/site_scons/site_tools/go_builder.py @@ -56,7 +56,7 @@ def get_min_go_version(): if line.startswith('go '): # e.g. "go 1.21" parts = line.split() return get_go_version("go" + parts[1]) - raise Exception("no minimum go version found in go.mod") + return None def get_go_version(output): @@ -91,8 +91,11 @@ def _check_go_version(context): context.Result(0) return 0 - context.Display(f'Getting minimum {env.d_go_bin} version... ') + context.Display('Getting minimum go version... ') min_go_version = get_min_go_version() + if min_go_version is None: + context.Result('no minimum go version found in go.mod') + return 0 context.Display(min_go_version + '\n') context.Display(f'Checking {env.d_go_bin} version... ') From 11ac499d2f773f97313c8c019cb904f1c53d1c1e Mon Sep 17 00:00:00 2001 From: Kris Jacque Date: Thu, 26 Sep 2024 18:14:32 -0600 Subject: [PATCH 8/8] Move add-apt-repository call to Dockerfile Signed-off-by: Kris Jacque --- utils/docker/Dockerfile.ubuntu | 3 +++ utils/scripts/install-ubuntu.sh | 5 ----- 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/utils/docker/Dockerfile.ubuntu b/utils/docker/Dockerfile.ubuntu index c7bf969e009..4f6c138db02 100644 --- a/utils/docker/Dockerfile.ubuntu +++ b/utils/docker/Dockerfile.ubuntu @@ -22,6 +22,9 @@ RUN echo "APT::Get::Assume-Yes \"true\";" > /etc/apt/apt.conf.d/no-prompt RUN echo "APT::Install-Recommends \"false\";" > /etc/apt/apt.conf.d/no-recommends RUN apt-get update && \ apt-get upgrade && \ + apt-get install gpg-agent software-properties-common && \ + add-apt-repository ppa:longsleep/golang-backports && \ + apt-get update && \ chmod +x /tmp/install.sh && \ /tmp/install.sh && \ apt-get clean all diff --git a/utils/scripts/install-ubuntu.sh b/utils/scripts/install-ubuntu.sh index 7d256bbba24..ec21c92f474 100755 --- a/utils/scripts/install-ubuntu.sh +++ b/utils/scripts/install-ubuntu.sh @@ -14,11 +14,6 @@ set -e arch=$(uname -i) -# DAOS requires newer golang version than the one available in core ubuntu repo -apt-get install gpg-agent software-properties-common -add-apt-repository ppa:longsleep/golang-backports -apt update - apt-get install \ autoconf \ build-essential \