DAOS-13386 ucx: Update to UCX v1.14.

debian/changelog

@@ -1,3 +1,9 @@
+ucx (1.14.0-1) unstable; urgency=medium
+
+  * Update to 1.14
+
+ -- Joseph Moore <joseph.moore@intel.com>  Wed, 10 May 2023 11:41:10 -0500
+
 ucx (1.13.0-1) unstable; urgency=medium
 
   * Update to 1.13

packaging/Dockerfile.mockbuild

@@ -34,14 +34,24 @@ RUN echo "$USER:$PASSWD" | chpasswd
 # add the user to the mock group so it can run mock
 RUN usermod -a -G mock $USER
 
-ARG CACHEBUST
+ARG CB0
 RUN dnf -y upgrade && \
     dnf clean all
 
 # Monkey-patch rpmlint until a new release is made with
 # https://github.com/rpm-software-management/rpmlint/pull/795 in it
+# But make sure to patch after dnf upgrade so that an upgraded rpmlint
+# RPM doesn't wipe out our patch
 COPY packaging/rpmlint--ignore-unused-rpmlintrc.patch .
-RUN (cd $(python3 -c 'import site; print(site.getsitepackages()[-1])') &&                    \
-     patch -p1 &&                                                                            \
-     rm -f rpmlint/__pycache__/{cli,lint}.*.pyc) < rpmlint--ignore-unused-rpmlintrc.patch && \
+RUN (cd $(python3 -c 'import site; print(site.getsitepackages()[-1])') &&                      \
+     if ! grep -e --ignore-unused-rpmlintrc rpmlint/cli.py; then                               \
+         if ! patch -p1; then                                                                  \
+             exit 1;                                                                           \
+         fi;                                                                                   \
+         rm -f rpmlint/__pycache__/{cli,lint}.*.pyc;                                           \
+     fi) < rpmlint--ignore-unused-rpmlintrc.patch;                                             \
     rm -f  rpmlint--ignore-unused-rpmlintrc.patch
+
+# show the release that was built
+ARG CACHEBUST
+RUN cat /etc/os-release

packaging/Dockerfile.ubuntu.20.04

@@ -7,33 +7,59 @@
 FROM ubuntu:20.04
 LABEL org.opencontainers.image.authors="daos@daos.groups.io"
 
-# use same UID as host and default value of 1000 if not specified
-ARG UID=1000
-ARG REPO_URL=""
-ARG REPO_UBUNTU_20_04=""
+RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y \
+    curl gpg
+
+ARG REPO_FILE_URL
+RUN if [ -n "$REPO_FILE_URL" ]; then                                 \
+        cd /etc/apt/sources.list.d &&                                \
+        curl -f -o daos_ci-ubuntu20.04-artifactory.list.tmp          \
+             "$REPO_FILE_URL"daos_ci-ubuntu20.04-artifactory.list && \
+        true > ../sources.list &&                                    \
+        mv daos_ci-ubuntu20.04-artifactory.list.tmp                  \
+           daos_ci-ubuntu20.04-artifactory.list;                     \
+    fi;                                                              \
+    cd -;                                                            \
+    curl -f -O "$REPO_FILE_URL"esad_repo.key;                        \
+    gpg --no-default-keyring --keyring ./temp-keyring.gpg            \
+        --import esad_repo.key;                                      \
+    mkdir -p /usr/local/share/keyrings/;                             \
+    gpg --no-default-keyring --keyring ./temp-keyring.gpg --export   \
+        --output /usr/local/share/keyrings/daos-stack-public.gpg;    \
+    rm ./temp-keyring.gpg;                                           \
+    url_prefix=https://downloads.linux.hpe.com/SDR/;                 \
+    for url in hpPublicKey2048.pub                                   \
+               hpPublicKey2048_key1.pub                              \
+               hpePublicKey2048_key1.pub; do                         \
+        curl -f -O "$url_prefix$url";                                \
+        gpg --no-default-keyring --keyring ./temp-keyring.gpg        \
+            --import "$(basename $url)";                             \
+    done;                                                            \
+    gpg --no-default-keyring --keyring ./temp-keyring.gpg --export   \
+        --output /usr/local/share/keyrings/hpe-sdr-public.gpg;       \
+    rm ./temp-keyring.gpg
 
 # Install basic tools
 RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y \
             autoconf bash ca-certificates curl debhelper dh-make        \
             dpkg-dev dh-python doxygen gcc git git-buildpackage locales \
             make patch pbuilder pkg-config python3-dev python3-distro   \
-            python3-distutils rpm scons wget cmake valgrind
+            python3-distutils rpm scons wget cmake valgrind rpmdevtools
 
-# rpmdevtools
-RUN echo "deb [trusted=yes] ${REPO_URL}${REPO_UBUNTU_20_04} focal main" > /etc/apt/sources.list.d/daos-stack-ubuntu-stable-local.list
-RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y \
-            rpmdevtools
+# use same UID as host and default value of 1000 if not specified
+ARG UID=1000
 
 # Add build user (to keep chrootbuild happy)
 ENV USER build
 RUN useradd -u $UID -ms /bin/bash $USER
 
 # need to run the build command as root, as it needs to chroot
-RUN if ! grep "^#includedir /etc/sudoers.d" /etc/sudoers; then                          \
-        echo "#includedir /etc/sudoers.d" >> /etc/sudoers;                              \
-    fi;                                                                                 \
-    echo "Defaults env_keep += \"DPKG_GENSYMBOLS_CHECK_LEVEL\"" > /etc/sudoers.d/build; \
-    echo "build ALL=(ALL) NOPASSWD: /usr/sbin/pbuilder" >> /etc/sudoers.d/build;        \
-    chmod 0440 /etc/sudoers.d/build;                                                    \
-    visudo -c;                                                                          \
+RUN if ! grep "^#includedir /etc/sudoers.d" /etc/sudoers; then                                     \
+        echo "#includedir /etc/sudoers.d" >> /etc/sudoers;                                         \
+    fi;                                                                                            \
+    echo "Defaults env_keep += \"DPKG_GENSYMBOLS_CHECK_LEVEL\"" > /etc/sudoers.d/build;            \
+    echo "build ALL=(ALL) NOPASSWD: /usr/bin/tee /root/.pbuilderrc" >> /etc/sudoers.d/build; \
+    echo "build ALL=(ALL) NOPASSWD: /usr/sbin/pbuilder" >> /etc/sudoers.d/build;                   \
+    chmod 0440 /etc/sudoers.d/build;                                                               \
+    visudo -c;                                                                                     \
     sudo -l -U build

packaging/Makefile_distro_vars.mk

@@ -36,16 +36,16 @@ ORIG_TARGET_VER := 7
 SED_EXPR        := 1s/$(DIST)//p
 endif
 ifeq ($(patsubst %epel-8-x86_64,,$(lastword $(subst +, ,$(CHROOT_NAME)))),)
-DIST                    := $(shell rpm $(COMMON_RPM_ARGS) --eval %{?dist})
-VERSION_ID              := 8.5
-DISTRO_ID               := el8
-DISTRO_BASE             := EL_8
+DIST            := $(shell rpm $(COMMON_RPM_ARGS) --eval %{?dist})
+VERSION_ID      := 8
+DISTRO_ID       := el8
+DISTRO_BASE     := EL_8
 ifneq ($(DISTRO_VERSION_EL8),)
 override DISTRO_VERSION := $(DISTRO_VERSION_EL8)
 endif
-DISTRO_VERSION          ?= $(VERSION_ID)
-ORIG_TARGET_VER         := 8.5
-SED_EXPR                := 1s/$(DIST)//p
+DISTRO_VERSION  ?= $(VERSION_ID)
+ORIG_TARGET_VER := 8
+SED_EXPR        := 1s/$(DIST)//p
 endif
 ifeq ($(CHROOT_NAME),opensuse-leap-15.2-x86_64)
 VERSION_ID      := 15.2
@@ -60,7 +60,15 @@ VERSION_ID      := 15.3
 DISTRO_ID       := sl15.3
 DISTRO_BASE     := LEAP_15
 DISTRO_VERSION  ?= $(VERSION_ID)
-ORIG_TARGET_VER := 15.2
+ORIG_TARGET_VER := 15.3
+SED_EXPR        := 1p
+endif
+ifeq ($(CHROOT_NAME),opensuse-leap-15.4-x86_64)
+VERSION_ID      := 15.4
+DISTRO_ID       := sl15.4
+DISTRO_BASE     := LEAP_15
+DISTRO_VERSION  ?= $(VERSION_ID)
+ORIG_TARGET_VER := 15.4
 SED_EXPR        := 1p
 endif
 endif

packaging/Makefile_packaging.mk

@@ -6,7 +6,7 @@
 # force bash (looking at you Ubuntu)
 SHELL=/bin/bash
 
-# Put site overrides (i.e. REPOSITORY_URL, DAOS_STACK_*_LOCAL_REPO) in here
+# Put site overrides (i.e. DAOS_STACK_*_LOCAL_REPO) in here
 -include Makefile.local
 
 # default to Leap 15 distro for chrootbuild
@@ -154,35 +154,10 @@ ifeq ($(DL_NAME),)
 DL_NAME = $(NAME)
 endif
 
-# this actually should replace all of the downloaders below
 $(notdir $(SOURCE)): $(SPEC) $(CALLING_MAKEFILE)
 	# TODO: need to clean up old ones
 	$(SPECTOOL) -g $(SPEC)
 
-$(DL_NAME)$(DL_VERSION).linux-amd64.tar.$(SRC_EXT): $(SPEC) $(CALLING_MAKEFILE)
-	rm -f ./$(DL_NAME)*.tar{gz,bz*,xz}
-	$(SPECTOOL) -g $(SPEC)
-
-$(DL_NAME)-$(DL_VERSION).tar.$(SRC_EXT).asc: $(SPEC) $(CALLING_MAKEFILE)
-	rm -f ./$(DL_NAME)-*.tar.{gz,bz*,xz}.asc
-	$(SPECTOOL) -g $(SPEC)
-
-$(DL_NAME)-$(DL_VERSION).tar.$(SRC_EXT).sig: $(SPEC) $(CALLING_MAKEFILE)
-	rm -f ./$(DL_NAME)-*.tar.{gz,bz*,xz}.sig
-	$(SPECTOOL) -g $(SPEC)
-
-$(DL_NAME)-$(DL_VERSION).tar.$(SRC_EXT): $(SPEC) $(CALLING_MAKEFILE)
-	rm -f ./$(DL_NAME)-*.tar.{gz,bz*,xz}
-	$(SPECTOOL) -g $(SPEC)
-
-v$(DL_VERSION).tar.$(SRC_EXT): $(SPEC) $(CALLING_MAKEFILE)
-	rm -f ./v*.tar.{gz,bz*,xz}
-	$(SPECTOOL) -g $(SPEC)
-
-$(DL_VERSION).tar.$(SRC_EXT): $(SPEC) $(CALLING_MAKEFILE)
-	rm -f ./*.tar.{gz,bz*,xz}
-	$(SPECTOOL) -g $(SPEC)
-
 $(DEB_TOP)/%: % | $(DEB_TOP)/
 
 $(DEB_BUILD)/%: % | $(DEB_BUILD)/
@@ -304,6 +279,10 @@ $(RPMS): $(SRPM) $(CALLING_MAKEFILE)
 
 rpms: $(RPMS)
 
+repo: rpms
+	rm -rf _topdir/RPMS/repodata/
+	createrepo _topdir/RPMS/
+
 $(DEBS): $(CALLING_MAKEFILE)
 
 debs: $(DEBS)
@@ -338,20 +317,6 @@ patch:
 	echo "PKG_GIT_COMMIT is not defined"
 endif
 
-# *_LOCAL_* repos are locally built packages.
-ifeq ($(LOCAL_REPOS),true)
-  ifneq ($(ARTIFACTORY_URL),)
-    ifneq ($(DAOS_STACK_$(DISTRO_BASE)_LOCAL_REPO),)
-      DISTRO_REPOS = disabled # any non-empty value here works and is not used beyond testing if the value is empty or not
-	  # convert to artifactory url
-      DAOS_STACK_$(DISTRO_BASE)_LOCAL_REPO := $(subst reposi,artifac,$(DAOS_STACK_$(DISTRO_BASE)_LOCAL_REPO))
-      # $(DISTRO_BASE)_LOCAL_REPOS is a list separated by | because you cannot pass lists
-      # of values with spaces as environment variables
-      $(DISTRO_BASE)_LOCAL_REPOS := [trusted=yes] $(ARTIFACTORY_URL)$(subst stack,stack-daos,$(DAOS_STACK_$(DISTRO_BASE)_LOCAL_REPO))
-      $(DISTRO_BASE)_LOCAL_REPOS += |[trusted=yes] $(ARTIFACTORY_URL)$(subst stack,stack-deps,$(DAOS_STACK_$(DISTRO_BASE)_LOCAL_REPO))
-    endif #ifneq ($(DAOS_STACK_$(DISTRO_BASE)_LOCAL_REPO),)
-  endif # ifneq ($(ARTIFACTORY_URL),)
-endif # ifeq ($(LOCAL_REPOS),true)
 ifeq ($(ID_LIKE),debian)
 chrootbuild: $(DEB_TOP)/$(DEB_DSC)
 	$(call distro_map)                                      \
@@ -367,6 +332,8 @@ chrootbuild: $(DEB_TOP)/$(DEB_DSC)
 	DEB_TOP="$(DEB_TOP)"                                    \
 	DEB_DSC="$(DEB_DSC)"                                    \
 	DISTRO_ID_OPT="$(DISTRO_ID_OPT)"                        \
+	LOCAL_REPOS='$(LOCAL_REPOS)'                            \
+	ARTIFACTORY_URL="$(ARTIFACTORY_URL)"                    \
 	packaging/debian_chrootbuild
 else
 chrootbuild: $(SRPM) $(CALLING_MAKEFILE)
@@ -381,9 +348,8 @@ chrootbuild: $(SRPM) $(CALLING_MAKEFILE)
 	REPO_FILE_URL="$(REPO_FILE_URL)"                        \
 	MOCK_OPTIONS="$(MOCK_OPTIONS)"                          \
 	RPM_BUILD_OPTIONS='$(RPM_BUILD_OPTIONS)'                \
-	DISTRO_REPOS='$(DISTRO_REPOS)'                          \
+	LOCAL_REPOS='$(LOCAL_REPOS)'                            \
 	ARTIFACTORY_URL="$(ARTIFACTORY_URL)"                    \
-	REPOSITORY_URL="$(REPOSITORY_URL)"                      \
 	DISTRO_VERSION="$(DISTRO_VERSION)"                      \
 	TARGET="$<"                                             \
 	packaging/rpm_chrootbuild
@@ -397,19 +363,18 @@ podman_chrootbuild:
 	    exit 1;                                                  \
 	fi
 	rm -f /var/lib/mock/$(CHROOT_NAME)/result/{root,build}.log
-	podman run --rm --privileged -w $(TOPDIR) -v=$(TOPDIR)/..:$(TOPDIR)/..                                                     \
-	           -it $(subst +,-,$(CHROOT_NAME))-chrootbuild                                                                     \
-	           bash -c 'if ! DISTRO_REPOS=false                                                                                \
-	                         REPO_FILE_URL=$(REPO_FILE_URL)                                                                    \
-	                         REPOSITORY_URL=$(REPOSITORY_URL)                                                                  \
-	                         make REPO_FILES_PR=$(REPO_FILES_PR)                                                               \
-	                              MOCK_OPTIONS=$(MOCK_OPTIONS)                                                                 \
-	                              CHROOT_NAME=$(CHROOT_NAME) -C $(CURDIR) chrootbuild; then                                    \
-	                            cat /var/lib/mock/$(CHROOT_NAME)/{result/{root,build},root/builddir/build/BUILD/*/config}.log; \
-	                            exit 1;                                                                                        \
-	                        fi;                                                                                                \
-	                        rpmlint $$(ls /var/lib/mock/$(CHROOT_NAME)/result/*.rpm |                                          \
-	                            grep -v -e debuginfo -e debugsource -e src.rpm)'
+	if ! podman run --rm --privileged -w $(TOPDIR) -v=$(TOPDIR)/..:$(TOPDIR)/..                                                     \
+	                -it $(subst +,-,$(CHROOT_NAME))-chrootbuild                                                                     \
+	                bash -c 'if ! DISTRO_REPOS=false                                                                                \
+	                              REPO_FILE_URL=$(REPO_FILE_URL)                                                                    \
+	                              make REPO_FILES_PR=$(REPO_FILES_PR)                                                               \
+	                                   MOCK_OPTIONS=$(MOCK_OPTIONS)                                                                 \
+	                                   CHROOT_NAME=$(CHROOT_NAME) -C $(CURDIR) chrootbuild; then                                    \
+	                                 cat /var/lib/mock/$(CHROOT_NAME)/{result/{root,build},root/builddir/build/BUILD/*/config}.log; \
+	                                 exit 1;                                                                                        \
+	                             fi;                                                                                                \
+	                             rpmlint $$(ls /var/lib/mock/$(CHROOT_NAME)/result/*.rpm |                                          \
+	                                 grep -v -e debuginfo -e debugsource -e src.rpm)'
 
 docker_chrootbuild:
 	if ! $(DOCKER) build --build-arg UID=$$(id -u) -t chrootbuild   \

packaging/debian_chrootbuild

@@ -2,14 +2,13 @@
 
 set -uex
 
-# shellcheck disable=SC2153
-IFS=\| read -r -a distro_base_local_repos <<< "$DISTRO_BASE_LOCAL_REPOS"
+if [ -n "${ARTIFACTORY_URL:-}" ] && "$LOCAL_REPOS"; then
+    echo "MIRRORSITE=${ARTIFACTORY_URL}artifactory/ubuntu-proxy" | sudo tee /root/.pbuilderrc
+fi
 
 # shellcheck disable=SC2086
-sudo pbuilder create                                                                                                                                                           \
-    --extrapackages "gnupg ca-certificates"                                                                                                                                    \
-    --othermirror                                                                                                                                                              \
-    "deb [arch=amd64] http://archive.ubuntu.com/ubuntu/ $VERSION_CODENAME universe|deb [arch=amd64] http://archive.ubuntu.com/ubuntu/ $VERSION_CODENAME-updates main universe" \
+sudo pbuilder create                        \
+    --extrapackages "gnupg ca-certificates" \
     $DISTRO_ID_OPT
 
 repo_args=""
@@ -26,9 +25,18 @@ for repo in $DISTRO_BASE_PR_REPOS $PR_REPOS; do
     fi
     repo_args="$repo_args|deb [trusted=yes] ${JENKINS_URL:-https://build.hpdd.intel.com/}job/daos-stack/job/$repo/job/$branch/$build_number/artifact/artifacts/$DISTRO/ ./"
 done
-for repo in $JOB_REPOS "${distro_base_local_repos[@]}"; do
-    repo_args="$repo_args|deb ${repo} $VERSION_CODENAME main"
+
+repo_args+="|$(curl -sSf "$REPO_FILE_URL"daos_ci-"$DISTRO"-artifactory.list |
+            sed -e 's/#.*//' -e '/ubuntu-proxy/d' -e '/^$/d' -e '/^$/d'     \
+                -e 's/signed-by=.*\.gpg/trusted=yes/'                       |
+            sed -e ':a; N; $!ba; s/\n/|/g')"
+for repo in $JOB_REPOS; do
+    repo_args+="|deb ${repo} $VERSION_CODENAME main"
 done
+# NB: This PPA is needed to support modern go toolchains on ubuntu 20.04.
+# After the build is updated to use 22.04, which supports go >= 1.18, it
+# should no longer be needed.
+repo_args="$repo_args|deb [trusted=yes] https://ppa.launchpadcontent.net/longsleep/golang-backports/ubuntu $VERSION_CODENAME main"
 echo "$repo_args"
 if [ "$repo_args" = "|" ]; then
     repo_args=""

packaging/rpm_chrootbuild

@@ -18,17 +18,28 @@ config_opts['module_setup_commands'] = [
   ('disable',  'go-toolset')
 ]
 EOF
+fi
 
+# Use dnf on CentOS 7
+if [[ $CHROOT_NAME == *epel-7-x86_64 ]]; then
+    MOCK_OPTIONS="--dnf --no-bootstrap-chroot${MOCK_OPTIONS:+ }$MOCK_OPTIONS"
 fi
 
+# Allow BR: foo-devel < 1.2 to work when foo-devel-1.3 is actually available
+cat <<EOF >> "$cfg_file"
+config_opts['dnf.conf'] += """
+[main]
+best=0
+"""
+EOF
+
 # shellcheck disable=SC2153
 repo_adds=()
 repo_dels=()
 
 echo -e "config_opts['yum.conf'] += \"\"\"\n" >> "$cfg_file"
 
-if { [ -n "${REPOSITORY_URL:-}" ] || [ -n "${ARTIFACTORY_URL:-}" ]; } \
-   && [ -n "$DISTRO_REPOS" ]; then
+if [ -n "${ARTIFACTORY_URL:-}" ] && "$LOCAL_REPOS"; then
     repo_dels+=("--disablerepo=\*")
 
     if [ -n "${REPO_FILE_URL:-}" ]; then

ucx.spec

@@ -20,7 +20,7 @@
 %bcond_with    vfs
 
 %global major 1
-%global minor 13
+%global minor 14
 %global bugrelease 0
 #%%global prerelease
 
@@ -34,7 +34,6 @@ Summary: UCX is a communication library implementing high-performance messaging
 License: BSD
 URL: http://www.openucx.org
 Source: https://github.com/openucx/ucx/releases/download/v%{dl_version}%{?prerelease:-%{prerelease}}/ucx-%{dl_version}.tar.gz
-Patch0: undo-upstream.patch
 
 BuildRoot: %(mktemp -ud %{_tmppath}/%{name}-%{version}-%{release}-XXXXXX)
 Prefix: %{_prefix}
@@ -327,6 +326,9 @@ library internals, protocol objects, transports status, and more.
 %endif
 
 %changelog
+* Wed May 10 2023 Joseph Moore <joseph.moore@intel.com> - 1.14.1-1
+- Update to 1.14
+
 * Tue Sep 06 2022 Joseph Moore <joseph.moore@intel.com> - 1.13.1-1
 - Update to 1.13