DAOS-13471 control: Add structured version info to utilities (#12544)

Allow consumers to grab structured build/version information.
Centralizes JSON output logic to a single implementation in
the cmdutil package. Includes updates to control/version.py
to use the structured version information instead of scraping
stdout. Includes changes made for DAOS-13236 to include
build info in version output and DAOS-13878 to install
golang >= 1.18 as a daos-tests dependency.

Signed-off-by: Michael MacDonald <mjmac.macdonald@intel.com>

ci/provisioning/post_provision_config_common_functions.sh

@@ -196,6 +196,13 @@ set_local_repo() {
  dnf -y config-manager \
  --disable daos-stack-daos-"${DISTRO_GENERIC}"-"$version"-x86_64-stable-local-artifactory
  fi
+
+ if [ "$repo_server" = "artifactory" ]; then
+ # Disable module filtering for our deps repo
+ deps_repo="daos-stack-deps-${DISTRO_GENERIC}-$version-x86_64-stable-local-artifactory"
+ dnf config-manager --save --setopt "$deps_repo.module_hotfixes=true" "$deps_repo"
+ fi
+
  dnf repolist
 }
 

ci/unit/required_packages.sh

@@ -12,19 +12,30 @@ elif [[ "$distro" = *8 ]]; then
  OPENMPI_VER=""
  PY_MINOR_VER=""
 fi
-pkgs="gotestsum openmpi$OPENMPI_VER \
- hwloc-devel argobots \
- fuse3-libs fuse3 \
- boost-python3$PY_MINOR_VER-devel \
- libisa-l-devel libpmem \
- libpmemobj protobuf-c \
- spdk-devel libfabric-devel \
- pmix numactl-devel \
- libipmctl-devel python3$PY_MINOR_VER-pyxattr \
- python3$PY_MINOR_VER-junit_xml \
- python3$PY_MINOR_VER-tabulate numactl \
- libyaml-devel \
- valgrind-devel patchelf capstone"
+pkgs="argobots \
+ boost-python3$PY_MINOR_VER-devel \
+ capstone \
+ fuse3 \
+ fuse3-libs \
+ gotestsum \
+ hwloc-devel \
+ libipmctl-devel \
+ libisa-l-devel \
+ libfabric-devel \
+ libpmem \
+ libpmemobj \
+ libyaml-devel \
+ numactl \
+ numactl-devel \
+ openmpi$OPENMPI_VER \
+ patchelf \
+ pmix \
+ protobuf-c \
+ python3$PY_MINOR_VER-junit_xml \
+ python3$PY_MINOR_VER-pyxattr \
+ python3$PY_MINOR_VER-tabulate \
+ spdk-devel \
+ valgrind-devel"
 
 if $quick_build; then
  if ! read -r mercury_version < "$distro"-required-mercury-rpm-version; then

debian/changelog

@@ -1,3 +1,9 @@
+daos (2.3.108-4) unstable; urgency=medium
+ [Michael MacDonald]
+ * Add golang-go as a tests dependency for dfuse/daos_build.py
+
+ -- Michael MacDonald <mjmac.macdonald@intel.com> Mon, 17 Jul 2023 10:10:00 -0400
+
 daos (2.3.108-3) unstable; urgency=medium
  [ Wang Shilong ]
  * Remove lmdb-devel for MD on SSD

debian/control

@@ -115,6 +115,7 @@ Depends: python (>=3.8), python3, python-yaml, python3-yaml,
  ${shlibs:Depends}, ${misc:Depends},
  daos-client (= ${binary:Version}),
  daos-admin (= ${binary:Version}),
+ golang-go (>=1.18),
  libcapstone-dev
 Description: The Distributed Asynchronous Object Storage (DAOS) is an open-source
  software-defined object store designed from the ground up for

site_scons/site_tools/go_builder.py

@@ -8,7 +8,7 @@
 from SCons.Script import Configure, GetOption, Scanner, Glob, Exit, File
 
 GO_COMPILER = 'go'
-MIN_GO_VERSION = '1.17.0'
+MIN_GO_VERSION = '1.18.0'
 include_re = re.compile(r'\#include [<"](\S+[>"])', re.M)
 
 

src/control/SConscript

@@ -18,6 +18,8 @@ def get_build_tags(benv):
  tags.append("firmware")
  if not is_release_build(benv):
  tags.append("pprof")
+ else:
+ tags.append("release")
  return f"-tags {','.join(tags)}"
 
 

src/control/build/info.go

@@ -0,0 +1,39 @@
+//
+// (C) Copyright 2023 Intel Corporation.
+//
+// SPDX-License-Identifier: BSD-2-Clause-Patent
+//
+
+//go:build go1.18
+
+package build
+
+import (
+ "runtime/debug"
+ "strings"
+ "time"
+)
+
+func init() {
+ info, ok := debug.ReadBuildInfo()
+ if !ok {
+ return
+ }
+
+ for _, setting := range info.Settings {
+ switch setting.Key {
+ case "vcs":
+ VCS = setting.Value
+ case "vcs.revision":
+ Revision = setting.Value
+ case "vcs.modified":
+ DirtyBuild = setting.Value == "true"
+ case "vcs.time":
+ LastCommit, _ = time.Parse(time.RFC3339, setting.Value)
+ case "-tags":
+ if strings.Contains(setting.Value, "release") {
+ ReleaseBuild = true
+ }
+ }
+ }
+}

src/control/build/string.go

@@ -0,0 +1,57 @@
+//
+// (C) Copyright 2023 Intel Corporation.
+//
+// SPDX-License-Identifier: BSD-2-Clause-Patent
+//
+
+package build
+
+import (
+ "encoding/json"
+ "fmt"
+ "strings"
+)
+
+func revString(version string) string {
+ if ReleaseBuild {
+ return version
+ }
+
+ revParts := []string{version}
+ if Revision != "" {
+ switch VCS {
+ case "git":
+ revParts = append(revParts, fmt.Sprintf("g%7s", Revision)[0:7])
+ default:
+ revParts = append(revParts, Revision)
+ }
+ if DirtyBuild {
+ revParts = append(revParts, "dirty")
+ }
+ }
+ return strings.Join(revParts, "-")
+}
+
+// String returns a string containing the name, version, and for non-release builds,
+// the revision of the binary.
+func String(name string) string {
+ return fmt.Sprintf("%s version %s", name, revString(DaosVersion))
+}
+
+// MarshalJSON returns a JSON string containing a structured representation of
+// the binary build info.
+func MarshalJSON(name string) ([]byte, error) {
+ return json.Marshal(&struct {
+ Name string `json:"name"`
+ Version string `json:"version"`
+ Revision string `json:"revision,omitempty"`
+ Dirty bool `json:"dirty,omitempty"`
+ Release bool `json:"release,omitempty"`
+ }{
+ Name: name,
+ Version: DaosVersion,
+ Revision: Revision,
+ Dirty: DirtyBuild,
+ Release: ReleaseBuild,
+ })
+}

src/control/build/variables.go

@@ -1,12 +1,14 @@
 //
-// (C) Copyright 2020-2021 Intel Corporation.
+// (C) Copyright 2020-2023 Intel Corporation.
 //
 // SPDX-License-Identifier: BSD-2-Clause-Patent
 //
 
 // Package build provides an importable repository of variables set at build time.
 package build
 
+import "time"
+
 var (
  // ConfigDir should be set via linker flag using the value of CONF_DIR.
  ConfigDir string = "./"
@@ -20,10 +22,25 @@ var (
  ManagementServiceName = "DAOS Management Service"
  // AgentName defines a consistent name for the compute node agent.
  AgentName = "DAOS Agent"
+ // CLIUtilName defines a consistent name for the daos CLI utility.
+ CLIUtilName = "DAOS CLI"
+ // AdminUtilName defines a consistent name for the dmg utility.
+ AdminUtilName = "DAOS Admin Tool"
 
  // DefaultControlPort defines the default control plane listener port.
  DefaultControlPort = 10001
 
  // DefaultSystemName defines the default DAOS system name.
  DefaultSystemName = "daos_server"
+
+ // VCS is the version control system used to build the binary.
+ VCS = ""
+ // Revision is the VCS revision of the binary.
+ Revision = ""
+ // LastCommit is the time of the last commit.
+ LastCommit time.Time
+ // ReleaseBuild is true if the binary was built with the release tag.
+ ReleaseBuild bool
+ // DirtyBuild is true if the binary was built with uncommitted changes.
+ DirtyBuild bool
 )

src/control/cmd/daos/acl.go

@@ -121,12 +121,11 @@ func (cmd *aclCmd) getACL(ap *C.struct_cmd_args_s) (*control.AccessControlList,
 }
 
 func (cmd *aclCmd) outputACL(out io.Writer, acl *control.AccessControlList, verbose bool) error {
- if cmd.jsonOutputEnabled() {
- cmd.wroteJSON.SetTrue()
- return outputJSON(out, acl, nil)
+ if cmd.JSONOutputEnabled() {
+ return cmd.OutputJSON(acl, nil)
  }
 
- _, err := fmt.Fprintf(out, control.FormatACL(acl, verbose))
+ _, err := fmt.Fprint(out, control.FormatACL(acl, verbose))
  return err
 }
 
@@ -406,9 +405,8 @@ func (cmd *containerSetOwnerCmd) Execute(args []string) error {
  cmd.ContainerID())
  }
 
- if cmd.jsonOutputEnabled() {
- cmd.wroteJSON.SetTrue()
- return outputJSON(os.Stdout, nil, nil)
+ if cmd.JSONOutputEnabled() {
+ return cmd.OutputJSON(nil, nil)
  }
 
  var contID string

src/control/cmd/daos/container.go

@@ -314,8 +314,8 @@ func (cmd *containerCreateCmd) Execute(_ []string) (err error) {
  ci.ContainerLabel = cmd.Args.Label
  }
 
- if cmd.jsonOutputEnabled() {
- return cmd.outputJSON(ci, nil)
+ if cmd.JSONOutputEnabled() {
+ return cmd.OutputJSON(ci, nil)
  }
 
  var bld strings.Builder
@@ -750,8 +750,8 @@ func (cmd *containerListCmd) Execute(_ []string) error {
  "unable to list containers for pool %s", cmd.PoolID())
  }
 
- if cmd.jsonOutputEnabled() {
- return cmd.outputJSON(contIDs, nil)
+ if cmd.JSONOutputEnabled() {
+ return cmd.OutputJSON(contIDs, nil)
  }
 
  var bld strings.Builder
@@ -903,16 +903,16 @@ func (cmd *containerListObjectsCmd) Execute(_ []string) error {
  for i := C.uint32_t(0); i < readOids; i++ {
  oid := fmt.Sprintf("%d.%d", oidArr[i].hi, oidArr[i].lo)
 
- if !cmd.jsonOutputEnabled() {
+ if !cmd.JSONOutputEnabled() {
  cmd.Infof("%s", oid)
  continue
  }
  oids = append(oids, oid)
  }
  }
 
- if cmd.jsonOutputEnabled() {
- return cmd.outputJSON(oids, nil)
+ if cmd.JSONOutputEnabled() {
+ return cmd.OutputJSON(oids, nil)
  }
 
  return nil
@@ -1035,8 +1035,8 @@ func (cmd *containerQueryCmd) Execute(_ []string) error {
  cmd.contUUID)
  }
 
- if cmd.jsonOutputEnabled() {
- return cmd.outputJSON(ci, nil)
+ if cmd.JSONOutputEnabled() {
+ return cmd.OutputJSON(ci, nil)
  }
 
  var bld strings.Builder
@@ -1080,8 +1080,8 @@ func (cmd *containerCloneCmd) Execute(_ []string) error {
  return errors.Wrapf(err, "failed to clone container %s", cmd.Source)
  }
 
- if cmd.shouldEmitJSON {
- return cmd.outputJSON(struct {
+ if cmd.JSONOutputEnabled() {
+ return cmd.OutputJSON(struct {
  SourcePool string `json:"src_pool"`
  SourceCont string `json:"src_cont"`
  DestPool string `json:"dst_pool"`
@@ -1160,11 +1160,11 @@ func (cmd *containerListAttrsCmd) Execute(args []string) error {
  cmd.ContainerID())
  }
 
- if cmd.jsonOutputEnabled() {
+ if cmd.JSONOutputEnabled() {
  if cmd.Verbose {
- return cmd.outputJSON(attrs.asMap(), nil)
+ return cmd.OutputJSON(attrs.asMap(), nil)
  }
- return cmd.outputJSON(attrs.asList(), nil)
+ return cmd.OutputJSON(attrs.asList(), nil)
  }
 
  var bld strings.Builder
@@ -1258,12 +1258,12 @@ func (cmd *containerGetAttrCmd) Execute(args []string) error {
  return errors.Wrapf(err, "failed to get attributes from container %s", cmd.ContainerID())
  }
 
- if cmd.jsonOutputEnabled() {
+ if cmd.JSONOutputEnabled() {
  // Maintain compatibility with older behavior.
  if len(cmd.Args.Attrs.ParsedProps) == 1 && len(attrs) == 1 {
- return cmd.outputJSON(attrs[0], nil)
+ return cmd.OutputJSON(attrs[0], nil)
  }
- return cmd.outputJSON(attrs, nil)
+ return cmd.OutputJSON(attrs, nil)
  }
 
  var bld strings.Builder
@@ -1398,8 +1398,8 @@ func (cmd *containerGetPropCmd) Execute(args []string) error {
  }
  }
 
- if cmd.jsonOutputEnabled() {
- return cmd.outputJSON(props, nil)
+ if cmd.JSONOutputEnabled() {
+ return cmd.OutputJSON(props, nil)
  }
 
  title := fmt.Sprintf("Properties for container %s", cmd.ContainerID())