Skip to content

Commit

Permalink
DAOS-13471 control: Add structured version info to utilities
Browse files Browse the repository at this point in the history
Allow consumers to grab structured build/version information.
Centralizes JSON output logic to a single implementation in
the cmdutil package. Includes updates to control/version.py
to use the structured version information instead of scraping
stdout. Includes changes made for DAOS-13236 to include
build info in version output and DAOS-13878 to install
golang >= 1.18 as a daos-tests dependency.

Required-githooks: true

Signed-off-by: Michael MacDonald <mjmac.macdonald@intel.com>
  • Loading branch information
mjmac committed Jul 2, 2023
1 parent 776249a commit 3fbd6d7
Show file tree
Hide file tree
Showing 42 changed files with 545 additions and 478 deletions.
7 changes: 7 additions & 0 deletions ci/provisioning/post_provision_config_common_functions.sh
Original file line number Diff line number Diff line change
Expand Up @@ -196,6 +196,13 @@ set_local_repo() {
dnf -y config-manager \
--disable daos-stack-daos-"${DISTRO_GENERIC}"-"$version"-x86_64-stable-local-artifactory
fi

if [ "$repo_server" = "artifactory" ]; then
# Disable module filtering for our deps repo
deps_repo="daos-stack-deps-${DISTRO_GENERIC}-$version-x86_64-stable-local-artifactory"
dnf config-manager --save --setopt "$deps_repo.module_hotfixes=true" "$deps_repo"
fi

dnf repolist
}

Expand Down
37 changes: 24 additions & 13 deletions ci/unit/required_packages.sh
Original file line number Diff line number Diff line change
Expand Up @@ -12,19 +12,30 @@ elif [[ "$distro" = *8 ]]; then
OPENMPI_VER=""
PY_MINOR_VER=""
fi
pkgs="gotestsum openmpi$OPENMPI_VER \
hwloc-devel argobots \
fuse3-libs fuse3 \
boost-python3$PY_MINOR_VER-devel \
libisa-l-devel libpmem \
libpmemobj protobuf-c \
spdk-devel libfabric-devel \
pmix numactl-devel \
libipmctl-devel python3$PY_MINOR_VER-pyxattr \
python3$PY_MINOR_VER-junit_xml \
python3$PY_MINOR_VER-tabulate numactl \
libyaml-devel \
valgrind-devel patchelf capstone"
pkgs="argobots \
boost-python3$PY_MINOR_VER-devel \
capstone \
fuse3 \
fuse3-libs \
gotestsum \
hwloc-devel \
libipmctl-devel \
libisa-l-devel \
libfabric-devel \
libpmem \
libpmemobj \
libyaml-devel \
numactl \
numactl-devel \
openmpi$OPENMPI_VER \
patchelf \
pmix \
protobuf-c \
python3$PY_MINOR_VER-junit_xml \
python3$PY_MINOR_VER-pyxattr \
python3$PY_MINOR_VER-tabulate \
spdk-devel \
valgrind-devel"

if $quick_build; then
if ! read -r mercury_version < "$distro"-required-mercury-rpm-version; then
Expand Down
6 changes: 6 additions & 0 deletions debian/changelog
Original file line number Diff line number Diff line change
@@ -1,3 +1,9 @@
daos (2.3.108-2) unstable; urgency=medium
[Michael MacDonald]
* Add golang-go as a tests dependency for dfuse/daos_build.py

-- Michael MacDonald <mjmac.macdonald@intel.com> Thu, 29 Jun 2023 10:10:00 -0400

daos (2.3.108-1) unstable; urgency=medium
[ Jeff Olivier ]
* Bump version to 2.3.108
Expand Down
1 change: 1 addition & 0 deletions debian/control
Original file line number Diff line number Diff line change
Expand Up @@ -116,6 +116,7 @@ Depends: python (>=3.8), python3, python-yaml, python3-yaml,
${shlibs:Depends}, ${misc:Depends},
daos-client (= ${binary:Version}),
daos-admin (= ${binary:Version}),
golang-go (>=1.18),
libcapstone-dev
Description: The Distributed Asynchronous Object Storage (DAOS) is an open-source
software-defined object store designed from the ground up for
Expand Down
2 changes: 1 addition & 1 deletion site_scons/site_tools/go_builder.py
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,7 @@
from SCons.Script import Configure, GetOption, Scanner, Glob, Exit, File

GO_COMPILER = 'go'
MIN_GO_VERSION = '1.17.0'
MIN_GO_VERSION = '1.18.0'
include_re = re.compile(r'\#include [<"](\S+[>"])', re.M)


Expand Down
2 changes: 2 additions & 0 deletions src/control/SConscript
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,8 @@ def get_build_tags(benv):
tags.append("firmware")
if not is_release_build(benv):
tags.append("pprof")
else:
tags.append("release")
return f"-tags {','.join(tags)}"


Expand Down
39 changes: 39 additions & 0 deletions src/control/build/info.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,39 @@
//
// (C) Copyright 2023 Intel Corporation.
//
// SPDX-License-Identifier: BSD-2-Clause-Patent
//

//go:build go1.18

package build

import (
"runtime/debug"
"strings"
"time"
)

func init() {
info, ok := debug.ReadBuildInfo()
if !ok {
return
}

for _, setting := range info.Settings {
switch setting.Key {
case "vcs":
VCS = setting.Value
case "vcs.revision":
Revision = setting.Value
case "vcs.modified":
DirtyBuild = setting.Value == "true"
case "vcs.time":
LastCommit, _ = time.Parse(time.RFC3339, setting.Value)
case "-tags":
if strings.Contains(setting.Value, "release") {
ReleaseBuild = true
}
}
}
}
57 changes: 57 additions & 0 deletions src/control/build/string.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,57 @@
//
// (C) Copyright 2023 Intel Corporation.
//
// SPDX-License-Identifier: BSD-2-Clause-Patent
//

package build

import (
"encoding/json"
"fmt"
"strings"
)

func revString(version string) string {
if ReleaseBuild {
return version
}

revParts := []string{version}
if Revision != "" {
switch VCS {
case "git":
revParts = append(revParts, fmt.Sprintf("g%7s", Revision)[0:7])
default:
revParts = append(revParts, Revision)
}
if DirtyBuild {
revParts = append(revParts, "dirty")
}
}
return strings.Join(revParts, "-")
}

// String returns a string containing the name, version, and for non-release builds,
// the revision of the binary.
func String(name string) string {
return fmt.Sprintf("%s version %s", name, revString(DaosVersion))
}

// MarshalJSON returns a JSON string containing a structured representation of
// the binary build info.
func MarshalJSON(name string) ([]byte, error) {
return json.Marshal(&struct {
Name string `json:"name"`
Version string `json:"version"`
Revision string `json:"revision,omitempty"`
Dirty bool `json:"dirty,omitempty"`
Release bool `json:"release,omitempty"`
}{
Name: name,
Version: DaosVersion,
Revision: Revision,
Dirty: DirtyBuild,
Release: ReleaseBuild,
})
}
19 changes: 18 additions & 1 deletion src/control/build/variables.go
Original file line number Diff line number Diff line change
@@ -1,12 +1,14 @@
//
// (C) Copyright 2020-2021 Intel Corporation.
// (C) Copyright 2020-2023 Intel Corporation.
//
// SPDX-License-Identifier: BSD-2-Clause-Patent
//

// Package build provides an importable repository of variables set at build time.
package build

import "time"

var (
// ConfigDir should be set via linker flag using the value of CONF_DIR.
ConfigDir string = "./"
Expand All @@ -20,10 +22,25 @@ var (
ManagementServiceName = "DAOS Management Service"
// AgentName defines a consistent name for the compute node agent.
AgentName = "DAOS Agent"
// CLIUtilName defines a consistent name for the daos CLI utility.
CLIUtilName = "DAOS CLI"
// AdminUtilName defines a consistent name for the dmg utility.
AdminUtilName = "DAOS Admin Tool"

// DefaultControlPort defines the default control plane listener port.
DefaultControlPort = 10001

// DefaultSystemName defines the default DAOS system name.
DefaultSystemName = "daos_server"

// VCS is the version control system used to build the binary.
VCS = ""
// Revision is the VCS revision of the binary.
Revision = ""
// LastCommit is the time of the last commit.
LastCommit time.Time
// ReleaseBuild is true if the binary was built with the release tag.
ReleaseBuild bool
// DirtyBuild is true if the binary was built with uncommitted changes.
DirtyBuild bool
)
12 changes: 5 additions & 7 deletions src/control/cmd/daos/acl.go
Original file line number Diff line number Diff line change
Expand Up @@ -121,12 +121,11 @@ func (cmd *aclCmd) getACL(ap *C.struct_cmd_args_s) (*control.AccessControlList,
}

func (cmd *aclCmd) outputACL(out io.Writer, acl *control.AccessControlList, verbose bool) error {
if cmd.jsonOutputEnabled() {
cmd.wroteJSON.SetTrue()
return outputJSON(out, acl, nil)
if cmd.JSONOutputEnabled() {
return cmd.OutputJSON(acl, nil)
}

_, err := fmt.Fprintf(out, control.FormatACL(acl, verbose))
_, err := fmt.Fprint(out, control.FormatACL(acl, verbose))
return err
}

Expand Down Expand Up @@ -400,9 +399,8 @@ func (cmd *containerSetOwnerCmd) Execute(args []string) error {
cmd.ContainerID())
}

if cmd.jsonOutputEnabled() {
cmd.wroteJSON.SetTrue()
return outputJSON(os.Stdout, nil, nil)
if cmd.JSONOutputEnabled() {
return cmd.OutputJSON(nil, nil)
}

var contID string
Expand Down
38 changes: 19 additions & 19 deletions src/control/cmd/daos/container.go
Original file line number Diff line number Diff line change
Expand Up @@ -314,8 +314,8 @@ func (cmd *containerCreateCmd) Execute(_ []string) (err error) {
ci.ContainerLabel = cmd.Args.Label
}

if cmd.jsonOutputEnabled() {
return cmd.outputJSON(ci, nil)
if cmd.JSONOutputEnabled() {
return cmd.OutputJSON(ci, nil)
}

var bld strings.Builder
Expand Down Expand Up @@ -750,8 +750,8 @@ func (cmd *containerListCmd) Execute(_ []string) error {
"unable to list containers for pool %s", cmd.PoolID())
}

if cmd.jsonOutputEnabled() {
return cmd.outputJSON(contIDs, nil)
if cmd.JSONOutputEnabled() {
return cmd.OutputJSON(contIDs, nil)
}

var bld strings.Builder
Expand Down Expand Up @@ -903,16 +903,16 @@ func (cmd *containerListObjectsCmd) Execute(_ []string) error {
for i := C.uint32_t(0); i < readOids; i++ {
oid := fmt.Sprintf("%d.%d", oidArr[i].hi, oidArr[i].lo)

if !cmd.jsonOutputEnabled() {
if !cmd.JSONOutputEnabled() {
cmd.Infof("%s", oid)
continue
}
oids = append(oids, oid)
}
}

if cmd.jsonOutputEnabled() {
return cmd.outputJSON(oids, nil)
if cmd.JSONOutputEnabled() {
return cmd.OutputJSON(oids, nil)
}

return nil
Expand Down Expand Up @@ -1035,8 +1035,8 @@ func (cmd *containerQueryCmd) Execute(_ []string) error {
cmd.contUUID)
}

if cmd.jsonOutputEnabled() {
return cmd.outputJSON(ci, nil)
if cmd.JSONOutputEnabled() {
return cmd.OutputJSON(ci, nil)
}

var bld strings.Builder
Expand Down Expand Up @@ -1080,8 +1080,8 @@ func (cmd *containerCloneCmd) Execute(_ []string) error {
return errors.Wrapf(err, "failed to clone container %s", cmd.Source)
}

if cmd.shouldEmitJSON {
return cmd.outputJSON(struct {
if cmd.JSONOutputEnabled() {
return cmd.OutputJSON(struct {
SourcePool string `json:"src_pool"`
SourceCont string `json:"src_cont"`
DestPool string `json:"dst_pool"`
Expand Down Expand Up @@ -1160,11 +1160,11 @@ func (cmd *containerListAttrsCmd) Execute(args []string) error {
cmd.ContainerID())
}

if cmd.jsonOutputEnabled() {
if cmd.JSONOutputEnabled() {
if cmd.Verbose {
return cmd.outputJSON(attrs.asMap(), nil)
return cmd.OutputJSON(attrs.asMap(), nil)
}
return cmd.outputJSON(attrs.asList(), nil)
return cmd.OutputJSON(attrs.asList(), nil)
}

var bld strings.Builder
Expand Down Expand Up @@ -1258,12 +1258,12 @@ func (cmd *containerGetAttrCmd) Execute(args []string) error {
return errors.Wrapf(err, "failed to get attributes from container %s", cmd.ContainerID())
}

if cmd.jsonOutputEnabled() {
if cmd.JSONOutputEnabled() {
// Maintain compatibility with older behavior.
if len(cmd.Args.Attrs.ParsedProps) == 1 && len(attrs) == 1 {
return cmd.outputJSON(attrs[0], nil)
return cmd.OutputJSON(attrs[0], nil)
}
return cmd.outputJSON(attrs, nil)
return cmd.OutputJSON(attrs, nil)
}

var bld strings.Builder
Expand Down Expand Up @@ -1398,8 +1398,8 @@ func (cmd *containerGetPropCmd) Execute(args []string) error {
}
}

if cmd.jsonOutputEnabled() {
return cmd.outputJSON(props, nil)
if cmd.JSONOutputEnabled() {
return cmd.OutputJSON(props, nil)
}

title := fmt.Sprintf("Properties for container %s", cmd.ContainerID())
Expand Down
Loading

0 comments on commit 3fbd6d7

Please sign in to comment.