Skip to content

Commit

Permalink
DAOS-15794 tools: Add --health-only flag to (dmg|daos) pool query
Browse files Browse the repository at this point in the history 
As a convenience, provide a "streamlined" version of the pool
query that only performs the minimum amount of work to query
the pool's health. Practically speaking, this means that it
will query for disabled ranks and omit the space query, which
is expensive.

Features: pool
Required-githooks: true
Change-Id: I5abe26c2a9a449a9d7c9c0867ae1fff1de9685d5
Signed-off-by: Michael MacDonald <mjmac@google.com>
  • Loading branch information
@mjmac
mjmac committed May 22, 2024
1 parent 32bfde0 commit 2ab6f22
Show file tree
Hide file tree
Showing 20 changed files with 1,240 additions and 729 deletions.
32 changes: 19 additions & 13 deletions src/control/cmd/daos/pool.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -194,6 +194,7 @@ type poolQueryCmd struct {
poolBaseCmd
ShowEnabledRanks bool `short:"e" long:"show-enabled" description:"Show engine unique identifiers (ranks) which are enabled"`
ShowDisabledRanks bool `short:"b" long:"show-disabled" description:"Show engine unique identifiers (ranks) which are disabled"`
HealthOnly bool `short:"t" long:"health-only" description:"Only perform pool health related queries"`
}

func convertPoolSpaceInfo(in *C.struct_daos_pool_space, mt C.uint) *daos.StorageUsageStats {
Expand Down Expand Up @@ -238,6 +239,7 @@ func convertPoolRebuildStatus(in *C.struct_daos_rebuild_status) *daos.PoolRebuil
func convertPoolInfo(pinfo *C.daos_pool_info_t) (*daos.PoolInfo, error) {
poolInfo := new(daos.PoolInfo)

poolInfo.QueryMask = daos.PoolQueryMask(pinfo.pi_bits)
poolInfo.UUID = uuid.Must(uuidFromC(pinfo.pi_uuid))
poolInfo.TotalTargets = uint32(pinfo.pi_ntargets)
poolInfo.DisabledTargets = uint32(pinfo.pi_ndisabled)
Expand All @@ -251,20 +253,16 @@ func convertPoolInfo(pinfo *C.daos_pool_info_t) (*daos.PoolInfo, error) {
}

poolInfo.Rebuild = convertPoolRebuildStatus(&pinfo.pi_rebuild_st)
poolInfo.TierStats = []*daos.StorageUsageStats{
convertPoolSpaceInfo(&pinfo.pi_space, C.DAOS_MEDIA_SCM),
convertPoolSpaceInfo(&pinfo.pi_space, C.DAOS_MEDIA_NVME),
if poolInfo.QueryMask.HasOption("space") {
poolInfo.TierStats = []*daos.StorageUsageStats{
convertPoolSpaceInfo(&pinfo.pi_space, C.DAOS_MEDIA_SCM),
convertPoolSpaceInfo(&pinfo.pi_space, C.DAOS_MEDIA_NVME),
}
}

return poolInfo, nil
}

const (
dpiQuerySpace = C.DPI_SPACE
dpiQueryRebuild = C.DPI_REBUILD_STATUS
dpiQueryAll = C.uint64_t(^uint64(0)) // DPI_ALL is -1
)

func generateRankSet(ranklist *C.d_rank_list_t) string {
if ranklist.rl_nr == 0 {
return ""
Expand All @@ -283,9 +281,19 @@ func generateRankSet(ranklist *C.d_rank_list_t) string {
}

func (cmd *poolQueryCmd) Execute(_ []string) error {
queryMask := daos.DefaultPoolQueryMask
if cmd.HealthOnly {
if !cmd.ShowEnabledRanks {
cmd.ShowDisabledRanks = true // enable for health queries
}
queryMask.SetQuerySpace(false)
}
if cmd.ShowEnabledRanks && cmd.ShowDisabledRanks {
return errors.New("show-enabled and show-disabled can't be used at the same time.")
}
queryMask.SetQueryDisabledEngines(cmd.ShowDisabledRanks)
queryMask.SetQueryEnabledEngines(cmd.ShowEnabledRanks)

var rlPtr **C.d_rank_list_t = nil
var rl *C.d_rank_list_t = nil

Expand All @@ -300,10 +308,7 @@ func (cmd *poolQueryCmd) Execute(_ []string) error {
defer cleanup()

cPoolInfo := C.daos_pool_info_t{
pi_bits: dpiQueryAll,
}
if cmd.ShowDisabledRanks {
cPoolInfo.pi_bits &= C.uint64_t(^(uint64(C.DPI_ENGINES_ENABLED)))
pi_bits: C.uint64_t(queryMask),
}

rc := C.daos_pool_query(cmd.cPoolHandle, rlPtr, &cPoolInfo, nil, nil)
Expand Down Expand Up @@ -336,6 +341,7 @@ func (cmd *poolQueryCmd) Execute(_ []string) error {
return err
}

cmd.Debugf("Pool query options: %s", poolInfo.QueryMask)
cmd.Info(bld.String())

return nil
Expand Down
29 changes: 15 additions & 14 deletions src/control/cmd/daos/pretty/pool.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -46,15 +46,25 @@ func PrintPoolInfo(pi *daos.PoolInfo, out io.Writer) error {
fmt.Fprintf(w, "Pool layout out of date (%d < %d) -- see `dmg pool upgrade` for details.\n",
pi.PoolLayoutVer, pi.UpgradeLayoutVer)
}
fmt.Fprintln(w, "Pool space info:")
if pi.EnabledRanks != nil {
fmt.Fprintln(w, "Pool health info:")
if pi.EnabledRanks != nil && pi.EnabledRanks.Count() > 0 {
fmt.Fprintf(w, "- Enabled ranks: %s\n", pi.EnabledRanks)
}
if pi.DisabledRanks != nil {
if pi.DisabledRanks != nil && pi.DisabledRanks.Count() > 0 {
fmt.Fprintf(w, "- Disabled ranks: %s\n", pi.DisabledRanks)
}
fmt.Fprintf(w, "- Target(VOS) count:%d\n", pi.ActiveTargets)
if pi.TierStats != nil {
if pi.Rebuild != nil {
if pi.Rebuild.Status == 0 {
fmt.Fprintf(w, "- Rebuild %s, %d objs, %d recs\n",
pi.Rebuild.State, pi.Rebuild.Objects, pi.Rebuild.Records)
} else {
fmt.Fprintf(w, "- Rebuild failed, status=%d\n", pi.Rebuild.Status)
}
}

if pi.QueryMask.HasOption("space") && pi.TierStats != nil {
fmt.Fprintln(w, "Pool space info:")
fmt.Fprintf(w, "- Target(VOS) count:%d\n", pi.ActiveTargets)
for tierIdx, tierStats := range pi.TierStats {
fmt.Fprintln(w, getTierNameText(tierIdx))
fmt.Fprintf(w, " Total size: %s\n", humanize.Bytes(tierStats.Total))
Expand All @@ -63,15 +73,6 @@ func PrintPoolInfo(pi *daos.PoolInfo, out io.Writer) error {
humanize.Bytes(tierStats.Max), humanize.Bytes(tierStats.Mean))
}
}
if pi.Rebuild != nil {
if pi.Rebuild.Status == 0 {
fmt.Fprintf(w, "Rebuild %s, %d objs, %d recs\n",
pi.Rebuild.State, pi.Rebuild.Objects, pi.Rebuild.Records)
} else {
fmt.Fprintf(w, "Rebuild failed, status=%d\n", pi.Rebuild.Status)
}
}

return w.Err
}

Expand Down
29 changes: 19 additions & 10 deletions src/control/cmd/daos/pretty/pool_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -31,12 +31,12 @@ func TestPretty_PrintPoolInfo(t *testing.T) {
pi: &daos.PoolInfo{},
expPrintStr: fmt.Sprintf(`
Pool %s, ntarget=0, disabled=0, leader=0, version=0, state=Creating
Pool space info:
- Target(VOS) count:0
Pool health info:
`, uuid.Nil.String()),
},
"normal response": {
pi: &daos.PoolInfo{
QueryMask: daos.DefaultPoolQueryMask,
State: daos.PoolServiceStateDegraded,
UUID: poolUUID,
TotalTargets: 2,
Expand Down Expand Up @@ -65,6 +65,8 @@ Pool space info:
expPrintStr: fmt.Sprintf(`
Pool %s, ntarget=2, disabled=1, leader=42, version=100, state=Degraded
Pool layout out of date (1 < 2) -- see `+backtickStr+` for details.
Pool health info:
- Rebuild busy, 42 objs, 21 recs
Pool space info:
- Target(VOS) count:1
- Storage tier 0 (SCM):
Expand All @@ -73,11 +75,11 @@ Pool space info:
- Storage tier 1 (NVMe):
Total size: 2 B
Free: 1 B, min:0 B, max:0 B, mean:0 B
Rebuild busy, 42 objs, 21 recs
`, poolUUID.String()),
},
"normal response; enabled ranks": {
pi: &daos.PoolInfo{
QueryMask: daos.DefaultPoolQueryMask,
State: daos.PoolServiceStateDegraded,
UUID: poolUUID,
TotalTargets: 2,
Expand Down Expand Up @@ -107,20 +109,22 @@ Rebuild busy, 42 objs, 21 recs
expPrintStr: fmt.Sprintf(`
Pool %s, ntarget=2, disabled=1, leader=42, version=100, state=Degraded
Pool layout out of date (1 < 2) -- see `+backtickStr+` for details.
Pool space info:
Pool health info:
- Enabled ranks: 0-2
- Rebuild busy, 42 objs, 21 recs
Pool space info:
- Target(VOS) count:1
- Storage tier 0 (SCM):
Total size: 2 B
Free: 1 B, min:0 B, max:0 B, mean:0 B
- Storage tier 1 (NVMe):
Total size: 2 B
Free: 1 B, min:0 B, max:0 B, mean:0 B
Rebuild busy, 42 objs, 21 recs
`, poolUUID.String()),
},
"normal response; disabled ranks": {
pi: &daos.PoolInfo{
QueryMask: daos.DefaultPoolQueryMask,
State: daos.PoolServiceStateDegraded,
UUID: poolUUID,
TotalTargets: 2,
Expand Down Expand Up @@ -150,20 +154,22 @@ Rebuild busy, 42 objs, 21 recs
expPrintStr: fmt.Sprintf(`
Pool %s, ntarget=2, disabled=1, leader=42, version=100, state=Degraded
Pool layout out of date (1 < 2) -- see `+backtickStr+` for details.
Pool space info:
Pool health info:
- Disabled ranks: 0-1,3
- Rebuild busy, 42 objs, 21 recs
Pool space info:
- Target(VOS) count:1
- Storage tier 0 (SCM):
Total size: 2 B
Free: 1 B, min:0 B, max:0 B, mean:0 B
- Storage tier 1 (NVMe):
Total size: 2 B
Free: 1 B, min:0 B, max:0 B, mean:0 B
Rebuild busy, 42 objs, 21 recs
`, poolUUID.String()),
},
"unknown/invalid rebuild state response": {
pi: &daos.PoolInfo{
QueryMask: daos.DefaultPoolQueryMask,
State: daos.PoolServiceStateDegraded,
UUID: poolUUID,
TotalTargets: 2,
Expand Down Expand Up @@ -193,20 +199,22 @@ Rebuild busy, 42 objs, 21 recs
expPrintStr: fmt.Sprintf(`
Pool %s, ntarget=2, disabled=1, leader=42, version=100, state=Degraded
Pool layout out of date (1 < 2) -- see `+backtickStr+` for details.
Pool space info:
Pool health info:
- Disabled ranks: 0-1,3
- Rebuild unknown, 42 objs, 21 recs
Pool space info:
- Target(VOS) count:1
- Storage tier 0 (SCM):
Total size: 2 B
Free: 1 B, min:0 B, max:0 B, mean:0 B
- Storage tier 1 (NVMe):
Total size: 2 B
Free: 1 B, min:0 B, max:0 B, mean:0 B
Rebuild unknown, 42 objs, 21 recs
`, poolUUID.String()),
},
"rebuild failed": {
pi: &daos.PoolInfo{
QueryMask: daos.DefaultPoolQueryMask,
State: daos.PoolServiceStateDegraded,
UUID: poolUUID,
TotalTargets: 2,
Expand Down Expand Up @@ -236,6 +244,8 @@ Rebuild unknown, 42 objs, 21 recs
expPrintStr: fmt.Sprintf(`
Pool %s, ntarget=2, disabled=1, leader=42, version=100, state=Degraded
Pool layout out of date (1 < 2) -- see `+backtickStr+` for details.
Pool health info:
- Rebuild failed, status=2
Pool space info:
- Target(VOS) count:1
- Storage tier 0 (SCM):
Expand All @@ -244,7 +254,6 @@ Pool space info:
- Storage tier 1 (NVMe):
Total size: 2 B
Free: 1 B, min:0 B, max:0 B, mean:0 B
Rebuild failed, status=2
`, poolUUID.String()),
},
} {
Expand Down
18 changes: 14 additions & 4 deletions src/control/cmd/dmg/pool.go
View file
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
//
// (C) Copyright 2019-2023 Intel Corporation.
// (C) Copyright 2019-2024 Intel Corporation.
//
// SPDX-License-Identifier: BSD-2-Clause-Patent
//
Expand Down Expand Up @@ -601,20 +601,28 @@ type PoolQueryCmd struct {
poolCmd
ShowEnabledRanks bool `short:"e" long:"show-enabled" description:"Show engine unique identifiers (ranks) which are enabled"`
ShowDisabledRanks bool `short:"b" long:"show-disabled" description:"Show engine unique identifiers (ranks) which are disabled"`
HealthOnly bool `short:"t" long:"health-only" description:"Only perform pool health related queries"`
}

// Execute is run when PoolQueryCmd subcommand is activated
func (cmd *PoolQueryCmd) Execute(args []string) error {
req := &control.PoolQueryReq{
ID: cmd.PoolID().String(),
ID: cmd.PoolID().String(),
QueryMask: daos.DefaultPoolQueryMask,
}

if cmd.HealthOnly {
if !cmd.ShowEnabledRanks {
cmd.ShowDisabledRanks = true // enable for health queries
}
req.QueryMask.SetQuerySpace(false)
}
// TODO (DAOS-10250) The two options should not be incompatible (i.e. engine limitation)
if cmd.ShowEnabledRanks && cmd.ShowDisabledRanks {
return errIncompatFlags("show-enabled-ranks", "show-disabled-ranks")
}
req.IncludeEnabledRanks = cmd.ShowEnabledRanks
req.IncludeDisabledRanks = cmd.ShowDisabledRanks
req.QueryMask.SetQueryEnabledEngines(cmd.ShowEnabledRanks)
req.QueryMask.SetQueryDisabledEngines(cmd.ShowDisabledRanks)

resp, err := control.PoolQuery(cmd.MustLogCtx(), cmd.ctlInvoker, req)

Expand All @@ -630,6 +638,8 @@ func (cmd *PoolQueryCmd) Execute(args []string) error {
if err := pretty.PrintPoolQueryResponse(resp, &bld); err != nil {
return err
}

cmd.Debugf("Pool query options: %s", resp.PoolInfo.QueryMask)
cmd.Info(bld.String())
return nil
}
Expand Down
39 changes: 29 additions & 10 deletions src/control/cmd/dmg/pool_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -185,6 +185,12 @@ func TestPoolCommands(t *testing.T) {
return prop
}

setQueryMask := func(xfrm func(qm *daos.PoolQueryMask)) daos.PoolQueryMask {
qm := daos.DefaultPoolQueryMask
xfrm(&qm)
return qm
}

runCmdTests(t, []cmdTest{
{
"Pool create with extra argument",
Expand Down Expand Up @@ -918,7 +924,8 @@ func TestPoolCommands(t *testing.T) {
"pool query 12345678-1234-1234-1234-1234567890ab",
strings.Join([]string{
printRequest(t, &control.PoolQueryReq{
ID: "12345678-1234-1234-1234-1234567890ab",
ID: "12345678-1234-1234-1234-1234567890ab",
QueryMask: daos.DefaultPoolQueryMask,
}),
}, " "),
nil,
Expand All @@ -928,8 +935,8 @@ func TestPoolCommands(t *testing.T) {
"pool query --show-enabled 12345678-1234-1234-1234-1234567890ab",
strings.Join([]string{
printRequest(t, &control.PoolQueryReq{
ID: "12345678-1234-1234-1234-1234567890ab",
IncludeEnabledRanks: true,
ID: "12345678-1234-1234-1234-1234567890ab",
QueryMask: setQueryMask(func(qm *daos.PoolQueryMask) { qm.SetQueryEnabledEngines(true) }),
}),
}, " "),
nil,
Expand All @@ -939,8 +946,8 @@ func TestPoolCommands(t *testing.T) {
"pool query -e 12345678-1234-1234-1234-1234567890ab",
strings.Join([]string{
printRequest(t, &control.PoolQueryReq{
ID: "12345678-1234-1234-1234-1234567890ab",
IncludeEnabledRanks: true,
ID: "12345678-1234-1234-1234-1234567890ab",
QueryMask: setQueryMask(func(qm *daos.PoolQueryMask) { qm.SetQueryEnabledEngines(true) }),
}),
}, " "),
nil,
Expand All @@ -950,8 +957,8 @@ func TestPoolCommands(t *testing.T) {
"pool query --show-disabled 12345678-1234-1234-1234-1234567890ab",
strings.Join([]string{
printRequest(t, &control.PoolQueryReq{
ID: "12345678-1234-1234-1234-1234567890ab",
IncludeDisabledRanks: true,
ID: "12345678-1234-1234-1234-1234567890ab",
QueryMask: setQueryMask(func(qm *daos.PoolQueryMask) { qm.SetQueryDisabledEngines(true) }),
}),
}, " "),
nil,
Expand All @@ -961,8 +968,19 @@ func TestPoolCommands(t *testing.T) {
"pool query -b 12345678-1234-1234-1234-1234567890ab",
strings.Join([]string{
printRequest(t, &control.PoolQueryReq{
ID: "12345678-1234-1234-1234-1234567890ab",
IncludeDisabledRanks: true,
ID: "12345678-1234-1234-1234-1234567890ab",
QueryMask: setQueryMask(func(qm *daos.PoolQueryMask) { qm.SetQueryDisabledEngines(true) }),
}),
}, " "),
nil,
},
{
"Query pool for health only",
"pool query --health-only 12345678-1234-1234-1234-1234567890ab",
strings.Join([]string{
printRequest(t, &control.PoolQueryReq{
ID: "12345678-1234-1234-1234-1234567890ab",
QueryMask: setQueryMask(func(qm *daos.PoolQueryMask) { qm.SetQueryDisabledEngines(true); qm.SetQuerySpace(false) }),
}),
}, " "),
nil,
Expand All @@ -972,7 +990,8 @@ func TestPoolCommands(t *testing.T) {
"pool query test_label",
strings.Join([]string{
printRequest(t, &control.PoolQueryReq{
ID: "test_label",
ID: "test_label",
QueryMask: daos.DefaultPoolQueryMask,
}),
}, " "),
nil,
Expand Down
Loading

0 comments on commit 2ab6f22

Please sign in to comment.