From e7db73a68a8b62a761b02a9c6eb0ecab122bac47 Mon Sep 17 00:00:00 2001 From: Michael MacDonald Date: Tue, 11 Jun 2024 20:16:51 +0000 Subject: [PATCH 01/10] DAOS-15982 pool: Enable pool list for clients Revive parts of the old client mgmt API to allow libdaos clients to list pools in the system, but only allow a given client to see the pools that it could connect to. Required-githooks: true Change-Id: I2b3c391ddf042b23811be8f3390ec290e92e4290 Signed-off-by: Michael MacDonald --- src/client/api/init.c | 2 +- src/client/api/mgmt.c | 26 +++ src/control/cmd/daos/pool.go | 77 +++++++ src/control/cmd/daos/pretty/pool.go | 184 +++++++++++++++ src/control/cmd/daos/pretty/pool_test.go | 261 ++++++++++++++++++++++ src/control/cmd/daos/pretty/ranks.go | 31 +++ src/control/cmd/daos/pretty/ranks_test.go | 60 +++++ src/control/cmd/daos/util.go | 15 ++ src/control/cmd/dmg/pretty/pool.go | 177 +-------------- src/control/cmd/dmg/pretty/pool_test.go | 215 ------------------ src/control/cmd/dmg/pretty/ranks.go | 16 -- src/control/common/proto/srv/srv.pb.go | 236 +++++++++++++++++-- src/control/drpc/modules.go | 11 +- src/control/go.mod | 2 +- src/control/go.sum | 4 +- src/control/server/mgmt_drpc.go | 39 +++- src/engine/drpc_client.c | 98 ++++++++ src/engine/srv.pb-c.c | 180 +++++++++++++++ src/engine/srv.pb-c.h | 88 +++++++- src/include/daos/drpc.pb-c.h | 75 +++---- src/include/daos/drpc_modules.h | 13 +- src/include/daos/mgmt.h | 2 + src/include/daos_mgmt.h | 6 + src/include/daos_srv/daos_engine.h | 3 + src/include/daos_srv/pool.h | 2 + src/include/daos_task.h | 15 +- src/mgmt/cli_mgmt.c | 157 ++++++++++++- src/mgmt/rpc.c | 41 ++++ src/mgmt/rpc.h | 29 +++ src/mgmt/srv.c | 87 +++++++- src/mgmt/srv_internal.h | 2 + src/pool/rpc.c | 2 + src/pool/rpc.h | 14 ++ src/pool/srv_internal.h | 2 + src/pool/srv_pool.c | 168 ++++++++++++++ src/proto/srv/srv.proto | 16 ++ 36 files changed, 1886 insertions(+), 470 deletions(-) create mode 100644 src/control/cmd/daos/pretty/ranks.go create mode 100644 src/control/cmd/daos/pretty/ranks_test.go delete mode 100644 src/control/cmd/dmg/pretty/ranks.go diff --git a/src/client/api/init.c b/src/client/api/init.c index 40e7aa5e0d9..a5ab9226701 100644 --- a/src/client/api/init.c +++ b/src/client/api/init.c @@ -37,7 +37,7 @@ const struct daos_task_api dc_funcs[] = { /** Management */ {dc_deprecated, 0}, {dc_deprecated, 0}, - {dc_deprecated, 0}, + {dc_mgmt_pool_list, sizeof(daos_mgmt_pool_list_t)}, {dc_debug_set_params, sizeof(daos_set_params_t)}, {dc_mgmt_get_bs_state, sizeof(daos_mgmt_get_bs_state_t)}, diff --git a/src/client/api/mgmt.c b/src/client/api/mgmt.c index ac494d15330..4d24f5ec106 100644 --- a/src/client/api/mgmt.c +++ b/src/client/api/mgmt.c @@ -79,3 +79,29 @@ daos_mgmt_put_sys_info(struct daos_sys_info *info) { dc_mgmt_put_sys_info(info); } + +int +daos_mgmt_list_pools(const char *group, daos_size_t *npools, daos_mgmt_pool_info_t *pools, + daos_event_t *ev) +{ + daos_mgmt_pool_list_t *args; + tse_task_t *task; + int rc; + + DAOS_API_ARG_ASSERT(*args, MGMT_LIST_POOLS); + + if (npools == NULL) { + D_ERROR("npools must be non-NULL\n"); + return -DER_INVAL; + } + + rc = dc_task_create(dc_mgmt_pool_list, NULL, ev, &task); + if (rc) + return rc; + args = dc_task_get_args(task); + args->grp = group; + args->pools = pools; + args->npools = npools; + + return dc_task_schedule(task, true); +} diff --git a/src/control/cmd/daos/pool.go b/src/control/cmd/daos/pool.go index 1d3b8ce9182..87554972bca 100644 --- a/src/control/cmd/daos/pool.go +++ b/src/control/cmd/daos/pool.go @@ -20,6 +20,7 @@ import ( "github.com/daos-stack/daos/src/control/lib/daos" "github.com/daos-stack/daos/src/control/lib/ranklist" "github.com/daos-stack/daos/src/control/lib/ui" + "github.com/daos-stack/daos/src/control/logging" ) /* @@ -180,6 +181,7 @@ func (cmd *poolBaseCmd) getAttr(name string) (*attribute, error) { } type poolCmd struct { + List poolListCmd `command:"list" description:"list pools to which this user has access"` Query poolQueryCmd `command:"query" description:"query pool info"` QueryTargets poolQueryTargetsCmd `command:"query-targets" description:"query pool target info"` ListConts containerListCmd `command:"list-containers" alias:"list-cont" description:"list all containers in pool"` @@ -601,3 +603,78 @@ func (cmd *poolAutoTestCmd) Execute(_ []string) error { return nil } + +func getPoolList(log logging.Logger, queryEnabled bool) ([]*daos.PoolInfo, error) { + var rc C.int + bufSize := C.size_t(0) + + // First, fetch the total number of pools in the system. + // We may not have access to all of them, so this is an upper bound. + rc = C.daos_mgmt_list_pools(nil, &bufSize, nil, nil) + if err := daosError(rc); err != nil { + return nil, err + } + + if bufSize < 1 { + return nil, nil + } + + // Now, we actually fetch the pools into the buffer that we've created. + cPools := make([]C.daos_mgmt_pool_info_t, bufSize) + rc = C.daos_mgmt_list_pools(nil, &bufSize, &cPools[0], nil) + if err := daosError(rc); err != nil { + return nil, err + } + pools := make([]*daos.PoolInfo, 0, bufSize) + for i := 0; i < int(bufSize); i++ { + poolUUID, err := uuidFromC(cPools[i].mgpi_uuid) + if err != nil { + return nil, err + } + svcRanks, err := rankSetFromC(cPools[i].mgpi_svc) + if err != nil { + return nil, err + } + + // Populate the basic info. + pool := &daos.PoolInfo{ + UUID: poolUUID, + Label: C.GoString(cPools[i].mgpi_label), + ServiceReplicas: svcRanks.Ranks(), + State: daos.PoolServiceStateReady, + } + + if queryEnabled { + // TODO: rework query logic to be callable standalone. + } + pools = append(pools, pool) + } + + log.Debugf("fetched %d/%d pools", len(pools), len(cPools)) + return pools, nil +} + +type poolListCmd struct { + daosCmd + Verbose bool `short:"v" long:"verbose" description:"Add pool UUIDs and service replica lists to display"` + NoQuery bool `short:"n" long:"no-query" description:"Disable query of listed pools"` +} + +func (cmd *poolListCmd) Execute(_ []string) error { + pools, err := getPoolList(cmd.Logger, !cmd.NoQuery) + if err != nil { + return err + } + + if cmd.JSONOutputEnabled() { + return cmd.OutputJSON(pools, nil) + } + + var buf strings.Builder + if err := pretty.PrintPoolList(pools, &buf, cmd.Verbose); err != nil { + return err + } + cmd.Info(buf.String()) + + return nil +} diff --git a/src/control/cmd/daos/pretty/pool.go b/src/control/cmd/daos/pretty/pool.go index 032a4e72319..abf46c52025 100644 --- a/src/control/cmd/daos/pretty/pool.go +++ b/src/control/cmd/daos/pretty/pool.go @@ -98,3 +98,187 @@ func PrintPoolQueryTargetInfo(pqti *daos.PoolQueryTargetInfo, out io.Writer) err return w.Err } + +func poolListCreateRow(pool *daos.PoolInfo, upgrade bool) txtfmt.TableRow { + // display size of the largest non-empty tier + var size uint64 + poolUsage := pool.Usage() + for ti := len(poolUsage) - 1; ti >= 0; ti-- { + if poolUsage[ti].Size != 0 { + size = poolUsage[ti].Size + break + } + } + + // display usage of the most used tier + var used int + for ti := 0; ti < len(poolUsage); ti++ { + t := poolUsage[ti] + u := float64(t.Size-t.Free) / float64(t.Size) + + if int(u*100) > used { + used = int(u * 100) + } + } + + // display imbalance of the most imbalanced tier + var imbalance uint32 + for ti := 0; ti < len(poolUsage); ti++ { + if poolUsage[ti].Imbalance > imbalance { + imbalance = poolUsage[ti].Imbalance + } + } + + row := txtfmt.TableRow{ + "Pool": pool.Name(), + "Size": humanize.Bytes(size), + "State": pool.State.String(), + "Used": fmt.Sprintf("%d%%", used), + "Imbalance": fmt.Sprintf("%d%%", imbalance), + "Disabled": fmt.Sprintf("%d/%d", pool.DisabledTargets, pool.TotalTargets), + } + + if upgrade { + upgradeString := "None" + + if pool.PoolLayoutVer != pool.UpgradeLayoutVer { + upgradeString = fmt.Sprintf("%d->%d", pool.PoolLayoutVer, pool.UpgradeLayoutVer) + } + row["UpgradeNeeded?"] = upgradeString + } + + return row +} + +func printPoolList(pools []*daos.PoolInfo, out io.Writer) error { + upgrade := false + for _, pool := range pools { + if pool.PoolLayoutVer != pool.UpgradeLayoutVer { + upgrade = true + } + } + + titles := []string{"Pool", "Size", "State", "Used", "Imbalance", "Disabled"} + if upgrade { + titles = append(titles, "UpgradeNeeded?") + } + formatter := txtfmt.NewTableFormatter(titles...) + + var table []txtfmt.TableRow + for _, pool := range pools { + table = append(table, poolListCreateRow(pool, upgrade)) + } + + fmt.Fprintln(out, formatter.Format(table)) + + return nil +} + +func addVerboseTierUsage(row txtfmt.TableRow, usage *daos.PoolTierUsage) txtfmt.TableRow { + row[usage.TierName+" Size"] = humanize.Bytes(usage.Size) + row[usage.TierName+" Used"] = humanize.Bytes(usage.Size - usage.Free) + row[usage.TierName+" Imbalance"] = fmt.Sprintf("%d%%", usage.Imbalance) + + return row +} + +func poolListCreateRowVerbose(pool *daos.PoolInfo, hasSpace, hasRebuild bool) txtfmt.TableRow { + label := pool.Label + if label == "" { + label = "-" + } + + svcReps := "N/A" + if len(pool.ServiceReplicas) != 0 { + svcReps = PrintRanks(pool.ServiceReplicas) + } + + upgrade := "None" + if pool.PoolLayoutVer != pool.UpgradeLayoutVer { + upgrade = fmt.Sprintf("%d->%d", pool.PoolLayoutVer, pool.UpgradeLayoutVer) + } + + row := txtfmt.TableRow{ + "Label": label, + "UUID": pool.UUID.String(), + "State": pool.State.String(), + "SvcReps": svcReps, + } + if hasSpace { + row["Disabled"] = fmt.Sprintf("%d/%d", pool.DisabledTargets, pool.TotalTargets) + row["UpgradeNeeded?"] = upgrade + } + if hasRebuild { + row["Rebuild State"] = pool.RebuildState() + } + + if hasSpace { + for _, tu := range pool.Usage() { + row = addVerboseTierUsage(row, tu) + } + } + + return row +} + +func printVerbosePoolList(pools []*daos.PoolInfo, out io.Writer) error { + // Basic pool info should be available without a query. + titles := []string{"Label", "UUID", "State", "SvcReps"} + + hasSpaceQuery := false + hasRebuildQuery := false + for _, pool := range pools { + if hasSpaceQuery && hasRebuildQuery { + break + } + + if pool.QueryMask.HasOption(daos.PoolQueryOptionSpace) { + hasSpaceQuery = true + } + if pool.QueryMask.HasOption(daos.PoolQueryOptionRebuild) { + hasRebuildQuery = true + } + } + + // If any of the pools was queried, then we'll need to show more fields. + if hasSpaceQuery { + for _, t := range pools[0].Usage() { + titles = append(titles, + t.TierName+" Size", + t.TierName+" Used", + t.TierName+" Imbalance") + } + titles = append(titles, "Disabled") + titles = append(titles, "UpgradeNeeded?") + } + + if hasRebuildQuery { + titles = append(titles, "Rebuild State") + } + + formatter := txtfmt.NewTableFormatter(titles...) + + var table []txtfmt.TableRow + for _, pool := range pools { + table = append(table, poolListCreateRowVerbose(pool, hasSpaceQuery, hasRebuildQuery)) + } + + fmt.Fprintln(out, formatter.Format(table)) + + return nil +} + +// PrintPoolList generates a human-readable representation of the supplied +// slice of daos.PoolInfo structs and writes it to the supplied io.Writer. +func PrintPoolList(pools []*daos.PoolInfo, out io.Writer, verbose bool) error { + if len(pools) == 0 { + fmt.Fprintln(out, "no pools in system") + return nil + } + + if verbose { + return printVerbosePoolList(pools, out) + } + + return printPoolList(pools, out) +} diff --git a/src/control/cmd/daos/pretty/pool_test.go b/src/control/cmd/daos/pretty/pool_test.go index cd3580e047d..e342be18e37 100644 --- a/src/control/cmd/daos/pretty/pool_test.go +++ b/src/control/cmd/daos/pretty/pool_test.go @@ -12,6 +12,7 @@ import ( "strings" "testing" + "github.com/dustin/go-humanize" "github.com/google/go-cmp/cmp" "github.com/google/uuid" @@ -445,3 +446,263 @@ Target: type unknown, state drain }) } } + +func TestPretty_PrintListPools(t *testing.T) { + exampleTierStats := []*daos.StorageUsageStats{ + { + MediaType: daos.StorageMediaTypeScm, + Total: 100 * humanize.GByte, + Free: 20 * humanize.GByte, + Min: 5 * humanize.GByte, + Max: 6 * humanize.GByte, + }, + { + MediaType: daos.StorageMediaTypeNvme, + Total: 6 * humanize.TByte, + Free: 1 * humanize.TByte, + Min: 20 * humanize.GByte, + Max: 50 * humanize.GByte, + }, + } + + for name, tc := range map[string]struct { + pools []*daos.PoolInfo + verbose bool + expErr error + expPrintStr string + }{ + "empty list": { + expPrintStr: ` +no pools in system +`, + }, + "one pool; no usage": { + pools: []*daos.PoolInfo{ + { + UUID: test.MockPoolUUID(1), + ServiceReplicas: []ranklist.Rank{0, 1, 2}, + State: daos.PoolServiceStateReady, + }, + }, + expPrintStr: ` +Pool Size State Used Imbalance Disabled +---- ---- ----- ---- --------- -------- +00000001 0 B Ready 0% 0% 0/0 + +`, + }, + "two pools; only one labeled": { + pools: []*daos.PoolInfo{ + { + UUID: test.MockPoolUUID(1), + ServiceReplicas: []ranklist.Rank{0, 1, 2}, + TierStats: exampleTierStats, + TotalTargets: 16, + ActiveTargets: 16, + DisabledTargets: 0, + State: daos.PoolServiceStateReady, + PoolLayoutVer: 1, + UpgradeLayoutVer: 2, + }, + { + Label: "two", + UUID: test.MockPoolUUID(2), + ServiceReplicas: []ranklist.Rank{3, 4, 5}, + TierStats: exampleTierStats, + TotalTargets: 64, + ActiveTargets: 56, + DisabledTargets: 8, + State: daos.PoolServiceStateReady, + PoolLayoutVer: 1, + UpgradeLayoutVer: 2, + }, + }, + expPrintStr: ` +Pool Size State Used Imbalance Disabled UpgradeNeeded? +---- ---- ----- ---- --------- -------- -------------- +00000001 6.0 TB Ready 83% 16% 0/16 1->2 +two 6.0 TB Ready 83% 56% 8/64 1->2 + +`, + }, + "two pools; one SCM only": { + pools: []*daos.PoolInfo{ + { + Label: "one", + UUID: test.MockPoolUUID(1), + ServiceReplicas: []ranklist.Rank{0, 1, 2}, + TierStats: exampleTierStats, + TotalTargets: 16, + ActiveTargets: 16, + DisabledTargets: 0, + State: daos.PoolServiceStateReady, + PoolLayoutVer: 1, + UpgradeLayoutVer: 2, + QueryMask: daos.DefaultPoolQueryMask, + }, + { + Label: "two", + UUID: test.MockPoolUUID(2), + ServiceReplicas: []ranklist.Rank{3, 4, 5}, + TierStats: []*daos.StorageUsageStats{ + exampleTierStats[0], + {MediaType: daos.StorageMediaTypeNvme}, + }, + TotalTargets: 64, + ActiveTargets: 56, + DisabledTargets: 8, + State: daos.PoolServiceStateReady, + PoolLayoutVer: 2, + UpgradeLayoutVer: 2, + QueryMask: daos.DefaultPoolQueryMask, + }, + }, + expPrintStr: ` +Pool Size State Used Imbalance Disabled UpgradeNeeded? +---- ---- ----- ---- --------- -------- -------------- +one 6.0 TB Ready 83% 16% 0/16 1->2 +two 100 GB Ready 80% 56% 8/64 None + +`, + }, + "verbose, empty response": { + verbose: true, + expPrintStr: ` +no pools in system +`, + }, + "verbose; zero svc replicas": { + pools: []*daos.PoolInfo{ + { + UUID: test.MockPoolUUID(1), + TierStats: exampleTierStats, + TotalTargets: 16, + ActiveTargets: 16, + DisabledTargets: 0, + State: daos.PoolServiceStateReady, + PoolLayoutVer: 1, + UpgradeLayoutVer: 2, + Rebuild: &daos.PoolRebuildStatus{ + State: daos.PoolRebuildStateIdle, + }, + QueryMask: daos.DefaultPoolQueryMask, + }, + }, + verbose: true, + expPrintStr: ` +Label UUID State SvcReps SCM Size SCM Used SCM Imbalance NVME Size NVME Used NVME Imbalance Disabled UpgradeNeeded? Rebuild State +----- ---- ----- ------- -------- -------- ------------- --------- --------- -------------- -------- -------------- ------------- +- 00000001-0001-0001-0001-000000000001 Ready N/A 100 GB 80 GB 16% 6.0 TB 5.0 TB 8% 0/16 1->2 idle + +`, + }, + "verbose; zero svc replicas with no query": { + pools: []*daos.PoolInfo{ + { + UUID: test.MockPoolUUID(1), + TierStats: exampleTierStats, + TotalTargets: 16, + ActiveTargets: 16, + DisabledTargets: 0, + State: daos.PoolServiceStateReady, + PoolLayoutVer: 1, + UpgradeLayoutVer: 2, + }, + }, + verbose: true, + expPrintStr: ` +Label UUID State SvcReps +----- ---- ----- ------- +- 00000001-0001-0001-0001-000000000001 Ready N/A + +`, + }, + "verbose; two pools; one destroying": { + pools: []*daos.PoolInfo{ + { + Label: "one", + UUID: test.MockPoolUUID(1), + ServiceReplicas: []ranklist.Rank{0, 1, 2}, + TierStats: exampleTierStats, + TotalTargets: 16, + ActiveTargets: 16, + DisabledTargets: 0, + State: daos.PoolServiceStateReady, + PoolLayoutVer: 1, + UpgradeLayoutVer: 2, + Rebuild: &daos.PoolRebuildStatus{ + State: daos.PoolRebuildStateIdle, + }, + QueryMask: daos.DefaultPoolQueryMask, + }, + { + Label: "two", + UUID: test.MockPoolUUID(2), + ServiceReplicas: []ranklist.Rank{3, 4, 5}, + TierStats: exampleTierStats, + TotalTargets: 64, + ActiveTargets: 56, + DisabledTargets: 8, + State: daos.PoolServiceStateDestroying, + PoolLayoutVer: 2, + UpgradeLayoutVer: 2, + Rebuild: &daos.PoolRebuildStatus{ + State: daos.PoolRebuildStateDone, + }, + QueryMask: daos.DefaultPoolQueryMask, + }, + }, + verbose: true, + expPrintStr: ` +Label UUID State SvcReps SCM Size SCM Used SCM Imbalance NVME Size NVME Used NVME Imbalance Disabled UpgradeNeeded? Rebuild State +----- ---- ----- ------- -------- -------- ------------- --------- --------- -------------- -------- -------------- ------------- +one 00000001-0001-0001-0001-000000000001 Ready [0-2] 100 GB 80 GB 16% 6.0 TB 5.0 TB 8% 0/16 1->2 idle +two 00000002-0002-0002-0002-000000000002 Destroying [3-5] 100 GB 80 GB 56% 6.0 TB 5.0 TB 27% 8/64 None done + +`, + }, + "verbose; one pool; rebuild state busy": { + pools: []*daos.PoolInfo{ + { + Label: "one", + UUID: test.MockPoolUUID(1), + ServiceReplicas: []ranklist.Rank{0, 1, 2}, + TierStats: exampleTierStats, + TotalTargets: 16, + ActiveTargets: 8, + DisabledTargets: 8, + State: daos.PoolServiceStateDegraded, + PoolLayoutVer: 1, + UpgradeLayoutVer: 2, + Rebuild: &daos.PoolRebuildStatus{ + State: daos.PoolRebuildStateBusy, + }, + QueryMask: daos.DefaultPoolQueryMask, + }, + }, + verbose: true, + expPrintStr: ` +Label UUID State SvcReps SCM Size SCM Used SCM Imbalance NVME Size NVME Used NVME Imbalance Disabled UpgradeNeeded? Rebuild State +----- ---- ----- ------- -------- -------- ------------- --------- --------- -------------- -------- -------------- ------------- +one 00000001-0001-0001-0001-000000000001 Degraded [0-2] 100 GB 80 GB 8% 6.0 TB 5.0 TB 4% 8/16 1->2 busy + +`, + }, + } { + t.Run(name, func(t *testing.T) { + var bld strings.Builder + + // pass the same io writer to standard and error stream + // parameters to mimic combined output seen on terminal + err := PrintPoolList(tc.pools, &bld, tc.verbose) + test.CmpErr(t, tc.expErr, err) + if tc.expErr != nil { + return + } + + if diff := cmp.Diff(strings.TrimLeft(tc.expPrintStr, "\n"), bld.String()); diff != "" { + t.Fatalf("unexpected format string (-want, +got):\n%s\n", diff) + } + }) + } +} diff --git a/src/control/cmd/daos/pretty/ranks.go b/src/control/cmd/daos/pretty/ranks.go new file mode 100644 index 00000000000..da63cd37583 --- /dev/null +++ b/src/control/cmd/daos/pretty/ranks.go @@ -0,0 +1,31 @@ +// +// (C) Copyright 2020-2024 Intel Corporation. +// +// SPDX-License-Identifier: BSD-2-Clause-Patent +// + +package pretty + +import ( + "fmt" + + "github.com/daos-stack/daos/src/control/lib/ranklist" +) + +// rankList defines a type constraint for acceptable rank list formats. +type rankList interface { + []uint32 | []ranklist.Rank +} + +// PrintRanks returns a string representation of the given ranks. +func PrintRanks[rlt rankList](ranks rlt) string { + switch rl := any(ranks).(type) { + case []uint32: + return ranklist.RankSetFromRanks(ranklist.RanksFromUint32(rl)).RangedString() + case []ranklist.Rank: + return ranklist.RankSetFromRanks(rl).RangedString() + default: + // NB: This should never happen, due to the compiler check of the type constraint. + return fmt.Sprintf("unknown rank list type: %T", rl) + } +} diff --git a/src/control/cmd/daos/pretty/ranks_test.go b/src/control/cmd/daos/pretty/ranks_test.go new file mode 100644 index 00000000000..42673e753d1 --- /dev/null +++ b/src/control/cmd/daos/pretty/ranks_test.go @@ -0,0 +1,60 @@ +// +// (C) Copyright 2024 Intel Corporation. +// +// SPDX-License-Identifier: BSD-2-Clause-Patent +// + +package pretty_test + +import ( + "testing" + + "github.com/daos-stack/daos/src/control/cmd/daos/pretty" + "github.com/daos-stack/daos/src/control/lib/ranklist" +) + +func TestPretty_PrintRanks_Uint32(t *testing.T) { + for name, tc := range map[string]struct { + ranks []uint32 + expString string + }{ + "nil": {}, + "empty": { + ranks: []uint32{}, + }, + "ranks": { + ranks: []uint32{0, 1, 2, 3}, + expString: "[0-3]", + }, + } { + t.Run(name, func(t *testing.T) { + gotString := pretty.PrintRanks(tc.ranks) + if gotString != tc.expString { + t.Fatalf("got %q, want %q", gotString, tc.expString) + } + }) + } +} + +func TestPretty_PrintRanks_RanklistRank(t *testing.T) { + for name, tc := range map[string]struct { + ranks []ranklist.Rank + expString string + }{ + "nil": {}, + "empty": { + ranks: []ranklist.Rank{}, + }, + "ranks": { + ranks: []ranklist.Rank{0, 1, 2, 3}, + expString: "[0-3]", + }, + } { + t.Run(name, func(t *testing.T) { + gotString := pretty.PrintRanks(tc.ranks) + if gotString != tc.expString { + t.Fatalf("got %q, want %q", gotString, tc.expString) + } + }) + } +} diff --git a/src/control/cmd/daos/util.go b/src/control/cmd/daos/util.go index e9334c0f110..0bcbd5ab3a8 100644 --- a/src/control/cmd/daos/util.go +++ b/src/control/cmd/daos/util.go @@ -19,6 +19,7 @@ import ( "github.com/daos-stack/daos/src/control/common/cmdutil" "github.com/daos-stack/daos/src/control/lib/daos" + "github.com/daos-stack/daos/src/control/lib/ranklist" "github.com/daos-stack/daos/src/control/logging" ) @@ -100,6 +101,20 @@ func uuidFromC(cUUID C.uuid_t) (uuid.UUID, error) { return uuid.FromBytes(C.GoBytes(unsafe.Pointer(&cUUID[0]), C.int(len(cUUID)))) } +func rankSetFromC(cRankList *C.d_rank_list_t) (*ranklist.RankSet, error) { + if cRankList == nil { + return nil, errors.New("nil ranklist") + } + + cRankSlice := unsafe.Slice(cRankList.rl_ranks, cRankList.rl_nr) + rs := ranklist.NewRankSet() + for _, cRank := range cRankSlice { + rs.Add(ranklist.Rank(cRank)) + } + + return rs, nil +} + func iterStringsBuf(cBuf unsafe.Pointer, expected C.size_t, cb func(string)) error { var curLen C.size_t diff --git a/src/control/cmd/dmg/pretty/pool.go b/src/control/cmd/dmg/pretty/pool.go index 0c697a54cb1..c2445dc61e3 100644 --- a/src/control/cmd/dmg/pretty/pool.go +++ b/src/control/cmd/dmg/pretty/pool.go @@ -16,7 +16,6 @@ import ( pretty "github.com/daos-stack/daos/src/control/cmd/daos/pretty" "github.com/daos-stack/daos/src/control/lib/control" "github.com/daos-stack/daos/src/control/lib/daos" - "github.com/daos-stack/daos/src/control/lib/ranklist" "github.com/daos-stack/daos/src/control/lib/txtfmt" ) @@ -90,8 +89,8 @@ func PrintPoolCreateResponse(pcr *control.PoolCreateResp, out io.Writer, opts .. fmtArgs := make([]txtfmt.TableRow, 0, 6) fmtArgs = append(fmtArgs, txtfmt.TableRow{"UUID": pcr.UUID}) fmtArgs = append(fmtArgs, txtfmt.TableRow{"Service Leader": fmt.Sprintf("%d", pcr.Leader)}) - fmtArgs = append(fmtArgs, txtfmt.TableRow{"Service Ranks": formatRanks(pcr.SvcReps)}) - fmtArgs = append(fmtArgs, txtfmt.TableRow{"Storage Ranks": formatRanks(pcr.TgtRanks)}) + fmtArgs = append(fmtArgs, txtfmt.TableRow{"Service Ranks": pretty.PrintRanks(pcr.SvcReps)}) + fmtArgs = append(fmtArgs, txtfmt.TableRow{"Storage Ranks": pretty.PrintRanks(pcr.TgtRanks)}) fmtArgs = append(fmtArgs, txtfmt.TableRow{"Total Size": humanize.Bytes(totalSize * numRanks)}) title := "Pool created with " @@ -112,167 +111,6 @@ func PrintPoolCreateResponse(pcr *control.PoolCreateResp, out io.Writer, opts .. return err } -func poolListCreateRow(pool *daos.PoolInfo, upgrade bool) txtfmt.TableRow { - // display size of the largest non-empty tier - var size uint64 - poolUsage := pool.Usage() - for ti := len(poolUsage) - 1; ti >= 0; ti-- { - if poolUsage[ti].Size != 0 { - size = poolUsage[ti].Size - break - } - } - - // display usage of the most used tier - var used int - for ti := 0; ti < len(poolUsage); ti++ { - t := poolUsage[ti] - u := float64(t.Size-t.Free) / float64(t.Size) - - if int(u*100) > used { - used = int(u * 100) - } - } - - // display imbalance of the most imbalanced tier - var imbalance uint32 - for ti := 0; ti < len(poolUsage); ti++ { - if poolUsage[ti].Imbalance > imbalance { - imbalance = poolUsage[ti].Imbalance - } - } - - row := txtfmt.TableRow{ - "Pool": pool.Name(), - "Size": humanize.Bytes(size), - "State": pool.State.String(), - "Used": fmt.Sprintf("%d%%", used), - "Imbalance": fmt.Sprintf("%d%%", imbalance), - "Disabled": fmt.Sprintf("%d/%d", pool.DisabledTargets, pool.TotalTargets), - } - - if upgrade { - upgradeString := "None" - - if pool.PoolLayoutVer != pool.UpgradeLayoutVer { - upgradeString = fmt.Sprintf("%d->%d", pool.PoolLayoutVer, pool.UpgradeLayoutVer) - } - row["UpgradeNeeded?"] = upgradeString - } - - return row -} - -func printListPoolsResp(out io.Writer, resp *control.ListPoolsResp) error { - if len(resp.Pools) == 0 { - fmt.Fprintln(out, "no pools in system") - return nil - } - upgrade := false - for _, pool := range resp.Pools { - if resp.PoolQueryError(pool.UUID) != nil { - continue - } - if pool.PoolLayoutVer != pool.UpgradeLayoutVer { - upgrade = true - } - } - - titles := []string{"Pool", "Size", "State", "Used", "Imbalance", "Disabled"} - if upgrade { - titles = append(titles, "UpgradeNeeded?") - } - formatter := txtfmt.NewTableFormatter(titles...) - - var table []txtfmt.TableRow - for _, pool := range resp.Pools { - if resp.PoolQueryError(pool.UUID) != nil { - continue - } - table = append(table, poolListCreateRow(pool, upgrade)) - } - - fmt.Fprintln(out, formatter.Format(table)) - - return nil -} - -func addVerboseTierUsage(row txtfmt.TableRow, usage *daos.PoolTierUsage) txtfmt.TableRow { - row[usage.TierName+" Size"] = humanize.Bytes(usage.Size) - row[usage.TierName+" Used"] = humanize.Bytes(usage.Size - usage.Free) - row[usage.TierName+" Imbalance"] = fmt.Sprintf("%d%%", usage.Imbalance) - - return row -} - -func poolListCreateRowVerbose(pool *daos.PoolInfo) txtfmt.TableRow { - label := pool.Label - if label == "" { - label = "-" - } - - svcReps := "N/A" - if len(pool.ServiceReplicas) != 0 { - rl := ranklist.RanksToUint32(pool.ServiceReplicas) - svcReps = formatRanks(rl) - } - - upgrade := "None" - if pool.PoolLayoutVer != pool.UpgradeLayoutVer { - upgrade = fmt.Sprintf("%d->%d", pool.PoolLayoutVer, pool.UpgradeLayoutVer) - } - - row := txtfmt.TableRow{ - "Label": label, - "UUID": pool.UUID.String(), - "State": pool.State.String(), - "SvcReps": svcReps, - "Disabled": fmt.Sprintf("%d/%d", pool.DisabledTargets, pool.TotalTargets), - "UpgradeNeeded?": upgrade, - "Rebuild State": pool.RebuildState(), - } - - for _, tu := range pool.Usage() { - row = addVerboseTierUsage(row, tu) - } - - return row -} - -func printListPoolsRespVerbose(noQuery bool, out io.Writer, resp *control.ListPoolsResp) error { - if len(resp.Pools) == 0 { - fmt.Fprintln(out, "no pools in system") - return nil - } - - titles := []string{"Label", "UUID", "State", "SvcReps"} - for _, t := range resp.Pools[0].Usage() { - titles = append(titles, - t.TierName+" Size", - t.TierName+" Used", - t.TierName+" Imbalance") - } - titles = append(titles, "Disabled") - titles = append(titles, "UpgradeNeeded?") - - if !noQuery { - titles = append(titles, "Rebuild State") - } - formatter := txtfmt.NewTableFormatter(titles...) - - var table []txtfmt.TableRow - for _, pool := range resp.Pools { - if resp.PoolQueryError(pool.UUID) != nil { - continue - } - table = append(table, poolListCreateRowVerbose(pool)) - } - - fmt.Fprintln(out, formatter.Format(table)) - - return nil -} - // PrintListPoolsResponse generates a human-readable representation of the // supplied ListPoolsResp struct and writes it to the supplied io.Writer. // Additional columns for pool UUID and service replicas if verbose is set. @@ -285,11 +123,16 @@ func PrintListPoolsResponse(out, outErr io.Writer, resp *control.ListPoolsResp, fmt.Fprintln(outErr, warn) } - if verbose { - return printListPoolsRespVerbose(noQuery, out, resp) + // Filter out any pools that had query errors. + queriedPools := make([]*daos.PoolInfo, 0, len(resp.Pools)) + for _, pool := range resp.Pools { + if _, found := resp.QueryErrors[pool.UUID]; found { + continue + } + queriedPools = append(queriedPools, pool) } - return printListPoolsResp(out, resp) + return pretty.PrintPoolList(queriedPools, out, verbose) } // PrintPoolProperties displays a two-column table of pool property names and values. diff --git a/src/control/cmd/dmg/pretty/pool_test.go b/src/control/cmd/dmg/pretty/pool_test.go index 9ecd6a4c174..a808cd63235 100644 --- a/src/control/cmd/dmg/pretty/pool_test.go +++ b/src/control/cmd/dmg/pretty/pool_test.go @@ -557,23 +557,6 @@ func TestPretty_PrintListPoolsResponse(t *testing.T) { resp: &control.ListPoolsResp{}, expPrintStr: ` no pools in system -`, - }, - "one pool; no usage": { - resp: &control.ListPoolsResp{ - Pools: []*daos.PoolInfo{ - { - UUID: test.MockPoolUUID(1), - ServiceReplicas: []ranklist.Rank{0, 1, 2}, - State: daos.PoolServiceStateReady, - }, - }, - }, - expPrintStr: ` -Pool Size State Used Imbalance Disabled ----- ---- ----- ---- --------- -------- -00000001 0 B Ready 0% 0% 0/0 - `, }, "one pool; no uuid": { @@ -615,82 +598,6 @@ Pool Size State Used Imbalance Disabled }, expErr: errors.New("has 1 storage tiers, want 2"), }, - "two pools; only one labeled": { - resp: &control.ListPoolsResp{ - Pools: []*daos.PoolInfo{ - { - UUID: test.MockPoolUUID(1), - ServiceReplicas: []ranklist.Rank{0, 1, 2}, - TierStats: exampleTierStats, - TotalTargets: 16, - ActiveTargets: 16, - DisabledTargets: 0, - State: daos.PoolServiceStateReady, - PoolLayoutVer: 1, - UpgradeLayoutVer: 2, - }, - { - Label: "two", - UUID: test.MockPoolUUID(2), - ServiceReplicas: []ranklist.Rank{3, 4, 5}, - TierStats: exampleTierStats, - TotalTargets: 64, - ActiveTargets: 56, - DisabledTargets: 8, - State: daos.PoolServiceStateReady, - PoolLayoutVer: 1, - UpgradeLayoutVer: 2, - }, - }, - }, - expPrintStr: ` -Pool Size State Used Imbalance Disabled UpgradeNeeded? ----- ---- ----- ---- --------- -------- -------------- -00000001 6.0 TB Ready 83% 16% 0/16 1->2 -two 6.0 TB Ready 83% 56% 8/64 1->2 - -`, - }, - "two pools; one SCM only": { - resp: &control.ListPoolsResp{ - Pools: []*daos.PoolInfo{ - { - Label: "one", - UUID: test.MockPoolUUID(1), - ServiceReplicas: []ranklist.Rank{0, 1, 2}, - TierStats: exampleTierStats, - TotalTargets: 16, - ActiveTargets: 16, - DisabledTargets: 0, - State: daos.PoolServiceStateReady, - PoolLayoutVer: 1, - UpgradeLayoutVer: 2, - }, - { - Label: "two", - UUID: test.MockPoolUUID(2), - ServiceReplicas: []ranklist.Rank{3, 4, 5}, - TierStats: []*daos.StorageUsageStats{ - exampleTierStats[0], - {MediaType: daos.StorageMediaTypeNvme}, - }, - TotalTargets: 64, - ActiveTargets: 56, - DisabledTargets: 8, - State: daos.PoolServiceStateReady, - PoolLayoutVer: 2, - UpgradeLayoutVer: 2, - }, - }, - }, - expPrintStr: ` -Pool Size State Used Imbalance Disabled UpgradeNeeded? ----- ---- ----- ---- --------- -------- -------------- -one 6.0 TB Ready 83% 16% 0/16 1->2 -two 100 GB Ready 80% 56% 8/64 None - -`, - }, "two pools; one failed query": { resp: &control.ListPoolsResp{ Pools: []*daos.PoolInfo{ @@ -776,128 +683,6 @@ one 6.0 TB Ready 83%% 16%% 0/16 verbose: true, expPrintStr: ` no pools in system -`, - }, - "verbose; zero svc replicas": { - resp: &control.ListPoolsResp{ - Pools: []*daos.PoolInfo{ - { - UUID: test.MockPoolUUID(1), - TierStats: exampleTierStats, - TotalTargets: 16, - ActiveTargets: 16, - DisabledTargets: 0, - State: daos.PoolServiceStateReady, - PoolLayoutVer: 1, - UpgradeLayoutVer: 2, - Rebuild: &daos.PoolRebuildStatus{ - State: daos.PoolRebuildStateIdle, - }, - }, - }, - }, - verbose: true, - expPrintStr: ` -Label UUID State SvcReps SCM Size SCM Used SCM Imbalance NVME Size NVME Used NVME Imbalance Disabled UpgradeNeeded? Rebuild State ------ ---- ----- ------- -------- -------- ------------- --------- --------- -------------- -------- -------------- ------------- -- 00000001-0001-0001-0001-000000000001 Ready N/A 100 GB 80 GB 16% 6.0 TB 5.0 TB 8% 0/16 1->2 idle - -`, - }, - "verbose; zero svc replicas with no query": { - resp: &control.ListPoolsResp{ - Pools: []*daos.PoolInfo{ - { - UUID: test.MockPoolUUID(1), - TierStats: exampleTierStats, - TotalTargets: 16, - ActiveTargets: 16, - DisabledTargets: 0, - State: daos.PoolServiceStateReady, - PoolLayoutVer: 1, - UpgradeLayoutVer: 2, - }, - }, - }, - verbose: true, - noQuery: true, - expPrintStr: ` -Label UUID State SvcReps SCM Size SCM Used SCM Imbalance NVME Size NVME Used NVME Imbalance Disabled UpgradeNeeded? ------ ---- ----- ------- -------- -------- ------------- --------- --------- -------------- -------- -------------- -- 00000001-0001-0001-0001-000000000001 Ready N/A 100 GB 80 GB 16% 6.0 TB 5.0 TB 8% 0/16 1->2 - -`, - }, - "verbose; two pools; one destroying": { - resp: &control.ListPoolsResp{ - Pools: []*daos.PoolInfo{ - { - Label: "one", - UUID: test.MockPoolUUID(1), - ServiceReplicas: []ranklist.Rank{0, 1, 2}, - TierStats: exampleTierStats, - TotalTargets: 16, - ActiveTargets: 16, - DisabledTargets: 0, - State: daos.PoolServiceStateReady, - PoolLayoutVer: 1, - UpgradeLayoutVer: 2, - Rebuild: &daos.PoolRebuildStatus{ - State: daos.PoolRebuildStateIdle, - }, - }, - { - Label: "two", - UUID: test.MockPoolUUID(2), - ServiceReplicas: []ranklist.Rank{3, 4, 5}, - TierStats: exampleTierStats, - TotalTargets: 64, - ActiveTargets: 56, - DisabledTargets: 8, - State: daos.PoolServiceStateDestroying, - PoolLayoutVer: 2, - UpgradeLayoutVer: 2, - Rebuild: &daos.PoolRebuildStatus{ - State: daos.PoolRebuildStateDone, - }, - }, - }, - }, - verbose: true, - expPrintStr: ` -Label UUID State SvcReps SCM Size SCM Used SCM Imbalance NVME Size NVME Used NVME Imbalance Disabled UpgradeNeeded? Rebuild State ------ ---- ----- ------- -------- -------- ------------- --------- --------- -------------- -------- -------------- ------------- -one 00000001-0001-0001-0001-000000000001 Ready [0-2] 100 GB 80 GB 16% 6.0 TB 5.0 TB 8% 0/16 1->2 idle -two 00000002-0002-0002-0002-000000000002 Destroying [3-5] 100 GB 80 GB 56% 6.0 TB 5.0 TB 27% 8/64 None done - -`, - }, - "verbose; one pools; rebuild state busy": { - resp: &control.ListPoolsResp{ - Pools: []*daos.PoolInfo{ - { - Label: "one", - UUID: test.MockPoolUUID(1), - ServiceReplicas: []ranklist.Rank{0, 1, 2}, - TierStats: exampleTierStats, - TotalTargets: 16, - ActiveTargets: 8, - DisabledTargets: 8, - State: daos.PoolServiceStateDegraded, - PoolLayoutVer: 1, - UpgradeLayoutVer: 2, - Rebuild: &daos.PoolRebuildStatus{ - State: daos.PoolRebuildStateBusy, - }, - }, - }, - }, - verbose: true, - expPrintStr: ` -Label UUID State SvcReps SCM Size SCM Used SCM Imbalance NVME Size NVME Used NVME Imbalance Disabled UpgradeNeeded? Rebuild State ------ ---- ----- ------- -------- -------- ------------- --------- --------- -------------- -------- -------------- ------------- -one 00000001-0001-0001-0001-000000000001 Degraded [0-2] 100 GB 80 GB 8% 6.0 TB 5.0 TB 4% 8/16 1->2 busy - `, }, } { diff --git a/src/control/cmd/dmg/pretty/ranks.go b/src/control/cmd/dmg/pretty/ranks.go deleted file mode 100644 index f13b678b58e..00000000000 --- a/src/control/cmd/dmg/pretty/ranks.go +++ /dev/null @@ -1,16 +0,0 @@ -// -// (C) Copyright 2020-2022 Intel Corporation. -// -// SPDX-License-Identifier: BSD-2-Clause-Patent -// - -package pretty - -import "github.com/daos-stack/daos/src/control/lib/ranklist" - -// formatRanks takes a slice of uint32 ranks and returns a string -// representation of the set created from the slice. -func formatRanks(ranks []uint32) string { - rs := ranklist.RankSetFromRanks(ranklist.RanksFromUint32(ranks)) - return rs.RangedString() -} diff --git a/src/control/common/proto/srv/srv.pb.go b/src/control/common/proto/srv/srv.pb.go index 3734e1883fc..20eafc21963 100644 --- a/src/control/common/proto/srv/srv.pb.go +++ b/src/control/common/proto/srv/srv.pb.go @@ -8,7 +8,7 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: -// protoc-gen-go v1.28.1 +// protoc-gen-go v1.31.0 // protoc v3.5.0 // source: srv/srv.proto @@ -422,6 +422,163 @@ func (x *PoolFindByLabelResp) GetSvcreps() []uint32 { return nil } +type ListPoolsReq struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + IncludeAll bool `protobuf:"varint,1,opt,name=includeAll,proto3" json:"includeAll,omitempty"` // Include all pools in response, regardless of state +} + +func (x *ListPoolsReq) Reset() { + *x = ListPoolsReq{} + if protoimpl.UnsafeEnabled { + mi := &file_srv_srv_proto_msgTypes[6] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *ListPoolsReq) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*ListPoolsReq) ProtoMessage() {} + +func (x *ListPoolsReq) ProtoReflect() protoreflect.Message { + mi := &file_srv_srv_proto_msgTypes[6] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use ListPoolsReq.ProtoReflect.Descriptor instead. +func (*ListPoolsReq) Descriptor() ([]byte, []int) { + return file_srv_srv_proto_rawDescGZIP(), []int{6} +} + +func (x *ListPoolsReq) GetIncludeAll() bool { + if x != nil { + return x.IncludeAll + } + return false +} + +type ListPoolsResp struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + Pools []*ListPoolsResp_Pool `protobuf:"bytes,1,rep,name=pools,proto3" json:"pools,omitempty"` // List of pools +} + +func (x *ListPoolsResp) Reset() { + *x = ListPoolsResp{} + if protoimpl.UnsafeEnabled { + mi := &file_srv_srv_proto_msgTypes[7] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *ListPoolsResp) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*ListPoolsResp) ProtoMessage() {} + +func (x *ListPoolsResp) ProtoReflect() protoreflect.Message { + mi := &file_srv_srv_proto_msgTypes[7] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use ListPoolsResp.ProtoReflect.Descriptor instead. +func (*ListPoolsResp) Descriptor() ([]byte, []int) { + return file_srv_srv_proto_rawDescGZIP(), []int{7} +} + +func (x *ListPoolsResp) GetPools() []*ListPoolsResp_Pool { + if x != nil { + return x.Pools + } + return nil +} + +type ListPoolsResp_Pool struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + Uuid string `protobuf:"bytes,1,opt,name=uuid,proto3" json:"uuid,omitempty"` // Pool UUID + Label string `protobuf:"bytes,2,opt,name=label,proto3" json:"label,omitempty"` // Pool label + Svcreps []uint32 `protobuf:"varint,3,rep,packed,name=svcreps,proto3" json:"svcreps,omitempty"` // Pool service ranks +} + +func (x *ListPoolsResp_Pool) Reset() { + *x = ListPoolsResp_Pool{} + if protoimpl.UnsafeEnabled { + mi := &file_srv_srv_proto_msgTypes[8] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *ListPoolsResp_Pool) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*ListPoolsResp_Pool) ProtoMessage() {} + +func (x *ListPoolsResp_Pool) ProtoReflect() protoreflect.Message { + mi := &file_srv_srv_proto_msgTypes[8] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use ListPoolsResp_Pool.ProtoReflect.Descriptor instead. +func (*ListPoolsResp_Pool) Descriptor() ([]byte, []int) { + return file_srv_srv_proto_rawDescGZIP(), []int{7, 0} +} + +func (x *ListPoolsResp_Pool) GetUuid() string { + if x != nil { + return x.Uuid + } + return "" +} + +func (x *ListPoolsResp_Pool) GetLabel() string { + if x != nil { + return x.Label + } + return "" +} + +func (x *ListPoolsResp_Pool) GetSvcreps() []uint32 { + if x != nil { + return x.Svcreps + } + return nil +} + var File_srv_srv_proto protoreflect.FileDescriptor var file_srv_srv_proto_rawDesc = []byte{ @@ -466,12 +623,23 @@ var file_srv_srv_proto_rawDesc = []byte{ 0x74, 0x75, 0x73, 0x18, 0x01, 0x20, 0x01, 0x28, 0x05, 0x52, 0x06, 0x73, 0x74, 0x61, 0x74, 0x75, 0x73, 0x12, 0x12, 0x0a, 0x04, 0x75, 0x75, 0x69, 0x64, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x75, 0x75, 0x69, 0x64, 0x12, 0x18, 0x0a, 0x07, 0x73, 0x76, 0x63, 0x72, 0x65, 0x70, 0x73, - 0x18, 0x03, 0x20, 0x03, 0x28, 0x0d, 0x52, 0x07, 0x73, 0x76, 0x63, 0x72, 0x65, 0x70, 0x73, 0x42, - 0x39, 0x5a, 0x37, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x64, 0x61, - 0x6f, 0x73, 0x2d, 0x73, 0x74, 0x61, 0x63, 0x6b, 0x2f, 0x64, 0x61, 0x6f, 0x73, 0x2f, 0x73, 0x72, - 0x63, 0x2f, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2f, 0x63, 0x6f, 0x6d, 0x6d, 0x6f, 0x6e, - 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2f, 0x73, 0x72, 0x76, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, - 0x6f, 0x33, + 0x18, 0x03, 0x20, 0x03, 0x28, 0x0d, 0x52, 0x07, 0x73, 0x76, 0x63, 0x72, 0x65, 0x70, 0x73, 0x22, + 0x2e, 0x0a, 0x0c, 0x4c, 0x69, 0x73, 0x74, 0x50, 0x6f, 0x6f, 0x6c, 0x73, 0x52, 0x65, 0x71, 0x12, + 0x1e, 0x0a, 0x0a, 0x69, 0x6e, 0x63, 0x6c, 0x75, 0x64, 0x65, 0x41, 0x6c, 0x6c, 0x18, 0x01, 0x20, + 0x01, 0x28, 0x08, 0x52, 0x0a, 0x69, 0x6e, 0x63, 0x6c, 0x75, 0x64, 0x65, 0x41, 0x6c, 0x6c, 0x22, + 0x8a, 0x01, 0x0a, 0x0d, 0x4c, 0x69, 0x73, 0x74, 0x50, 0x6f, 0x6f, 0x6c, 0x73, 0x52, 0x65, 0x73, + 0x70, 0x12, 0x2d, 0x0a, 0x05, 0x70, 0x6f, 0x6f, 0x6c, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, + 0x32, 0x17, 0x2e, 0x73, 0x72, 0x76, 0x2e, 0x4c, 0x69, 0x73, 0x74, 0x50, 0x6f, 0x6f, 0x6c, 0x73, + 0x52, 0x65, 0x73, 0x70, 0x2e, 0x50, 0x6f, 0x6f, 0x6c, 0x52, 0x05, 0x70, 0x6f, 0x6f, 0x6c, 0x73, + 0x1a, 0x4a, 0x0a, 0x04, 0x50, 0x6f, 0x6f, 0x6c, 0x12, 0x12, 0x0a, 0x04, 0x75, 0x75, 0x69, 0x64, + 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x75, 0x75, 0x69, 0x64, 0x12, 0x14, 0x0a, 0x05, + 0x6c, 0x61, 0x62, 0x65, 0x6c, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x6c, 0x61, 0x62, + 0x65, 0x6c, 0x12, 0x18, 0x0a, 0x07, 0x73, 0x76, 0x63, 0x72, 0x65, 0x70, 0x73, 0x18, 0x03, 0x20, + 0x03, 0x28, 0x0d, 0x52, 0x07, 0x73, 0x76, 0x63, 0x72, 0x65, 0x70, 0x73, 0x42, 0x39, 0x5a, 0x37, + 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x64, 0x61, 0x6f, 0x73, 0x2d, + 0x73, 0x74, 0x61, 0x63, 0x6b, 0x2f, 0x64, 0x61, 0x6f, 0x73, 0x2f, 0x73, 0x72, 0x63, 0x2f, 0x63, + 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2f, 0x63, 0x6f, 0x6d, 0x6d, 0x6f, 0x6e, 0x2f, 0x70, 0x72, + 0x6f, 0x74, 0x6f, 0x2f, 0x73, 0x72, 0x76, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, } var ( @@ -486,7 +654,7 @@ func file_srv_srv_proto_rawDescGZIP() []byte { return file_srv_srv_proto_rawDescData } -var file_srv_srv_proto_msgTypes = make([]protoimpl.MessageInfo, 6) +var file_srv_srv_proto_msgTypes = make([]protoimpl.MessageInfo, 9) var file_srv_srv_proto_goTypes = []interface{}{ (*NotifyReadyReq)(nil), // 0: srv.NotifyReadyReq (*BioErrorReq)(nil), // 1: srv.BioErrorReq @@ -494,13 +662,17 @@ var file_srv_srv_proto_goTypes = []interface{}{ (*GetPoolSvcResp)(nil), // 3: srv.GetPoolSvcResp (*PoolFindByLabelReq)(nil), // 4: srv.PoolFindByLabelReq (*PoolFindByLabelResp)(nil), // 5: srv.PoolFindByLabelResp + (*ListPoolsReq)(nil), // 6: srv.ListPoolsReq + (*ListPoolsResp)(nil), // 7: srv.ListPoolsResp + (*ListPoolsResp_Pool)(nil), // 8: srv.ListPoolsResp.Pool } var file_srv_srv_proto_depIdxs = []int32{ - 0, // [0:0] is the sub-list for method output_type - 0, // [0:0] is the sub-list for method input_type - 0, // [0:0] is the sub-list for extension type_name - 0, // [0:0] is the sub-list for extension extendee - 0, // [0:0] is the sub-list for field type_name + 8, // 0: srv.ListPoolsResp.pools:type_name -> srv.ListPoolsResp.Pool + 1, // [1:1] is the sub-list for method output_type + 1, // [1:1] is the sub-list for method input_type + 1, // [1:1] is the sub-list for extension type_name + 1, // [1:1] is the sub-list for extension extendee + 0, // [0:1] is the sub-list for field type_name } func init() { file_srv_srv_proto_init() } @@ -581,6 +753,42 @@ func file_srv_srv_proto_init() { return nil } } + file_srv_srv_proto_msgTypes[6].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*ListPoolsReq); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_srv_srv_proto_msgTypes[7].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*ListPoolsResp); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_srv_srv_proto_msgTypes[8].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*ListPoolsResp_Pool); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } } type x struct{} out := protoimpl.TypeBuilder{ @@ -588,7 +796,7 @@ func file_srv_srv_proto_init() { GoPackagePath: reflect.TypeOf(x{}).PkgPath(), RawDescriptor: file_srv_srv_proto_rawDesc, NumEnums: 0, - NumMessages: 6, + NumMessages: 9, NumExtensions: 0, NumServices: 0, }, diff --git a/src/control/drpc/modules.go b/src/control/drpc/modules.go index 0aacbae1d4a..84165ad3d01 100644 --- a/src/control/drpc/modules.go +++ b/src/control/drpc/modules.go @@ -261,9 +261,12 @@ func (m srvMethod) ID() int32 { func (m srvMethod) String() string { if s, ok := map[srvMethod]string{ - MethodNotifyReady: "notify ready", - MethodBIOError: "block i/o error", - MethodClusterEvent: "cluster event", + MethodNotifyReady: "notify ready", + MethodBIOError: "block i/o error", + MethodClusterEvent: "cluster event", + MethodGetPoolServiceRanks: "get pool service ranks", + MethodPoolFindByLabel: "find pool by label", + MethodListPools: "list pools", }[m]; ok { return s } @@ -293,6 +296,8 @@ const ( MethodPoolFindByLabel srvMethod = C.DRPC_METHOD_SRV_POOL_FIND_BYLABEL // MethodClusterEvent notifies of a cluster event in the I/O Engine. MethodClusterEvent srvMethod = C.DRPC_METHOD_SRV_CLUSTER_EVENT + // MethodListPools requests the list of pools in the system + MethodListPools srvMethod = C.DRPC_METHOD_SRV_LIST_POOLS ) type securityMethod int32 diff --git a/src/control/go.mod b/src/control/go.mod index 298769214b3..248e35b6817 100644 --- a/src/control/go.mod +++ b/src/control/go.mod @@ -1,6 +1,6 @@ module github.com/daos-stack/daos/src/control -go 1.17 +go 1.20 require ( github.com/Jille/raft-grpc-transport v1.2.0 diff --git a/src/control/go.sum b/src/control/go.sum index 48980442ea1..31fa37b995d 100644 --- a/src/control/go.sum +++ b/src/control/go.sum @@ -356,8 +356,8 @@ golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81R golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= golang.org/x/net v0.0.0-20210525063256-abc453219eb5/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20210907225631-ff17edfbf26d/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= -golang.org/x/net v0.22.0 h1:9sGLhx7iRIHEiX0oAJ3MRZMUCElJgy7Br1nO+AMN3Tc= -golang.org/x/net v0.22.0/go.mod h1:JKghWKKOSdJwpW2GEx0Ja7fmaKnMsbu+MWVZTokSYmg= +golang.org/x/net v0.23.0 h1:7EYJ93RZ9vYSZAIb2x3lnuvqO5zneoD6IvWjuhfxjTs= +golang.org/x/net v0.23.0/go.mod h1:JKghWKKOSdJwpW2GEx0Ja7fmaKnMsbu+MWVZTokSYmg= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= diff --git a/src/control/server/mgmt_drpc.go b/src/control/server/mgmt_drpc.go index b45570ecc57..70ada0c493e 100644 --- a/src/control/server/mgmt_drpc.go +++ b/src/control/server/mgmt_drpc.go @@ -47,6 +47,7 @@ func (mod *mgmtModule) ID() drpc.ModuleID { type poolResolver interface { FindPoolServiceByLabel(string) (*system.PoolService, error) FindPoolServiceByUUID(uuid.UUID) (*system.PoolService, error) + PoolServiceList(bool) ([]*system.PoolService, error) } // srvModule represents the daos_server dRPC module. It handles dRPCs sent by @@ -70,7 +71,15 @@ func newSrvModule(log logging.Logger, sysdb poolResolver, engines []Engine, even } // HandleCall is the handler for calls to the srvModule. -func (mod *srvModule) HandleCall(_ context.Context, session *drpc.Session, method drpc.Method, req []byte) ([]byte, error) { +func (mod *srvModule) HandleCall(_ context.Context, session *drpc.Session, method drpc.Method, req []byte) (_ []byte, err error) { + defer func() { + msg := ": success" + if err != nil { + msg = ", failed: " + err.Error() + } + mod.log.Tracef("srv upcall: %s%s", method, msg) + }() + switch method { case drpc.MethodNotifyReady: return nil, mod.handleNotifyReady(req) @@ -82,6 +91,8 @@ func (mod *srvModule) HandleCall(_ context.Context, session *drpc.Session, metho return mod.handlePoolFindByLabel(req) case drpc.MethodClusterEvent: return mod.handleClusterEvent(req) + case drpc.MethodListPools: + return mod.handleListPools(req) default: return nil, drpc.UnknownMethodFailure() } @@ -198,3 +209,29 @@ func (mod *srvModule) handleClusterEvent(reqb []byte) ([]byte, error) { return proto.Marshal(resp) } + +func (mod *srvModule) handleListPools(reqb []byte) ([]byte, error) { + req := new(srvpb.ListPoolsReq) + if err := proto.Unmarshal(reqb, req); err != nil { + return nil, drpc.UnmarshalingPayloadFailure() + } + + mod.log.Tracef("%T: %+v", req, req) + pools, err := mod.sysdb.PoolServiceList(req.IncludeAll) + if err != nil { + return nil, errors.Wrap(err, "failed to list system pools") + } + + resp := new(srvpb.ListPoolsResp) + resp.Pools = make([]*srvpb.ListPoolsResp_Pool, len(pools)) + for i, ps := range pools { + resp.Pools[i] = &srvpb.ListPoolsResp_Pool{ + Uuid: ps.PoolUUID.String(), + Label: ps.PoolLabel, + Svcreps: ranklist.RanksToUint32(ps.Replicas), + } + } + mod.log.Tracef("%T %+v", resp, resp) + + return proto.Marshal(resp) +} diff --git a/src/engine/drpc_client.c b/src/engine/drpc_client.c index 9be829e0f11..54db5c15e0e 100644 --- a/src/engine/drpc_client.c +++ b/src/engine/drpc_client.c @@ -414,6 +414,104 @@ ds_pool_find_bylabel(d_const_string_t label, uuid_t pool_uuid, return rc; } +int +ds_get_pool_list(uint64_t *npools, daos_mgmt_pool_info_t *pools) +{ + struct drpc_alloc alloc = PROTO_ALLOCATOR_INIT(alloc); + Srv__ListPoolsReq lp_req = SRV__LIST_POOLS_REQ__INIT; + Srv__ListPoolsResp *lp_resp = NULL; + Drpc__Response *dresp; + uint8_t *req; + size_t req_size; + d_rank_list_t *svc_ranks; + int i, rc; + + if (npools == NULL) { + D_ERROR("npools may not be NULL\n"); + D_GOTO(out, rc = -DER_INVAL); + } + + lp_req.includeall = false; /* only list Ready pools */ + + req_size = srv__list_pools_req__get_packed_size(&lp_req); + D_ALLOC(req, req_size); + if (req == NULL) + D_GOTO(out, rc = -DER_NOMEM); + srv__list_pools_req__pack(&lp_req, req); + + rc = dss_drpc_call(DRPC_MODULE_SRV, DRPC_METHOD_SRV_LIST_POOLS, req, req_size, + 0 /* flags */, &dresp); + if (rc != 0) + goto out_req; + if (dresp->status != DRPC__STATUS__SUCCESS) { + D_ERROR("received erroneous dRPC response: %d\n", dresp->status); + D_GOTO(out_dresp, rc = -DER_IO); + } + + lp_resp = srv__list_pools_resp__unpack(&alloc.alloc, dresp->body.len, dresp->body.data); + if (alloc.oom) { + D_GOTO(out_dresp, rc = -DER_NOMEM); + } else if (lp_resp == NULL) { + D_ERROR("failed to unpack resp (list pools)\n"); + D_GOTO(out_dresp, rc = -DER_NOMEM); + } + + if (*npools > 0 && lp_resp->n_pools > *npools) { + D_ERROR("pool list exceeds request buffer (req: %lu, actual: %lu)", *npools, + lp_resp->n_pools); + D_GOTO(out_resp, rc = -DER_TRUNC); + } + + *npools = lp_resp->n_pools; + if (pools == NULL) { + /* caller just needs the # of pools */ + D_GOTO(out_resp, rc); + } + + for (i = 0; i < lp_resp->n_pools; i++) { + daos_mgmt_pool_info_t *mgmt_pool = &pools[i]; + Srv__ListPoolsResp__Pool *resp_pool = lp_resp->pools[i]; + + rc = uuid_parse(resp_pool->uuid, mgmt_pool->mgpi_uuid); + if (rc != 0) { + D_ERROR("failed to parse pool uuid: %d\n", rc); + D_GOTO(out_free_pools, rc = -DER_INVAL); + } + + D_STRNDUP(mgmt_pool->mgpi_label, resp_pool->label, DAOS_PROP_LABEL_MAX_LEN); + if (mgmt_pool->mgpi_label == NULL) { + D_ERROR("failed to copy pool label\n"); + D_GOTO(out_free_pools, rc = -DER_NOMEM); + } + + svc_ranks = uint32_array_to_rank_list(resp_pool->svcreps, resp_pool->n_svcreps); + if (svc_ranks == NULL) { + D_ERROR("failed to create svc ranks list\n"); + D_GOTO(out_free_pools, rc = -DER_NOMEM); + } + mgmt_pool->mgpi_svc = svc_ranks; + } + +out_free_pools: + if (rc != 0 && pools != NULL) { + for (i = 0; i < lp_resp->n_pools; i++) { + daos_mgmt_pool_info_t *mgmt_pool = &pools[i]; + if (mgmt_pool->mgpi_label) + D_FREE(mgmt_pool->mgpi_label); + if (mgmt_pool->mgpi_svc) + d_rank_list_free(mgmt_pool->mgpi_svc); + } + } +out_resp: + srv__list_pools_resp__free_unpacked(lp_resp, &alloc.alloc); +out_dresp: + drpc_response_free(dresp); +out_req: + D_FREE(req); +out: + return rc; +} + int drpc_init(void) { diff --git a/src/engine/srv.pb-c.c b/src/engine/srv.pb-c.c index 8db1c142a9f..9db07ec8187 100644 --- a/src/engine/srv.pb-c.c +++ b/src/engine/srv.pb-c.c @@ -277,6 +277,88 @@ void srv__pool_find_by_label_resp__free_unpacked assert(message->base.descriptor == &srv__pool_find_by_label_resp__descriptor); protobuf_c_message_free_unpacked ((ProtobufCMessage*)message, allocator); } +void +srv__list_pools_req__init(Srv__ListPoolsReq *message) +{ + static const Srv__ListPoolsReq init_value = SRV__LIST_POOLS_REQ__INIT; + *message = init_value; +} +size_t +srv__list_pools_req__get_packed_size(const Srv__ListPoolsReq *message) +{ + assert(message->base.descriptor == &srv__list_pools_req__descriptor); + return protobuf_c_message_get_packed_size((const ProtobufCMessage *)(message)); +} +size_t +srv__list_pools_req__pack(const Srv__ListPoolsReq *message, uint8_t *out) +{ + assert(message->base.descriptor == &srv__list_pools_req__descriptor); + return protobuf_c_message_pack((const ProtobufCMessage *)message, out); +} +size_t +srv__list_pools_req__pack_to_buffer(const Srv__ListPoolsReq *message, ProtobufCBuffer *buffer) +{ + assert(message->base.descriptor == &srv__list_pools_req__descriptor); + return protobuf_c_message_pack_to_buffer((const ProtobufCMessage *)message, buffer); +} +Srv__ListPoolsReq * +srv__list_pools_req__unpack(ProtobufCAllocator *allocator, size_t len, const uint8_t *data) +{ + return (Srv__ListPoolsReq *)protobuf_c_message_unpack(&srv__list_pools_req__descriptor, allocator, + len, data); +} +void +srv__list_pools_req__free_unpacked(Srv__ListPoolsReq *message, ProtobufCAllocator *allocator) +{ + if (!message) + return; + assert(message->base.descriptor == &srv__list_pools_req__descriptor); + protobuf_c_message_free_unpacked((ProtobufCMessage *)message, allocator); +} +void +srv__list_pools_resp__pool__init(Srv__ListPoolsResp__Pool *message) +{ + static const Srv__ListPoolsResp__Pool init_value = SRV__LIST_POOLS_RESP__POOL__INIT; + *message = init_value; +} +void +srv__list_pools_resp__init(Srv__ListPoolsResp *message) +{ + static const Srv__ListPoolsResp init_value = SRV__LIST_POOLS_RESP__INIT; + *message = init_value; +} +size_t +srv__list_pools_resp__get_packed_size(const Srv__ListPoolsResp *message) +{ + assert(message->base.descriptor == &srv__list_pools_resp__descriptor); + return protobuf_c_message_get_packed_size((const ProtobufCMessage *)(message)); +} +size_t +srv__list_pools_resp__pack(const Srv__ListPoolsResp *message, uint8_t *out) +{ + assert(message->base.descriptor == &srv__list_pools_resp__descriptor); + return protobuf_c_message_pack((const ProtobufCMessage *)message, out); +} +size_t +srv__list_pools_resp__pack_to_buffer(const Srv__ListPoolsResp *message, ProtobufCBuffer *buffer) +{ + assert(message->base.descriptor == &srv__list_pools_resp__descriptor); + return protobuf_c_message_pack_to_buffer((const ProtobufCMessage *)message, buffer); +} +Srv__ListPoolsResp * +srv__list_pools_resp__unpack(ProtobufCAllocator *allocator, size_t len, const uint8_t *data) +{ + return (Srv__ListPoolsResp *)protobuf_c_message_unpack(&srv__list_pools_resp__descriptor, + allocator, len, data); +} +void +srv__list_pools_resp__free_unpacked(Srv__ListPoolsResp *message, ProtobufCAllocator *allocator) +{ + if (!message) + return; + assert(message->base.descriptor == &srv__list_pools_resp__descriptor); + protobuf_c_message_free_unpacked((ProtobufCMessage *)message, allocator); +} static const ProtobufCFieldDescriptor srv__notify_ready_req__field_descriptors[6] = { { @@ -687,3 +769,101 @@ const ProtobufCMessageDescriptor srv__pool_find_by_label_resp__descriptor = (ProtobufCMessageInit) srv__pool_find_by_label_resp__init, NULL,NULL,NULL /* reserved[123] */ }; +static const ProtobufCFieldDescriptor srv__list_pools_req__field_descriptors[1] = { + { + "includeAll", 1, PROTOBUF_C_LABEL_NONE, PROTOBUF_C_TYPE_BOOL, 0, /* quantifier_offset */ + offsetof(Srv__ListPoolsReq, includeall), NULL, NULL, 0, /* flags */ + 0, NULL, NULL /* reserved1,reserved2, etc */ + }, +}; +static const unsigned srv__list_pools_req__field_indices_by_name[] = { + 0, /* field[0] = includeAll */ +}; +static const ProtobufCIntRange srv__list_pools_req__number_ranges[1 + 1] = {{1, 0}, {0, 1}}; +const ProtobufCMessageDescriptor srv__list_pools_req__descriptor = { + PROTOBUF_C__MESSAGE_DESCRIPTOR_MAGIC, + "srv.ListPoolsReq", + "ListPoolsReq", + "Srv__ListPoolsReq", + "srv", + sizeof(Srv__ListPoolsReq), + 1, + srv__list_pools_req__field_descriptors, + srv__list_pools_req__field_indices_by_name, + 1, + srv__list_pools_req__number_ranges, + (ProtobufCMessageInit)srv__list_pools_req__init, + NULL, + NULL, + NULL /* reserved[123] */ +}; +static const ProtobufCFieldDescriptor srv__list_pools_resp__pool__field_descriptors[3] = { + { + "uuid", 1, PROTOBUF_C_LABEL_NONE, PROTOBUF_C_TYPE_STRING, 0, /* quantifier_offset */ + offsetof(Srv__ListPoolsResp__Pool, uuid), NULL, &protobuf_c_empty_string, 0, /* flags */ + 0, NULL, NULL /* reserved1,reserved2, etc */ + }, + { + "label", 2, PROTOBUF_C_LABEL_NONE, PROTOBUF_C_TYPE_STRING, 0, /* quantifier_offset */ + offsetof(Srv__ListPoolsResp__Pool, label), NULL, &protobuf_c_empty_string, 0, /* flags */ + 0, NULL, NULL /* reserved1,reserved2, etc */ + }, + { + "svcreps", 3, PROTOBUF_C_LABEL_REPEATED, PROTOBUF_C_TYPE_UINT32, + offsetof(Srv__ListPoolsResp__Pool, n_svcreps), offsetof(Srv__ListPoolsResp__Pool, svcreps), + NULL, NULL, 0, /* flags */ + 0, NULL, NULL /* reserved1,reserved2, etc */ + }, +}; +static const unsigned srv__list_pools_resp__pool__field_indices_by_name[] = { + 1, /* field[1] = label */ + 2, /* field[2] = svcreps */ + 0, /* field[0] = uuid */ +}; +static const ProtobufCIntRange srv__list_pools_resp__pool__number_ranges[1 + 1] = {{1, 0}, {0, 3}}; +const ProtobufCMessageDescriptor srv__list_pools_resp__pool__descriptor = { + PROTOBUF_C__MESSAGE_DESCRIPTOR_MAGIC, + "srv.ListPoolsResp.Pool", + "Pool", + "Srv__ListPoolsResp__Pool", + "srv", + sizeof(Srv__ListPoolsResp__Pool), + 3, + srv__list_pools_resp__pool__field_descriptors, + srv__list_pools_resp__pool__field_indices_by_name, + 1, + srv__list_pools_resp__pool__number_ranges, + (ProtobufCMessageInit)srv__list_pools_resp__pool__init, + NULL, + NULL, + NULL /* reserved[123] */ +}; +static const ProtobufCFieldDescriptor srv__list_pools_resp__field_descriptors[1] = { + { + "pools", 1, PROTOBUF_C_LABEL_REPEATED, PROTOBUF_C_TYPE_MESSAGE, + offsetof(Srv__ListPoolsResp, n_pools), offsetof(Srv__ListPoolsResp, pools), + &srv__list_pools_resp__pool__descriptor, NULL, 0, /* flags */ + 0, NULL, NULL /* reserved1,reserved2, etc */ + }, +}; +static const unsigned srv__list_pools_resp__field_indices_by_name[] = { + 0, /* field[0] = pools */ +}; +static const ProtobufCIntRange srv__list_pools_resp__number_ranges[1 + 1] = {{1, 0}, {0, 1}}; +const ProtobufCMessageDescriptor srv__list_pools_resp__descriptor = { + PROTOBUF_C__MESSAGE_DESCRIPTOR_MAGIC, + "srv.ListPoolsResp", + "ListPoolsResp", + "Srv__ListPoolsResp", + "srv", + sizeof(Srv__ListPoolsResp), + 1, + srv__list_pools_resp__field_descriptors, + srv__list_pools_resp__field_indices_by_name, + 1, + srv__list_pools_resp__number_ranges, + (ProtobufCMessageInit)srv__list_pools_resp__init, + NULL, + NULL, + NULL /* reserved[123] */ +}; diff --git a/src/engine/srv.pb-c.h b/src/engine/srv.pb-c.h index 4f65ce7b43c..8a6006b1faa 100644 --- a/src/engine/srv.pb-c.h +++ b/src/engine/srv.pb-c.h @@ -21,7 +21,9 @@ typedef struct _Srv__GetPoolSvcReq Srv__GetPoolSvcReq; typedef struct _Srv__GetPoolSvcResp Srv__GetPoolSvcResp; typedef struct _Srv__PoolFindByLabelReq Srv__PoolFindByLabelReq; typedef struct _Srv__PoolFindByLabelResp Srv__PoolFindByLabelResp; - +typedef struct _Srv__ListPoolsReq Srv__ListPoolsReq; +typedef struct _Srv__ListPoolsResp Srv__ListPoolsResp; +typedef struct _Srv__ListPoolsResp__Pool Srv__ListPoolsResp__Pool; /* --- enums --- */ @@ -163,6 +165,54 @@ struct _Srv__PoolFindByLabelResp { PROTOBUF_C_MESSAGE_INIT (&srv__pool_find_by_label_resp__descriptor) \ , 0, (char *)protobuf_c_empty_string, 0,NULL } +struct _Srv__ListPoolsReq { + ProtobufCMessage base; + /* + * Include all pools in response, regardless of state + */ + protobuf_c_boolean includeall; +}; +#define SRV__LIST_POOLS_REQ__INIT \ + { \ + PROTOBUF_C_MESSAGE_INIT(&srv__list_pools_req__descriptor) \ + , 0 \ + } + +struct _Srv__ListPoolsResp__Pool { + ProtobufCMessage base; + /* + * Pool UUID + */ + char *uuid; + /* + * Pool label + */ + char *label; + /* + * Pool service ranks + */ + size_t n_svcreps; + uint32_t *svcreps; +}; +#define SRV__LIST_POOLS_RESP__POOL__INIT \ + { \ + PROTOBUF_C_MESSAGE_INIT(&srv__list_pools_resp__pool__descriptor) \ + , (char *)protobuf_c_empty_string, (char *)protobuf_c_empty_string, 0, NULL \ + } + +struct _Srv__ListPoolsResp { + ProtobufCMessage base; + /* + * List of pools + */ + size_t n_pools; + Srv__ListPoolsResp__Pool **pools; +}; +#define SRV__LIST_POOLS_RESP__INIT \ + { \ + PROTOBUF_C_MESSAGE_INIT(&srv__list_pools_resp__descriptor) \ + , 0, NULL \ + } /* Srv__NotifyReadyReq methods */ void srv__notify_ready_req__init @@ -278,6 +328,35 @@ Srv__PoolFindByLabelResp * void srv__pool_find_by_label_resp__free_unpacked (Srv__PoolFindByLabelResp *message, ProtobufCAllocator *allocator); +/* Srv__ListPoolsReq methods */ +void +srv__list_pools_req__init(Srv__ListPoolsReq *message); +size_t +srv__list_pools_req__get_packed_size(const Srv__ListPoolsReq *message); +size_t +srv__list_pools_req__pack(const Srv__ListPoolsReq *message, uint8_t *out); +size_t +srv__list_pools_req__pack_to_buffer(const Srv__ListPoolsReq *message, ProtobufCBuffer *buffer); +Srv__ListPoolsReq * +srv__list_pools_req__unpack(ProtobufCAllocator *allocator, size_t len, const uint8_t *data); +void +srv__list_pools_req__free_unpacked(Srv__ListPoolsReq *message, ProtobufCAllocator *allocator); +/* Srv__ListPoolsResp__Pool methods */ +void +srv__list_pools_resp__pool__init(Srv__ListPoolsResp__Pool *message); +/* Srv__ListPoolsResp methods */ +void +srv__list_pools_resp__init(Srv__ListPoolsResp *message); +size_t +srv__list_pools_resp__get_packed_size(const Srv__ListPoolsResp *message); +size_t +srv__list_pools_resp__pack(const Srv__ListPoolsResp *message, uint8_t *out); +size_t +srv__list_pools_resp__pack_to_buffer(const Srv__ListPoolsResp *message, ProtobufCBuffer *buffer); +Srv__ListPoolsResp * +srv__list_pools_resp__unpack(ProtobufCAllocator *allocator, size_t len, const uint8_t *data); +void +srv__list_pools_resp__free_unpacked(Srv__ListPoolsResp *message, ProtobufCAllocator *allocator); /* --- per-message closures --- */ typedef void (*Srv__NotifyReadyReq_Closure) @@ -298,6 +377,10 @@ typedef void (*Srv__PoolFindByLabelReq_Closure) typedef void (*Srv__PoolFindByLabelResp_Closure) (const Srv__PoolFindByLabelResp *message, void *closure_data); +typedef void (*Srv__ListPoolsReq_Closure)(const Srv__ListPoolsReq *message, void *closure_data); +typedef void (*Srv__ListPoolsResp__Pool_Closure)(const Srv__ListPoolsResp__Pool *message, + void *closure_data); +typedef void (*Srv__ListPoolsResp_Closure)(const Srv__ListPoolsResp *message, void *closure_data); /* --- services --- */ @@ -310,6 +393,9 @@ extern const ProtobufCMessageDescriptor srv__get_pool_svc_req__descriptor; extern const ProtobufCMessageDescriptor srv__get_pool_svc_resp__descriptor; extern const ProtobufCMessageDescriptor srv__pool_find_by_label_req__descriptor; extern const ProtobufCMessageDescriptor srv__pool_find_by_label_resp__descriptor; +extern const ProtobufCMessageDescriptor srv__list_pools_req__descriptor; +extern const ProtobufCMessageDescriptor srv__list_pools_resp__descriptor; +extern const ProtobufCMessageDescriptor srv__list_pools_resp__pool__descriptor; PROTOBUF_C__END_DECLS diff --git a/src/include/daos/drpc.pb-c.h b/src/include/daos/drpc.pb-c.h index 6083131d433..24cfdcfcf7f 100644 --- a/src/include/daos/drpc.pb-c.h +++ b/src/include/daos/drpc.pb-c.h @@ -10,14 +10,12 @@ PROTOBUF_C__BEGIN_DECLS #if PROTOBUF_C_VERSION_NUMBER < 1003000 # error This file was generated by a newer version of protoc-c which is incompatible with your libprotobuf-c headers. Please update your headers. -#elif 1003000 < PROTOBUF_C_MIN_COMPILER_VERSION +#elif 1004001 < PROTOBUF_C_MIN_COMPILER_VERSION # error This file was generated by an older version of protoc-c which is incompatible with your libprotobuf-c headers. Please regenerate this file with a newer version of protoc-c. #endif - -typedef struct _Drpc__Call Drpc__Call; -typedef struct _Drpc__Response Drpc__Response; - +typedef struct Drpc__Call Drpc__Call; +typedef struct Drpc__Response Drpc__Response; /* --- enums --- */ @@ -65,25 +63,24 @@ typedef enum _Drpc__Status { /* * Call describes a function call to be executed over the dRPC channel. */ -struct _Drpc__Call -{ - ProtobufCMessage base; - /* - * ID of the module to process the call. - */ - int32_t module; - /* - * ID of the method to be executed. - */ - int32_t method; - /* - * Sequence number for matching a response to this call. - */ - int64_t sequence; - /* - * Input payload to be used by the method. - */ - ProtobufCBinaryData body; +struct Drpc__Call { + ProtobufCMessage base; + /* + * ID of the module to process the call. + */ + int32_t module; + /* + * ID of the method to be executed. + */ + int32_t method; + /* + * Sequence number for matching a response to this call. + */ + int64_t sequence; + /* + * Input payload to be used by the method. + */ + ProtobufCBinaryData body; }; #define DRPC__CALL__INIT \ { PROTOBUF_C_MESSAGE_INIT (&drpc__call__descriptor) \ @@ -93,21 +90,21 @@ struct _Drpc__Call /* * Response describes the result of a dRPC call. */ -struct _Drpc__Response -{ - ProtobufCMessage base; - /* - * Sequence number of the Call that triggered this response. - */ - int64_t sequence; - /* - * High-level status of the RPC. If SUCCESS, method-specific status may be included in the body. - */ - Drpc__Status status; - /* - * Output payload produced by the method. - */ - ProtobufCBinaryData body; +struct Drpc__Response { + ProtobufCMessage base; + /* + * Sequence number of the Call that triggered this response. + */ + int64_t sequence; + /* + * High-level status of the RPC. If SUCCESS, method-specific status may be included in the + * body. + */ + Drpc__Status status; + /* + * Output payload produced by the method. + */ + ProtobufCBinaryData body; }; #define DRPC__RESPONSE__INIT \ { PROTOBUF_C_MESSAGE_INIT (&drpc__response__descriptor) \ diff --git a/src/include/daos/drpc_modules.h b/src/include/daos/drpc_modules.h index a8821d9f079..67e1a50c3b9 100644 --- a/src/include/daos/drpc_modules.h +++ b/src/include/daos/drpc_modules.h @@ -74,13 +74,14 @@ enum drpc_mgmt_method { }; enum drpc_srv_method { - DRPC_METHOD_SRV_NOTIFY_READY = 301, - DRPC_METHOD_SRV_BIO_ERR = 302, - DRPC_METHOD_SRV_GET_POOL_SVC = 303, - DRPC_METHOD_SRV_CLUSTER_EVENT = 304, - DRPC_METHOD_SRV_POOL_FIND_BYLABEL = 305, + DRPC_METHOD_SRV_NOTIFY_READY = 301, + DRPC_METHOD_SRV_BIO_ERR = 302, + DRPC_METHOD_SRV_GET_POOL_SVC = 303, + DRPC_METHOD_SRV_CLUSTER_EVENT = 304, + DRPC_METHOD_SRV_POOL_FIND_BYLABEL = 305, + DRPC_METHOD_SRV_LIST_POOLS = 306, - NUM_DRPC_SRV_METHODS /* Must be last */ + NUM_DRPC_SRV_METHODS /* Must be last */ }; enum drpc_sec_method { diff --git a/src/include/daos/mgmt.h b/src/include/daos/mgmt.h index 8572b18b8b2..5020fe84af7 100644 --- a/src/include/daos/mgmt.h +++ b/src/include/daos/mgmt.h @@ -64,6 +64,8 @@ int dc_mgmt_get_pool_svc_ranks(struct dc_mgmt_sys *sys, const uuid_t puuid, d_rank_list_t **svcranksp); int dc_mgmt_pool_find(struct dc_mgmt_sys *sys, const char *label, uuid_t puuid, d_rank_list_t **svcranksp); +int + dc_mgmt_pool_list(tse_task_t *task); int dc_mgmt_notify_pool_connect(struct dc_pool *pool); int dc_mgmt_notify_pool_disconnect(struct dc_pool *pool); int dc_mgmt_notify_exit(void); diff --git a/src/include/daos_mgmt.h b/src/include/daos_mgmt.h index 5a9e0e49274..291bda0b820 100644 --- a/src/include/daos_mgmt.h +++ b/src/include/daos_mgmt.h @@ -53,6 +53,8 @@ typedef struct { * daos_pool_info_t mgpi_info; */ uuid_t mgpi_uuid; + /** Pool label */ + d_string_t mgpi_label; /** List of current pool service replica ranks */ d_rank_list_t *mgpi_svc; /** Current pool service leader */ @@ -77,6 +79,10 @@ typedef struct { int daos_pool_stop_svc(daos_handle_t poh, daos_event_t *ev); +int +daos_mgmt_list_pools(const char *group, daos_size_t *npools, daos_mgmt_pool_info_t *pools, + daos_event_t *ev); + /** * The operation code for DAOS client to set different parameters globally * on all servers. diff --git a/src/include/daos_srv/daos_engine.h b/src/include/daos_srv/daos_engine.h index a7588e36e3f..a5407084ab8 100644 --- a/src/include/daos_srv/daos_engine.h +++ b/src/include/daos_srv/daos_engine.h @@ -22,6 +22,7 @@ #include #include #include +#include #include #include #include @@ -711,6 +712,8 @@ ds_notify_bio_error(int media_err_type, int tgt_id); int ds_get_pool_svc_ranks(uuid_t pool_uuid, d_rank_list_t **svc_ranks); int ds_pool_find_bylabel(d_const_string_t label, uuid_t pool_uuid, d_rank_list_t **svc_ranks); +int +ds_get_pool_list(uint64_t *npools, daos_mgmt_pool_info_t *pools); /** Flags for dss_drpc_call */ enum dss_drpc_call_flag { diff --git a/src/include/daos_srv/pool.h b/src/include/daos_srv/pool.h index 576e547ef10..44855afc9ca 100644 --- a/src/include/daos_srv/pool.h +++ b/src/include/daos_srv/pool.h @@ -243,6 +243,8 @@ int ds_pool_svc_upgrade(uuid_t pool_uuid, d_rank_list_t *ranks); int ds_pool_failed_add(uuid_t uuid, int rc); void ds_pool_failed_remove(uuid_t uuid); int ds_pool_failed_lookup(uuid_t uuid); +int +ds_pool_check_access(uuid_t pool_uuid, d_rank_list_t *ps_ranks, unsigned int flags, d_iov_t *cred); /* * Called by dmg on the pool service leader to list all pool handles of a pool. diff --git a/src/include/daos_task.h b/src/include/daos_task.h index e6e00af1f7a..26f3126110f 100644 --- a/src/include/daos_task.h +++ b/src/include/daos_task.h @@ -27,12 +27,13 @@ extern "C" { /** DAOS operation codes for task creation */ typedef enum { - DAOS_OPC_INVALID = -1, + DAOS_OPC_INVALID = -1, /** Management APIs */ /* Starting at 0 will break Application Binary Interface backward * compatibility */ - DAOS_OPC_SET_PARAMS = 3, + DAOS_OPC_MGMT_LIST_POOLS = 2, + DAOS_OPC_SET_PARAMS, DAOS_OPC_MGMT_GET_BS_STATE, /** Pool APIs */ @@ -186,6 +187,16 @@ struct daos_obj_register_class_t { struct daos_oclass_attr *cattr; }; +/** pool management pool list args */ +typedef struct { + /** Process set name of the DAOS servers managing the pool */ + const char *grp; + /** Array of pool mgmt information structures. */ + daos_mgmt_pool_info_t *pools; + /** length of array */ + daos_size_t *npools; +} daos_mgmt_pool_list_t; + /** pool query args */ typedef struct { /** Pool open handle. */ diff --git a/src/mgmt/cli_mgmt.c b/src/mgmt/cli_mgmt.c index bc5da348369..354b0ef794b 100644 --- a/src/mgmt/cli_mgmt.c +++ b/src/mgmt/cli_mgmt.c @@ -1,5 +1,5 @@ /* - * (C) Copyright 2016-2023 Intel Corporation. + * (C) Copyright 2016-2024 Intel Corporation. * * SPDX-License-Identifier: BSD-2-Clause-Patent */ @@ -20,6 +20,7 @@ #include #include #include +#include #include "svc.pb-c.h" #include "rpc.h" #include @@ -1282,6 +1283,160 @@ dc_mgmt_tm_register(const char *sys, const char *jobid, key_t shm_key, uid_t *ow return rc; } +int +dc_mgmt_pool_list(tse_task_t *task) +{ + daos_mgmt_pool_list_t *args; + d_rank_list_t *ms_ranks; + struct rsvc_client ms_client; + crt_endpoint_t ep; + crt_rpc_t *rpc = NULL; + crt_opcode_t opc; + struct mgmt_pool_list_in *in = NULL; + struct mgmt_pool_list_out *out = NULL; + struct dc_mgmt_sys *sys; + int rc, pidx; + + args = dc_task_get_args(task); + if (args->npools == NULL) { + D_ERROR("npools argument must not be NULL"); + D_GOTO(out, rc = -DER_INVAL); + } + + rc = dc_mgmt_sys_attach(args->grp, &sys); + if (rc != 0) { + DL_ERROR(rc, "cannot attach to DAOS system: %s", args->grp); + D_GOTO(out, rc); + } + + ms_ranks = sys->sy_info.ms_ranks; + D_ASSERT(ms_ranks->rl_nr > 0); + + rc = rsvc_client_init(&ms_client, ms_ranks); + if (rc != 0) { + DL_ERROR(rc, "failed to init ms client"); + D_GOTO(out_grp, rc); + } + + ep.ep_grp = sys->sy_group; + ep.ep_tag = daos_rpc_tag(DAOS_REQ_MGMT, 0); + opc = DAOS_RPC_OPCODE(MGMT_POOL_LIST, DAOS_MGMT_MODULE, DAOS_MGMT_VERSION); + +rechoose: + rc = rsvc_client_choose(&ms_client, &ep); + if (rc != 0) { + DL_ERROR(rc, "failed to choose MS rank"); + D_GOTO(out_client, rc); + } + + rc = crt_req_create(daos_task2ctx(task), &ep, opc, &rpc); + if (rc != 0) { + DL_ERROR(rc, "crt_req_create(MGMT_POOL_LIST) failed"); + D_GOTO(out_client, rc); + } + + in = crt_req_get(rpc); + in->pli_grp = (d_string_t)args->grp; + /* If provided pools is NULL, caller needs the number of pools + * to be returned in npools. Set npools=0 in the request in this case + * (caller value may be uninitialized). + */ + if (args->pools == NULL) + in->pli_npools = 0; + else + in->pli_npools = *args->npools; + + /* Now fill in the client credential for the pool access checks. */ + rc = dc_sec_request_creds(&in->pli_cred); + if (rc != 0) { + DL_ERROR(rc, "failed to obtain security credential"); + D_GOTO(out_put_req, rc); + } + + D_DEBUG(DB_MGMT, "req_npools=" DF_U64 " (pools=%p, *npools=" DF_U64 "\n", in->pli_npools, + args->pools, *args->npools); + + crt_req_addref(rpc); + rc = daos_rpc_send_wait(rpc); + if (rc != 0) { + DL_ERROR(rc, "rpc send failed"); + crt_req_decref(rpc); + goto rechoose; + } + + out = crt_reply_get(rpc); + D_ASSERT(out != NULL); + + rc = rsvc_client_complete_rpc(&ms_client, &ep, rc, out->plo_op.mo_rc, &out->plo_op.mo_hint); + if (rc == RSVC_CLIENT_RECHOOSE) { + crt_req_decref(rpc); + goto rechoose; + } + + rc = out->plo_op.mo_rc; + if (rc != 0) { + DL_ERROR(rc, "failed to list pools"); + D_GOTO(out_put_req, rc); + } + + *args->npools = out->plo_npools; + + /* copy RPC response pools info to client buffer, if provided */ + if (args->pools) { + /* Response ca_count expected <= client-specified npools */ + for (pidx = 0; pidx < out->plo_pools.ca_count; pidx++) { + struct mgmt_pool_list_pool *rpc_pool = &out->plo_pools.ca_arrays[pidx]; + daos_mgmt_pool_info_t *cli_pool = &args->pools[pidx]; + + uuid_copy(cli_pool->mgpi_uuid, rpc_pool->plp_uuid); + + D_STRNDUP(cli_pool->mgpi_label, rpc_pool->plp_label, + DAOS_PROP_LABEL_MAX_LEN); + if (cli_pool->mgpi_label == NULL) { + D_ERROR("copy RPC reply label failed\n"); + D_GOTO(out_free_args_pools, rc = -DER_NOMEM); + } + + /* allocate rank list for caller (simplifies API) */ + rc = d_rank_list_dup(&cli_pool->mgpi_svc, rpc_pool->plp_svc_list); + if (rc != 0) { + D_ERROR("copy RPC reply svc list failed\n"); + D_GOTO(out_free_args_pools, rc = -DER_NOMEM); + } + } + } + +out_free_args_pools: + if (args->pools && (rc != 0)) { + for (pidx = 0; pidx < out->plo_pools.ca_count; pidx++) { + daos_mgmt_pool_info_t *pool = &args->pools[pidx]; + + if (pool->mgpi_label) + D_FREE(pool->mgpi_label); + if (pool->mgpi_svc) + d_rank_list_free(pool->mgpi_svc); + } + } +out_put_req: + if (rc != 0) + DL_ERROR(rc, "failed to list pools"); + + crt_req_decref(rpc); + + if (in != NULL) { + /* Ensure credential memory is wiped clean */ + explicit_bzero(in->pli_cred.iov_buf, in->pli_cred.iov_buf_len); + daos_iov_free(&in->pli_cred); + } +out_client: + rsvc_client_fini(&ms_client); +out_grp: + dc_mgmt_sys_detach(sys); +out: + tse_task_complete(task, rc); + return rc; +} + /** * Initialize management interface */ diff --git a/src/mgmt/rpc.c b/src/mgmt/rpc.c index 59d4cb0d646..5a5a9b3873a 100644 --- a/src/mgmt/rpc.c +++ b/src/mgmt/rpc.c @@ -9,9 +9,11 @@ #define D_LOGFAC DD_FAC(mgmt) #include +#include #include "rpc.h" CRT_GEN_PROC_FUNC(server_entry, DAOS_SEQ_SERVER_ENTRY); +CRT_GEN_PROC_FUNC(mgmt_pool_list_pool, DAOS_SEQ_MGMT_POOL_LIST_POOL); static int crt_proc_struct_server_entry(crt_proc_t proc, crt_proc_op_t proc_op, @@ -20,6 +22,44 @@ crt_proc_struct_server_entry(crt_proc_t proc, crt_proc_op_t proc_op, return crt_proc_server_entry(proc, data); } +static int +crt_proc_struct_mgmt_pool_list_pool(crt_proc_t proc, crt_proc_op_t proc_op, + struct mgmt_pool_list_pool *data) +{ + return crt_proc_mgmt_pool_list_pool(proc, data); +} + +/* FIXME: dupe of pool/rpc.c:36 */ +static int +crt_proc_struct_rsvc_hint(crt_proc_t proc, crt_proc_op_t proc_op, + struct rsvc_hint *hint) +{ + int rc; + + rc = crt_proc_uint32_t(proc, proc_op, &hint->sh_flags); + if (rc != 0) + return -DER_HG; + + rc = crt_proc_uint32_t(proc, proc_op, &hint->sh_rank); + if (rc != 0) + return -DER_HG; + + rc = crt_proc_uint64_t(proc, proc_op, &hint->sh_term); + if (rc != 0) + return -DER_HG; + + return 0; +} + +CRT_GEN_PROC_FUNC(mgmt_op_out, DAOS_OSEQ_MGMT_OP); + +static int +crt_proc_struct_mgmt_op_out(crt_proc_t proc, crt_proc_op_t proc_op, + struct mgmt_op_out *data) +{ + return crt_proc_mgmt_op_out(proc, data); +} + CRT_RPC_DEFINE(mgmt_svc_rip, DAOS_ISEQ_MGMT_SVR_RIP, DAOS_OSEQ_MGMT_SVR_RIP) CRT_RPC_DEFINE(mgmt_params_set, DAOS_ISEQ_MGMT_PARAMS_SET, DAOS_OSEQ_MGMT_PARAMS_SET) @@ -29,6 +69,7 @@ CRT_RPC_DEFINE(mgmt_pool_get_svcranks, DAOS_ISEQ_MGMT_POOL_GET_SVCRANKS, DAOS_OSEQ_MGMT_POOL_GET_SVCRANKS) CRT_RPC_DEFINE(mgmt_pool_find, DAOS_ISEQ_MGMT_POOL_FIND, DAOS_OSEQ_MGMT_POOL_FIND) +CRT_RPC_DEFINE(mgmt_pool_list, DAOS_ISEQ_MGMT_POOL_LIST, DAOS_OSEQ_MGMT_POOL_LIST) CRT_RPC_DEFINE(mgmt_mark, DAOS_ISEQ_MGMT_MARK, DAOS_OSEQ_MGMT_MARK) diff --git a/src/mgmt/rpc.h b/src/mgmt/rpc.h index f10840cc6c9..0d9824d84ba 100644 --- a/src/mgmt/rpc.h +++ b/src/mgmt/rpc.h @@ -38,6 +38,9 @@ X(MGMT_POOL_FIND, \ 0, &CQF_mgmt_pool_find, \ ds_mgmt_pool_find_hdlr, NULL), \ + X(MGMT_POOL_LIST, \ + 0, &CQF_mgmt_pool_list, \ + ds_mgmt_pool_list_hdlr, NULL), \ X(MGMT_MARK, \ 0, &CQF_mgmt_mark, \ ds_mgmt_mark_hdlr, NULL), \ @@ -87,6 +90,12 @@ enum mgmt_profile_op { extern struct crt_proto_format mgmt_proto_fmt; +#define DAOS_OSEQ_MGMT_OP /* output fields */ \ + ((int32_t) (mo_rc) CRT_VAR) \ + ((struct rsvc_hint) (mo_hint) CRT_VAR) + +CRT_GEN_STRUCT(mgmt_op_out, DAOS_OSEQ_MGMT_OP); + #define DAOS_ISEQ_MGMT_SVR_RIP /* input fields */ \ ((uint32_t) (rip_flags) CRT_VAR) @@ -143,6 +152,26 @@ CRT_RPC_DECLARE(mgmt_pool_find, DAOS_ISEQ_MGMT_POOL_FIND, #define MGMT_POOL_FIND_DUMMY_LABEL "NO LABEL, FINDING BY UUID" +#define DAOS_SEQ_MGMT_POOL_LIST_POOL /* listed pool */ \ + ((uuid_t) (plp_uuid) CRT_VAR) \ + ((d_string_t) (plp_label) CRT_VAR) \ + ((d_rank_t) (plp_svc_ldr) CRT_VAR) \ + ((d_rank_list_t) (plp_svc_list) CRT_PTR) + +CRT_GEN_STRUCT(mgmt_pool_list_pool, DAOS_SEQ_MGMT_POOL_LIST_POOL); + +#define DAOS_ISEQ_MGMT_POOL_LIST /* input fields */ \ + ((d_string_t) (pli_grp) CRT_VAR) \ + ((d_iov_t) (pli_cred) CRT_VAR) \ + ((uint64_t) (pli_npools) CRT_VAR) + +#define DAOS_OSEQ_MGMT_POOL_LIST /* output fields */ \ + ((struct mgmt_op_out) (plo_op) CRT_VAR) \ + ((struct mgmt_pool_list_pool) (plo_pools) CRT_ARRAY) \ + ((uint64_t) (plo_npools) CRT_VAR) + +CRT_RPC_DECLARE(mgmt_pool_list, DAOS_ISEQ_MGMT_POOL_LIST, DAOS_OSEQ_MGMT_POOL_LIST) + #define DAOS_ISEQ_MGMT_TGT_CREATE /* input fields */ \ ((uuid_t) (tc_pool_uuid) CRT_VAR) \ ((d_string_t) (tc_tgt_dev) CRT_VAR) \ diff --git a/src/mgmt/srv.c b/src/mgmt/srv.c index ed00b2c0de5..f0c52e3e4d7 100644 --- a/src/mgmt/srv.c +++ b/src/mgmt/srv.c @@ -1,5 +1,5 @@ /** - * (C) Copyright 2016-2022 Intel Corporation. + * (C) Copyright 2016-2024 Intel Corporation. * * SPDX-License-Identifier: BSD-2-Clause-Patent */ @@ -19,6 +19,7 @@ #include #include #include +#include #include #include @@ -412,6 +413,90 @@ void ds_mgmt_pool_find_hdlr(crt_rpc_t *rpc) d_rank_list_free(out->pfo_ranks); } +void +ds_mgmt_pool_list_hdlr(crt_rpc_t *rpc) +{ + struct mgmt_pool_list_in *in; + struct mgmt_pool_list_out *out; + size_t n_mgmt = 0, n_rpc = 0; + daos_mgmt_pool_info_t *mgmt_pools = NULL; + struct mgmt_pool_list_pool *rpc_pools = NULL; + int i, rc, chk_rc; + + in = crt_req_get(rpc); + D_ASSERT(in != NULL); + + out = crt_reply_get(rpc); + D_ASSERT(out != NULL); + + if (in->pli_npools > 0) { + D_ALLOC_ARRAY(mgmt_pools, in->pli_npools); + if (mgmt_pools == NULL) { + D_ERROR("failed to alloc mgmt pools\n"); + D_GOTO(send_resp, rc = -DER_NOMEM); + } + } + + n_mgmt = in->pli_npools; + rc = ds_get_pool_list(&n_mgmt, mgmt_pools); + if (rc != 0) { + DL_ERROR(rc, "ds_get_pool_list() failed"); + D_GOTO(send_resp, rc); + } + + /* caller just needs the number of pools */ + if (in->pli_npools == 0) { + out->plo_npools = n_mgmt; + D_GOTO(send_resp, rc); + } + + D_ALLOC_ARRAY(rpc_pools, n_mgmt); + if (rpc_pools == NULL) { + D_ERROR("failed to alloc response pools\n"); + D_GOTO(send_resp, rc = -DER_NOMEM); + } + + for (i = 0; i < n_mgmt; i++) { + daos_mgmt_pool_info_t *mgmt_pool = &mgmt_pools[i]; + struct mgmt_pool_list_pool *rpc_pool = &rpc_pools[n_rpc]; + chk_rc = ds_pool_check_access(mgmt_pool->mgpi_uuid, mgmt_pool->mgpi_svc, DAOS_PC_RO, + &in->pli_cred); + if (chk_rc != 0) { + if (chk_rc == -DER_NO_PERM) + continue; + else + D_GOTO(send_resp, rc = chk_rc); + } + uuid_copy(rpc_pool->plp_uuid, mgmt_pool->mgpi_uuid); + rpc_pool->plp_label = mgmt_pool->mgpi_label; + rpc_pool->plp_svc_list = mgmt_pool->mgpi_svc; + n_rpc++; + } + out->plo_pools.ca_arrays = rpc_pools; + out->plo_pools.ca_count = n_rpc; + out->plo_npools = n_rpc; + +send_resp: + out->plo_op.mo_rc = rc; + + rc = crt_reply_send(rpc); + if (rc != 0) + DL_ERROR(rc, "crt_reply_send() failed"); + + if (n_mgmt > 0 && mgmt_pools != NULL) { + for (i = 0; i < n_mgmt; i++) { + daos_mgmt_pool_info_t *mgmt_pool = &mgmt_pools[i]; + if (mgmt_pool->mgpi_label) + D_FREE(mgmt_pool->mgpi_label); + if (mgmt_pool->mgpi_svc) + d_rank_list_free(mgmt_pool->mgpi_svc); + } + D_FREE(mgmt_pools); + } + if (rpc_pools) + D_FREE(rpc_pools); +} + static int ds_mgmt_init() { diff --git a/src/mgmt/srv_internal.h b/src/mgmt/srv_internal.h index b22341fb35b..3692677f565 100644 --- a/src/mgmt/srv_internal.h +++ b/src/mgmt/srv_internal.h @@ -36,6 +36,8 @@ void ds_mgmt_tgt_params_set_hdlr(crt_rpc_t *rpc); void ds_mgmt_profile_hdlr(crt_rpc_t *rpc); void ds_mgmt_pool_get_svcranks_hdlr(crt_rpc_t *rpc); void ds_mgmt_pool_find_hdlr(crt_rpc_t *rpc); +void + ds_mgmt_pool_list_hdlr(crt_rpc_t *rpc); void ds_mgmt_mark_hdlr(crt_rpc_t *rpc); void dss_bind_to_xstream_cpuset(int tgt_id); diff --git a/src/pool/rpc.c b/src/pool/rpc.c index 08117f41942..9655c4e0597 100644 --- a/src/pool/rpc.c +++ b/src/pool/rpc.c @@ -74,6 +74,8 @@ CRT_RPC_DEFINE(pool_connect_v4, DAOS_ISEQ_POOL_CONNECT_V4, DAOS_OSEQ_POOL_CONNEC CRT_RPC_DEFINE(pool_connect_v5, DAOS_ISEQ_POOL_CONNECT_V5, DAOS_OSEQ_POOL_CONNECT) CRT_RPC_DEFINE(pool_disconnect, DAOS_ISEQ_POOL_DISCONNECT, DAOS_OSEQ_POOL_DISCONNECT) +CRT_RPC_DEFINE(pool_check_access, DAOS_ISEQ_POOL_CHECK_ACCESS, + DAOS_OSEQ_POOL_CHECK_ACCESS) CRT_RPC_DEFINE(pool_query_v4, DAOS_ISEQ_POOL_QUERY, DAOS_OSEQ_POOL_QUERY_V4) CRT_RPC_DEFINE(pool_query_v5, DAOS_ISEQ_POOL_QUERY, DAOS_OSEQ_POOL_QUERY_V5) CRT_RPC_DEFINE(pool_attr_list, DAOS_ISEQ_POOL_ATTR_LIST, diff --git a/src/pool/rpc.h b/src/pool/rpc.h index c462c84ce88..3cf14858594 100644 --- a/src/pool/rpc.h +++ b/src/pool/rpc.h @@ -104,6 +104,9 @@ ds_pool_filter_cont_handler, NULL) #define POOL_PROTO_SRV_RPC_LIST \ + X(POOL_CHECK_ACCESS, \ + 0, &CQF_pool_check_access, \ + ds_pool_check_access_handler, NULL), \ X(POOL_TGT_DISCONNECT, \ 0, &CQF_pool_tgt_disconnect, \ ds_pool_tgt_disconnect_handler, \ @@ -262,6 +265,17 @@ CRT_RPC_DECLARE(pool_connect_v5, DAOS_ISEQ_POOL_CONNECT_V5, DAOS_OSEQ_POOL_CONNE CRT_RPC_DECLARE(pool_disconnect, DAOS_ISEQ_POOL_DISCONNECT, DAOS_OSEQ_POOL_DISCONNECT) +#define DAOS_ISEQ_POOL_CHECK_ACCESS /* input fields */ \ + ((struct pool_op_in) (pcai_op) CRT_VAR) \ + ((d_iov_t) (pcai_cred) CRT_VAR) \ + ((uint64_t) (pcai_flags) CRT_VAR) + +#define DAOS_OSEQ_POOL_CHECK_ACCESS /* output fields */ \ + ((struct pool_op_out) (pcao_op) CRT_VAR) \ + ((int32_t) (pcao_rc) CRT_VAR) + +CRT_RPC_DECLARE(pool_check_access, DAOS_ISEQ_POOL_CHECK_ACCESS, DAOS_OSEQ_POOL_CHECK_ACCESS) + #define DAOS_ISEQ_POOL_QUERY /* input fields */ \ ((struct pool_op_in) (pqi_op) CRT_VAR) \ ((crt_bulk_t) (pqi_map_bulk) CRT_VAR) \ diff --git a/src/pool/srv_internal.h b/src/pool/srv_internal.h index 0ffb55cb4c7..a0780cb06b2 100644 --- a/src/pool/srv_internal.h +++ b/src/pool/srv_internal.h @@ -162,6 +162,8 @@ void ds_pool_connect_handler_v5(crt_rpc_t *rpc); void ds_pool_disconnect_handler(crt_rpc_t *rpc); void ds_pool_query_handler_v4(crt_rpc_t *rpc); void ds_pool_query_handler_v5(crt_rpc_t *rpc); +void + ds_pool_check_access_handler(crt_rpc_t *rpc); void ds_pool_prop_get_handler(crt_rpc_t *rpc); void ds_pool_prop_set_handler(crt_rpc_t *rpc); void ds_pool_acl_update_handler(crt_rpc_t *rpc); diff --git a/src/pool/srv_pool.c b/src/pool/srv_pool.c index 20aca49f867..13f4370471f 100644 --- a/src/pool/srv_pool.c +++ b/src/pool/srv_pool.c @@ -2907,6 +2907,174 @@ bulk_cb(const struct crt_bulk_cb_info *cb_info) return 0; } +int +ds_pool_check_access(uuid_t pool_uuid, d_rank_list_t *ps_ranks, unsigned int flags, d_iov_t *cred) +{ + struct rsvc_client client; + crt_endpoint_t ep; + struct dss_module_info *info = dss_get_module_info(); + crt_rpc_t *rpc; + struct pool_check_access_in *in; + struct pool_check_access_out *out; + int rc; + + D_DEBUG(DB_MD, DF_UUID ": checking pool access\n", DP_UUID(pool_uuid)); + + rc = rsvc_client_init(&client, ps_ranks); + if (rc != 0) + goto out; + +rechoose: + ep.ep_grp = NULL; /* primary group */ + rc = rsvc_client_choose(&client, &ep); + if (rc != 0) { + D_ERROR(DF_UUID ": cannot find pool service: " DF_RC "\n", DP_UUID(pool_uuid), + DP_RC(rc)); + goto out_client; + } + + rc = pool_req_create(info->dmi_ctx, &ep, POOL_CHECK_ACCESS, &rpc); + if (rc != 0) { + D_ERROR(DF_UUID ": failed to create pool check access rpc, " DF_RC "\n", + DP_UUID(pool_uuid), DP_RC(rc)); + goto out_client; + } + + in = crt_req_get(rpc); + in->pcai_flags = flags; + uuid_copy(in->pcai_op.pi_uuid, pool_uuid); + uuid_clear(in->pcai_op.pi_hdl); + + rc = daos_iov_copy(&in->pcai_cred, cred); + if (rc != 0) { + D_ERROR(DF_UUID ": failed to copy credential IOV, " DF_RC "\n", DP_UUID(pool_uuid), + DP_RC(rc)); + goto out_client; + } + + rc = dss_rpc_send(rpc); + out = crt_reply_get(rpc); + D_ASSERT(out != NULL); + + rc = pool_rsvc_client_complete_rpc(&client, &ep, rc, &out->pcao_op); + if (rc == RSVC_CLIENT_RECHOOSE) { + crt_req_decref(rpc); + dss_sleep(RECHOOSE_SLEEP_MS); + goto rechoose; + } + + rc = out->pcao_op.po_rc; + if (rc != 0 && rc != -DER_NO_PERM) { + D_ERROR(DF_UUID ": failed to check pool access, " DF_RC "\n", DP_UUID(pool_uuid), + DP_RC(rc)); + } + + crt_req_decref(rpc); +out_client: + rsvc_client_fini(&client); +out: + return rc; +} + +static int +pool_check_cred_access(d_iov_t *cred, int flags, daos_prop_t *prop) +{ + struct daos_prop_entry *acl_entry; + struct d_ownership owner; + struct daos_prop_entry *owner_entry; + struct daos_prop_entry *owner_grp_entry; + uint64_t sec_capas = 0; + int rc; + + D_ASSERT(prop != NULL); + + acl_entry = daos_prop_entry_get(prop, DAOS_PROP_PO_ACL); + D_ASSERT(acl_entry != NULL); + D_ASSERT(acl_entry->dpe_val_ptr != NULL); + + owner_entry = daos_prop_entry_get(prop, DAOS_PROP_PO_OWNER); + D_ASSERT(owner_entry != NULL); + D_ASSERT(owner_entry->dpe_str != NULL); + + owner_grp_entry = daos_prop_entry_get(prop, DAOS_PROP_PO_OWNER_GROUP); + D_ASSERT(owner_grp_entry != NULL); + D_ASSERT(owner_grp_entry->dpe_str != NULL); + + owner.user = owner_entry->dpe_str; + owner.group = owner_grp_entry->dpe_str; + + /* + * Security capabilities determine the access control policy on this + * pool handle. + */ + rc = ds_sec_pool_get_capabilities(flags, cred, &owner, acl_entry->dpe_val_ptr, &sec_capas); + if (rc != 0) { + DL_ERROR(rc, "failed to read sec capabilities"); + return rc; + } + + if (!ds_sec_pool_can_connect(sec_capas)) { + return -DER_NO_PERM; + } + + return 0; +} + +void +ds_pool_check_access_handler(crt_rpc_t *rpc) +{ + struct pool_check_access_in *in = crt_req_get(rpc); + struct pool_check_access_out *out = crt_reply_get(rpc); + struct pool_svc *svc; + struct rdb_tx tx; + daos_prop_t *prop = NULL; + uint64_t prop_bits; + int rc; + + D_DEBUG(DB_MD, DF_UUID ": processing rpc: %p hdl=" DF_UUID "\n", + DP_UUID(in->pcai_op.pi_uuid), rpc, DP_UUID(in->pcai_op.pi_hdl)); + + rc = pool_svc_lookup_leader(in->pcai_op.pi_uuid, &svc, &out->pcao_op.po_hint); + if (rc != 0) + D_GOTO(out, rc); + + rc = rdb_tx_begin(svc->ps_rsvc.s_db, svc->ps_rsvc.s_term, &tx); + if (rc != 0) + D_GOTO(out_svc, rc); + + ABT_rwlock_rdlock(svc->ps_lock); + + prop_bits = DAOS_PO_QUERY_PROP_ALL; + rc = pool_prop_read(&tx, svc, prop_bits, &prop); + if (rc != 0) { + D_ERROR(DF_UUID ": cannot get access data for pool, " + "rc=" DF_RC "\n", + DP_UUID(in->pcai_op.pi_uuid), DP_RC(rc)); + D_GOTO(out_lock, rc); + } + D_ASSERT(prop != NULL); + + rc = pool_check_cred_access(&in->pcai_cred, in->pcai_flags, prop); + if (rc != 0) { + D_INFO(DF_UUID ": rejecting access based on credential, " + "rc=" DF_RC "\n", + DP_UUID(in->pcai_op.pi_uuid), DP_RC(rc)); + } + + out->pcao_op.po_map_version = ds_pool_get_version(svc->ps_pool); +out_lock: + ABT_rwlock_unlock(svc->ps_lock); + rdb_tx_end(&tx); + if (prop) + daos_prop_free(prop); +out_svc: + ds_rsvc_set_hint(&svc->ps_rsvc, &out->pcao_op.po_hint); + pool_svc_put_leader(svc); +out: + out->pcao_op.po_rc = rc; + crt_reply_send(rpc); +} + /* Currently we only maintain compatibility between 2 versions */ #define NUM_POOL_VERSIONS 2 diff --git a/src/proto/srv/srv.proto b/src/proto/srv/srv.proto index c63a402501c..5cc4b9bc3b0 100644 --- a/src/proto/srv/srv.proto +++ b/src/proto/srv/srv.proto @@ -52,3 +52,19 @@ message PoolFindByLabelResp { string uuid = 2; // Pool UUID repeated uint32 svcreps = 3; // Pool service replica ranks } + +message ListPoolsReq +{ + bool includeAll = 1; // Include all pools in response, regardless of state +} + +message ListPoolsResp +{ + message Pool + { + string uuid = 1; // Pool UUID + string label = 2; // Pool label + repeated uint32 svcreps = 3; // Pool service ranks + } + repeated Pool pools = 1; // List of pools +} From 257c1b7976bbe142b6542378fdec15ab50c30f81 Mon Sep 17 00:00:00 2001 From: Michael MacDonald Date: Tue, 25 Jun 2024 22:20:09 +0000 Subject: [PATCH 02/10] Address review feedback, add pool query to list Signed-off-by: Michael MacDonald --- src/control/cmd/daos/pool.go | 148 +++++++++++++++++++++-------------- src/include/daos_mgmt.h | 14 +++- 2 files changed, 101 insertions(+), 61 deletions(-) diff --git a/src/control/cmd/daos/pool.go b/src/control/cmd/daos/pool.go index 87554972bca..6c48429d577 100644 --- a/src/control/cmd/daos/pool.go +++ b/src/control/cmd/daos/pool.go @@ -18,7 +18,6 @@ import ( "github.com/daos-stack/daos/src/control/cmd/daos/pretty" "github.com/daos-stack/daos/src/control/common" "github.com/daos-stack/daos/src/control/lib/daos" - "github.com/daos-stack/daos/src/control/lib/ranklist" "github.com/daos-stack/daos/src/control/lib/ui" "github.com/daos-stack/daos/src/control/logging" ) @@ -265,21 +264,43 @@ func convertPoolInfo(pinfo *C.daos_pool_info_t) (*daos.PoolInfo, error) { return poolInfo, nil } -func generateRankSet(ranklist *C.d_rank_list_t) string { - if ranklist.rl_nr == 0 { - return "" +func queryPool(poolHdl C.daos_handle_t, queryMask daos.PoolQueryMask) (*daos.PoolInfo, error) { + var rlPtr **C.d_rank_list_t = nil + var rl *C.d_rank_list_t = nil + + if queryMask.HasOption(daos.PoolQueryOptionEnabledEngines) || queryMask.HasOption(daos.PoolQueryOptionDisabledEngines) { + rlPtr = &rl + } + + cPoolInfo := C.daos_pool_info_t{ + pi_bits: C.uint64_t(queryMask), + } + + rc := C.daos_pool_query(poolHdl, rlPtr, &cPoolInfo, nil, nil) + defer C.d_rank_list_free(rl) + if err := daosError(rc); err != nil { + return nil, err + } + + poolInfo, err := convertPoolInfo(&cPoolInfo) + if err != nil { + return nil, err } - ranks := uintptr(unsafe.Pointer(ranklist.rl_ranks)) - const size = unsafe.Sizeof(uint32(0)) - rankset := "[" - for i := 0; i < int(ranklist.rl_nr); i++ { - if i > 0 { - rankset += "," + + if rlPtr != nil { + rs, err := rankSetFromC(rl) + if err != nil { + return nil, err + } + if queryMask.HasOption(daos.PoolQueryOptionEnabledEngines) { + poolInfo.EnabledRanks = rs + } + if queryMask.HasOption(daos.PoolQueryOptionDisabledEngines) { + poolInfo.DisabledRanks = rs } - rankset += fmt.Sprint(*(*uint32)(unsafe.Pointer(ranks + uintptr(i)*size))) } - rankset += "]" - return rankset + + return poolInfo, nil } func (cmd *poolQueryCmd) Execute(_ []string) error { @@ -297,42 +318,15 @@ func (cmd *poolQueryCmd) Execute(_ []string) error { queryMask.SetOptions(daos.PoolQueryOptionDisabledEngines) } - var rlPtr **C.d_rank_list_t = nil - var rl *C.d_rank_list_t = nil - - if cmd.ShowEnabledRanks || cmd.ShowDisabledRanks { - rlPtr = &rl - } - cleanup, err := cmd.resolveAndConnect(C.DAOS_PC_RO, nil) if err != nil { return err } defer cleanup() - cPoolInfo := C.daos_pool_info_t{ - pi_bits: C.uint64_t(queryMask), - } - - rc := C.daos_pool_query(cmd.cPoolHandle, rlPtr, &cPoolInfo, nil, nil) - defer C.d_rank_list_free(rl) - if err := daosError(rc); err != nil { - return errors.Wrapf(err, - "failed to query pool %s", cmd.poolUUID) - } - - poolInfo, err := convertPoolInfo(&cPoolInfo) + poolInfo, err := queryPool(cmd.cPoolHandle, queryMask) if err != nil { - return err - } - - if rlPtr != nil { - if cmd.ShowEnabledRanks { - poolInfo.EnabledRanks = ranklist.MustCreateRankSet(generateRankSet(rl)) - } - if cmd.ShowDisabledRanks { - poolInfo.DisabledRanks = ranklist.MustCreateRankSet(generateRankSet(rl)) - } + return errors.Wrapf(err, "failed to query pool %q", cmd.PoolID()) } if cmd.JSONOutputEnabled() { @@ -604,7 +598,7 @@ func (cmd *poolAutoTestCmd) Execute(_ []string) error { return nil } -func getPoolList(log logging.Logger, queryEnabled bool) ([]*daos.PoolInfo, error) { +func getPoolList(log logging.Logger, sysName string, queryEnabled bool) ([]*daos.PoolInfo, error) { var rc C.int bufSize := C.size_t(0) @@ -619,6 +613,12 @@ func getPoolList(log logging.Logger, queryEnabled bool) ([]*daos.PoolInfo, error return nil, nil } + var cSysName *C.char + if sysName != "" { + cSysName := C.CString(sysName) + defer freeString(cSysName) + } + // Now, we actually fetch the pools into the buffer that we've created. cPools := make([]C.daos_mgmt_pool_info_t, bufSize) rc = C.daos_mgmt_list_pools(nil, &bufSize, &cPools[0], nil) @@ -627,26 +627,53 @@ func getPoolList(log logging.Logger, queryEnabled bool) ([]*daos.PoolInfo, error } pools := make([]*daos.PoolInfo, 0, bufSize) for i := 0; i < int(bufSize); i++ { - poolUUID, err := uuidFromC(cPools[i].mgpi_uuid) - if err != nil { - return nil, err - } - svcRanks, err := rankSetFromC(cPools[i].mgpi_svc) + cPool := &cPools[i] + + svcRanks, err := rankSetFromC(cPool.mgpi_svc) if err != nil { return nil, err } + poolLabel := C.GoString(cPool.mgpi_label) - // Populate the basic info. - pool := &daos.PoolInfo{ - UUID: poolUUID, - Label: C.GoString(cPools[i].mgpi_label), - ServiceReplicas: svcRanks.Ranks(), - State: daos.PoolServiceStateReady, - } - + var pool *daos.PoolInfo if queryEnabled { - // TODO: rework query logic to be callable standalone. + var poolInfo C.daos_pool_info_t + var poolHandle C.daos_handle_t + if err := daosError(C.daos_pool_connect(cPool.mgpi_label, cSysName, C.DAOS_PC_RO, + &poolHandle, &poolInfo, nil)); err != nil { + log.Errorf("failed to connect to pool %q: %s", poolLabel, err) + continue + } + + var qErr error + pool, qErr = queryPool(poolHandle, daos.DefaultPoolQueryMask) + if qErr != nil { + log.Errorf("failed to query pool %q: %s", poolLabel, qErr) + } + pool.Label = poolLabel + pool.ServiceReplicas = svcRanks.Ranks() + + if err := daosError(C.daos_pool_disconnect(poolHandle, nil)); err != nil { + log.Errorf("failed to disconnect from pool %q: %s", poolLabel, err) + } + if qErr != nil { + continue + } + } else { + // Just populate the basic info. + poolUUID, err := uuidFromC(cPool.mgpi_uuid) + if err != nil { + return nil, err + } + + pool = &daos.PoolInfo{ + UUID: poolUUID, + Label: poolLabel, + ServiceReplicas: svcRanks.Ranks(), + State: daos.PoolServiceStateReady, + } } + pools = append(pools, pool) } @@ -656,12 +683,13 @@ func getPoolList(log logging.Logger, queryEnabled bool) ([]*daos.PoolInfo, error type poolListCmd struct { daosCmd - Verbose bool `short:"v" long:"verbose" description:"Add pool UUIDs and service replica lists to display"` - NoQuery bool `short:"n" long:"no-query" description:"Disable query of listed pools"` + SysName string `long:"sys-name" short:"G" description:"DAOS system name"` + Verbose bool `short:"v" long:"verbose" description:"Add pool UUIDs and service replica lists to display"` + NoQuery bool `short:"n" long:"no-query" description:"Disable query of listed pools"` } func (cmd *poolListCmd) Execute(_ []string) error { - pools, err := getPoolList(cmd.Logger, !cmd.NoQuery) + pools, err := getPoolList(cmd.Logger, cmd.SysName, !cmd.NoQuery) if err != nil { return err } diff --git a/src/include/daos_mgmt.h b/src/include/daos_mgmt.h index 291bda0b820..cffc89dce24 100644 --- a/src/include/daos_mgmt.h +++ b/src/include/daos_mgmt.h @@ -1,5 +1,5 @@ /** - * (C) Copyright 2016-2023 Intel Corporation. + * (C) Copyright 2016-2024 Intel Corporation. * * SPDX-License-Identifier: BSD-2-Clause-Patent */ @@ -79,6 +79,18 @@ typedef struct { int daos_pool_stop_svc(daos_handle_t poh, daos_event_t *ev); +/** + * List the pools to which the requesting user has access. + * + * \param[in] group Name of DAOS system managing the pool. + * \param[in,out] npools [in] Pool array size. [out] If pools is NULL, + * only the number of pools is retrieved. + * \param[out] pools Array of \a n pool info structs. If NULL, + * only the number of pools is retrieved. + * \param[in] ev Completion event, it is optional and can be + * NULL. The function will run in blocking mode if + * \a ev is NULL. + */ int daos_mgmt_list_pools(const char *group, daos_size_t *npools, daos_mgmt_pool_info_t *pools, daos_event_t *ev); From 26f2b99cbe4e2b36f83b5a7928557b29174d6120 Mon Sep 17 00:00:00 2001 From: Michael MacDonald Date: Tue, 25 Jun 2024 22:30:50 +0000 Subject: [PATCH 03/10] use pool UUID instead of label in query --- src/control/cmd/daos/pool.go | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/src/control/cmd/daos/pool.go b/src/control/cmd/daos/pool.go index 6c48429d577..42b69cab520 100644 --- a/src/control/cmd/daos/pool.go +++ b/src/control/cmd/daos/pool.go @@ -633,13 +633,21 @@ func getPoolList(log logging.Logger, sysName string, queryEnabled bool) ([]*daos if err != nil { return nil, err } + poolUUID, err := uuidFromC(cPool.mgpi_uuid) + if err != nil { + return nil, err + } + poolLabel := C.GoString(cPool.mgpi_label) var pool *daos.PoolInfo if queryEnabled { var poolInfo C.daos_pool_info_t var poolHandle C.daos_handle_t - if err := daosError(C.daos_pool_connect(cPool.mgpi_label, cSysName, C.DAOS_PC_RO, + cPoolUUID := C.CString(poolUUID.String()) + defer freeString(cPoolUUID) + + if err := daosError(C.daos_pool_connect(cPoolUUID, cSysName, C.DAOS_PC_RO, &poolHandle, &poolInfo, nil)); err != nil { log.Errorf("failed to connect to pool %q: %s", poolLabel, err) continue @@ -661,11 +669,6 @@ func getPoolList(log logging.Logger, sysName string, queryEnabled bool) ([]*daos } } else { // Just populate the basic info. - poolUUID, err := uuidFromC(cPool.mgpi_uuid) - if err != nil { - return nil, err - } - pool = &daos.PoolInfo{ UUID: poolUUID, Label: poolLabel, From d7200240d6a137d3e4b87ca01f5c21c72a45a41b Mon Sep 17 00:00:00 2001 From: Michael MacDonald Date: Wed, 26 Jun 2024 18:02:58 +0000 Subject: [PATCH 04/10] revert pool svc RPC additions --- src/pool/rpc.c | 2 -- src/pool/rpc.h | 14 -------------- src/pool/srv_internal.h | 2 -- 3 files changed, 18 deletions(-) diff --git a/src/pool/rpc.c b/src/pool/rpc.c index 9655c4e0597..08117f41942 100644 --- a/src/pool/rpc.c +++ b/src/pool/rpc.c @@ -74,8 +74,6 @@ CRT_RPC_DEFINE(pool_connect_v4, DAOS_ISEQ_POOL_CONNECT_V4, DAOS_OSEQ_POOL_CONNEC CRT_RPC_DEFINE(pool_connect_v5, DAOS_ISEQ_POOL_CONNECT_V5, DAOS_OSEQ_POOL_CONNECT) CRT_RPC_DEFINE(pool_disconnect, DAOS_ISEQ_POOL_DISCONNECT, DAOS_OSEQ_POOL_DISCONNECT) -CRT_RPC_DEFINE(pool_check_access, DAOS_ISEQ_POOL_CHECK_ACCESS, - DAOS_OSEQ_POOL_CHECK_ACCESS) CRT_RPC_DEFINE(pool_query_v4, DAOS_ISEQ_POOL_QUERY, DAOS_OSEQ_POOL_QUERY_V4) CRT_RPC_DEFINE(pool_query_v5, DAOS_ISEQ_POOL_QUERY, DAOS_OSEQ_POOL_QUERY_V5) CRT_RPC_DEFINE(pool_attr_list, DAOS_ISEQ_POOL_ATTR_LIST, diff --git a/src/pool/rpc.h b/src/pool/rpc.h index 3cf14858594..c462c84ce88 100644 --- a/src/pool/rpc.h +++ b/src/pool/rpc.h @@ -104,9 +104,6 @@ ds_pool_filter_cont_handler, NULL) #define POOL_PROTO_SRV_RPC_LIST \ - X(POOL_CHECK_ACCESS, \ - 0, &CQF_pool_check_access, \ - ds_pool_check_access_handler, NULL), \ X(POOL_TGT_DISCONNECT, \ 0, &CQF_pool_tgt_disconnect, \ ds_pool_tgt_disconnect_handler, \ @@ -265,17 +262,6 @@ CRT_RPC_DECLARE(pool_connect_v5, DAOS_ISEQ_POOL_CONNECT_V5, DAOS_OSEQ_POOL_CONNE CRT_RPC_DECLARE(pool_disconnect, DAOS_ISEQ_POOL_DISCONNECT, DAOS_OSEQ_POOL_DISCONNECT) -#define DAOS_ISEQ_POOL_CHECK_ACCESS /* input fields */ \ - ((struct pool_op_in) (pcai_op) CRT_VAR) \ - ((d_iov_t) (pcai_cred) CRT_VAR) \ - ((uint64_t) (pcai_flags) CRT_VAR) - -#define DAOS_OSEQ_POOL_CHECK_ACCESS /* output fields */ \ - ((struct pool_op_out) (pcao_op) CRT_VAR) \ - ((int32_t) (pcao_rc) CRT_VAR) - -CRT_RPC_DECLARE(pool_check_access, DAOS_ISEQ_POOL_CHECK_ACCESS, DAOS_OSEQ_POOL_CHECK_ACCESS) - #define DAOS_ISEQ_POOL_QUERY /* input fields */ \ ((struct pool_op_in) (pqi_op) CRT_VAR) \ ((crt_bulk_t) (pqi_map_bulk) CRT_VAR) \ diff --git a/src/pool/srv_internal.h b/src/pool/srv_internal.h index a0780cb06b2..0ffb55cb4c7 100644 --- a/src/pool/srv_internal.h +++ b/src/pool/srv_internal.h @@ -162,8 +162,6 @@ void ds_pool_connect_handler_v5(crt_rpc_t *rpc); void ds_pool_disconnect_handler(crt_rpc_t *rpc); void ds_pool_query_handler_v4(crt_rpc_t *rpc); void ds_pool_query_handler_v5(crt_rpc_t *rpc); -void - ds_pool_check_access_handler(crt_rpc_t *rpc); void ds_pool_prop_get_handler(crt_rpc_t *rpc); void ds_pool_prop_set_handler(crt_rpc_t *rpc); void ds_pool_acl_update_handler(crt_rpc_t *rpc); From b4a14deeb5005ee6c974815ea3af5833ed21b9c7 Mon Sep 17 00:00:00 2001 From: Michael MacDonald Date: Wed, 26 Jun 2024 18:05:37 +0000 Subject: [PATCH 05/10] Perform pool ACL evaluation in pool list handler Signed-off-by: Michael MacDonald --- src/mgmt/srv.c | 60 ++++++++++++++-- src/pool/srv_pool.c | 168 -------------------------------------------- 2 files changed, 54 insertions(+), 174 deletions(-) diff --git a/src/mgmt/srv.c b/src/mgmt/srv.c index 45245c397a3..cea08b359ac 100644 --- a/src/mgmt/srv.c +++ b/src/mgmt/srv.c @@ -20,6 +20,7 @@ #include #include #include +#include #include #include @@ -416,6 +417,53 @@ void ds_mgmt_pool_find_hdlr(crt_rpc_t *rpc) d_rank_list_free(out->pfo_ranks); } +static int +check_cred_pool_access(uuid_t pool_uuid, d_rank_list_t *svc_ranks, int flags, d_iov_t *cred) +{ + daos_prop_t *props = NULL; + struct daos_prop_entry *acl_entry = NULL; + struct daos_prop_entry *owner_entry = NULL; + struct daos_prop_entry *owner_grp_entry = NULL; + struct d_ownership owner; + uint64_t sec_capas; + int rc; + + rc = ds_mgmt_pool_get_acl(pool_uuid, svc_ranks, &props); + if (rc != 0) { + DL_ERROR(rc, DF_UUID ": failed to retrieve ACL props", DP_UUID(pool_uuid)); + return rc; + } + + acl_entry = daos_prop_entry_get(props, DAOS_PROP_PO_ACL); + D_ASSERT(acl_entry != NULL); + D_ASSERT(acl_entry->dpe_val_ptr != NULL); + + owner_entry = daos_prop_entry_get(props, DAOS_PROP_PO_OWNER); + D_ASSERT(owner_entry != NULL); + D_ASSERT(owner_entry->dpe_str != NULL); + + owner_grp_entry = daos_prop_entry_get(props, DAOS_PROP_PO_OWNER_GROUP); + D_ASSERT(owner_grp_entry != NULL); + D_ASSERT(owner_grp_entry->dpe_str != NULL); + + owner.user = owner_entry->dpe_str; + owner.group = owner_grp_entry->dpe_str; + + rc = ds_sec_pool_get_capabilities(flags, cred, &owner, acl_entry->dpe_val_ptr, &sec_capas); + if (rc != 0) { + DL_ERROR(rc, DF_UUID ": failed to read sec capabilities", DP_UUID(pool_uuid)); + D_GOTO(out_props, rc); + } + + if (!ds_sec_pool_can_connect(sec_capas)) { + D_GOTO(out_props, rc = -DER_NO_PERM); + } + +out_props: + daos_prop_free(props); + return rc; +} + void ds_mgmt_pool_list_hdlr(crt_rpc_t *rpc) { @@ -462,13 +510,13 @@ ds_mgmt_pool_list_hdlr(crt_rpc_t *rpc) for (i = 0; i < n_mgmt; i++) { daos_mgmt_pool_info_t *mgmt_pool = &mgmt_pools[i]; struct mgmt_pool_list_pool *rpc_pool = &rpc_pools[n_rpc]; - chk_rc = ds_pool_check_access(mgmt_pool->mgpi_uuid, mgmt_pool->mgpi_svc, DAOS_PC_RO, - &in->pli_cred); + chk_rc = check_cred_pool_access(mgmt_pool->mgpi_uuid, mgmt_pool->mgpi_svc, + DAOS_PC_RO, &in->pli_cred); if (chk_rc != 0) { - if (chk_rc == -DER_NO_PERM) - continue; - else - D_GOTO(send_resp, rc = chk_rc); + if (chk_rc != -DER_NO_PERM) + DL_ERROR(chk_rc, DF_UUID ": failed to check pool access", + DP_UUID(mgmt_pool->mgpi_uuid)); + continue; } uuid_copy(rpc_pool->plp_uuid, mgmt_pool->mgpi_uuid); rpc_pool->plp_label = mgmt_pool->mgpi_label; diff --git a/src/pool/srv_pool.c b/src/pool/srv_pool.c index 64030a7f4c5..213521b0952 100644 --- a/src/pool/srv_pool.c +++ b/src/pool/srv_pool.c @@ -2907,174 +2907,6 @@ bulk_cb(const struct crt_bulk_cb_info *cb_info) return 0; } -int -ds_pool_check_access(uuid_t pool_uuid, d_rank_list_t *ps_ranks, unsigned int flags, d_iov_t *cred) -{ - struct rsvc_client client; - crt_endpoint_t ep; - struct dss_module_info *info = dss_get_module_info(); - crt_rpc_t *rpc; - struct pool_check_access_in *in; - struct pool_check_access_out *out; - int rc; - - D_DEBUG(DB_MD, DF_UUID ": checking pool access\n", DP_UUID(pool_uuid)); - - rc = rsvc_client_init(&client, ps_ranks); - if (rc != 0) - goto out; - -rechoose: - ep.ep_grp = NULL; /* primary group */ - rc = rsvc_client_choose(&client, &ep); - if (rc != 0) { - D_ERROR(DF_UUID ": cannot find pool service: " DF_RC "\n", DP_UUID(pool_uuid), - DP_RC(rc)); - goto out_client; - } - - rc = pool_req_create(info->dmi_ctx, &ep, POOL_CHECK_ACCESS, &rpc); - if (rc != 0) { - D_ERROR(DF_UUID ": failed to create pool check access rpc, " DF_RC "\n", - DP_UUID(pool_uuid), DP_RC(rc)); - goto out_client; - } - - in = crt_req_get(rpc); - in->pcai_flags = flags; - uuid_copy(in->pcai_op.pi_uuid, pool_uuid); - uuid_clear(in->pcai_op.pi_hdl); - - rc = daos_iov_copy(&in->pcai_cred, cred); - if (rc != 0) { - D_ERROR(DF_UUID ": failed to copy credential IOV, " DF_RC "\n", DP_UUID(pool_uuid), - DP_RC(rc)); - goto out_client; - } - - rc = dss_rpc_send(rpc); - out = crt_reply_get(rpc); - D_ASSERT(out != NULL); - - rc = pool_rsvc_client_complete_rpc(&client, &ep, rc, &out->pcao_op); - if (rc == RSVC_CLIENT_RECHOOSE) { - crt_req_decref(rpc); - dss_sleep(RECHOOSE_SLEEP_MS); - goto rechoose; - } - - rc = out->pcao_op.po_rc; - if (rc != 0 && rc != -DER_NO_PERM) { - D_ERROR(DF_UUID ": failed to check pool access, " DF_RC "\n", DP_UUID(pool_uuid), - DP_RC(rc)); - } - - crt_req_decref(rpc); -out_client: - rsvc_client_fini(&client); -out: - return rc; -} - -static int -pool_check_cred_access(d_iov_t *cred, int flags, daos_prop_t *prop) -{ - struct daos_prop_entry *acl_entry; - struct d_ownership owner; - struct daos_prop_entry *owner_entry; - struct daos_prop_entry *owner_grp_entry; - uint64_t sec_capas = 0; - int rc; - - D_ASSERT(prop != NULL); - - acl_entry = daos_prop_entry_get(prop, DAOS_PROP_PO_ACL); - D_ASSERT(acl_entry != NULL); - D_ASSERT(acl_entry->dpe_val_ptr != NULL); - - owner_entry = daos_prop_entry_get(prop, DAOS_PROP_PO_OWNER); - D_ASSERT(owner_entry != NULL); - D_ASSERT(owner_entry->dpe_str != NULL); - - owner_grp_entry = daos_prop_entry_get(prop, DAOS_PROP_PO_OWNER_GROUP); - D_ASSERT(owner_grp_entry != NULL); - D_ASSERT(owner_grp_entry->dpe_str != NULL); - - owner.user = owner_entry->dpe_str; - owner.group = owner_grp_entry->dpe_str; - - /* - * Security capabilities determine the access control policy on this - * pool handle. - */ - rc = ds_sec_pool_get_capabilities(flags, cred, &owner, acl_entry->dpe_val_ptr, &sec_capas); - if (rc != 0) { - DL_ERROR(rc, "failed to read sec capabilities"); - return rc; - } - - if (!ds_sec_pool_can_connect(sec_capas)) { - return -DER_NO_PERM; - } - - return 0; -} - -void -ds_pool_check_access_handler(crt_rpc_t *rpc) -{ - struct pool_check_access_in *in = crt_req_get(rpc); - struct pool_check_access_out *out = crt_reply_get(rpc); - struct pool_svc *svc; - struct rdb_tx tx; - daos_prop_t *prop = NULL; - uint64_t prop_bits; - int rc; - - D_DEBUG(DB_MD, DF_UUID ": processing rpc: %p hdl=" DF_UUID "\n", - DP_UUID(in->pcai_op.pi_uuid), rpc, DP_UUID(in->pcai_op.pi_hdl)); - - rc = pool_svc_lookup_leader(in->pcai_op.pi_uuid, &svc, &out->pcao_op.po_hint); - if (rc != 0) - D_GOTO(out, rc); - - rc = rdb_tx_begin(svc->ps_rsvc.s_db, svc->ps_rsvc.s_term, &tx); - if (rc != 0) - D_GOTO(out_svc, rc); - - ABT_rwlock_rdlock(svc->ps_lock); - - prop_bits = DAOS_PO_QUERY_PROP_ALL; - rc = pool_prop_read(&tx, svc, prop_bits, &prop); - if (rc != 0) { - D_ERROR(DF_UUID ": cannot get access data for pool, " - "rc=" DF_RC "\n", - DP_UUID(in->pcai_op.pi_uuid), DP_RC(rc)); - D_GOTO(out_lock, rc); - } - D_ASSERT(prop != NULL); - - rc = pool_check_cred_access(&in->pcai_cred, in->pcai_flags, prop); - if (rc != 0) { - D_INFO(DF_UUID ": rejecting access based on credential, " - "rc=" DF_RC "\n", - DP_UUID(in->pcai_op.pi_uuid), DP_RC(rc)); - } - - out->pcao_op.po_map_version = ds_pool_get_version(svc->ps_pool); -out_lock: - ABT_rwlock_unlock(svc->ps_lock); - rdb_tx_end(&tx); - if (prop) - daos_prop_free(prop); -out_svc: - ds_rsvc_set_hint(&svc->ps_rsvc, &out->pcao_op.po_hint); - pool_svc_put_leader(svc); -out: - out->pcao_op.po_rc = rc; - crt_reply_send(rpc); -} - /* Currently we only maintain compatibility between 2 versions */ #define NUM_POOL_VERSIONS 2 From 054c3cd33a2a8bf075d43392d5284b9165dda904 Mon Sep 17 00:00:00 2001 From: Michael MacDonald Date: Wed, 26 Jun 2024 21:51:03 +0000 Subject: [PATCH 06/10] handle fetch retries on pool count changes Signed-off-by: Michael MacDonald --- src/control/cmd/daos/pool.go | 61 ++++++++++++++++++++++-------------- src/engine/drpc_client.c | 4 +-- src/mgmt/cli_mgmt.c | 4 +-- src/mgmt/srv.c | 9 ++++++ 4 files changed, 50 insertions(+), 28 deletions(-) diff --git a/src/control/cmd/daos/pool.go b/src/control/cmd/daos/pool.go index 42b69cab520..6c3f2b4a8a6 100644 --- a/src/control/cmd/daos/pool.go +++ b/src/control/cmd/daos/pool.go @@ -599,34 +599,48 @@ func (cmd *poolAutoTestCmd) Execute(_ []string) error { } func getPoolList(log logging.Logger, sysName string, queryEnabled bool) ([]*daos.PoolInfo, error) { - var rc C.int - bufSize := C.size_t(0) - - // First, fetch the total number of pools in the system. - // We may not have access to all of them, so this is an upper bound. - rc = C.daos_mgmt_list_pools(nil, &bufSize, nil, nil) - if err := daosError(rc); err != nil { - return nil, err - } - - if bufSize < 1 { - return nil, nil - } - var cSysName *C.char if sysName != "" { cSysName := C.CString(sysName) defer freeString(cSysName) } - // Now, we actually fetch the pools into the buffer that we've created. - cPools := make([]C.daos_mgmt_pool_info_t, bufSize) - rc = C.daos_mgmt_list_pools(nil, &bufSize, &cPools[0], nil) - if err := daosError(rc); err != nil { + var cPools []C.daos_mgmt_pool_info_t + for { + var rc C.int + var poolCount C.size_t + + // First, fetch the total number of pools in the system. + // We may not have access to all of them, so this is an upper bound. + rc = C.daos_mgmt_list_pools(cSysName, &poolCount, nil, nil) + if err := daosError(rc); err != nil { + return nil, err + } + log.Debugf("pools in system: %d", poolCount) + + if poolCount < 1 { + return nil, nil + } + + // Now, we actually fetch the pools into the buffer that we've created. + cPools = make([]C.daos_mgmt_pool_info_t, poolCount) + rc = C.daos_mgmt_list_pools(cSysName, &poolCount, &cPools[0], nil) + err := daosError(rc) + if err == nil { + cPools = cPools[:poolCount] // adjust the slice to the number of pools retrieved + log.Debugf("fetched %d pools", len(cPools)) + break + } + if err == daos.StructTooSmall { + log.Notice("server-side pool list changed; re-fetching") + continue + } + log.Errorf("failed to fetch pool list: %s", err) return nil, err } - pools := make([]*daos.PoolInfo, 0, bufSize) - for i := 0; i < int(bufSize); i++ { + + pools := make([]*daos.PoolInfo, 0, len(cPools)) + for i := 0; i < len(cPools); i++ { cPool := &cPools[i] svcRanks, err := rankSetFromC(cPool.mgpi_svc) @@ -658,15 +672,16 @@ func getPoolList(log logging.Logger, sysName string, queryEnabled bool) ([]*daos if qErr != nil { log.Errorf("failed to query pool %q: %s", poolLabel, qErr) } - pool.Label = poolLabel - pool.ServiceReplicas = svcRanks.Ranks() - if err := daosError(C.daos_pool_disconnect(poolHandle, nil)); err != nil { log.Errorf("failed to disconnect from pool %q: %s", poolLabel, err) } if qErr != nil { continue } + + // Add a few missing pieces that the query doesn't fill in. + pool.Label = poolLabel + pool.ServiceReplicas = svcRanks.Ranks() } else { // Just populate the basic info. pool = &daos.PoolInfo{ diff --git a/src/engine/drpc_client.c b/src/engine/drpc_client.c index 54db5c15e0e..ddf33bd6dfd 100644 --- a/src/engine/drpc_client.c +++ b/src/engine/drpc_client.c @@ -1,5 +1,5 @@ /* - * (C) Copyright 2019-2021 Intel Corporation. + * (C) Copyright 2019-2024 Intel Corporation. * * SPDX-License-Identifier: BSD-2-Clause-Patent */ @@ -459,7 +459,7 @@ ds_get_pool_list(uint64_t *npools, daos_mgmt_pool_info_t *pools) if (*npools > 0 && lp_resp->n_pools > *npools) { D_ERROR("pool list exceeds request buffer (req: %lu, actual: %lu)", *npools, lp_resp->n_pools); - D_GOTO(out_resp, rc = -DER_TRUNC); + D_GOTO(out_resp, rc = -DER_OVERFLOW); } *npools = lp_resp->n_pools; diff --git a/src/mgmt/cli_mgmt.c b/src/mgmt/cli_mgmt.c index 354b0ef794b..95e76d0e4e5 100644 --- a/src/mgmt/cli_mgmt.c +++ b/src/mgmt/cli_mgmt.c @@ -1374,10 +1374,8 @@ dc_mgmt_pool_list(tse_task_t *task) } rc = out->plo_op.mo_rc; - if (rc != 0) { - DL_ERROR(rc, "failed to list pools"); + if (rc != 0) D_GOTO(out_put_req, rc); - } *args->npools = out->plo_npools; diff --git a/src/mgmt/srv.c b/src/mgmt/srv.c index cea08b359ac..db738a88ed1 100644 --- a/src/mgmt/srv.c +++ b/src/mgmt/srv.c @@ -530,6 +530,15 @@ ds_mgmt_pool_list_hdlr(crt_rpc_t *rpc) send_resp: out->plo_op.mo_rc = rc; + if (rc == 0) { + if (n_rpc > 0) + D_DEBUG(DB_MGMT, "returning %zu/%zu pools\n", n_rpc, n_mgmt); + else + D_DEBUG(DB_MGMT, "returning %zu pools\n", n_mgmt); + } else { + DL_ERROR(rc, "failed to list pools"); + } + rc = crt_reply_send(rpc); if (rc != 0) DL_ERROR(rc, "crt_reply_send() failed"); From 3943270022c69178e27b1095fa93771046fc1b16 Mon Sep 17 00:00:00 2001 From: Michael MacDonald Date: Thu, 27 Jun 2024 18:19:23 +0000 Subject: [PATCH 07/10] update task.h --- src/include/daos/task.h | 1 + 1 file changed, 1 insertion(+) diff --git a/src/include/daos/task.h b/src/include/daos/task.h index ce62c3c3aec..f5df2b99047 100644 --- a/src/include/daos/task.h +++ b/src/include/daos/task.h @@ -26,6 +26,7 @@ struct daos_task_args { daos_pool_replicas_t pool_add_replicas; daos_pool_replicas_t pool_remove_replicas; daos_mgmt_get_bs_state_t mgmt_get_bs_state; + daos_mgmt_pool_list_t mgmt_pool_list; /** Pool */ daos_pool_connect_t pool_connect; From 5300e812c5d606756d746c1e352dfe0f80e22f7a Mon Sep 17 00:00:00 2001 From: Michael MacDonald Date: Thu, 27 Jun 2024 18:20:15 +0000 Subject: [PATCH 08/10] update mgmt suite test to use client pool list Quick-functional: true Test-tag: ListClientPoolsTest Signed-off-by: Michael MacDonald --- src/tests/suite/daos_mgmt.c | 36 ++++++++++++------------------------ 1 file changed, 12 insertions(+), 24 deletions(-) diff --git a/src/tests/suite/daos_mgmt.c b/src/tests/suite/daos_mgmt.c index 8adc4ad9125..0dc0959ca03 100644 --- a/src/tests/suite/daos_mgmt.c +++ b/src/tests/suite/daos_mgmt.c @@ -212,9 +212,9 @@ find_pool(void **state, daos_mgmt_pool_info_t *pool) } /* Verify pool info returned by DAOS API - * rc_ret: return code from dmg_list_pool() - * npools_in: npools input argument to dmg_list_pool() - * npools_out: npools output argument value after dmg_list_pool() + * rc_ret: return code from daos_mgmt_list_pools() + * npools_in: npools input argument to daos_mgmt_list_pools() + * npools_out: npools output argument value after daos_mgmt_list_pools() */ static void verify_pool_info(void **state, int rc_ret, daos_size_t npools_in, @@ -272,13 +272,13 @@ list_pools_test(void **state) /***** Test: retrieve number of pools in system *****/ npools = npools_orig = 0xABC0; /* Junk value (e.g., uninitialized) */ /* test only */ - rc = dmg_pool_list(dmg_config_file, arg->group, &npools, NULL); + rc = daos_mgmt_list_pools(arg->group, &npools, NULL, NULL); assert_rc_equal(rc, 0); verify_pool_info(state, rc, npools_orig, NULL /* pools */, npools); print_message("success t%d: output npools=%zu\n", tnum++, lparg->nsyspools); - /* Setup for next 2 tests: alloc pools[] */ + /* Setup for next test: alloc pools[] */ npools_alloc = lparg->nsyspools + 10; D_ALLOC_ARRAY(pools, npools_alloc); assert_ptr_not_equal(pools, NULL); @@ -287,29 +287,18 @@ list_pools_test(void **state) * and that many items in pools[] filled *****/ npools = npools_alloc; - rc = dmg_pool_list(dmg_config_file, arg->group, &npools, pools); + rc = daos_mgmt_list_pools(arg->group, &npools, pools, NULL); assert_rc_equal(rc, 0); verify_pool_info(state, rc, npools_alloc, pools, npools); clean_pool_info(npools_alloc, pools); print_message("success t%d: pools[] over-sized\n", tnum++); - /***** Test: provide npools=0, non-NULL pools ****/ - npools = 0; - rc = dmg_pool_list(dmg_config_file, arg->group, &npools, pools); - if (lparg->nsyspools > 0) - assert_rc_equal(rc, -DER_TRUNC); - else - assert_rc_equal(rc, 0); - assert_int_equal(npools, lparg->nsyspools); - print_message("success t%d: npools=0, non-NULL pools[] rc=%d\n", - tnum++, rc); - - /* Teardown for above 2 tests */ + /* Teardown for above test */ D_FREE(pools); /* clean_pool_info() freed mgpi_svc */ pools = NULL; /***** Test: invalid npools=NULL *****/ - rc = dmg_pool_list(dmg_config_file, arg->group, NULL, NULL); + rc = daos_mgmt_list_pools(arg->group, NULL, NULL, NULL); assert_rc_equal(rc, -DER_INVAL); print_message("success t%d: in &npools NULL, -DER_INVAL\n", tnum++); @@ -324,7 +313,7 @@ list_pools_test(void **state) /* Test: Exact size buffer */ npools = npools_alloc; - rc = dmg_pool_list(dmg_config_file, arg->group, &npools, pools); + rc = daos_mgmt_list_pools(arg->group, &npools, pools, NULL); assert_rc_equal(rc, 0); verify_pool_info(state, rc, npools_alloc, pools, npools); @@ -333,7 +322,7 @@ list_pools_test(void **state) pools = NULL; print_message("success t%d: pools[] exact length\n", tnum++); - /***** Test: Under-sized buffer (negative) -DER_TRUNC *****/ + /***** Test: Under-sized buffer (negative) -DER_OVERFLOW *****/ /* Setup */ npools_alloc = lparg->nsyspools - 1; D_ALLOC_ARRAY(pools, npools_alloc); @@ -341,9 +330,8 @@ list_pools_test(void **state) /* Test: Under-sized buffer */ npools = npools_alloc; - rc = dmg_pool_list(dmg_config_file, arg->group, &npools, pools); - assert_rc_equal(rc, -DER_TRUNC); - verify_pool_info(state, rc, npools_alloc, pools, npools); + rc = daos_mgmt_list_pools(arg->group, &npools, pools, NULL); + assert_rc_equal(rc, -DER_OVERFLOW); print_message("success t%d: pools[] under-sized\n", tnum++); /* Teardown */ From d147fb3c46fccb190d0213ac8ed18b809e084c9d Mon Sep 17 00:00:00 2001 From: Michael MacDonald Date: Fri, 28 Jun 2024 23:23:50 +0000 Subject: [PATCH 09/10] adjust daos list pools JSON output to match dmg --- src/control/cmd/daos/pool.go | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/control/cmd/daos/pool.go b/src/control/cmd/daos/pool.go index 6c3f2b4a8a6..f6f82dafee1 100644 --- a/src/control/cmd/daos/pool.go +++ b/src/control/cmd/daos/pool.go @@ -713,7 +713,11 @@ func (cmd *poolListCmd) Execute(_ []string) error { } if cmd.JSONOutputEnabled() { - return cmd.OutputJSON(pools, nil) + return cmd.OutputJSON(struct { + Pools []*daos.PoolInfo `json:"pools"` // compatibility with dmg + }{ + Pools: pools, + }, nil) } var buf strings.Builder From ad2d9488a1d15f22747612e2f66deb65255f8a28 Mon Sep 17 00:00:00 2001 From: Michael MacDonald Date: Thu, 27 Jun 2024 16:37:04 +0000 Subject: [PATCH 10/10] ftest for daos pool list Skip-NLT: true Skip-unit-test: true Quick-functional: true Test-tag: test_list_pools Required-githooks: true Change-Id: I829dd6e5d26ea8ff3f5c9ff0da758dcc92e90774 Signed-off-by: Michael MacDonald --- src/tests/ftest/pool/list_pools.py | 77 +++++++++++------ src/tests/ftest/util/command_utils.py | 6 +- src/tests/ftest/util/daos_utils.py | 107 ++++++++++++++++++++++++ src/tests/ftest/util/daos_utils_base.py | 18 +++- 4 files changed, 178 insertions(+), 30 deletions(-) diff --git a/src/tests/ftest/pool/list_pools.py b/src/tests/ftest/pool/list_pools.py index b1a85a0125d..265b3d39089 100644 --- a/src/tests/ftest/pool/list_pools.py +++ b/src/tests/ftest/pool/list_pools.py @@ -1,5 +1,5 @@ """ - (C) Copyright 2018-2023 Intel Corporation. + (C) Copyright 2018-2024 Intel Corporation. SPDX-License-Identifier: BSD-2-Clause-Patent """ @@ -9,7 +9,7 @@ class ListPoolsTest(TestWithServers): - """Test class for dmg pool list tests. + """Test class for dmg and daos pool list tests. Test Class Description: This class contains tests for list pool. @@ -20,8 +20,9 @@ class ListPoolsTest(TestWithServers): def run_case(self, rank_lists, svcn=None): """Run test case. - Create pools, call dmg pool list to get the list, and compare against - the UUIDs and service replicas returned at create time. + Create pools, then verify that both dmg and daos tools are able to + list the pools with the same information that was returned when the + pools were created. Args: rank_lists (list): Rank lists. List of list of int. @@ -35,47 +36,71 @@ def run_case(self, rank_lists, svcn=None): # Iterate rank lists to create pools. Store the created pool information # as a dictionary of pool UUID keys with a service replica list value. self.pool = [] - expected_uuids = {} + expected_admin_uuids = {} for rank_list in rank_lists: self.pool.append(self.get_pool(target_list=rank_list, svcn=svcn)) - expected_uuids[self.pool[-1].uuid.lower()] = self.pool[-1].svc_ranks - - # Verify the 'dmg pool info' command lists the correct created pool - # information. The DmgCommand.pool_info() method returns the command - # output as a dictionary of pool UUID keys with service replica list - # values. - detected_uuids = {} + expected_admin_uuids[self.pool[-1].uuid.lower()] = self.pool[-1].svc_ranks + + # Remove the default ACLs that allow the creator to access the pool. + # These ACLs don't apply to dmg, but do apply to users. + offlimits = self.pool[0] + self.get_dmg_command().pool_delete_acl(offlimits.uuid, 'OWNER@') + self.get_dmg_command().pool_delete_acl(offlimits.uuid, 'GROUP@') + expected_user_uuids = expected_admin_uuids.copy() + del expected_user_uuids[offlimits.uuid.lower()] + + # Verify the 'dmg pool list' command lists the correct created pool + # information. + detected_admin_uuids = {} try: for data in self.get_dmg_command().get_pool_list_all(): - detected_uuids[data["uuid"]] = data["svc_reps"] + detected_admin_uuids[data["uuid"]] = data["svc_reps"] except KeyError as error: self.fail("Error parsing dmg pool list output: {}".format(error)) - self.log.info("Expected pool info: %s", str(expected_uuids)) - self.log.info("Detected pool info: %s", str(detected_uuids)) - - # Destroy all the pools - if self.destroy_pools(self.pool): - self.fail("Error destroying pools") - self.pool = [] + self.log.info("Expected admin pool info: %s", str(expected_admin_uuids)) + self.log.info("Detected admin pool info: %s", str(detected_admin_uuids)) # Compare the expected and detected pool information self.assertEqual( - expected_uuids, detected_uuids, + expected_admin_uuids, detected_admin_uuids, "dmg pool info does not list all expected pool UUIDs and their " "service replicas") + # Verify the 'daos pool list' command lists the correct created pool + # information. + detected_user_uuids = {} + try: + for data in self.get_daos_command().get_pool_list_all(): + detected_user_uuids[data["uuid"]] = data["svc_reps"] + except KeyError as error: + self.fail("Error parsing dmg pool list output: {}".format(error)) + + self.log.info("Expected user pool info: %s", str(expected_user_uuids)) + self.log.info("Detected user pool info: %s", str(detected_user_uuids)) + + # Compare the expected and detected pool information + self.assertEqual( + expected_user_uuids, detected_user_uuids, + "daos pool info does not list all expected pool UUIDs and their " + "service replicas") + + # Destroy all the pools + if self.destroy_pools(self.pool): + self.fail("Error destroying pools") + self.pool = [] + def test_list_pools(self): - """JIRA ID: DAOS-3459. + """JIRA ID: DAOS-3459, DAOS-16038. Test Description: - Create pools in different ranks, call dmg pool list and verify the - output list matches the output returned when the pools were - created. + Create pools in different ranks, then verify that both dmg and + daos tools are able to list the pools and that the listed output + matches what was returned when the pools were created. :avocado: tags=all,full_regression :avocado: tags=vm - :avocado: tags=pool + :avocado: tags=pool,daos_cmd :avocado: tags=list_pools,test_list_pools """ ranks = list(range(len(self.hostlist_servers))) diff --git a/src/tests/ftest/util/command_utils.py b/src/tests/ftest/util/command_utils.py index 9c21c21feeb..2d278cc697b 100644 --- a/src/tests/ftest/util/command_utils.py +++ b/src/tests/ftest/util/command_utils.py @@ -637,7 +637,11 @@ def set_command(self, sub_command_list=None, **kwargs): if sub_command_list is not None: for sub_command in sub_command_list: full_command.set_sub_command(sub_command) - full_command = full_command.sub_command_class + + if full_command.sub_command_class is not None: + full_command = full_command.sub_command_class + else: + raise CommandFailure(f"invalid sub command: {sub_command}") # Update any argument values for the full command full_command.update_params(**kwargs) diff --git a/src/tests/ftest/util/daos_utils.py b/src/tests/ftest/util/daos_utils.py index 679ca432414..13dfdbdd9f3 100644 --- a/src/tests/ftest/util/daos_utils.py +++ b/src/tests/ftest/util/daos_utils.py @@ -369,6 +369,113 @@ def pool_list_attrs(self, pool, sys_name=None, verbose=False): ("pool", "list-attrs"), pool=pool, sys_name=sys_name, verbose=verbose) + def pool_list_containers(self, pool, sys_name=None): + """List containers in the pool. + + Args: + pool (str): pool label or UUID + sys_name (str): DAOS system name. Defaults to None. + + Returns: + dict: JSON output + + Raises: + CommandFailure: if the daos pool list-containers command fails. + + """ + return self._get_json_result( + ("pool", "list-containers"), pool=pool, sys_name=sys_name) + + def pool_list(self, no_query=False, verbose=False): + """List pools. + + Args: + no_query (bool, optional): If True, do not query for pool stats. + verbose (bool, optional): If True, use verbose mode. + + Raises: + CommandFailure: if the daos pool list command fails. + + Returns: + dict: the daos json command output converted to a python dictionary + + """ + # Sample verbose JSON Output: + # { + # "response": { + # "status": 0, + # "pools": [ + # { + # "uuid": "517217db-47c4-4bb9-aae5-e38ca7b3dafc", + # "label": "mkp1", + # "svc_reps": [ + # 0 + # ], + # "total_targets": 8, + # "disabled_targets": 0, + # "usage": [ + # { + # "tier_name": "SCM", + # "size": 3000000000, + # "free": 2995801112, + # "imbalance": 0 + # }, + # { + # "tier_name": "NVME", + # "size": 47000000000, + # "free": 26263322624, + # "imbalance": 36 + # } + # ] + # } + # ] + # }, + # "error": null, + # "status": 0 + # } + return self._get_json_result( + ("pool", "list"), no_query=no_query, verbose=verbose) + + def _parse_pool_list(self, key=None, **kwargs): + """Parse the daos pool list json output for the requested information. + + Args: + key (str, optional): pool list json dictionary key in + ["response"]["pools"]. Defaults to None. + + Raises: + CommandFailure: if the daos pool list command fails. + + Returns: + list: list of all the pool items in the daos pool list json output + for the requested json dictionary key. This will be an empty + list if the key does not exist or the json output was not in + the expected format. + + """ + pool_list = self.pool_list(**kwargs) + try: + if pool_list["response"]["pools"] is None: + return [] + if key: + return [pool[key] for pool in pool_list["response"]["pools"]] + return pool_list["response"]["pools"] + except KeyError: + return [] + + def get_pool_list_all(self, **kwargs): + """Get a list of all the pool information from daos pool list. + + Raises: + CommandFailure: if the daos pool list command fails. + + Returns: + list: a list of dictionaries containing information for each pool + from the daos pool list json output + + """ + return self._parse_pool_list(**kwargs) + def container_query(self, pool, cont, sys_name=None): """Query a container. diff --git a/src/tests/ftest/util/daos_utils_base.py b/src/tests/ftest/util/daos_utils_base.py index 9130b214dac..9f76d97e10e 100644 --- a/src/tests/ftest/util/daos_utils_base.py +++ b/src/tests/ftest/util/daos_utils_base.py @@ -1,5 +1,5 @@ """ - (C) Copyright 2020-2023 Intel Corporation. + (C) Copyright 2020-2024 Intel Corporation. SPDX-License-Identifier: BSD-2-Clause-Patent """ @@ -47,8 +47,10 @@ def __init__(self): def get_sub_command_class(self): # pylint: disable=redefined-variable-type - """Get the dmg network sub command object.""" - if self.sub_command.value in ("list-containers", "list"): + """Get the daos pool sub command object.""" + if self.sub_command.value == "list": + self.sub_command_class = self.ListSubCommand() + elif self.sub_command.value == "list-containers": self.sub_command_class = self.ListContainersSubCommand() elif self.sub_command.value == "query": self.sub_command_class = self.QuerySubCommand() @@ -65,6 +67,16 @@ def get_sub_command_class(self): else: self.sub_command_class = None + class ListSubCommand(CommandWithParameters): + """Defines an object for the daos pool list command.""" + + def __init__(self): + """Create a daos pool list command object.""" + super().__init__("/run/daos/pool/list/*", "list") + self.sys_name = FormattedParameter("--sys-name={}") + self.no_query = FormattedParameter("--no-query", False) + self.verbose = FormattedParameter("--verbose", False) + class CommonPoolSubCommand(CommandWithParameters): """Defines an object for the common daos pool sub-command.