From e7db73a68a8b62a761b02a9c6eb0ecab122bac47 Mon Sep 17 00:00:00 2001 From: Michael MacDonald Date: Tue, 11 Jun 2024 20:16:51 +0000 Subject: [PATCH] DAOS-15982 pool: Enable pool list for clients Revive parts of the old client mgmt API to allow libdaos clients to list pools in the system, but only allow a given client to see the pools that it could connect to. Required-githooks: true Change-Id: I2b3c391ddf042b23811be8f3390ec290e92e4290 Signed-off-by: Michael MacDonald --- src/client/api/init.c | 2 +- src/client/api/mgmt.c | 26 +++ src/control/cmd/daos/pool.go | 77 +++++++ src/control/cmd/daos/pretty/pool.go | 184 +++++++++++++++ src/control/cmd/daos/pretty/pool_test.go | 261 ++++++++++++++++++++++ src/control/cmd/daos/pretty/ranks.go | 31 +++ src/control/cmd/daos/pretty/ranks_test.go | 60 +++++ src/control/cmd/daos/util.go | 15 ++ src/control/cmd/dmg/pretty/pool.go | 177 +-------------- src/control/cmd/dmg/pretty/pool_test.go | 215 ------------------ src/control/cmd/dmg/pretty/ranks.go | 16 -- src/control/common/proto/srv/srv.pb.go | 236 +++++++++++++++++-- src/control/drpc/modules.go | 11 +- src/control/go.mod | 2 +- src/control/go.sum | 4 +- src/control/server/mgmt_drpc.go | 39 +++- src/engine/drpc_client.c | 98 ++++++++ src/engine/srv.pb-c.c | 180 +++++++++++++++ src/engine/srv.pb-c.h | 88 +++++++- src/include/daos/drpc.pb-c.h | 75 +++---- src/include/daos/drpc_modules.h | 13 +- src/include/daos/mgmt.h | 2 + src/include/daos_mgmt.h | 6 + src/include/daos_srv/daos_engine.h | 3 + src/include/daos_srv/pool.h | 2 + src/include/daos_task.h | 15 +- src/mgmt/cli_mgmt.c | 157 ++++++++++++- src/mgmt/rpc.c | 41 ++++ src/mgmt/rpc.h | 29 +++ src/mgmt/srv.c | 87 +++++++- src/mgmt/srv_internal.h | 2 + src/pool/rpc.c | 2 + src/pool/rpc.h | 14 ++ src/pool/srv_internal.h | 2 + src/pool/srv_pool.c | 168 ++++++++++++++ src/proto/srv/srv.proto | 16 ++ 36 files changed, 1886 insertions(+), 470 deletions(-) create mode 100644 src/control/cmd/daos/pretty/ranks.go create mode 100644 src/control/cmd/daos/pretty/ranks_test.go delete mode 100644 src/control/cmd/dmg/pretty/ranks.go diff --git a/src/client/api/init.c b/src/client/api/init.c index 40e7aa5e0d9..a5ab9226701 100644 --- a/src/client/api/init.c +++ b/src/client/api/init.c @@ -37,7 +37,7 @@ const struct daos_task_api dc_funcs[] = { /** Management */ {dc_deprecated, 0}, {dc_deprecated, 0}, - {dc_deprecated, 0}, + {dc_mgmt_pool_list, sizeof(daos_mgmt_pool_list_t)}, {dc_debug_set_params, sizeof(daos_set_params_t)}, {dc_mgmt_get_bs_state, sizeof(daos_mgmt_get_bs_state_t)}, diff --git a/src/client/api/mgmt.c b/src/client/api/mgmt.c index ac494d15330..4d24f5ec106 100644 --- a/src/client/api/mgmt.c +++ b/src/client/api/mgmt.c @@ -79,3 +79,29 @@ daos_mgmt_put_sys_info(struct daos_sys_info *info) { dc_mgmt_put_sys_info(info); } + +int +daos_mgmt_list_pools(const char *group, daos_size_t *npools, daos_mgmt_pool_info_t *pools, + daos_event_t *ev) +{ + daos_mgmt_pool_list_t *args; + tse_task_t *task; + int rc; + + DAOS_API_ARG_ASSERT(*args, MGMT_LIST_POOLS); + + if (npools == NULL) { + D_ERROR("npools must be non-NULL\n"); + return -DER_INVAL; + } + + rc = dc_task_create(dc_mgmt_pool_list, NULL, ev, &task); + if (rc) + return rc; + args = dc_task_get_args(task); + args->grp = group; + args->pools = pools; + args->npools = npools; + + return dc_task_schedule(task, true); +} diff --git a/src/control/cmd/daos/pool.go b/src/control/cmd/daos/pool.go index 1d3b8ce9182..87554972bca 100644 --- a/src/control/cmd/daos/pool.go +++ b/src/control/cmd/daos/pool.go @@ -20,6 +20,7 @@ import ( "github.com/daos-stack/daos/src/control/lib/daos" "github.com/daos-stack/daos/src/control/lib/ranklist" "github.com/daos-stack/daos/src/control/lib/ui" + "github.com/daos-stack/daos/src/control/logging" ) /* @@ -180,6 +181,7 @@ func (cmd *poolBaseCmd) getAttr(name string) (*attribute, error) { } type poolCmd struct { + List poolListCmd `command:"list" description:"list pools to which this user has access"` Query poolQueryCmd `command:"query" description:"query pool info"` QueryTargets poolQueryTargetsCmd `command:"query-targets" description:"query pool target info"` ListConts containerListCmd `command:"list-containers" alias:"list-cont" description:"list all containers in pool"` @@ -601,3 +603,78 @@ func (cmd *poolAutoTestCmd) Execute(_ []string) error { return nil } + +func getPoolList(log logging.Logger, queryEnabled bool) ([]*daos.PoolInfo, error) { + var rc C.int + bufSize := C.size_t(0) + + // First, fetch the total number of pools in the system. + // We may not have access to all of them, so this is an upper bound. + rc = C.daos_mgmt_list_pools(nil, &bufSize, nil, nil) + if err := daosError(rc); err != nil { + return nil, err + } + + if bufSize < 1 { + return nil, nil + } + + // Now, we actually fetch the pools into the buffer that we've created. + cPools := make([]C.daos_mgmt_pool_info_t, bufSize) + rc = C.daos_mgmt_list_pools(nil, &bufSize, &cPools[0], nil) + if err := daosError(rc); err != nil { + return nil, err + } + pools := make([]*daos.PoolInfo, 0, bufSize) + for i := 0; i < int(bufSize); i++ { + poolUUID, err := uuidFromC(cPools[i].mgpi_uuid) + if err != nil { + return nil, err + } + svcRanks, err := rankSetFromC(cPools[i].mgpi_svc) + if err != nil { + return nil, err + } + + // Populate the basic info. + pool := &daos.PoolInfo{ + UUID: poolUUID, + Label: C.GoString(cPools[i].mgpi_label), + ServiceReplicas: svcRanks.Ranks(), + State: daos.PoolServiceStateReady, + } + + if queryEnabled { + // TODO: rework query logic to be callable standalone. + } + pools = append(pools, pool) + } + + log.Debugf("fetched %d/%d pools", len(pools), len(cPools)) + return pools, nil +} + +type poolListCmd struct { + daosCmd + Verbose bool `short:"v" long:"verbose" description:"Add pool UUIDs and service replica lists to display"` + NoQuery bool `short:"n" long:"no-query" description:"Disable query of listed pools"` +} + +func (cmd *poolListCmd) Execute(_ []string) error { + pools, err := getPoolList(cmd.Logger, !cmd.NoQuery) + if err != nil { + return err + } + + if cmd.JSONOutputEnabled() { + return cmd.OutputJSON(pools, nil) + } + + var buf strings.Builder + if err := pretty.PrintPoolList(pools, &buf, cmd.Verbose); err != nil { + return err + } + cmd.Info(buf.String()) + + return nil +} diff --git a/src/control/cmd/daos/pretty/pool.go b/src/control/cmd/daos/pretty/pool.go index 032a4e72319..abf46c52025 100644 --- a/src/control/cmd/daos/pretty/pool.go +++ b/src/control/cmd/daos/pretty/pool.go @@ -98,3 +98,187 @@ func PrintPoolQueryTargetInfo(pqti *daos.PoolQueryTargetInfo, out io.Writer) err return w.Err } + +func poolListCreateRow(pool *daos.PoolInfo, upgrade bool) txtfmt.TableRow { + // display size of the largest non-empty tier + var size uint64 + poolUsage := pool.Usage() + for ti := len(poolUsage) - 1; ti >= 0; ti-- { + if poolUsage[ti].Size != 0 { + size = poolUsage[ti].Size + break + } + } + + // display usage of the most used tier + var used int + for ti := 0; ti < len(poolUsage); ti++ { + t := poolUsage[ti] + u := float64(t.Size-t.Free) / float64(t.Size) + + if int(u*100) > used { + used = int(u * 100) + } + } + + // display imbalance of the most imbalanced tier + var imbalance uint32 + for ti := 0; ti < len(poolUsage); ti++ { + if poolUsage[ti].Imbalance > imbalance { + imbalance = poolUsage[ti].Imbalance + } + } + + row := txtfmt.TableRow{ + "Pool": pool.Name(), + "Size": humanize.Bytes(size), + "State": pool.State.String(), + "Used": fmt.Sprintf("%d%%", used), + "Imbalance": fmt.Sprintf("%d%%", imbalance), + "Disabled": fmt.Sprintf("%d/%d", pool.DisabledTargets, pool.TotalTargets), + } + + if upgrade { + upgradeString := "None" + + if pool.PoolLayoutVer != pool.UpgradeLayoutVer { + upgradeString = fmt.Sprintf("%d->%d", pool.PoolLayoutVer, pool.UpgradeLayoutVer) + } + row["UpgradeNeeded?"] = upgradeString + } + + return row +} + +func printPoolList(pools []*daos.PoolInfo, out io.Writer) error { + upgrade := false + for _, pool := range pools { + if pool.PoolLayoutVer != pool.UpgradeLayoutVer { + upgrade = true + } + } + + titles := []string{"Pool", "Size", "State", "Used", "Imbalance", "Disabled"} + if upgrade { + titles = append(titles, "UpgradeNeeded?") + } + formatter := txtfmt.NewTableFormatter(titles...) + + var table []txtfmt.TableRow + for _, pool := range pools { + table = append(table, poolListCreateRow(pool, upgrade)) + } + + fmt.Fprintln(out, formatter.Format(table)) + + return nil +} + +func addVerboseTierUsage(row txtfmt.TableRow, usage *daos.PoolTierUsage) txtfmt.TableRow { + row[usage.TierName+" Size"] = humanize.Bytes(usage.Size) + row[usage.TierName+" Used"] = humanize.Bytes(usage.Size - usage.Free) + row[usage.TierName+" Imbalance"] = fmt.Sprintf("%d%%", usage.Imbalance) + + return row +} + +func poolListCreateRowVerbose(pool *daos.PoolInfo, hasSpace, hasRebuild bool) txtfmt.TableRow { + label := pool.Label + if label == "" { + label = "-" + } + + svcReps := "N/A" + if len(pool.ServiceReplicas) != 0 { + svcReps = PrintRanks(pool.ServiceReplicas) + } + + upgrade := "None" + if pool.PoolLayoutVer != pool.UpgradeLayoutVer { + upgrade = fmt.Sprintf("%d->%d", pool.PoolLayoutVer, pool.UpgradeLayoutVer) + } + + row := txtfmt.TableRow{ + "Label": label, + "UUID": pool.UUID.String(), + "State": pool.State.String(), + "SvcReps": svcReps, + } + if hasSpace { + row["Disabled"] = fmt.Sprintf("%d/%d", pool.DisabledTargets, pool.TotalTargets) + row["UpgradeNeeded?"] = upgrade + } + if hasRebuild { + row["Rebuild State"] = pool.RebuildState() + } + + if hasSpace { + for _, tu := range pool.Usage() { + row = addVerboseTierUsage(row, tu) + } + } + + return row +} + +func printVerbosePoolList(pools []*daos.PoolInfo, out io.Writer) error { + // Basic pool info should be available without a query. + titles := []string{"Label", "UUID", "State", "SvcReps"} + + hasSpaceQuery := false + hasRebuildQuery := false + for _, pool := range pools { + if hasSpaceQuery && hasRebuildQuery { + break + } + + if pool.QueryMask.HasOption(daos.PoolQueryOptionSpace) { + hasSpaceQuery = true + } + if pool.QueryMask.HasOption(daos.PoolQueryOptionRebuild) { + hasRebuildQuery = true + } + } + + // If any of the pools was queried, then we'll need to show more fields. + if hasSpaceQuery { + for _, t := range pools[0].Usage() { + titles = append(titles, + t.TierName+" Size", + t.TierName+" Used", + t.TierName+" Imbalance") + } + titles = append(titles, "Disabled") + titles = append(titles, "UpgradeNeeded?") + } + + if hasRebuildQuery { + titles = append(titles, "Rebuild State") + } + + formatter := txtfmt.NewTableFormatter(titles...) + + var table []txtfmt.TableRow + for _, pool := range pools { + table = append(table, poolListCreateRowVerbose(pool, hasSpaceQuery, hasRebuildQuery)) + } + + fmt.Fprintln(out, formatter.Format(table)) + + return nil +} + +// PrintPoolList generates a human-readable representation of the supplied +// slice of daos.PoolInfo structs and writes it to the supplied io.Writer. +func PrintPoolList(pools []*daos.PoolInfo, out io.Writer, verbose bool) error { + if len(pools) == 0 { + fmt.Fprintln(out, "no pools in system") + return nil + } + + if verbose { + return printVerbosePoolList(pools, out) + } + + return printPoolList(pools, out) +} diff --git a/src/control/cmd/daos/pretty/pool_test.go b/src/control/cmd/daos/pretty/pool_test.go index cd3580e047d..e342be18e37 100644 --- a/src/control/cmd/daos/pretty/pool_test.go +++ b/src/control/cmd/daos/pretty/pool_test.go @@ -12,6 +12,7 @@ import ( "strings" "testing" + "github.com/dustin/go-humanize" "github.com/google/go-cmp/cmp" "github.com/google/uuid" @@ -445,3 +446,263 @@ Target: type unknown, state drain }) } } + +func TestPretty_PrintListPools(t *testing.T) { + exampleTierStats := []*daos.StorageUsageStats{ + { + MediaType: daos.StorageMediaTypeScm, + Total: 100 * humanize.GByte, + Free: 20 * humanize.GByte, + Min: 5 * humanize.GByte, + Max: 6 * humanize.GByte, + }, + { + MediaType: daos.StorageMediaTypeNvme, + Total: 6 * humanize.TByte, + Free: 1 * humanize.TByte, + Min: 20 * humanize.GByte, + Max: 50 * humanize.GByte, + }, + } + + for name, tc := range map[string]struct { + pools []*daos.PoolInfo + verbose bool + expErr error + expPrintStr string + }{ + "empty list": { + expPrintStr: ` +no pools in system +`, + }, + "one pool; no usage": { + pools: []*daos.PoolInfo{ + { + UUID: test.MockPoolUUID(1), + ServiceReplicas: []ranklist.Rank{0, 1, 2}, + State: daos.PoolServiceStateReady, + }, + }, + expPrintStr: ` +Pool Size State Used Imbalance Disabled +---- ---- ----- ---- --------- -------- +00000001 0 B Ready 0% 0% 0/0 + +`, + }, + "two pools; only one labeled": { + pools: []*daos.PoolInfo{ + { + UUID: test.MockPoolUUID(1), + ServiceReplicas: []ranklist.Rank{0, 1, 2}, + TierStats: exampleTierStats, + TotalTargets: 16, + ActiveTargets: 16, + DisabledTargets: 0, + State: daos.PoolServiceStateReady, + PoolLayoutVer: 1, + UpgradeLayoutVer: 2, + }, + { + Label: "two", + UUID: test.MockPoolUUID(2), + ServiceReplicas: []ranklist.Rank{3, 4, 5}, + TierStats: exampleTierStats, + TotalTargets: 64, + ActiveTargets: 56, + DisabledTargets: 8, + State: daos.PoolServiceStateReady, + PoolLayoutVer: 1, + UpgradeLayoutVer: 2, + }, + }, + expPrintStr: ` +Pool Size State Used Imbalance Disabled UpgradeNeeded? +---- ---- ----- ---- --------- -------- -------------- +00000001 6.0 TB Ready 83% 16% 0/16 1->2 +two 6.0 TB Ready 83% 56% 8/64 1->2 + +`, + }, + "two pools; one SCM only": { + pools: []*daos.PoolInfo{ + { + Label: "one", + UUID: test.MockPoolUUID(1), + ServiceReplicas: []ranklist.Rank{0, 1, 2}, + TierStats: exampleTierStats, + TotalTargets: 16, + ActiveTargets: 16, + DisabledTargets: 0, + State: daos.PoolServiceStateReady, + PoolLayoutVer: 1, + UpgradeLayoutVer: 2, + QueryMask: daos.DefaultPoolQueryMask, + }, + { + Label: "two", + UUID: test.MockPoolUUID(2), + ServiceReplicas: []ranklist.Rank{3, 4, 5}, + TierStats: []*daos.StorageUsageStats{ + exampleTierStats[0], + {MediaType: daos.StorageMediaTypeNvme}, + }, + TotalTargets: 64, + ActiveTargets: 56, + DisabledTargets: 8, + State: daos.PoolServiceStateReady, + PoolLayoutVer: 2, + UpgradeLayoutVer: 2, + QueryMask: daos.DefaultPoolQueryMask, + }, + }, + expPrintStr: ` +Pool Size State Used Imbalance Disabled UpgradeNeeded? +---- ---- ----- ---- --------- -------- -------------- +one 6.0 TB Ready 83% 16% 0/16 1->2 +two 100 GB Ready 80% 56% 8/64 None + +`, + }, + "verbose, empty response": { + verbose: true, + expPrintStr: ` +no pools in system +`, + }, + "verbose; zero svc replicas": { + pools: []*daos.PoolInfo{ + { + UUID: test.MockPoolUUID(1), + TierStats: exampleTierStats, + TotalTargets: 16, + ActiveTargets: 16, + DisabledTargets: 0, + State: daos.PoolServiceStateReady, + PoolLayoutVer: 1, + UpgradeLayoutVer: 2, + Rebuild: &daos.PoolRebuildStatus{ + State: daos.PoolRebuildStateIdle, + }, + QueryMask: daos.DefaultPoolQueryMask, + }, + }, + verbose: true, + expPrintStr: ` +Label UUID State SvcReps SCM Size SCM Used SCM Imbalance NVME Size NVME Used NVME Imbalance Disabled UpgradeNeeded? Rebuild State +----- ---- ----- ------- -------- -------- ------------- --------- --------- -------------- -------- -------------- ------------- +- 00000001-0001-0001-0001-000000000001 Ready N/A 100 GB 80 GB 16% 6.0 TB 5.0 TB 8% 0/16 1->2 idle + +`, + }, + "verbose; zero svc replicas with no query": { + pools: []*daos.PoolInfo{ + { + UUID: test.MockPoolUUID(1), + TierStats: exampleTierStats, + TotalTargets: 16, + ActiveTargets: 16, + DisabledTargets: 0, + State: daos.PoolServiceStateReady, + PoolLayoutVer: 1, + UpgradeLayoutVer: 2, + }, + }, + verbose: true, + expPrintStr: ` +Label UUID State SvcReps +----- ---- ----- ------- +- 00000001-0001-0001-0001-000000000001 Ready N/A + +`, + }, + "verbose; two pools; one destroying": { + pools: []*daos.PoolInfo{ + { + Label: "one", + UUID: test.MockPoolUUID(1), + ServiceReplicas: []ranklist.Rank{0, 1, 2}, + TierStats: exampleTierStats, + TotalTargets: 16, + ActiveTargets: 16, + DisabledTargets: 0, + State: daos.PoolServiceStateReady, + PoolLayoutVer: 1, + UpgradeLayoutVer: 2, + Rebuild: &daos.PoolRebuildStatus{ + State: daos.PoolRebuildStateIdle, + }, + QueryMask: daos.DefaultPoolQueryMask, + }, + { + Label: "two", + UUID: test.MockPoolUUID(2), + ServiceReplicas: []ranklist.Rank{3, 4, 5}, + TierStats: exampleTierStats, + TotalTargets: 64, + ActiveTargets: 56, + DisabledTargets: 8, + State: daos.PoolServiceStateDestroying, + PoolLayoutVer: 2, + UpgradeLayoutVer: 2, + Rebuild: &daos.PoolRebuildStatus{ + State: daos.PoolRebuildStateDone, + }, + QueryMask: daos.DefaultPoolQueryMask, + }, + }, + verbose: true, + expPrintStr: ` +Label UUID State SvcReps SCM Size SCM Used SCM Imbalance NVME Size NVME Used NVME Imbalance Disabled UpgradeNeeded? Rebuild State +----- ---- ----- ------- -------- -------- ------------- --------- --------- -------------- -------- -------------- ------------- +one 00000001-0001-0001-0001-000000000001 Ready [0-2] 100 GB 80 GB 16% 6.0 TB 5.0 TB 8% 0/16 1->2 idle +two 00000002-0002-0002-0002-000000000002 Destroying [3-5] 100 GB 80 GB 56% 6.0 TB 5.0 TB 27% 8/64 None done + +`, + }, + "verbose; one pool; rebuild state busy": { + pools: []*daos.PoolInfo{ + { + Label: "one", + UUID: test.MockPoolUUID(1), + ServiceReplicas: []ranklist.Rank{0, 1, 2}, + TierStats: exampleTierStats, + TotalTargets: 16, + ActiveTargets: 8, + DisabledTargets: 8, + State: daos.PoolServiceStateDegraded, + PoolLayoutVer: 1, + UpgradeLayoutVer: 2, + Rebuild: &daos.PoolRebuildStatus{ + State: daos.PoolRebuildStateBusy, + }, + QueryMask: daos.DefaultPoolQueryMask, + }, + }, + verbose: true, + expPrintStr: ` +Label UUID State SvcReps SCM Size SCM Used SCM Imbalance NVME Size NVME Used NVME Imbalance Disabled UpgradeNeeded? Rebuild State +----- ---- ----- ------- -------- -------- ------------- --------- --------- -------------- -------- -------------- ------------- +one 00000001-0001-0001-0001-000000000001 Degraded [0-2] 100 GB 80 GB 8% 6.0 TB 5.0 TB 4% 8/16 1->2 busy + +`, + }, + } { + t.Run(name, func(t *testing.T) { + var bld strings.Builder + + // pass the same io writer to standard and error stream + // parameters to mimic combined output seen on terminal + err := PrintPoolList(tc.pools, &bld, tc.verbose) + test.CmpErr(t, tc.expErr, err) + if tc.expErr != nil { + return + } + + if diff := cmp.Diff(strings.TrimLeft(tc.expPrintStr, "\n"), bld.String()); diff != "" { + t.Fatalf("unexpected format string (-want, +got):\n%s\n", diff) + } + }) + } +} diff --git a/src/control/cmd/daos/pretty/ranks.go b/src/control/cmd/daos/pretty/ranks.go new file mode 100644 index 00000000000..da63cd37583 --- /dev/null +++ b/src/control/cmd/daos/pretty/ranks.go @@ -0,0 +1,31 @@ +// +// (C) Copyright 2020-2024 Intel Corporation. +// +// SPDX-License-Identifier: BSD-2-Clause-Patent +// + +package pretty + +import ( + "fmt" + + "github.com/daos-stack/daos/src/control/lib/ranklist" +) + +// rankList defines a type constraint for acceptable rank list formats. +type rankList interface { + []uint32 | []ranklist.Rank +} + +// PrintRanks returns a string representation of the given ranks. +func PrintRanks[rlt rankList](ranks rlt) string { + switch rl := any(ranks).(type) { + case []uint32: + return ranklist.RankSetFromRanks(ranklist.RanksFromUint32(rl)).RangedString() + case []ranklist.Rank: + return ranklist.RankSetFromRanks(rl).RangedString() + default: + // NB: This should never happen, due to the compiler check of the type constraint. + return fmt.Sprintf("unknown rank list type: %T", rl) + } +} diff --git a/src/control/cmd/daos/pretty/ranks_test.go b/src/control/cmd/daos/pretty/ranks_test.go new file mode 100644 index 00000000000..42673e753d1 --- /dev/null +++ b/src/control/cmd/daos/pretty/ranks_test.go @@ -0,0 +1,60 @@ +// +// (C) Copyright 2024 Intel Corporation. +// +// SPDX-License-Identifier: BSD-2-Clause-Patent +// + +package pretty_test + +import ( + "testing" + + "github.com/daos-stack/daos/src/control/cmd/daos/pretty" + "github.com/daos-stack/daos/src/control/lib/ranklist" +) + +func TestPretty_PrintRanks_Uint32(t *testing.T) { + for name, tc := range map[string]struct { + ranks []uint32 + expString string + }{ + "nil": {}, + "empty": { + ranks: []uint32{}, + }, + "ranks": { + ranks: []uint32{0, 1, 2, 3}, + expString: "[0-3]", + }, + } { + t.Run(name, func(t *testing.T) { + gotString := pretty.PrintRanks(tc.ranks) + if gotString != tc.expString { + t.Fatalf("got %q, want %q", gotString, tc.expString) + } + }) + } +} + +func TestPretty_PrintRanks_RanklistRank(t *testing.T) { + for name, tc := range map[string]struct { + ranks []ranklist.Rank + expString string + }{ + "nil": {}, + "empty": { + ranks: []ranklist.Rank{}, + }, + "ranks": { + ranks: []ranklist.Rank{0, 1, 2, 3}, + expString: "[0-3]", + }, + } { + t.Run(name, func(t *testing.T) { + gotString := pretty.PrintRanks(tc.ranks) + if gotString != tc.expString { + t.Fatalf("got %q, want %q", gotString, tc.expString) + } + }) + } +} diff --git a/src/control/cmd/daos/util.go b/src/control/cmd/daos/util.go index e9334c0f110..0bcbd5ab3a8 100644 --- a/src/control/cmd/daos/util.go +++ b/src/control/cmd/daos/util.go @@ -19,6 +19,7 @@ import ( "github.com/daos-stack/daos/src/control/common/cmdutil" "github.com/daos-stack/daos/src/control/lib/daos" + "github.com/daos-stack/daos/src/control/lib/ranklist" "github.com/daos-stack/daos/src/control/logging" ) @@ -100,6 +101,20 @@ func uuidFromC(cUUID C.uuid_t) (uuid.UUID, error) { return uuid.FromBytes(C.GoBytes(unsafe.Pointer(&cUUID[0]), C.int(len(cUUID)))) } +func rankSetFromC(cRankList *C.d_rank_list_t) (*ranklist.RankSet, error) { + if cRankList == nil { + return nil, errors.New("nil ranklist") + } + + cRankSlice := unsafe.Slice(cRankList.rl_ranks, cRankList.rl_nr) + rs := ranklist.NewRankSet() + for _, cRank := range cRankSlice { + rs.Add(ranklist.Rank(cRank)) + } + + return rs, nil +} + func iterStringsBuf(cBuf unsafe.Pointer, expected C.size_t, cb func(string)) error { var curLen C.size_t diff --git a/src/control/cmd/dmg/pretty/pool.go b/src/control/cmd/dmg/pretty/pool.go index 0c697a54cb1..c2445dc61e3 100644 --- a/src/control/cmd/dmg/pretty/pool.go +++ b/src/control/cmd/dmg/pretty/pool.go @@ -16,7 +16,6 @@ import ( pretty "github.com/daos-stack/daos/src/control/cmd/daos/pretty" "github.com/daos-stack/daos/src/control/lib/control" "github.com/daos-stack/daos/src/control/lib/daos" - "github.com/daos-stack/daos/src/control/lib/ranklist" "github.com/daos-stack/daos/src/control/lib/txtfmt" ) @@ -90,8 +89,8 @@ func PrintPoolCreateResponse(pcr *control.PoolCreateResp, out io.Writer, opts .. fmtArgs := make([]txtfmt.TableRow, 0, 6) fmtArgs = append(fmtArgs, txtfmt.TableRow{"UUID": pcr.UUID}) fmtArgs = append(fmtArgs, txtfmt.TableRow{"Service Leader": fmt.Sprintf("%d", pcr.Leader)}) - fmtArgs = append(fmtArgs, txtfmt.TableRow{"Service Ranks": formatRanks(pcr.SvcReps)}) - fmtArgs = append(fmtArgs, txtfmt.TableRow{"Storage Ranks": formatRanks(pcr.TgtRanks)}) + fmtArgs = append(fmtArgs, txtfmt.TableRow{"Service Ranks": pretty.PrintRanks(pcr.SvcReps)}) + fmtArgs = append(fmtArgs, txtfmt.TableRow{"Storage Ranks": pretty.PrintRanks(pcr.TgtRanks)}) fmtArgs = append(fmtArgs, txtfmt.TableRow{"Total Size": humanize.Bytes(totalSize * numRanks)}) title := "Pool created with " @@ -112,167 +111,6 @@ func PrintPoolCreateResponse(pcr *control.PoolCreateResp, out io.Writer, opts .. return err } -func poolListCreateRow(pool *daos.PoolInfo, upgrade bool) txtfmt.TableRow { - // display size of the largest non-empty tier - var size uint64 - poolUsage := pool.Usage() - for ti := len(poolUsage) - 1; ti >= 0; ti-- { - if poolUsage[ti].Size != 0 { - size = poolUsage[ti].Size - break - } - } - - // display usage of the most used tier - var used int - for ti := 0; ti < len(poolUsage); ti++ { - t := poolUsage[ti] - u := float64(t.Size-t.Free) / float64(t.Size) - - if int(u*100) > used { - used = int(u * 100) - } - } - - // display imbalance of the most imbalanced tier - var imbalance uint32 - for ti := 0; ti < len(poolUsage); ti++ { - if poolUsage[ti].Imbalance > imbalance { - imbalance = poolUsage[ti].Imbalance - } - } - - row := txtfmt.TableRow{ - "Pool": pool.Name(), - "Size": humanize.Bytes(size), - "State": pool.State.String(), - "Used": fmt.Sprintf("%d%%", used), - "Imbalance": fmt.Sprintf("%d%%", imbalance), - "Disabled": fmt.Sprintf("%d/%d", pool.DisabledTargets, pool.TotalTargets), - } - - if upgrade { - upgradeString := "None" - - if pool.PoolLayoutVer != pool.UpgradeLayoutVer { - upgradeString = fmt.Sprintf("%d->%d", pool.PoolLayoutVer, pool.UpgradeLayoutVer) - } - row["UpgradeNeeded?"] = upgradeString - } - - return row -} - -func printListPoolsResp(out io.Writer, resp *control.ListPoolsResp) error { - if len(resp.Pools) == 0 { - fmt.Fprintln(out, "no pools in system") - return nil - } - upgrade := false - for _, pool := range resp.Pools { - if resp.PoolQueryError(pool.UUID) != nil { - continue - } - if pool.PoolLayoutVer != pool.UpgradeLayoutVer { - upgrade = true - } - } - - titles := []string{"Pool", "Size", "State", "Used", "Imbalance", "Disabled"} - if upgrade { - titles = append(titles, "UpgradeNeeded?") - } - formatter := txtfmt.NewTableFormatter(titles...) - - var table []txtfmt.TableRow - for _, pool := range resp.Pools { - if resp.PoolQueryError(pool.UUID) != nil { - continue - } - table = append(table, poolListCreateRow(pool, upgrade)) - } - - fmt.Fprintln(out, formatter.Format(table)) - - return nil -} - -func addVerboseTierUsage(row txtfmt.TableRow, usage *daos.PoolTierUsage) txtfmt.TableRow { - row[usage.TierName+" Size"] = humanize.Bytes(usage.Size) - row[usage.TierName+" Used"] = humanize.Bytes(usage.Size - usage.Free) - row[usage.TierName+" Imbalance"] = fmt.Sprintf("%d%%", usage.Imbalance) - - return row -} - -func poolListCreateRowVerbose(pool *daos.PoolInfo) txtfmt.TableRow { - label := pool.Label - if label == "" { - label = "-" - } - - svcReps := "N/A" - if len(pool.ServiceReplicas) != 0 { - rl := ranklist.RanksToUint32(pool.ServiceReplicas) - svcReps = formatRanks(rl) - } - - upgrade := "None" - if pool.PoolLayoutVer != pool.UpgradeLayoutVer { - upgrade = fmt.Sprintf("%d->%d", pool.PoolLayoutVer, pool.UpgradeLayoutVer) - } - - row := txtfmt.TableRow{ - "Label": label, - "UUID": pool.UUID.String(), - "State": pool.State.String(), - "SvcReps": svcReps, - "Disabled": fmt.Sprintf("%d/%d", pool.DisabledTargets, pool.TotalTargets), - "UpgradeNeeded?": upgrade, - "Rebuild State": pool.RebuildState(), - } - - for _, tu := range pool.Usage() { - row = addVerboseTierUsage(row, tu) - } - - return row -} - -func printListPoolsRespVerbose(noQuery bool, out io.Writer, resp *control.ListPoolsResp) error { - if len(resp.Pools) == 0 { - fmt.Fprintln(out, "no pools in system") - return nil - } - - titles := []string{"Label", "UUID", "State", "SvcReps"} - for _, t := range resp.Pools[0].Usage() { - titles = append(titles, - t.TierName+" Size", - t.TierName+" Used", - t.TierName+" Imbalance") - } - titles = append(titles, "Disabled") - titles = append(titles, "UpgradeNeeded?") - - if !noQuery { - titles = append(titles, "Rebuild State") - } - formatter := txtfmt.NewTableFormatter(titles...) - - var table []txtfmt.TableRow - for _, pool := range resp.Pools { - if resp.PoolQueryError(pool.UUID) != nil { - continue - } - table = append(table, poolListCreateRowVerbose(pool)) - } - - fmt.Fprintln(out, formatter.Format(table)) - - return nil -} - // PrintListPoolsResponse generates a human-readable representation of the // supplied ListPoolsResp struct and writes it to the supplied io.Writer. // Additional columns for pool UUID and service replicas if verbose is set. @@ -285,11 +123,16 @@ func PrintListPoolsResponse(out, outErr io.Writer, resp *control.ListPoolsResp, fmt.Fprintln(outErr, warn) } - if verbose { - return printListPoolsRespVerbose(noQuery, out, resp) + // Filter out any pools that had query errors. + queriedPools := make([]*daos.PoolInfo, 0, len(resp.Pools)) + for _, pool := range resp.Pools { + if _, found := resp.QueryErrors[pool.UUID]; found { + continue + } + queriedPools = append(queriedPools, pool) } - return printListPoolsResp(out, resp) + return pretty.PrintPoolList(queriedPools, out, verbose) } // PrintPoolProperties displays a two-column table of pool property names and values. diff --git a/src/control/cmd/dmg/pretty/pool_test.go b/src/control/cmd/dmg/pretty/pool_test.go index 9ecd6a4c174..a808cd63235 100644 --- a/src/control/cmd/dmg/pretty/pool_test.go +++ b/src/control/cmd/dmg/pretty/pool_test.go @@ -557,23 +557,6 @@ func TestPretty_PrintListPoolsResponse(t *testing.T) { resp: &control.ListPoolsResp{}, expPrintStr: ` no pools in system -`, - }, - "one pool; no usage": { - resp: &control.ListPoolsResp{ - Pools: []*daos.PoolInfo{ - { - UUID: test.MockPoolUUID(1), - ServiceReplicas: []ranklist.Rank{0, 1, 2}, - State: daos.PoolServiceStateReady, - }, - }, - }, - expPrintStr: ` -Pool Size State Used Imbalance Disabled ----- ---- ----- ---- --------- -------- -00000001 0 B Ready 0% 0% 0/0 - `, }, "one pool; no uuid": { @@ -615,82 +598,6 @@ Pool Size State Used Imbalance Disabled }, expErr: errors.New("has 1 storage tiers, want 2"), }, - "two pools; only one labeled": { - resp: &control.ListPoolsResp{ - Pools: []*daos.PoolInfo{ - { - UUID: test.MockPoolUUID(1), - ServiceReplicas: []ranklist.Rank{0, 1, 2}, - TierStats: exampleTierStats, - TotalTargets: 16, - ActiveTargets: 16, - DisabledTargets: 0, - State: daos.PoolServiceStateReady, - PoolLayoutVer: 1, - UpgradeLayoutVer: 2, - }, - { - Label: "two", - UUID: test.MockPoolUUID(2), - ServiceReplicas: []ranklist.Rank{3, 4, 5}, - TierStats: exampleTierStats, - TotalTargets: 64, - ActiveTargets: 56, - DisabledTargets: 8, - State: daos.PoolServiceStateReady, - PoolLayoutVer: 1, - UpgradeLayoutVer: 2, - }, - }, - }, - expPrintStr: ` -Pool Size State Used Imbalance Disabled UpgradeNeeded? ----- ---- ----- ---- --------- -------- -------------- -00000001 6.0 TB Ready 83% 16% 0/16 1->2 -two 6.0 TB Ready 83% 56% 8/64 1->2 - -`, - }, - "two pools; one SCM only": { - resp: &control.ListPoolsResp{ - Pools: []*daos.PoolInfo{ - { - Label: "one", - UUID: test.MockPoolUUID(1), - ServiceReplicas: []ranklist.Rank{0, 1, 2}, - TierStats: exampleTierStats, - TotalTargets: 16, - ActiveTargets: 16, - DisabledTargets: 0, - State: daos.PoolServiceStateReady, - PoolLayoutVer: 1, - UpgradeLayoutVer: 2, - }, - { - Label: "two", - UUID: test.MockPoolUUID(2), - ServiceReplicas: []ranklist.Rank{3, 4, 5}, - TierStats: []*daos.StorageUsageStats{ - exampleTierStats[0], - {MediaType: daos.StorageMediaTypeNvme}, - }, - TotalTargets: 64, - ActiveTargets: 56, - DisabledTargets: 8, - State: daos.PoolServiceStateReady, - PoolLayoutVer: 2, - UpgradeLayoutVer: 2, - }, - }, - }, - expPrintStr: ` -Pool Size State Used Imbalance Disabled UpgradeNeeded? ----- ---- ----- ---- --------- -------- -------------- -one 6.0 TB Ready 83% 16% 0/16 1->2 -two 100 GB Ready 80% 56% 8/64 None - -`, - }, "two pools; one failed query": { resp: &control.ListPoolsResp{ Pools: []*daos.PoolInfo{ @@ -776,128 +683,6 @@ one 6.0 TB Ready 83%% 16%% 0/16 verbose: true, expPrintStr: ` no pools in system -`, - }, - "verbose; zero svc replicas": { - resp: &control.ListPoolsResp{ - Pools: []*daos.PoolInfo{ - { - UUID: test.MockPoolUUID(1), - TierStats: exampleTierStats, - TotalTargets: 16, - ActiveTargets: 16, - DisabledTargets: 0, - State: daos.PoolServiceStateReady, - PoolLayoutVer: 1, - UpgradeLayoutVer: 2, - Rebuild: &daos.PoolRebuildStatus{ - State: daos.PoolRebuildStateIdle, - }, - }, - }, - }, - verbose: true, - expPrintStr: ` -Label UUID State SvcReps SCM Size SCM Used SCM Imbalance NVME Size NVME Used NVME Imbalance Disabled UpgradeNeeded? Rebuild State ------ ---- ----- ------- -------- -------- ------------- --------- --------- -------------- -------- -------------- ------------- -- 00000001-0001-0001-0001-000000000001 Ready N/A 100 GB 80 GB 16% 6.0 TB 5.0 TB 8% 0/16 1->2 idle - -`, - }, - "verbose; zero svc replicas with no query": { - resp: &control.ListPoolsResp{ - Pools: []*daos.PoolInfo{ - { - UUID: test.MockPoolUUID(1), - TierStats: exampleTierStats, - TotalTargets: 16, - ActiveTargets: 16, - DisabledTargets: 0, - State: daos.PoolServiceStateReady, - PoolLayoutVer: 1, - UpgradeLayoutVer: 2, - }, - }, - }, - verbose: true, - noQuery: true, - expPrintStr: ` -Label UUID State SvcReps SCM Size SCM Used SCM Imbalance NVME Size NVME Used NVME Imbalance Disabled UpgradeNeeded? ------ ---- ----- ------- -------- -------- ------------- --------- --------- -------------- -------- -------------- -- 00000001-0001-0001-0001-000000000001 Ready N/A 100 GB 80 GB 16% 6.0 TB 5.0 TB 8% 0/16 1->2 - -`, - }, - "verbose; two pools; one destroying": { - resp: &control.ListPoolsResp{ - Pools: []*daos.PoolInfo{ - { - Label: "one", - UUID: test.MockPoolUUID(1), - ServiceReplicas: []ranklist.Rank{0, 1, 2}, - TierStats: exampleTierStats, - TotalTargets: 16, - ActiveTargets: 16, - DisabledTargets: 0, - State: daos.PoolServiceStateReady, - PoolLayoutVer: 1, - UpgradeLayoutVer: 2, - Rebuild: &daos.PoolRebuildStatus{ - State: daos.PoolRebuildStateIdle, - }, - }, - { - Label: "two", - UUID: test.MockPoolUUID(2), - ServiceReplicas: []ranklist.Rank{3, 4, 5}, - TierStats: exampleTierStats, - TotalTargets: 64, - ActiveTargets: 56, - DisabledTargets: 8, - State: daos.PoolServiceStateDestroying, - PoolLayoutVer: 2, - UpgradeLayoutVer: 2, - Rebuild: &daos.PoolRebuildStatus{ - State: daos.PoolRebuildStateDone, - }, - }, - }, - }, - verbose: true, - expPrintStr: ` -Label UUID State SvcReps SCM Size SCM Used SCM Imbalance NVME Size NVME Used NVME Imbalance Disabled UpgradeNeeded? Rebuild State ------ ---- ----- ------- -------- -------- ------------- --------- --------- -------------- -------- -------------- ------------- -one 00000001-0001-0001-0001-000000000001 Ready [0-2] 100 GB 80 GB 16% 6.0 TB 5.0 TB 8% 0/16 1->2 idle -two 00000002-0002-0002-0002-000000000002 Destroying [3-5] 100 GB 80 GB 56% 6.0 TB 5.0 TB 27% 8/64 None done - -`, - }, - "verbose; one pools; rebuild state busy": { - resp: &control.ListPoolsResp{ - Pools: []*daos.PoolInfo{ - { - Label: "one", - UUID: test.MockPoolUUID(1), - ServiceReplicas: []ranklist.Rank{0, 1, 2}, - TierStats: exampleTierStats, - TotalTargets: 16, - ActiveTargets: 8, - DisabledTargets: 8, - State: daos.PoolServiceStateDegraded, - PoolLayoutVer: 1, - UpgradeLayoutVer: 2, - Rebuild: &daos.PoolRebuildStatus{ - State: daos.PoolRebuildStateBusy, - }, - }, - }, - }, - verbose: true, - expPrintStr: ` -Label UUID State SvcReps SCM Size SCM Used SCM Imbalance NVME Size NVME Used NVME Imbalance Disabled UpgradeNeeded? Rebuild State ------ ---- ----- ------- -------- -------- ------------- --------- --------- -------------- -------- -------------- ------------- -one 00000001-0001-0001-0001-000000000001 Degraded [0-2] 100 GB 80 GB 8% 6.0 TB 5.0 TB 4% 8/16 1->2 busy - `, }, } { diff --git a/src/control/cmd/dmg/pretty/ranks.go b/src/control/cmd/dmg/pretty/ranks.go deleted file mode 100644 index f13b678b58e..00000000000 --- a/src/control/cmd/dmg/pretty/ranks.go +++ /dev/null @@ -1,16 +0,0 @@ -// -// (C) Copyright 2020-2022 Intel Corporation. -// -// SPDX-License-Identifier: BSD-2-Clause-Patent -// - -package pretty - -import "github.com/daos-stack/daos/src/control/lib/ranklist" - -// formatRanks takes a slice of uint32 ranks and returns a string -// representation of the set created from the slice. -func formatRanks(ranks []uint32) string { - rs := ranklist.RankSetFromRanks(ranklist.RanksFromUint32(ranks)) - return rs.RangedString() -} diff --git a/src/control/common/proto/srv/srv.pb.go b/src/control/common/proto/srv/srv.pb.go index 3734e1883fc..20eafc21963 100644 --- a/src/control/common/proto/srv/srv.pb.go +++ b/src/control/common/proto/srv/srv.pb.go @@ -8,7 +8,7 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: -// protoc-gen-go v1.28.1 +// protoc-gen-go v1.31.0 // protoc v3.5.0 // source: srv/srv.proto @@ -422,6 +422,163 @@ func (x *PoolFindByLabelResp) GetSvcreps() []uint32 { return nil } +type ListPoolsReq struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + IncludeAll bool `protobuf:"varint,1,opt,name=includeAll,proto3" json:"includeAll,omitempty"` // Include all pools in response, regardless of state +} + +func (x *ListPoolsReq) Reset() { + *x = ListPoolsReq{} + if protoimpl.UnsafeEnabled { + mi := &file_srv_srv_proto_msgTypes[6] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *ListPoolsReq) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*ListPoolsReq) ProtoMessage() {} + +func (x *ListPoolsReq) ProtoReflect() protoreflect.Message { + mi := &file_srv_srv_proto_msgTypes[6] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use ListPoolsReq.ProtoReflect.Descriptor instead. +func (*ListPoolsReq) Descriptor() ([]byte, []int) { + return file_srv_srv_proto_rawDescGZIP(), []int{6} +} + +func (x *ListPoolsReq) GetIncludeAll() bool { + if x != nil { + return x.IncludeAll + } + return false +} + +type ListPoolsResp struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + Pools []*ListPoolsResp_Pool `protobuf:"bytes,1,rep,name=pools,proto3" json:"pools,omitempty"` // List of pools +} + +func (x *ListPoolsResp) Reset() { + *x = ListPoolsResp{} + if protoimpl.UnsafeEnabled { + mi := &file_srv_srv_proto_msgTypes[7] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *ListPoolsResp) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*ListPoolsResp) ProtoMessage() {} + +func (x *ListPoolsResp) ProtoReflect() protoreflect.Message { + mi := &file_srv_srv_proto_msgTypes[7] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use ListPoolsResp.ProtoReflect.Descriptor instead. +func (*ListPoolsResp) Descriptor() ([]byte, []int) { + return file_srv_srv_proto_rawDescGZIP(), []int{7} +} + +func (x *ListPoolsResp) GetPools() []*ListPoolsResp_Pool { + if x != nil { + return x.Pools + } + return nil +} + +type ListPoolsResp_Pool struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + Uuid string `protobuf:"bytes,1,opt,name=uuid,proto3" json:"uuid,omitempty"` // Pool UUID + Label string `protobuf:"bytes,2,opt,name=label,proto3" json:"label,omitempty"` // Pool label + Svcreps []uint32 `protobuf:"varint,3,rep,packed,name=svcreps,proto3" json:"svcreps,omitempty"` // Pool service ranks +} + +func (x *ListPoolsResp_Pool) Reset() { + *x = ListPoolsResp_Pool{} + if protoimpl.UnsafeEnabled { + mi := &file_srv_srv_proto_msgTypes[8] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *ListPoolsResp_Pool) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*ListPoolsResp_Pool) ProtoMessage() {} + +func (x *ListPoolsResp_Pool) ProtoReflect() protoreflect.Message { + mi := &file_srv_srv_proto_msgTypes[8] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use ListPoolsResp_Pool.ProtoReflect.Descriptor instead. +func (*ListPoolsResp_Pool) Descriptor() ([]byte, []int) { + return file_srv_srv_proto_rawDescGZIP(), []int{7, 0} +} + +func (x *ListPoolsResp_Pool) GetUuid() string { + if x != nil { + return x.Uuid + } + return "" +} + +func (x *ListPoolsResp_Pool) GetLabel() string { + if x != nil { + return x.Label + } + return "" +} + +func (x *ListPoolsResp_Pool) GetSvcreps() []uint32 { + if x != nil { + return x.Svcreps + } + return nil +} + var File_srv_srv_proto protoreflect.FileDescriptor var file_srv_srv_proto_rawDesc = []byte{ @@ -466,12 +623,23 @@ var file_srv_srv_proto_rawDesc = []byte{ 0x74, 0x75, 0x73, 0x18, 0x01, 0x20, 0x01, 0x28, 0x05, 0x52, 0x06, 0x73, 0x74, 0x61, 0x74, 0x75, 0x73, 0x12, 0x12, 0x0a, 0x04, 0x75, 0x75, 0x69, 0x64, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x75, 0x75, 0x69, 0x64, 0x12, 0x18, 0x0a, 0x07, 0x73, 0x76, 0x63, 0x72, 0x65, 0x70, 0x73, - 0x18, 0x03, 0x20, 0x03, 0x28, 0x0d, 0x52, 0x07, 0x73, 0x76, 0x63, 0x72, 0x65, 0x70, 0x73, 0x42, - 0x39, 0x5a, 0x37, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x64, 0x61, - 0x6f, 0x73, 0x2d, 0x73, 0x74, 0x61, 0x63, 0x6b, 0x2f, 0x64, 0x61, 0x6f, 0x73, 0x2f, 0x73, 0x72, - 0x63, 0x2f, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2f, 0x63, 0x6f, 0x6d, 0x6d, 0x6f, 0x6e, - 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2f, 0x73, 0x72, 0x76, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, - 0x6f, 0x33, + 0x18, 0x03, 0x20, 0x03, 0x28, 0x0d, 0x52, 0x07, 0x73, 0x76, 0x63, 0x72, 0x65, 0x70, 0x73, 0x22, + 0x2e, 0x0a, 0x0c, 0x4c, 0x69, 0x73, 0x74, 0x50, 0x6f, 0x6f, 0x6c, 0x73, 0x52, 0x65, 0x71, 0x12, + 0x1e, 0x0a, 0x0a, 0x69, 0x6e, 0x63, 0x6c, 0x75, 0x64, 0x65, 0x41, 0x6c, 0x6c, 0x18, 0x01, 0x20, + 0x01, 0x28, 0x08, 0x52, 0x0a, 0x69, 0x6e, 0x63, 0x6c, 0x75, 0x64, 0x65, 0x41, 0x6c, 0x6c, 0x22, + 0x8a, 0x01, 0x0a, 0x0d, 0x4c, 0x69, 0x73, 0x74, 0x50, 0x6f, 0x6f, 0x6c, 0x73, 0x52, 0x65, 0x73, + 0x70, 0x12, 0x2d, 0x0a, 0x05, 0x70, 0x6f, 0x6f, 0x6c, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, + 0x32, 0x17, 0x2e, 0x73, 0x72, 0x76, 0x2e, 0x4c, 0x69, 0x73, 0x74, 0x50, 0x6f, 0x6f, 0x6c, 0x73, + 0x52, 0x65, 0x73, 0x70, 0x2e, 0x50, 0x6f, 0x6f, 0x6c, 0x52, 0x05, 0x70, 0x6f, 0x6f, 0x6c, 0x73, + 0x1a, 0x4a, 0x0a, 0x04, 0x50, 0x6f, 0x6f, 0x6c, 0x12, 0x12, 0x0a, 0x04, 0x75, 0x75, 0x69, 0x64, + 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x75, 0x75, 0x69, 0x64, 0x12, 0x14, 0x0a, 0x05, + 0x6c, 0x61, 0x62, 0x65, 0x6c, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x6c, 0x61, 0x62, + 0x65, 0x6c, 0x12, 0x18, 0x0a, 0x07, 0x73, 0x76, 0x63, 0x72, 0x65, 0x70, 0x73, 0x18, 0x03, 0x20, + 0x03, 0x28, 0x0d, 0x52, 0x07, 0x73, 0x76, 0x63, 0x72, 0x65, 0x70, 0x73, 0x42, 0x39, 0x5a, 0x37, + 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x64, 0x61, 0x6f, 0x73, 0x2d, + 0x73, 0x74, 0x61, 0x63, 0x6b, 0x2f, 0x64, 0x61, 0x6f, 0x73, 0x2f, 0x73, 0x72, 0x63, 0x2f, 0x63, + 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2f, 0x63, 0x6f, 0x6d, 0x6d, 0x6f, 0x6e, 0x2f, 0x70, 0x72, + 0x6f, 0x74, 0x6f, 0x2f, 0x73, 0x72, 0x76, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, } var ( @@ -486,7 +654,7 @@ func file_srv_srv_proto_rawDescGZIP() []byte { return file_srv_srv_proto_rawDescData } -var file_srv_srv_proto_msgTypes = make([]protoimpl.MessageInfo, 6) +var file_srv_srv_proto_msgTypes = make([]protoimpl.MessageInfo, 9) var file_srv_srv_proto_goTypes = []interface{}{ (*NotifyReadyReq)(nil), // 0: srv.NotifyReadyReq (*BioErrorReq)(nil), // 1: srv.BioErrorReq @@ -494,13 +662,17 @@ var file_srv_srv_proto_goTypes = []interface{}{ (*GetPoolSvcResp)(nil), // 3: srv.GetPoolSvcResp (*PoolFindByLabelReq)(nil), // 4: srv.PoolFindByLabelReq (*PoolFindByLabelResp)(nil), // 5: srv.PoolFindByLabelResp + (*ListPoolsReq)(nil), // 6: srv.ListPoolsReq + (*ListPoolsResp)(nil), // 7: srv.ListPoolsResp + (*ListPoolsResp_Pool)(nil), // 8: srv.ListPoolsResp.Pool } var file_srv_srv_proto_depIdxs = []int32{ - 0, // [0:0] is the sub-list for method output_type - 0, // [0:0] is the sub-list for method input_type - 0, // [0:0] is the sub-list for extension type_name - 0, // [0:0] is the sub-list for extension extendee - 0, // [0:0] is the sub-list for field type_name + 8, // 0: srv.ListPoolsResp.pools:type_name -> srv.ListPoolsResp.Pool + 1, // [1:1] is the sub-list for method output_type + 1, // [1:1] is the sub-list for method input_type + 1, // [1:1] is the sub-list for extension type_name + 1, // [1:1] is the sub-list for extension extendee + 0, // [0:1] is the sub-list for field type_name } func init() { file_srv_srv_proto_init() } @@ -581,6 +753,42 @@ func file_srv_srv_proto_init() { return nil } } + file_srv_srv_proto_msgTypes[6].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*ListPoolsReq); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_srv_srv_proto_msgTypes[7].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*ListPoolsResp); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_srv_srv_proto_msgTypes[8].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*ListPoolsResp_Pool); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } } type x struct{} out := protoimpl.TypeBuilder{ @@ -588,7 +796,7 @@ func file_srv_srv_proto_init() { GoPackagePath: reflect.TypeOf(x{}).PkgPath(), RawDescriptor: file_srv_srv_proto_rawDesc, NumEnums: 0, - NumMessages: 6, + NumMessages: 9, NumExtensions: 0, NumServices: 0, }, diff --git a/src/control/drpc/modules.go b/src/control/drpc/modules.go index 0aacbae1d4a..84165ad3d01 100644 --- a/src/control/drpc/modules.go +++ b/src/control/drpc/modules.go @@ -261,9 +261,12 @@ func (m srvMethod) ID() int32 { func (m srvMethod) String() string { if s, ok := map[srvMethod]string{ - MethodNotifyReady: "notify ready", - MethodBIOError: "block i/o error", - MethodClusterEvent: "cluster event", + MethodNotifyReady: "notify ready", + MethodBIOError: "block i/o error", + MethodClusterEvent: "cluster event", + MethodGetPoolServiceRanks: "get pool service ranks", + MethodPoolFindByLabel: "find pool by label", + MethodListPools: "list pools", }[m]; ok { return s } @@ -293,6 +296,8 @@ const ( MethodPoolFindByLabel srvMethod = C.DRPC_METHOD_SRV_POOL_FIND_BYLABEL // MethodClusterEvent notifies of a cluster event in the I/O Engine. MethodClusterEvent srvMethod = C.DRPC_METHOD_SRV_CLUSTER_EVENT + // MethodListPools requests the list of pools in the system + MethodListPools srvMethod = C.DRPC_METHOD_SRV_LIST_POOLS ) type securityMethod int32 diff --git a/src/control/go.mod b/src/control/go.mod index 298769214b3..248e35b6817 100644 --- a/src/control/go.mod +++ b/src/control/go.mod @@ -1,6 +1,6 @@ module github.com/daos-stack/daos/src/control -go 1.17 +go 1.20 require ( github.com/Jille/raft-grpc-transport v1.2.0 diff --git a/src/control/go.sum b/src/control/go.sum index 48980442ea1..31fa37b995d 100644 --- a/src/control/go.sum +++ b/src/control/go.sum @@ -356,8 +356,8 @@ golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81R golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= golang.org/x/net v0.0.0-20210525063256-abc453219eb5/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20210907225631-ff17edfbf26d/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= -golang.org/x/net v0.22.0 h1:9sGLhx7iRIHEiX0oAJ3MRZMUCElJgy7Br1nO+AMN3Tc= -golang.org/x/net v0.22.0/go.mod h1:JKghWKKOSdJwpW2GEx0Ja7fmaKnMsbu+MWVZTokSYmg= +golang.org/x/net v0.23.0 h1:7EYJ93RZ9vYSZAIb2x3lnuvqO5zneoD6IvWjuhfxjTs= +golang.org/x/net v0.23.0/go.mod h1:JKghWKKOSdJwpW2GEx0Ja7fmaKnMsbu+MWVZTokSYmg= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= diff --git a/src/control/server/mgmt_drpc.go b/src/control/server/mgmt_drpc.go index b45570ecc57..70ada0c493e 100644 --- a/src/control/server/mgmt_drpc.go +++ b/src/control/server/mgmt_drpc.go @@ -47,6 +47,7 @@ func (mod *mgmtModule) ID() drpc.ModuleID { type poolResolver interface { FindPoolServiceByLabel(string) (*system.PoolService, error) FindPoolServiceByUUID(uuid.UUID) (*system.PoolService, error) + PoolServiceList(bool) ([]*system.PoolService, error) } // srvModule represents the daos_server dRPC module. It handles dRPCs sent by @@ -70,7 +71,15 @@ func newSrvModule(log logging.Logger, sysdb poolResolver, engines []Engine, even } // HandleCall is the handler for calls to the srvModule. -func (mod *srvModule) HandleCall(_ context.Context, session *drpc.Session, method drpc.Method, req []byte) ([]byte, error) { +func (mod *srvModule) HandleCall(_ context.Context, session *drpc.Session, method drpc.Method, req []byte) (_ []byte, err error) { + defer func() { + msg := ": success" + if err != nil { + msg = ", failed: " + err.Error() + } + mod.log.Tracef("srv upcall: %s%s", method, msg) + }() + switch method { case drpc.MethodNotifyReady: return nil, mod.handleNotifyReady(req) @@ -82,6 +91,8 @@ func (mod *srvModule) HandleCall(_ context.Context, session *drpc.Session, metho return mod.handlePoolFindByLabel(req) case drpc.MethodClusterEvent: return mod.handleClusterEvent(req) + case drpc.MethodListPools: + return mod.handleListPools(req) default: return nil, drpc.UnknownMethodFailure() } @@ -198,3 +209,29 @@ func (mod *srvModule) handleClusterEvent(reqb []byte) ([]byte, error) { return proto.Marshal(resp) } + +func (mod *srvModule) handleListPools(reqb []byte) ([]byte, error) { + req := new(srvpb.ListPoolsReq) + if err := proto.Unmarshal(reqb, req); err != nil { + return nil, drpc.UnmarshalingPayloadFailure() + } + + mod.log.Tracef("%T: %+v", req, req) + pools, err := mod.sysdb.PoolServiceList(req.IncludeAll) + if err != nil { + return nil, errors.Wrap(err, "failed to list system pools") + } + + resp := new(srvpb.ListPoolsResp) + resp.Pools = make([]*srvpb.ListPoolsResp_Pool, len(pools)) + for i, ps := range pools { + resp.Pools[i] = &srvpb.ListPoolsResp_Pool{ + Uuid: ps.PoolUUID.String(), + Label: ps.PoolLabel, + Svcreps: ranklist.RanksToUint32(ps.Replicas), + } + } + mod.log.Tracef("%T %+v", resp, resp) + + return proto.Marshal(resp) +} diff --git a/src/engine/drpc_client.c b/src/engine/drpc_client.c index 9be829e0f11..54db5c15e0e 100644 --- a/src/engine/drpc_client.c +++ b/src/engine/drpc_client.c @@ -414,6 +414,104 @@ ds_pool_find_bylabel(d_const_string_t label, uuid_t pool_uuid, return rc; } +int +ds_get_pool_list(uint64_t *npools, daos_mgmt_pool_info_t *pools) +{ + struct drpc_alloc alloc = PROTO_ALLOCATOR_INIT(alloc); + Srv__ListPoolsReq lp_req = SRV__LIST_POOLS_REQ__INIT; + Srv__ListPoolsResp *lp_resp = NULL; + Drpc__Response *dresp; + uint8_t *req; + size_t req_size; + d_rank_list_t *svc_ranks; + int i, rc; + + if (npools == NULL) { + D_ERROR("npools may not be NULL\n"); + D_GOTO(out, rc = -DER_INVAL); + } + + lp_req.includeall = false; /* only list Ready pools */ + + req_size = srv__list_pools_req__get_packed_size(&lp_req); + D_ALLOC(req, req_size); + if (req == NULL) + D_GOTO(out, rc = -DER_NOMEM); + srv__list_pools_req__pack(&lp_req, req); + + rc = dss_drpc_call(DRPC_MODULE_SRV, DRPC_METHOD_SRV_LIST_POOLS, req, req_size, + 0 /* flags */, &dresp); + if (rc != 0) + goto out_req; + if (dresp->status != DRPC__STATUS__SUCCESS) { + D_ERROR("received erroneous dRPC response: %d\n", dresp->status); + D_GOTO(out_dresp, rc = -DER_IO); + } + + lp_resp = srv__list_pools_resp__unpack(&alloc.alloc, dresp->body.len, dresp->body.data); + if (alloc.oom) { + D_GOTO(out_dresp, rc = -DER_NOMEM); + } else if (lp_resp == NULL) { + D_ERROR("failed to unpack resp (list pools)\n"); + D_GOTO(out_dresp, rc = -DER_NOMEM); + } + + if (*npools > 0 && lp_resp->n_pools > *npools) { + D_ERROR("pool list exceeds request buffer (req: %lu, actual: %lu)", *npools, + lp_resp->n_pools); + D_GOTO(out_resp, rc = -DER_TRUNC); + } + + *npools = lp_resp->n_pools; + if (pools == NULL) { + /* caller just needs the # of pools */ + D_GOTO(out_resp, rc); + } + + for (i = 0; i < lp_resp->n_pools; i++) { + daos_mgmt_pool_info_t *mgmt_pool = &pools[i]; + Srv__ListPoolsResp__Pool *resp_pool = lp_resp->pools[i]; + + rc = uuid_parse(resp_pool->uuid, mgmt_pool->mgpi_uuid); + if (rc != 0) { + D_ERROR("failed to parse pool uuid: %d\n", rc); + D_GOTO(out_free_pools, rc = -DER_INVAL); + } + + D_STRNDUP(mgmt_pool->mgpi_label, resp_pool->label, DAOS_PROP_LABEL_MAX_LEN); + if (mgmt_pool->mgpi_label == NULL) { + D_ERROR("failed to copy pool label\n"); + D_GOTO(out_free_pools, rc = -DER_NOMEM); + } + + svc_ranks = uint32_array_to_rank_list(resp_pool->svcreps, resp_pool->n_svcreps); + if (svc_ranks == NULL) { + D_ERROR("failed to create svc ranks list\n"); + D_GOTO(out_free_pools, rc = -DER_NOMEM); + } + mgmt_pool->mgpi_svc = svc_ranks; + } + +out_free_pools: + if (rc != 0 && pools != NULL) { + for (i = 0; i < lp_resp->n_pools; i++) { + daos_mgmt_pool_info_t *mgmt_pool = &pools[i]; + if (mgmt_pool->mgpi_label) + D_FREE(mgmt_pool->mgpi_label); + if (mgmt_pool->mgpi_svc) + d_rank_list_free(mgmt_pool->mgpi_svc); + } + } +out_resp: + srv__list_pools_resp__free_unpacked(lp_resp, &alloc.alloc); +out_dresp: + drpc_response_free(dresp); +out_req: + D_FREE(req); +out: + return rc; +} + int drpc_init(void) { diff --git a/src/engine/srv.pb-c.c b/src/engine/srv.pb-c.c index 8db1c142a9f..9db07ec8187 100644 --- a/src/engine/srv.pb-c.c +++ b/src/engine/srv.pb-c.c @@ -277,6 +277,88 @@ void srv__pool_find_by_label_resp__free_unpacked assert(message->base.descriptor == &srv__pool_find_by_label_resp__descriptor); protobuf_c_message_free_unpacked ((ProtobufCMessage*)message, allocator); } +void +srv__list_pools_req__init(Srv__ListPoolsReq *message) +{ + static const Srv__ListPoolsReq init_value = SRV__LIST_POOLS_REQ__INIT; + *message = init_value; +} +size_t +srv__list_pools_req__get_packed_size(const Srv__ListPoolsReq *message) +{ + assert(message->base.descriptor == &srv__list_pools_req__descriptor); + return protobuf_c_message_get_packed_size((const ProtobufCMessage *)(message)); +} +size_t +srv__list_pools_req__pack(const Srv__ListPoolsReq *message, uint8_t *out) +{ + assert(message->base.descriptor == &srv__list_pools_req__descriptor); + return protobuf_c_message_pack((const ProtobufCMessage *)message, out); +} +size_t +srv__list_pools_req__pack_to_buffer(const Srv__ListPoolsReq *message, ProtobufCBuffer *buffer) +{ + assert(message->base.descriptor == &srv__list_pools_req__descriptor); + return protobuf_c_message_pack_to_buffer((const ProtobufCMessage *)message, buffer); +} +Srv__ListPoolsReq * +srv__list_pools_req__unpack(ProtobufCAllocator *allocator, size_t len, const uint8_t *data) +{ + return (Srv__ListPoolsReq *)protobuf_c_message_unpack(&srv__list_pools_req__descriptor, allocator, + len, data); +} +void +srv__list_pools_req__free_unpacked(Srv__ListPoolsReq *message, ProtobufCAllocator *allocator) +{ + if (!message) + return; + assert(message->base.descriptor == &srv__list_pools_req__descriptor); + protobuf_c_message_free_unpacked((ProtobufCMessage *)message, allocator); +} +void +srv__list_pools_resp__pool__init(Srv__ListPoolsResp__Pool *message) +{ + static const Srv__ListPoolsResp__Pool init_value = SRV__LIST_POOLS_RESP__POOL__INIT; + *message = init_value; +} +void +srv__list_pools_resp__init(Srv__ListPoolsResp *message) +{ + static const Srv__ListPoolsResp init_value = SRV__LIST_POOLS_RESP__INIT; + *message = init_value; +} +size_t +srv__list_pools_resp__get_packed_size(const Srv__ListPoolsResp *message) +{ + assert(message->base.descriptor == &srv__list_pools_resp__descriptor); + return protobuf_c_message_get_packed_size((const ProtobufCMessage *)(message)); +} +size_t +srv__list_pools_resp__pack(const Srv__ListPoolsResp *message, uint8_t *out) +{ + assert(message->base.descriptor == &srv__list_pools_resp__descriptor); + return protobuf_c_message_pack((const ProtobufCMessage *)message, out); +} +size_t +srv__list_pools_resp__pack_to_buffer(const Srv__ListPoolsResp *message, ProtobufCBuffer *buffer) +{ + assert(message->base.descriptor == &srv__list_pools_resp__descriptor); + return protobuf_c_message_pack_to_buffer((const ProtobufCMessage *)message, buffer); +} +Srv__ListPoolsResp * +srv__list_pools_resp__unpack(ProtobufCAllocator *allocator, size_t len, const uint8_t *data) +{ + return (Srv__ListPoolsResp *)protobuf_c_message_unpack(&srv__list_pools_resp__descriptor, + allocator, len, data); +} +void +srv__list_pools_resp__free_unpacked(Srv__ListPoolsResp *message, ProtobufCAllocator *allocator) +{ + if (!message) + return; + assert(message->base.descriptor == &srv__list_pools_resp__descriptor); + protobuf_c_message_free_unpacked((ProtobufCMessage *)message, allocator); +} static const ProtobufCFieldDescriptor srv__notify_ready_req__field_descriptors[6] = { { @@ -687,3 +769,101 @@ const ProtobufCMessageDescriptor srv__pool_find_by_label_resp__descriptor = (ProtobufCMessageInit) srv__pool_find_by_label_resp__init, NULL,NULL,NULL /* reserved[123] */ }; +static const ProtobufCFieldDescriptor srv__list_pools_req__field_descriptors[1] = { + { + "includeAll", 1, PROTOBUF_C_LABEL_NONE, PROTOBUF_C_TYPE_BOOL, 0, /* quantifier_offset */ + offsetof(Srv__ListPoolsReq, includeall), NULL, NULL, 0, /* flags */ + 0, NULL, NULL /* reserved1,reserved2, etc */ + }, +}; +static const unsigned srv__list_pools_req__field_indices_by_name[] = { + 0, /* field[0] = includeAll */ +}; +static const ProtobufCIntRange srv__list_pools_req__number_ranges[1 + 1] = {{1, 0}, {0, 1}}; +const ProtobufCMessageDescriptor srv__list_pools_req__descriptor = { + PROTOBUF_C__MESSAGE_DESCRIPTOR_MAGIC, + "srv.ListPoolsReq", + "ListPoolsReq", + "Srv__ListPoolsReq", + "srv", + sizeof(Srv__ListPoolsReq), + 1, + srv__list_pools_req__field_descriptors, + srv__list_pools_req__field_indices_by_name, + 1, + srv__list_pools_req__number_ranges, + (ProtobufCMessageInit)srv__list_pools_req__init, + NULL, + NULL, + NULL /* reserved[123] */ +}; +static const ProtobufCFieldDescriptor srv__list_pools_resp__pool__field_descriptors[3] = { + { + "uuid", 1, PROTOBUF_C_LABEL_NONE, PROTOBUF_C_TYPE_STRING, 0, /* quantifier_offset */ + offsetof(Srv__ListPoolsResp__Pool, uuid), NULL, &protobuf_c_empty_string, 0, /* flags */ + 0, NULL, NULL /* reserved1,reserved2, etc */ + }, + { + "label", 2, PROTOBUF_C_LABEL_NONE, PROTOBUF_C_TYPE_STRING, 0, /* quantifier_offset */ + offsetof(Srv__ListPoolsResp__Pool, label), NULL, &protobuf_c_empty_string, 0, /* flags */ + 0, NULL, NULL /* reserved1,reserved2, etc */ + }, + { + "svcreps", 3, PROTOBUF_C_LABEL_REPEATED, PROTOBUF_C_TYPE_UINT32, + offsetof(Srv__ListPoolsResp__Pool, n_svcreps), offsetof(Srv__ListPoolsResp__Pool, svcreps), + NULL, NULL, 0, /* flags */ + 0, NULL, NULL /* reserved1,reserved2, etc */ + }, +}; +static const unsigned srv__list_pools_resp__pool__field_indices_by_name[] = { + 1, /* field[1] = label */ + 2, /* field[2] = svcreps */ + 0, /* field[0] = uuid */ +}; +static const ProtobufCIntRange srv__list_pools_resp__pool__number_ranges[1 + 1] = {{1, 0}, {0, 3}}; +const ProtobufCMessageDescriptor srv__list_pools_resp__pool__descriptor = { + PROTOBUF_C__MESSAGE_DESCRIPTOR_MAGIC, + "srv.ListPoolsResp.Pool", + "Pool", + "Srv__ListPoolsResp__Pool", + "srv", + sizeof(Srv__ListPoolsResp__Pool), + 3, + srv__list_pools_resp__pool__field_descriptors, + srv__list_pools_resp__pool__field_indices_by_name, + 1, + srv__list_pools_resp__pool__number_ranges, + (ProtobufCMessageInit)srv__list_pools_resp__pool__init, + NULL, + NULL, + NULL /* reserved[123] */ +}; +static const ProtobufCFieldDescriptor srv__list_pools_resp__field_descriptors[1] = { + { + "pools", 1, PROTOBUF_C_LABEL_REPEATED, PROTOBUF_C_TYPE_MESSAGE, + offsetof(Srv__ListPoolsResp, n_pools), offsetof(Srv__ListPoolsResp, pools), + &srv__list_pools_resp__pool__descriptor, NULL, 0, /* flags */ + 0, NULL, NULL /* reserved1,reserved2, etc */ + }, +}; +static const unsigned srv__list_pools_resp__field_indices_by_name[] = { + 0, /* field[0] = pools */ +}; +static const ProtobufCIntRange srv__list_pools_resp__number_ranges[1 + 1] = {{1, 0}, {0, 1}}; +const ProtobufCMessageDescriptor srv__list_pools_resp__descriptor = { + PROTOBUF_C__MESSAGE_DESCRIPTOR_MAGIC, + "srv.ListPoolsResp", + "ListPoolsResp", + "Srv__ListPoolsResp", + "srv", + sizeof(Srv__ListPoolsResp), + 1, + srv__list_pools_resp__field_descriptors, + srv__list_pools_resp__field_indices_by_name, + 1, + srv__list_pools_resp__number_ranges, + (ProtobufCMessageInit)srv__list_pools_resp__init, + NULL, + NULL, + NULL /* reserved[123] */ +}; diff --git a/src/engine/srv.pb-c.h b/src/engine/srv.pb-c.h index 4f65ce7b43c..8a6006b1faa 100644 --- a/src/engine/srv.pb-c.h +++ b/src/engine/srv.pb-c.h @@ -21,7 +21,9 @@ typedef struct _Srv__GetPoolSvcReq Srv__GetPoolSvcReq; typedef struct _Srv__GetPoolSvcResp Srv__GetPoolSvcResp; typedef struct _Srv__PoolFindByLabelReq Srv__PoolFindByLabelReq; typedef struct _Srv__PoolFindByLabelResp Srv__PoolFindByLabelResp; - +typedef struct _Srv__ListPoolsReq Srv__ListPoolsReq; +typedef struct _Srv__ListPoolsResp Srv__ListPoolsResp; +typedef struct _Srv__ListPoolsResp__Pool Srv__ListPoolsResp__Pool; /* --- enums --- */ @@ -163,6 +165,54 @@ struct _Srv__PoolFindByLabelResp { PROTOBUF_C_MESSAGE_INIT (&srv__pool_find_by_label_resp__descriptor) \ , 0, (char *)protobuf_c_empty_string, 0,NULL } +struct _Srv__ListPoolsReq { + ProtobufCMessage base; + /* + * Include all pools in response, regardless of state + */ + protobuf_c_boolean includeall; +}; +#define SRV__LIST_POOLS_REQ__INIT \ + { \ + PROTOBUF_C_MESSAGE_INIT(&srv__list_pools_req__descriptor) \ + , 0 \ + } + +struct _Srv__ListPoolsResp__Pool { + ProtobufCMessage base; + /* + * Pool UUID + */ + char *uuid; + /* + * Pool label + */ + char *label; + /* + * Pool service ranks + */ + size_t n_svcreps; + uint32_t *svcreps; +}; +#define SRV__LIST_POOLS_RESP__POOL__INIT \ + { \ + PROTOBUF_C_MESSAGE_INIT(&srv__list_pools_resp__pool__descriptor) \ + , (char *)protobuf_c_empty_string, (char *)protobuf_c_empty_string, 0, NULL \ + } + +struct _Srv__ListPoolsResp { + ProtobufCMessage base; + /* + * List of pools + */ + size_t n_pools; + Srv__ListPoolsResp__Pool **pools; +}; +#define SRV__LIST_POOLS_RESP__INIT \ + { \ + PROTOBUF_C_MESSAGE_INIT(&srv__list_pools_resp__descriptor) \ + , 0, NULL \ + } /* Srv__NotifyReadyReq methods */ void srv__notify_ready_req__init @@ -278,6 +328,35 @@ Srv__PoolFindByLabelResp * void srv__pool_find_by_label_resp__free_unpacked (Srv__PoolFindByLabelResp *message, ProtobufCAllocator *allocator); +/* Srv__ListPoolsReq methods */ +void +srv__list_pools_req__init(Srv__ListPoolsReq *message); +size_t +srv__list_pools_req__get_packed_size(const Srv__ListPoolsReq *message); +size_t +srv__list_pools_req__pack(const Srv__ListPoolsReq *message, uint8_t *out); +size_t +srv__list_pools_req__pack_to_buffer(const Srv__ListPoolsReq *message, ProtobufCBuffer *buffer); +Srv__ListPoolsReq * +srv__list_pools_req__unpack(ProtobufCAllocator *allocator, size_t len, const uint8_t *data); +void +srv__list_pools_req__free_unpacked(Srv__ListPoolsReq *message, ProtobufCAllocator *allocator); +/* Srv__ListPoolsResp__Pool methods */ +void +srv__list_pools_resp__pool__init(Srv__ListPoolsResp__Pool *message); +/* Srv__ListPoolsResp methods */ +void +srv__list_pools_resp__init(Srv__ListPoolsResp *message); +size_t +srv__list_pools_resp__get_packed_size(const Srv__ListPoolsResp *message); +size_t +srv__list_pools_resp__pack(const Srv__ListPoolsResp *message, uint8_t *out); +size_t +srv__list_pools_resp__pack_to_buffer(const Srv__ListPoolsResp *message, ProtobufCBuffer *buffer); +Srv__ListPoolsResp * +srv__list_pools_resp__unpack(ProtobufCAllocator *allocator, size_t len, const uint8_t *data); +void +srv__list_pools_resp__free_unpacked(Srv__ListPoolsResp *message, ProtobufCAllocator *allocator); /* --- per-message closures --- */ typedef void (*Srv__NotifyReadyReq_Closure) @@ -298,6 +377,10 @@ typedef void (*Srv__PoolFindByLabelReq_Closure) typedef void (*Srv__PoolFindByLabelResp_Closure) (const Srv__PoolFindByLabelResp *message, void *closure_data); +typedef void (*Srv__ListPoolsReq_Closure)(const Srv__ListPoolsReq *message, void *closure_data); +typedef void (*Srv__ListPoolsResp__Pool_Closure)(const Srv__ListPoolsResp__Pool *message, + void *closure_data); +typedef void (*Srv__ListPoolsResp_Closure)(const Srv__ListPoolsResp *message, void *closure_data); /* --- services --- */ @@ -310,6 +393,9 @@ extern const ProtobufCMessageDescriptor srv__get_pool_svc_req__descriptor; extern const ProtobufCMessageDescriptor srv__get_pool_svc_resp__descriptor; extern const ProtobufCMessageDescriptor srv__pool_find_by_label_req__descriptor; extern const ProtobufCMessageDescriptor srv__pool_find_by_label_resp__descriptor; +extern const ProtobufCMessageDescriptor srv__list_pools_req__descriptor; +extern const ProtobufCMessageDescriptor srv__list_pools_resp__descriptor; +extern const ProtobufCMessageDescriptor srv__list_pools_resp__pool__descriptor; PROTOBUF_C__END_DECLS diff --git a/src/include/daos/drpc.pb-c.h b/src/include/daos/drpc.pb-c.h index 6083131d433..24cfdcfcf7f 100644 --- a/src/include/daos/drpc.pb-c.h +++ b/src/include/daos/drpc.pb-c.h @@ -10,14 +10,12 @@ PROTOBUF_C__BEGIN_DECLS #if PROTOBUF_C_VERSION_NUMBER < 1003000 # error This file was generated by a newer version of protoc-c which is incompatible with your libprotobuf-c headers. Please update your headers. -#elif 1003000 < PROTOBUF_C_MIN_COMPILER_VERSION +#elif 1004001 < PROTOBUF_C_MIN_COMPILER_VERSION # error This file was generated by an older version of protoc-c which is incompatible with your libprotobuf-c headers. Please regenerate this file with a newer version of protoc-c. #endif - -typedef struct _Drpc__Call Drpc__Call; -typedef struct _Drpc__Response Drpc__Response; - +typedef struct Drpc__Call Drpc__Call; +typedef struct Drpc__Response Drpc__Response; /* --- enums --- */ @@ -65,25 +63,24 @@ typedef enum _Drpc__Status { /* * Call describes a function call to be executed over the dRPC channel. */ -struct _Drpc__Call -{ - ProtobufCMessage base; - /* - * ID of the module to process the call. - */ - int32_t module; - /* - * ID of the method to be executed. - */ - int32_t method; - /* - * Sequence number for matching a response to this call. - */ - int64_t sequence; - /* - * Input payload to be used by the method. - */ - ProtobufCBinaryData body; +struct Drpc__Call { + ProtobufCMessage base; + /* + * ID of the module to process the call. + */ + int32_t module; + /* + * ID of the method to be executed. + */ + int32_t method; + /* + * Sequence number for matching a response to this call. + */ + int64_t sequence; + /* + * Input payload to be used by the method. + */ + ProtobufCBinaryData body; }; #define DRPC__CALL__INIT \ { PROTOBUF_C_MESSAGE_INIT (&drpc__call__descriptor) \ @@ -93,21 +90,21 @@ struct _Drpc__Call /* * Response describes the result of a dRPC call. */ -struct _Drpc__Response -{ - ProtobufCMessage base; - /* - * Sequence number of the Call that triggered this response. - */ - int64_t sequence; - /* - * High-level status of the RPC. If SUCCESS, method-specific status may be included in the body. - */ - Drpc__Status status; - /* - * Output payload produced by the method. - */ - ProtobufCBinaryData body; +struct Drpc__Response { + ProtobufCMessage base; + /* + * Sequence number of the Call that triggered this response. + */ + int64_t sequence; + /* + * High-level status of the RPC. If SUCCESS, method-specific status may be included in the + * body. + */ + Drpc__Status status; + /* + * Output payload produced by the method. + */ + ProtobufCBinaryData body; }; #define DRPC__RESPONSE__INIT \ { PROTOBUF_C_MESSAGE_INIT (&drpc__response__descriptor) \ diff --git a/src/include/daos/drpc_modules.h b/src/include/daos/drpc_modules.h index a8821d9f079..67e1a50c3b9 100644 --- a/src/include/daos/drpc_modules.h +++ b/src/include/daos/drpc_modules.h @@ -74,13 +74,14 @@ enum drpc_mgmt_method { }; enum drpc_srv_method { - DRPC_METHOD_SRV_NOTIFY_READY = 301, - DRPC_METHOD_SRV_BIO_ERR = 302, - DRPC_METHOD_SRV_GET_POOL_SVC = 303, - DRPC_METHOD_SRV_CLUSTER_EVENT = 304, - DRPC_METHOD_SRV_POOL_FIND_BYLABEL = 305, + DRPC_METHOD_SRV_NOTIFY_READY = 301, + DRPC_METHOD_SRV_BIO_ERR = 302, + DRPC_METHOD_SRV_GET_POOL_SVC = 303, + DRPC_METHOD_SRV_CLUSTER_EVENT = 304, + DRPC_METHOD_SRV_POOL_FIND_BYLABEL = 305, + DRPC_METHOD_SRV_LIST_POOLS = 306, - NUM_DRPC_SRV_METHODS /* Must be last */ + NUM_DRPC_SRV_METHODS /* Must be last */ }; enum drpc_sec_method { diff --git a/src/include/daos/mgmt.h b/src/include/daos/mgmt.h index 8572b18b8b2..5020fe84af7 100644 --- a/src/include/daos/mgmt.h +++ b/src/include/daos/mgmt.h @@ -64,6 +64,8 @@ int dc_mgmt_get_pool_svc_ranks(struct dc_mgmt_sys *sys, const uuid_t puuid, d_rank_list_t **svcranksp); int dc_mgmt_pool_find(struct dc_mgmt_sys *sys, const char *label, uuid_t puuid, d_rank_list_t **svcranksp); +int + dc_mgmt_pool_list(tse_task_t *task); int dc_mgmt_notify_pool_connect(struct dc_pool *pool); int dc_mgmt_notify_pool_disconnect(struct dc_pool *pool); int dc_mgmt_notify_exit(void); diff --git a/src/include/daos_mgmt.h b/src/include/daos_mgmt.h index 5a9e0e49274..291bda0b820 100644 --- a/src/include/daos_mgmt.h +++ b/src/include/daos_mgmt.h @@ -53,6 +53,8 @@ typedef struct { * daos_pool_info_t mgpi_info; */ uuid_t mgpi_uuid; + /** Pool label */ + d_string_t mgpi_label; /** List of current pool service replica ranks */ d_rank_list_t *mgpi_svc; /** Current pool service leader */ @@ -77,6 +79,10 @@ typedef struct { int daos_pool_stop_svc(daos_handle_t poh, daos_event_t *ev); +int +daos_mgmt_list_pools(const char *group, daos_size_t *npools, daos_mgmt_pool_info_t *pools, + daos_event_t *ev); + /** * The operation code for DAOS client to set different parameters globally * on all servers. diff --git a/src/include/daos_srv/daos_engine.h b/src/include/daos_srv/daos_engine.h index a7588e36e3f..a5407084ab8 100644 --- a/src/include/daos_srv/daos_engine.h +++ b/src/include/daos_srv/daos_engine.h @@ -22,6 +22,7 @@ #include #include #include +#include #include #include #include @@ -711,6 +712,8 @@ ds_notify_bio_error(int media_err_type, int tgt_id); int ds_get_pool_svc_ranks(uuid_t pool_uuid, d_rank_list_t **svc_ranks); int ds_pool_find_bylabel(d_const_string_t label, uuid_t pool_uuid, d_rank_list_t **svc_ranks); +int +ds_get_pool_list(uint64_t *npools, daos_mgmt_pool_info_t *pools); /** Flags for dss_drpc_call */ enum dss_drpc_call_flag { diff --git a/src/include/daos_srv/pool.h b/src/include/daos_srv/pool.h index 576e547ef10..44855afc9ca 100644 --- a/src/include/daos_srv/pool.h +++ b/src/include/daos_srv/pool.h @@ -243,6 +243,8 @@ int ds_pool_svc_upgrade(uuid_t pool_uuid, d_rank_list_t *ranks); int ds_pool_failed_add(uuid_t uuid, int rc); void ds_pool_failed_remove(uuid_t uuid); int ds_pool_failed_lookup(uuid_t uuid); +int +ds_pool_check_access(uuid_t pool_uuid, d_rank_list_t *ps_ranks, unsigned int flags, d_iov_t *cred); /* * Called by dmg on the pool service leader to list all pool handles of a pool. diff --git a/src/include/daos_task.h b/src/include/daos_task.h index e6e00af1f7a..26f3126110f 100644 --- a/src/include/daos_task.h +++ b/src/include/daos_task.h @@ -27,12 +27,13 @@ extern "C" { /** DAOS operation codes for task creation */ typedef enum { - DAOS_OPC_INVALID = -1, + DAOS_OPC_INVALID = -1, /** Management APIs */ /* Starting at 0 will break Application Binary Interface backward * compatibility */ - DAOS_OPC_SET_PARAMS = 3, + DAOS_OPC_MGMT_LIST_POOLS = 2, + DAOS_OPC_SET_PARAMS, DAOS_OPC_MGMT_GET_BS_STATE, /** Pool APIs */ @@ -186,6 +187,16 @@ struct daos_obj_register_class_t { struct daos_oclass_attr *cattr; }; +/** pool management pool list args */ +typedef struct { + /** Process set name of the DAOS servers managing the pool */ + const char *grp; + /** Array of pool mgmt information structures. */ + daos_mgmt_pool_info_t *pools; + /** length of array */ + daos_size_t *npools; +} daos_mgmt_pool_list_t; + /** pool query args */ typedef struct { /** Pool open handle. */ diff --git a/src/mgmt/cli_mgmt.c b/src/mgmt/cli_mgmt.c index bc5da348369..354b0ef794b 100644 --- a/src/mgmt/cli_mgmt.c +++ b/src/mgmt/cli_mgmt.c @@ -1,5 +1,5 @@ /* - * (C) Copyright 2016-2023 Intel Corporation. + * (C) Copyright 2016-2024 Intel Corporation. * * SPDX-License-Identifier: BSD-2-Clause-Patent */ @@ -20,6 +20,7 @@ #include #include #include +#include #include "svc.pb-c.h" #include "rpc.h" #include @@ -1282,6 +1283,160 @@ dc_mgmt_tm_register(const char *sys, const char *jobid, key_t shm_key, uid_t *ow return rc; } +int +dc_mgmt_pool_list(tse_task_t *task) +{ + daos_mgmt_pool_list_t *args; + d_rank_list_t *ms_ranks; + struct rsvc_client ms_client; + crt_endpoint_t ep; + crt_rpc_t *rpc = NULL; + crt_opcode_t opc; + struct mgmt_pool_list_in *in = NULL; + struct mgmt_pool_list_out *out = NULL; + struct dc_mgmt_sys *sys; + int rc, pidx; + + args = dc_task_get_args(task); + if (args->npools == NULL) { + D_ERROR("npools argument must not be NULL"); + D_GOTO(out, rc = -DER_INVAL); + } + + rc = dc_mgmt_sys_attach(args->grp, &sys); + if (rc != 0) { + DL_ERROR(rc, "cannot attach to DAOS system: %s", args->grp); + D_GOTO(out, rc); + } + + ms_ranks = sys->sy_info.ms_ranks; + D_ASSERT(ms_ranks->rl_nr > 0); + + rc = rsvc_client_init(&ms_client, ms_ranks); + if (rc != 0) { + DL_ERROR(rc, "failed to init ms client"); + D_GOTO(out_grp, rc); + } + + ep.ep_grp = sys->sy_group; + ep.ep_tag = daos_rpc_tag(DAOS_REQ_MGMT, 0); + opc = DAOS_RPC_OPCODE(MGMT_POOL_LIST, DAOS_MGMT_MODULE, DAOS_MGMT_VERSION); + +rechoose: + rc = rsvc_client_choose(&ms_client, &ep); + if (rc != 0) { + DL_ERROR(rc, "failed to choose MS rank"); + D_GOTO(out_client, rc); + } + + rc = crt_req_create(daos_task2ctx(task), &ep, opc, &rpc); + if (rc != 0) { + DL_ERROR(rc, "crt_req_create(MGMT_POOL_LIST) failed"); + D_GOTO(out_client, rc); + } + + in = crt_req_get(rpc); + in->pli_grp = (d_string_t)args->grp; + /* If provided pools is NULL, caller needs the number of pools + * to be returned in npools. Set npools=0 in the request in this case + * (caller value may be uninitialized). + */ + if (args->pools == NULL) + in->pli_npools = 0; + else + in->pli_npools = *args->npools; + + /* Now fill in the client credential for the pool access checks. */ + rc = dc_sec_request_creds(&in->pli_cred); + if (rc != 0) { + DL_ERROR(rc, "failed to obtain security credential"); + D_GOTO(out_put_req, rc); + } + + D_DEBUG(DB_MGMT, "req_npools=" DF_U64 " (pools=%p, *npools=" DF_U64 "\n", in->pli_npools, + args->pools, *args->npools); + + crt_req_addref(rpc); + rc = daos_rpc_send_wait(rpc); + if (rc != 0) { + DL_ERROR(rc, "rpc send failed"); + crt_req_decref(rpc); + goto rechoose; + } + + out = crt_reply_get(rpc); + D_ASSERT(out != NULL); + + rc = rsvc_client_complete_rpc(&ms_client, &ep, rc, out->plo_op.mo_rc, &out->plo_op.mo_hint); + if (rc == RSVC_CLIENT_RECHOOSE) { + crt_req_decref(rpc); + goto rechoose; + } + + rc = out->plo_op.mo_rc; + if (rc != 0) { + DL_ERROR(rc, "failed to list pools"); + D_GOTO(out_put_req, rc); + } + + *args->npools = out->plo_npools; + + /* copy RPC response pools info to client buffer, if provided */ + if (args->pools) { + /* Response ca_count expected <= client-specified npools */ + for (pidx = 0; pidx < out->plo_pools.ca_count; pidx++) { + struct mgmt_pool_list_pool *rpc_pool = &out->plo_pools.ca_arrays[pidx]; + daos_mgmt_pool_info_t *cli_pool = &args->pools[pidx]; + + uuid_copy(cli_pool->mgpi_uuid, rpc_pool->plp_uuid); + + D_STRNDUP(cli_pool->mgpi_label, rpc_pool->plp_label, + DAOS_PROP_LABEL_MAX_LEN); + if (cli_pool->mgpi_label == NULL) { + D_ERROR("copy RPC reply label failed\n"); + D_GOTO(out_free_args_pools, rc = -DER_NOMEM); + } + + /* allocate rank list for caller (simplifies API) */ + rc = d_rank_list_dup(&cli_pool->mgpi_svc, rpc_pool->plp_svc_list); + if (rc != 0) { + D_ERROR("copy RPC reply svc list failed\n"); + D_GOTO(out_free_args_pools, rc = -DER_NOMEM); + } + } + } + +out_free_args_pools: + if (args->pools && (rc != 0)) { + for (pidx = 0; pidx < out->plo_pools.ca_count; pidx++) { + daos_mgmt_pool_info_t *pool = &args->pools[pidx]; + + if (pool->mgpi_label) + D_FREE(pool->mgpi_label); + if (pool->mgpi_svc) + d_rank_list_free(pool->mgpi_svc); + } + } +out_put_req: + if (rc != 0) + DL_ERROR(rc, "failed to list pools"); + + crt_req_decref(rpc); + + if (in != NULL) { + /* Ensure credential memory is wiped clean */ + explicit_bzero(in->pli_cred.iov_buf, in->pli_cred.iov_buf_len); + daos_iov_free(&in->pli_cred); + } +out_client: + rsvc_client_fini(&ms_client); +out_grp: + dc_mgmt_sys_detach(sys); +out: + tse_task_complete(task, rc); + return rc; +} + /** * Initialize management interface */ diff --git a/src/mgmt/rpc.c b/src/mgmt/rpc.c index 59d4cb0d646..5a5a9b3873a 100644 --- a/src/mgmt/rpc.c +++ b/src/mgmt/rpc.c @@ -9,9 +9,11 @@ #define D_LOGFAC DD_FAC(mgmt) #include +#include #include "rpc.h" CRT_GEN_PROC_FUNC(server_entry, DAOS_SEQ_SERVER_ENTRY); +CRT_GEN_PROC_FUNC(mgmt_pool_list_pool, DAOS_SEQ_MGMT_POOL_LIST_POOL); static int crt_proc_struct_server_entry(crt_proc_t proc, crt_proc_op_t proc_op, @@ -20,6 +22,44 @@ crt_proc_struct_server_entry(crt_proc_t proc, crt_proc_op_t proc_op, return crt_proc_server_entry(proc, data); } +static int +crt_proc_struct_mgmt_pool_list_pool(crt_proc_t proc, crt_proc_op_t proc_op, + struct mgmt_pool_list_pool *data) +{ + return crt_proc_mgmt_pool_list_pool(proc, data); +} + +/* FIXME: dupe of pool/rpc.c:36 */ +static int +crt_proc_struct_rsvc_hint(crt_proc_t proc, crt_proc_op_t proc_op, + struct rsvc_hint *hint) +{ + int rc; + + rc = crt_proc_uint32_t(proc, proc_op, &hint->sh_flags); + if (rc != 0) + return -DER_HG; + + rc = crt_proc_uint32_t(proc, proc_op, &hint->sh_rank); + if (rc != 0) + return -DER_HG; + + rc = crt_proc_uint64_t(proc, proc_op, &hint->sh_term); + if (rc != 0) + return -DER_HG; + + return 0; +} + +CRT_GEN_PROC_FUNC(mgmt_op_out, DAOS_OSEQ_MGMT_OP); + +static int +crt_proc_struct_mgmt_op_out(crt_proc_t proc, crt_proc_op_t proc_op, + struct mgmt_op_out *data) +{ + return crt_proc_mgmt_op_out(proc, data); +} + CRT_RPC_DEFINE(mgmt_svc_rip, DAOS_ISEQ_MGMT_SVR_RIP, DAOS_OSEQ_MGMT_SVR_RIP) CRT_RPC_DEFINE(mgmt_params_set, DAOS_ISEQ_MGMT_PARAMS_SET, DAOS_OSEQ_MGMT_PARAMS_SET) @@ -29,6 +69,7 @@ CRT_RPC_DEFINE(mgmt_pool_get_svcranks, DAOS_ISEQ_MGMT_POOL_GET_SVCRANKS, DAOS_OSEQ_MGMT_POOL_GET_SVCRANKS) CRT_RPC_DEFINE(mgmt_pool_find, DAOS_ISEQ_MGMT_POOL_FIND, DAOS_OSEQ_MGMT_POOL_FIND) +CRT_RPC_DEFINE(mgmt_pool_list, DAOS_ISEQ_MGMT_POOL_LIST, DAOS_OSEQ_MGMT_POOL_LIST) CRT_RPC_DEFINE(mgmt_mark, DAOS_ISEQ_MGMT_MARK, DAOS_OSEQ_MGMT_MARK) diff --git a/src/mgmt/rpc.h b/src/mgmt/rpc.h index f10840cc6c9..0d9824d84ba 100644 --- a/src/mgmt/rpc.h +++ b/src/mgmt/rpc.h @@ -38,6 +38,9 @@ X(MGMT_POOL_FIND, \ 0, &CQF_mgmt_pool_find, \ ds_mgmt_pool_find_hdlr, NULL), \ + X(MGMT_POOL_LIST, \ + 0, &CQF_mgmt_pool_list, \ + ds_mgmt_pool_list_hdlr, NULL), \ X(MGMT_MARK, \ 0, &CQF_mgmt_mark, \ ds_mgmt_mark_hdlr, NULL), \ @@ -87,6 +90,12 @@ enum mgmt_profile_op { extern struct crt_proto_format mgmt_proto_fmt; +#define DAOS_OSEQ_MGMT_OP /* output fields */ \ + ((int32_t) (mo_rc) CRT_VAR) \ + ((struct rsvc_hint) (mo_hint) CRT_VAR) + +CRT_GEN_STRUCT(mgmt_op_out, DAOS_OSEQ_MGMT_OP); + #define DAOS_ISEQ_MGMT_SVR_RIP /* input fields */ \ ((uint32_t) (rip_flags) CRT_VAR) @@ -143,6 +152,26 @@ CRT_RPC_DECLARE(mgmt_pool_find, DAOS_ISEQ_MGMT_POOL_FIND, #define MGMT_POOL_FIND_DUMMY_LABEL "NO LABEL, FINDING BY UUID" +#define DAOS_SEQ_MGMT_POOL_LIST_POOL /* listed pool */ \ + ((uuid_t) (plp_uuid) CRT_VAR) \ + ((d_string_t) (plp_label) CRT_VAR) \ + ((d_rank_t) (plp_svc_ldr) CRT_VAR) \ + ((d_rank_list_t) (plp_svc_list) CRT_PTR) + +CRT_GEN_STRUCT(mgmt_pool_list_pool, DAOS_SEQ_MGMT_POOL_LIST_POOL); + +#define DAOS_ISEQ_MGMT_POOL_LIST /* input fields */ \ + ((d_string_t) (pli_grp) CRT_VAR) \ + ((d_iov_t) (pli_cred) CRT_VAR) \ + ((uint64_t) (pli_npools) CRT_VAR) + +#define DAOS_OSEQ_MGMT_POOL_LIST /* output fields */ \ + ((struct mgmt_op_out) (plo_op) CRT_VAR) \ + ((struct mgmt_pool_list_pool) (plo_pools) CRT_ARRAY) \ + ((uint64_t) (plo_npools) CRT_VAR) + +CRT_RPC_DECLARE(mgmt_pool_list, DAOS_ISEQ_MGMT_POOL_LIST, DAOS_OSEQ_MGMT_POOL_LIST) + #define DAOS_ISEQ_MGMT_TGT_CREATE /* input fields */ \ ((uuid_t) (tc_pool_uuid) CRT_VAR) \ ((d_string_t) (tc_tgt_dev) CRT_VAR) \ diff --git a/src/mgmt/srv.c b/src/mgmt/srv.c index ed00b2c0de5..f0c52e3e4d7 100644 --- a/src/mgmt/srv.c +++ b/src/mgmt/srv.c @@ -1,5 +1,5 @@ /** - * (C) Copyright 2016-2022 Intel Corporation. + * (C) Copyright 2016-2024 Intel Corporation. * * SPDX-License-Identifier: BSD-2-Clause-Patent */ @@ -19,6 +19,7 @@ #include #include #include +#include #include #include @@ -412,6 +413,90 @@ void ds_mgmt_pool_find_hdlr(crt_rpc_t *rpc) d_rank_list_free(out->pfo_ranks); } +void +ds_mgmt_pool_list_hdlr(crt_rpc_t *rpc) +{ + struct mgmt_pool_list_in *in; + struct mgmt_pool_list_out *out; + size_t n_mgmt = 0, n_rpc = 0; + daos_mgmt_pool_info_t *mgmt_pools = NULL; + struct mgmt_pool_list_pool *rpc_pools = NULL; + int i, rc, chk_rc; + + in = crt_req_get(rpc); + D_ASSERT(in != NULL); + + out = crt_reply_get(rpc); + D_ASSERT(out != NULL); + + if (in->pli_npools > 0) { + D_ALLOC_ARRAY(mgmt_pools, in->pli_npools); + if (mgmt_pools == NULL) { + D_ERROR("failed to alloc mgmt pools\n"); + D_GOTO(send_resp, rc = -DER_NOMEM); + } + } + + n_mgmt = in->pli_npools; + rc = ds_get_pool_list(&n_mgmt, mgmt_pools); + if (rc != 0) { + DL_ERROR(rc, "ds_get_pool_list() failed"); + D_GOTO(send_resp, rc); + } + + /* caller just needs the number of pools */ + if (in->pli_npools == 0) { + out->plo_npools = n_mgmt; + D_GOTO(send_resp, rc); + } + + D_ALLOC_ARRAY(rpc_pools, n_mgmt); + if (rpc_pools == NULL) { + D_ERROR("failed to alloc response pools\n"); + D_GOTO(send_resp, rc = -DER_NOMEM); + } + + for (i = 0; i < n_mgmt; i++) { + daos_mgmt_pool_info_t *mgmt_pool = &mgmt_pools[i]; + struct mgmt_pool_list_pool *rpc_pool = &rpc_pools[n_rpc]; + chk_rc = ds_pool_check_access(mgmt_pool->mgpi_uuid, mgmt_pool->mgpi_svc, DAOS_PC_RO, + &in->pli_cred); + if (chk_rc != 0) { + if (chk_rc == -DER_NO_PERM) + continue; + else + D_GOTO(send_resp, rc = chk_rc); + } + uuid_copy(rpc_pool->plp_uuid, mgmt_pool->mgpi_uuid); + rpc_pool->plp_label = mgmt_pool->mgpi_label; + rpc_pool->plp_svc_list = mgmt_pool->mgpi_svc; + n_rpc++; + } + out->plo_pools.ca_arrays = rpc_pools; + out->plo_pools.ca_count = n_rpc; + out->plo_npools = n_rpc; + +send_resp: + out->plo_op.mo_rc = rc; + + rc = crt_reply_send(rpc); + if (rc != 0) + DL_ERROR(rc, "crt_reply_send() failed"); + + if (n_mgmt > 0 && mgmt_pools != NULL) { + for (i = 0; i < n_mgmt; i++) { + daos_mgmt_pool_info_t *mgmt_pool = &mgmt_pools[i]; + if (mgmt_pool->mgpi_label) + D_FREE(mgmt_pool->mgpi_label); + if (mgmt_pool->mgpi_svc) + d_rank_list_free(mgmt_pool->mgpi_svc); + } + D_FREE(mgmt_pools); + } + if (rpc_pools) + D_FREE(rpc_pools); +} + static int ds_mgmt_init() { diff --git a/src/mgmt/srv_internal.h b/src/mgmt/srv_internal.h index b22341fb35b..3692677f565 100644 --- a/src/mgmt/srv_internal.h +++ b/src/mgmt/srv_internal.h @@ -36,6 +36,8 @@ void ds_mgmt_tgt_params_set_hdlr(crt_rpc_t *rpc); void ds_mgmt_profile_hdlr(crt_rpc_t *rpc); void ds_mgmt_pool_get_svcranks_hdlr(crt_rpc_t *rpc); void ds_mgmt_pool_find_hdlr(crt_rpc_t *rpc); +void + ds_mgmt_pool_list_hdlr(crt_rpc_t *rpc); void ds_mgmt_mark_hdlr(crt_rpc_t *rpc); void dss_bind_to_xstream_cpuset(int tgt_id); diff --git a/src/pool/rpc.c b/src/pool/rpc.c index 08117f41942..9655c4e0597 100644 --- a/src/pool/rpc.c +++ b/src/pool/rpc.c @@ -74,6 +74,8 @@ CRT_RPC_DEFINE(pool_connect_v4, DAOS_ISEQ_POOL_CONNECT_V4, DAOS_OSEQ_POOL_CONNEC CRT_RPC_DEFINE(pool_connect_v5, DAOS_ISEQ_POOL_CONNECT_V5, DAOS_OSEQ_POOL_CONNECT) CRT_RPC_DEFINE(pool_disconnect, DAOS_ISEQ_POOL_DISCONNECT, DAOS_OSEQ_POOL_DISCONNECT) +CRT_RPC_DEFINE(pool_check_access, DAOS_ISEQ_POOL_CHECK_ACCESS, + DAOS_OSEQ_POOL_CHECK_ACCESS) CRT_RPC_DEFINE(pool_query_v4, DAOS_ISEQ_POOL_QUERY, DAOS_OSEQ_POOL_QUERY_V4) CRT_RPC_DEFINE(pool_query_v5, DAOS_ISEQ_POOL_QUERY, DAOS_OSEQ_POOL_QUERY_V5) CRT_RPC_DEFINE(pool_attr_list, DAOS_ISEQ_POOL_ATTR_LIST, diff --git a/src/pool/rpc.h b/src/pool/rpc.h index c462c84ce88..3cf14858594 100644 --- a/src/pool/rpc.h +++ b/src/pool/rpc.h @@ -104,6 +104,9 @@ ds_pool_filter_cont_handler, NULL) #define POOL_PROTO_SRV_RPC_LIST \ + X(POOL_CHECK_ACCESS, \ + 0, &CQF_pool_check_access, \ + ds_pool_check_access_handler, NULL), \ X(POOL_TGT_DISCONNECT, \ 0, &CQF_pool_tgt_disconnect, \ ds_pool_tgt_disconnect_handler, \ @@ -262,6 +265,17 @@ CRT_RPC_DECLARE(pool_connect_v5, DAOS_ISEQ_POOL_CONNECT_V5, DAOS_OSEQ_POOL_CONNE CRT_RPC_DECLARE(pool_disconnect, DAOS_ISEQ_POOL_DISCONNECT, DAOS_OSEQ_POOL_DISCONNECT) +#define DAOS_ISEQ_POOL_CHECK_ACCESS /* input fields */ \ + ((struct pool_op_in) (pcai_op) CRT_VAR) \ + ((d_iov_t) (pcai_cred) CRT_VAR) \ + ((uint64_t) (pcai_flags) CRT_VAR) + +#define DAOS_OSEQ_POOL_CHECK_ACCESS /* output fields */ \ + ((struct pool_op_out) (pcao_op) CRT_VAR) \ + ((int32_t) (pcao_rc) CRT_VAR) + +CRT_RPC_DECLARE(pool_check_access, DAOS_ISEQ_POOL_CHECK_ACCESS, DAOS_OSEQ_POOL_CHECK_ACCESS) + #define DAOS_ISEQ_POOL_QUERY /* input fields */ \ ((struct pool_op_in) (pqi_op) CRT_VAR) \ ((crt_bulk_t) (pqi_map_bulk) CRT_VAR) \ diff --git a/src/pool/srv_internal.h b/src/pool/srv_internal.h index 0ffb55cb4c7..a0780cb06b2 100644 --- a/src/pool/srv_internal.h +++ b/src/pool/srv_internal.h @@ -162,6 +162,8 @@ void ds_pool_connect_handler_v5(crt_rpc_t *rpc); void ds_pool_disconnect_handler(crt_rpc_t *rpc); void ds_pool_query_handler_v4(crt_rpc_t *rpc); void ds_pool_query_handler_v5(crt_rpc_t *rpc); +void + ds_pool_check_access_handler(crt_rpc_t *rpc); void ds_pool_prop_get_handler(crt_rpc_t *rpc); void ds_pool_prop_set_handler(crt_rpc_t *rpc); void ds_pool_acl_update_handler(crt_rpc_t *rpc); diff --git a/src/pool/srv_pool.c b/src/pool/srv_pool.c index 20aca49f867..13f4370471f 100644 --- a/src/pool/srv_pool.c +++ b/src/pool/srv_pool.c @@ -2907,6 +2907,174 @@ bulk_cb(const struct crt_bulk_cb_info *cb_info) return 0; } +int +ds_pool_check_access(uuid_t pool_uuid, d_rank_list_t *ps_ranks, unsigned int flags, d_iov_t *cred) +{ + struct rsvc_client client; + crt_endpoint_t ep; + struct dss_module_info *info = dss_get_module_info(); + crt_rpc_t *rpc; + struct pool_check_access_in *in; + struct pool_check_access_out *out; + int rc; + + D_DEBUG(DB_MD, DF_UUID ": checking pool access\n", DP_UUID(pool_uuid)); + + rc = rsvc_client_init(&client, ps_ranks); + if (rc != 0) + goto out; + +rechoose: + ep.ep_grp = NULL; /* primary group */ + rc = rsvc_client_choose(&client, &ep); + if (rc != 0) { + D_ERROR(DF_UUID ": cannot find pool service: " DF_RC "\n", DP_UUID(pool_uuid), + DP_RC(rc)); + goto out_client; + } + + rc = pool_req_create(info->dmi_ctx, &ep, POOL_CHECK_ACCESS, &rpc); + if (rc != 0) { + D_ERROR(DF_UUID ": failed to create pool check access rpc, " DF_RC "\n", + DP_UUID(pool_uuid), DP_RC(rc)); + goto out_client; + } + + in = crt_req_get(rpc); + in->pcai_flags = flags; + uuid_copy(in->pcai_op.pi_uuid, pool_uuid); + uuid_clear(in->pcai_op.pi_hdl); + + rc = daos_iov_copy(&in->pcai_cred, cred); + if (rc != 0) { + D_ERROR(DF_UUID ": failed to copy credential IOV, " DF_RC "\n", DP_UUID(pool_uuid), + DP_RC(rc)); + goto out_client; + } + + rc = dss_rpc_send(rpc); + out = crt_reply_get(rpc); + D_ASSERT(out != NULL); + + rc = pool_rsvc_client_complete_rpc(&client, &ep, rc, &out->pcao_op); + if (rc == RSVC_CLIENT_RECHOOSE) { + crt_req_decref(rpc); + dss_sleep(RECHOOSE_SLEEP_MS); + goto rechoose; + } + + rc = out->pcao_op.po_rc; + if (rc != 0 && rc != -DER_NO_PERM) { + D_ERROR(DF_UUID ": failed to check pool access, " DF_RC "\n", DP_UUID(pool_uuid), + DP_RC(rc)); + } + + crt_req_decref(rpc); +out_client: + rsvc_client_fini(&client); +out: + return rc; +} + +static int +pool_check_cred_access(d_iov_t *cred, int flags, daos_prop_t *prop) +{ + struct daos_prop_entry *acl_entry; + struct d_ownership owner; + struct daos_prop_entry *owner_entry; + struct daos_prop_entry *owner_grp_entry; + uint64_t sec_capas = 0; + int rc; + + D_ASSERT(prop != NULL); + + acl_entry = daos_prop_entry_get(prop, DAOS_PROP_PO_ACL); + D_ASSERT(acl_entry != NULL); + D_ASSERT(acl_entry->dpe_val_ptr != NULL); + + owner_entry = daos_prop_entry_get(prop, DAOS_PROP_PO_OWNER); + D_ASSERT(owner_entry != NULL); + D_ASSERT(owner_entry->dpe_str != NULL); + + owner_grp_entry = daos_prop_entry_get(prop, DAOS_PROP_PO_OWNER_GROUP); + D_ASSERT(owner_grp_entry != NULL); + D_ASSERT(owner_grp_entry->dpe_str != NULL); + + owner.user = owner_entry->dpe_str; + owner.group = owner_grp_entry->dpe_str; + + /* + * Security capabilities determine the access control policy on this + * pool handle. + */ + rc = ds_sec_pool_get_capabilities(flags, cred, &owner, acl_entry->dpe_val_ptr, &sec_capas); + if (rc != 0) { + DL_ERROR(rc, "failed to read sec capabilities"); + return rc; + } + + if (!ds_sec_pool_can_connect(sec_capas)) { + return -DER_NO_PERM; + } + + return 0; +} + +void +ds_pool_check_access_handler(crt_rpc_t *rpc) +{ + struct pool_check_access_in *in = crt_req_get(rpc); + struct pool_check_access_out *out = crt_reply_get(rpc); + struct pool_svc *svc; + struct rdb_tx tx; + daos_prop_t *prop = NULL; + uint64_t prop_bits; + int rc; + + D_DEBUG(DB_MD, DF_UUID ": processing rpc: %p hdl=" DF_UUID "\n", + DP_UUID(in->pcai_op.pi_uuid), rpc, DP_UUID(in->pcai_op.pi_hdl)); + + rc = pool_svc_lookup_leader(in->pcai_op.pi_uuid, &svc, &out->pcao_op.po_hint); + if (rc != 0) + D_GOTO(out, rc); + + rc = rdb_tx_begin(svc->ps_rsvc.s_db, svc->ps_rsvc.s_term, &tx); + if (rc != 0) + D_GOTO(out_svc, rc); + + ABT_rwlock_rdlock(svc->ps_lock); + + prop_bits = DAOS_PO_QUERY_PROP_ALL; + rc = pool_prop_read(&tx, svc, prop_bits, &prop); + if (rc != 0) { + D_ERROR(DF_UUID ": cannot get access data for pool, " + "rc=" DF_RC "\n", + DP_UUID(in->pcai_op.pi_uuid), DP_RC(rc)); + D_GOTO(out_lock, rc); + } + D_ASSERT(prop != NULL); + + rc = pool_check_cred_access(&in->pcai_cred, in->pcai_flags, prop); + if (rc != 0) { + D_INFO(DF_UUID ": rejecting access based on credential, " + "rc=" DF_RC "\n", + DP_UUID(in->pcai_op.pi_uuid), DP_RC(rc)); + } + + out->pcao_op.po_map_version = ds_pool_get_version(svc->ps_pool); +out_lock: + ABT_rwlock_unlock(svc->ps_lock); + rdb_tx_end(&tx); + if (prop) + daos_prop_free(prop); +out_svc: + ds_rsvc_set_hint(&svc->ps_rsvc, &out->pcao_op.po_hint); + pool_svc_put_leader(svc); +out: + out->pcao_op.po_rc = rc; + crt_reply_send(rpc); +} + /* Currently we only maintain compatibility between 2 versions */ #define NUM_POOL_VERSIONS 2 diff --git a/src/proto/srv/srv.proto b/src/proto/srv/srv.proto index c63a402501c..5cc4b9bc3b0 100644 --- a/src/proto/srv/srv.proto +++ b/src/proto/srv/srv.proto @@ -52,3 +52,19 @@ message PoolFindByLabelResp { string uuid = 2; // Pool UUID repeated uint32 svcreps = 3; // Pool service replica ranks } + +message ListPoolsReq +{ + bool includeAll = 1; // Include all pools in response, regardless of state +} + +message ListPoolsResp +{ + message Pool + { + string uuid = 1; // Pool UUID + string label = 2; // Pool label + repeated uint32 svcreps = 3; // Pool service ranks + } + repeated Pool pools = 1; // List of pools +}