diff --git a/src/control/lib/control/mocks.go b/src/control/lib/control/mocks.go index 60f6c6b0626..f80478b31e8 100644 --- a/src/control/lib/control/mocks.go +++ b/src/control/lib/control/mocks.go @@ -601,6 +601,7 @@ type ( AvailBytes uint64 // Available raw storage UsableBytes uint64 // Effective storage available for data NvmeState *storage.NvmeDevState + NvmeRole *storage.BdevRoles } MockScmConfig struct { @@ -673,6 +674,9 @@ func MockStorageScanResp(t *testing.T, if mockNvmeConfig.NvmeState != nil { smdDevice.NvmeState = *mockNvmeConfig.NvmeState } + if mockNvmeConfig.NvmeRole != nil { + smdDevice.Roles = *mockNvmeConfig.NvmeRole + } smdDevice.Rank = mockNvmeConfig.Rank nvmeControllers = append(nvmeControllers, nvmeController) } diff --git a/src/control/lib/control/pool.go b/src/control/lib/control/pool.go index 7d2fdb4d746..c241b09b45a 100644 --- a/src/control/lib/control/pool.go +++ b/src/control/lib/control/pool.go @@ -1266,6 +1266,12 @@ func processSCMSpaceStats(log logging.Logger, filterRank filterRankFn, scmNamesp func processNVMeSpaceStats(log logging.Logger, filterRank filterRankFn, nvmeControllers storage.NvmeControllers, rankNVMeFreeSpace rankFreeSpaceMap) error { for _, nvmeController := range nvmeControllers { for _, smdDevice := range nvmeController.SmdDevices { + if !smdDevice.Roles.IsEmpty() && (smdDevice.Roles.OptionBits&storage.BdevRoleData) == 0 { + log.Debugf("Skipping SMD device %s (rank %d, ctrlr %s) not used for storing data", + smdDevice.UUID, smdDevice.Rank, smdDevice.TrAddr, smdDevice.Rank) + continue + } + if smdDevice.NvmeState != storage.NvmeStateNormal { log.Noticef("SMD device %s (rank %d, ctrlr %s) not usable (device state %q)", smdDevice.UUID, smdDevice.Rank, smdDevice.TrAddr, smdDevice.NvmeState.String()) diff --git a/src/control/lib/control/pool_test.go b/src/control/lib/control/pool_test.go index c2668343111..2f824ed75d8 100644 --- a/src/control/lib/control/pool_test.go +++ b/src/control/lib/control/pool_test.go @@ -1696,6 +1696,83 @@ func TestControl_GetMaxPoolSize(t *testing.T) { NvmeBytes: uint64(1) * uint64(humanize.TByte), }, }, + "single MD-on-SSD server": { + HostsConfigArray: []MockHostStorageConfig{ + { + HostName: "foo", + ScmConfig: []MockScmConfig{ + { + MockStorageConfig: MockStorageConfig{ + TotalBytes: uint64(100) * uint64(humanize.GByte), + AvailBytes: uint64(100) * uint64(humanize.GByte), + UsableBytes: uint64(100) * uint64(humanize.GByte), + }, + Rank: 0, + }, + }, + NvmeConfig: []MockNvmeConfig{ + { + MockStorageConfig: MockStorageConfig{ + TotalBytes: uint64(1) * uint64(humanize.TByte), + AvailBytes: uint64(1) * uint64(humanize.TByte), + UsableBytes: uint64(1) * uint64(humanize.TByte), + NvmeRole: &storage.BdevRoles{ + storage.OptionBits(storage.BdevRoleData), + }, + }, + Rank: 0, + }, + { + MockStorageConfig: MockStorageConfig{ + TotalBytes: uint64(2) * uint64(humanize.TByte), + AvailBytes: uint64(2) * uint64(humanize.TByte), + UsableBytes: uint64(2) * uint64(humanize.TByte), + NvmeRole: &storage.BdevRoles{ + storage.OptionBits(storage.BdevRoleWAL | storage.BdevRoleMeta), + }, + }, + Rank: 0, + }, + }, + }, + }, + ExpectedOutput: ExpectedOutput{ + ScmBytes: uint64(100) * uint64(humanize.GByte), + NvmeBytes: uint64(1) * uint64(humanize.TByte), + }, + }, + "single Ephemeral server": { + HostsConfigArray: []MockHostStorageConfig{ + { + HostName: "foo", + ScmConfig: []MockScmConfig{ + { + MockStorageConfig: MockStorageConfig{ + TotalBytes: uint64(100) * uint64(humanize.GByte), + AvailBytes: uint64(100) * uint64(humanize.GByte), + UsableBytes: uint64(100) * uint64(humanize.GByte), + }, + Rank: 0, + }, + }, + NvmeConfig: []MockNvmeConfig{ + { + MockStorageConfig: MockStorageConfig{ + TotalBytes: uint64(1) * uint64(humanize.TByte), + AvailBytes: uint64(1) * uint64(humanize.TByte), + UsableBytes: uint64(1) * uint64(humanize.TByte), + NvmeRole: &storage.BdevRoles{storage.OptionBits(0)}, + }, + Rank: 0, + }, + }, + }, + }, + ExpectedOutput: ExpectedOutput{ + ScmBytes: uint64(100) * uint64(humanize.GByte), + NvmeBytes: uint64(1) * uint64(humanize.TByte), + }, + }, "double server": { HostsConfigArray: []MockHostStorageConfig{ { diff --git a/src/control/server/ctl_storage_rpc.go b/src/control/server/ctl_storage_rpc.go index 3cdec1b8a51..c949ac927c0 100644 --- a/src/control/server/ctl_storage_rpc.go +++ b/src/control/server/ctl_storage_rpc.go @@ -355,6 +355,15 @@ func (c *ControlService) adjustNvmeSize(resp *ctlpb.ScanNvmeResp) { for idx, dev := range ctlr.GetSmdDevices() { rank := dev.GetRank() + if dev.GetRoleBits() != 0 && (dev.GetRoleBits()&storage.BdevRoleData) == 0 { + c.log.Debugf("SMD device %s (rank %d, ctlr %s) not used to store data (Role bits 0x%X)", + dev.GetUuid(), rank, ctlr.GetPciAddr(), dev.GetRoleBits()) + dev.TotalBytes = 0 + dev.AvailBytes = 0 + dev.UsableBytes = 0 + continue + } + if dev.GetDevState() != ctlpb.NvmeDevState_NORMAL { c.log.Debugf("SMD device %s (rank %d, ctlr %s) not usable: device state %q", dev.GetUuid(), rank, ctlr.GetPciAddr(), ctlpb.NvmeDevState_name[int32(dev.DevState)]) @@ -371,14 +380,6 @@ func (c *ControlService) adjustNvmeSize(resp *ctlpb.ScanNvmeResp) { continue } - if dev.GetRoleBits() != 0 && (dev.GetRoleBits()&storage.BdevRoleData) == 0 { - c.log.Debugf("SMD device %s (rank %d, ctlr %s) not used to store data (Role bits 0x%X)", - dev.GetUuid(), rank, ctlr.GetPciAddr(), dev.GetRoleBits()) - dev.AvailBytes = 0 - dev.UsableBytes = 0 - continue - } - c.log.Tracef("Initial available size of SMD device %s (rank %d, ctlr %s): %s (%d bytes)", dev.GetUuid(), rank, ctlr.GetPciAddr(), humanize.Bytes(dev.GetAvailBytes()), dev.GetAvailBytes()) diff --git a/src/control/server/ctl_storage_rpc_test.go b/src/control/server/ctl_storage_rpc_test.go index f2a402cfd5e..f4f2237bc4e 100644 --- a/src/control/server/ctl_storage_rpc_test.go +++ b/src/control/server/ctl_storage_rpc_test.go @@ -2837,6 +2837,7 @@ func TestServer_CtlSvc_adjustNvmeSize(t *testing.T) { scanNvmeResp *ctlpb.ScanNvmeResp } type ExpectedOutput struct { + totalBytes []uint64 availableBytes []uint64 usableBytes []uint64 message string @@ -2891,6 +2892,7 @@ func TestServer_CtlSvc_adjustNvmeSize(t *testing.T) { { Uuid: "nvme0", TgtIds: []int32{0, 1, 2, 3}, + TotalBytes: 10 * hugeClusterSize, AvailBytes: 10 * hugeClusterSize, ClusterSize: hugeClusterSize, DevState: devStateNormal, @@ -2905,6 +2907,7 @@ func TestServer_CtlSvc_adjustNvmeSize(t *testing.T) { { Uuid: "nvme1", TgtIds: []int32{4, 5, 6, 7}, + TotalBytes: 10 * hugeClusterSize, AvailBytes: 10 * hugeClusterSize, ClusterSize: hugeClusterSize, DevState: devStateNormal, @@ -2918,6 +2921,7 @@ func TestServer_CtlSvc_adjustNvmeSize(t *testing.T) { { Uuid: "nvme2", TgtIds: []int32{8, 9, 10, 11}, + TotalBytes: 20 * hugeClusterSize, AvailBytes: 20 * hugeClusterSize, ClusterSize: hugeClusterSize, DevState: devStateNormal, @@ -2932,6 +2936,7 @@ func TestServer_CtlSvc_adjustNvmeSize(t *testing.T) { { Uuid: "nvme3", TgtIds: []int32{0, 1, 2}, + TotalBytes: 20 * hugeClusterSize, AvailBytes: 20 * hugeClusterSize, ClusterSize: hugeClusterSize, DevState: devStateNormal, @@ -2945,6 +2950,7 @@ func TestServer_CtlSvc_adjustNvmeSize(t *testing.T) { { Uuid: "nvme4", TgtIds: []int32{3, 4, 5}, + TotalBytes: 20 * hugeClusterSize, AvailBytes: 20 * hugeClusterSize, ClusterSize: hugeClusterSize, DevState: devStateNormal, @@ -2957,6 +2963,13 @@ func TestServer_CtlSvc_adjustNvmeSize(t *testing.T) { }, }, output: ExpectedOutput{ + totalBytes: []uint64{ + 10 * hugeClusterSize, + 10 * hugeClusterSize, + 20 * hugeClusterSize, + 20 * hugeClusterSize, + 20 * hugeClusterSize, + }, availableBytes: []uint64{ 10 * hugeClusterSize, 10 * hugeClusterSize, @@ -3000,6 +3013,7 @@ func TestServer_CtlSvc_adjustNvmeSize(t *testing.T) { { Uuid: "nvme0", TgtIds: []int32{0, 1, 2, 3}, + TotalBytes: 10 * hugeClusterSize, AvailBytes: 10 * hugeClusterSize, ClusterSize: hugeClusterSize, DevState: devStateNormal, @@ -3014,6 +3028,7 @@ func TestServer_CtlSvc_adjustNvmeSize(t *testing.T) { { Uuid: "nvme1", TgtIds: []int32{4, 5, 6}, + TotalBytes: 10 * hugeClusterSize, AvailBytes: 10 * hugeClusterSize, ClusterSize: hugeClusterSize, DevState: devStateNormal, @@ -3028,6 +3043,7 @@ func TestServer_CtlSvc_adjustNvmeSize(t *testing.T) { { Uuid: "nvme2", TgtIds: []int32{7, 8, 9, 10}, + TotalBytes: 20 * hugeClusterSize, AvailBytes: 20 * hugeClusterSize, ClusterSize: hugeClusterSize, DevState: devStateNormal, @@ -3042,6 +3058,7 @@ func TestServer_CtlSvc_adjustNvmeSize(t *testing.T) { { Uuid: "nvme3", TgtIds: []int32{0, 1, 2}, + TotalBytes: 20 * hugeClusterSize, AvailBytes: 20 * hugeClusterSize, ClusterSize: hugeClusterSize, DevState: devStateNormal, @@ -3056,6 +3073,7 @@ func TestServer_CtlSvc_adjustNvmeSize(t *testing.T) { { Uuid: "nvme4", TgtIds: []int32{3, 4}, + TotalBytes: 20 * hugeClusterSize, AvailBytes: 20 * hugeClusterSize, ClusterSize: hugeClusterSize, DevState: devStateNormal, @@ -3068,6 +3086,13 @@ func TestServer_CtlSvc_adjustNvmeSize(t *testing.T) { }, }, output: ExpectedOutput{ + totalBytes: []uint64{ + 10 * hugeClusterSize, + 10 * hugeClusterSize, + 20 * hugeClusterSize, + 20 * hugeClusterSize, + 20 * hugeClusterSize, + }, availableBytes: []uint64{ 10 * hugeClusterSize, 10 * hugeClusterSize, @@ -3103,6 +3128,7 @@ func TestServer_CtlSvc_adjustNvmeSize(t *testing.T) { { Uuid: "nvme0", TgtIds: []int32{0, 1, 2, 3}, + TotalBytes: 10 * hugeClusterSize, AvailBytes: 10 * hugeClusterSize, ClusterSize: hugeClusterSize, DevState: devStateNormal, @@ -3117,6 +3143,7 @@ func TestServer_CtlSvc_adjustNvmeSize(t *testing.T) { { Uuid: "nvme1", TgtIds: []int32{0, 1, 2}, + TotalBytes: 10 * hugeClusterSize, AvailBytes: 10 * hugeClusterSize, ClusterSize: hugeClusterSize, DevState: devStateNew, @@ -3129,6 +3156,10 @@ func TestServer_CtlSvc_adjustNvmeSize(t *testing.T) { }, }, output: ExpectedOutput{ + totalBytes: []uint64{ + 10 * hugeClusterSize, + 10 * hugeClusterSize, + }, availableBytes: []uint64{ 10 * hugeClusterSize, 0, @@ -3159,6 +3190,7 @@ func TestServer_CtlSvc_adjustNvmeSize(t *testing.T) { { Uuid: "nvme0", TgtIds: []int32{0, 1, 2, 3}, + TotalBytes: 10 * hugeClusterSize, AvailBytes: 10 * hugeClusterSize, ClusterSize: hugeClusterSize, DevState: devStateNormal, @@ -3173,6 +3205,7 @@ func TestServer_CtlSvc_adjustNvmeSize(t *testing.T) { { Uuid: "nvme1", TgtIds: []int32{0, 1, 2}, + TotalBytes: 10 * hugeClusterSize, AvailBytes: 10 * hugeClusterSize, ClusterSize: hugeClusterSize, DevState: devStateFaulty, @@ -3185,6 +3218,10 @@ func TestServer_CtlSvc_adjustNvmeSize(t *testing.T) { }, }, output: ExpectedOutput{ + totalBytes: []uint64{ + 10 * hugeClusterSize, + 10 * hugeClusterSize, + }, availableBytes: []uint64{ 10 * hugeClusterSize, 0, @@ -3215,6 +3252,7 @@ func TestServer_CtlSvc_adjustNvmeSize(t *testing.T) { { Uuid: "nvme0", TgtIds: []int32{0, 1, 2, 3}, + TotalBytes: 10 * hugeClusterSize, AvailBytes: 10 * hugeClusterSize, ClusterSize: hugeClusterSize, DevState: devStateNormal, @@ -3229,6 +3267,7 @@ func TestServer_CtlSvc_adjustNvmeSize(t *testing.T) { { Uuid: "nvme1", TgtIds: []int32{}, + TotalBytes: 10 * hugeClusterSize, AvailBytes: 10 * hugeClusterSize, ClusterSize: hugeClusterSize, DevState: devStateNormal, @@ -3241,6 +3280,10 @@ func TestServer_CtlSvc_adjustNvmeSize(t *testing.T) { }, }, output: ExpectedOutput{ + totalBytes: []uint64{ + 10 * hugeClusterSize, + 10 * hugeClusterSize, + }, availableBytes: []uint64{ 10 * hugeClusterSize, 0, @@ -3271,6 +3314,7 @@ func TestServer_CtlSvc_adjustNvmeSize(t *testing.T) { { Uuid: "nvme0", TgtIds: []int32{0, 1, 2, 3}, + TotalBytes: 10 * hugeClusterSize, AvailBytes: 10 * hugeClusterSize, ClusterSize: hugeClusterSize, DevState: devStateNormal, @@ -3285,6 +3329,7 @@ func TestServer_CtlSvc_adjustNvmeSize(t *testing.T) { { Uuid: "nvme1", TgtIds: []int32{0, 1, 2}, + TotalBytes: 10 * hugeClusterSize, AvailBytes: 10 * hugeClusterSize, DevState: devStateNormal, Rank: 0, @@ -3296,6 +3341,10 @@ func TestServer_CtlSvc_adjustNvmeSize(t *testing.T) { }, }, output: ExpectedOutput{ + totalBytes: []uint64{ + 10 * hugeClusterSize, + 10 * hugeClusterSize, + }, availableBytes: []uint64{ 10 * hugeClusterSize, 0, @@ -3343,6 +3392,7 @@ func TestServer_CtlSvc_adjustNvmeSize(t *testing.T) { { Uuid: "nvme0", TgtIds: []int32{0, 1, 2, 3}, + TotalBytes: 10 * humanize.GiByte, AvailBytes: 10 * humanize.GiByte, ClusterSize: clusterSize, DevState: devStateNormal, @@ -3357,6 +3407,7 @@ func TestServer_CtlSvc_adjustNvmeSize(t *testing.T) { { Uuid: "nvme1", TgtIds: []int32{0, 1, 2, 3}, + TotalBytes: 10 * humanize.GiByte, AvailBytes: 10 * humanize.GiByte, ClusterSize: clusterSize, DevState: devStateNormal, @@ -3371,6 +3422,7 @@ func TestServer_CtlSvc_adjustNvmeSize(t *testing.T) { { Uuid: "nvme2", TgtIds: []int32{0, 1, 2, 3}, + TotalBytes: 10 * humanize.GiByte, AvailBytes: 10 * humanize.GiByte, ClusterSize: clusterSize, DevState: devStateNormal, @@ -3385,6 +3437,7 @@ func TestServer_CtlSvc_adjustNvmeSize(t *testing.T) { { Uuid: "nvme3", TgtIds: []int32{0, 1, 2, 3}, + TotalBytes: 10 * humanize.GiByte, AvailBytes: 10 * humanize.GiByte, ClusterSize: clusterSize, DevState: devStateNormal, @@ -3399,6 +3452,7 @@ func TestServer_CtlSvc_adjustNvmeSize(t *testing.T) { { Uuid: "nvme4", TgtIds: []int32{0, 1, 2, 3}, + TotalBytes: 10 * humanize.GiByte, AvailBytes: 10 * humanize.GiByte, ClusterSize: clusterSize, DevState: devStateNormal, @@ -3413,6 +3467,7 @@ func TestServer_CtlSvc_adjustNvmeSize(t *testing.T) { { Uuid: "nvme5", TgtIds: []int32{0, 1, 2, 3}, + TotalBytes: 10 * humanize.GiByte, AvailBytes: 10 * humanize.GiByte, ClusterSize: clusterSize, DevState: devStateNormal, @@ -3425,6 +3480,14 @@ func TestServer_CtlSvc_adjustNvmeSize(t *testing.T) { }, }, output: ExpectedOutput{ + totalBytes: []uint64{ + 320 * clusterSize, + 320 * clusterSize, + 320 * clusterSize, + 0 * humanize.GiByte, + 0 * humanize.GiByte, + 0 * humanize.GiByte, + }, availableBytes: []uint64{ 320 * clusterSize, 320 * clusterSize, @@ -3462,6 +3525,9 @@ func TestServer_CtlSvc_adjustNvmeSize(t *testing.T) { for idx, ctlr := range tc.input.scanNvmeResp.GetCtrlrs() { dev := ctlr.GetSmdDevices()[0] + test.AssertEqual(t, tc.output.totalBytes[idx], dev.GetTotalBytes(), + fmt.Sprintf("Invalid total bytes with ctlr %s (index=%d): wait=%d, got=%d", + ctlr.GetPciAddr(), idx, tc.output.totalBytes[idx], dev.GetTotalBytes())) test.AssertEqual(t, tc.output.availableBytes[idx], dev.GetAvailBytes(), fmt.Sprintf("Invalid available bytes with ctlr %s (index=%d): wait=%d, got=%d", ctlr.GetPciAddr(), idx, tc.output.availableBytes[idx], dev.GetAvailBytes()))