diff --git a/.github/workflows/landing-builds.yml b/.github/workflows/landing-builds.yml index afd1c74f233..10370c73f1a 100644 --- a/.github/workflows/landing-builds.yml +++ b/.github/workflows/landing-builds.yml @@ -383,4 +383,4 @@ jobs: - name: Run NLT run: docker run --mount type=tmpfs,destination=/mnt/daos_0,tmpfs-mode=1777 --user root:root build-image ./daos/utils/node_local_test.py --no-root - --memcheck no --test cont_copy + --memcheck no --test cont_copy --system-ram-reserved 6 diff --git a/src/control/cmd/daos_server/auto_test.go b/src/control/cmd/daos_server/auto_test.go index 28cd936a49b..e416baf4f4b 100644 --- a/src/control/cmd/daos_server/auto_test.go +++ b/src/control/cmd/daos_server/auto_test.go @@ -151,9 +151,19 @@ func TestDaosServer_Auto_confGen(t *testing.T) { controlMetadata := storage.ControlMetadata{ Path: metadataMountPath, } - // SCM tmpfs 5GiB size calculated after subtracting reservations from MemTotalKiB. + // Nr hugepages expected with 18+1 (extra MD-on-SSD sys-xstream) targets * 2 engines * 512 + // pages-per-target. + tmpfsNrHugepages := 19 * 2 * 512 + tmpfsHugeMemGiB := (humanize.MiByte * 2 * tmpfsNrHugepages) / humanize.GiByte + tmpfsEngRsvdGiB := 2 /* calculated based on 18 targets */ + tmpfsSysRsvdGiB := storage.DefaultSysMemRsvd / humanize.GiByte + tmpfsRamdiskGiB := storage.MinRamdiskMem / humanize.GiByte + // Total mem to meet requirements 39GiB hugeMem, 2GiB per engine rsvd, 6GiB sys rsvd, 4GiB + // per engine RAM-disk. + tmpfsMemTotalGiB := humanize.GiByte * (tmpfsHugeMemGiB + (2 * tmpfsEngRsvdGiB) + tmpfsSysRsvdGiB + + (2 * tmpfsRamdiskGiB) + 1 /* add 1GiB buffer */) tmpfsEngineCfgs := []*engine.Config{ - control.MockEngineCfgTmpfs(0, 5, + control.MockEngineCfgTmpfs(0, tmpfsRamdiskGiB, control.MockBdevTierWithRole(0, storage.BdevRoleWAL, 2), control.MockBdevTierWithRole(0, storage.BdevRoleMeta|storage.BdevRoleData, 4)). WithStorageControlMetadataPath(metadataMountPath). @@ -161,7 +171,7 @@ func TestDaosServer_Auto_confGen(t *testing.T) { filepath.Join(controlMetadata.EngineDirectory(0), storage.BdevOutConfName), ). WithTargetCount(18).WithHelperStreamCount(4), - control.MockEngineCfgTmpfs(1, 5, + control.MockEngineCfgTmpfs(1, tmpfsRamdiskGiB, control.MockBdevTierWithRole(1, storage.BdevRoleWAL, 1), control.MockBdevTierWithRole(1, storage.BdevRoleMeta|storage.BdevRoleData, 3)). WithStorageControlMetadataPath(metadataMountPath). @@ -368,12 +378,7 @@ func TestDaosServer_Auto_confGen(t *testing.T) { storage.MockScmNamespace(0), storage.MockScmNamespace(1), }, - MemInfo: &common.MemInfo{ - HugepageSizeKiB: 2048, - // Total mem to meet requirements 39GiB hugeMem, 1GiB per - // engine rsvd, 6GiB sys rsvd, 5GiB per engine for tmpfs. - MemTotalKiB: (humanize.GiByte * (39 + 2 + 6 + 10)) / humanize.KiByte, - }, + MemInfo: &defMemInfo, NvmeDevices: storage.NvmeControllers{ storage.MockNvmeController(1), storage.MockNvmeController(2), @@ -428,9 +433,7 @@ func TestDaosServer_Auto_confGen(t *testing.T) { }, MemInfo: &common.MemInfo{ HugepageSizeKiB: 2048, - // Total mem to meet requirements 39GiB hugeMem, 1GiB per - // engine rsvd, 6GiB sys rsvd, 5GiB per engine for tmpfs. - MemTotalKiB: (humanize.GiByte * (39 + 2 + 6 + 10)) / humanize.KiByte, + MemTotalKiB: tmpfsMemTotalGiB / humanize.KiByte, }, NvmeDevices: storage.NvmeControllers{ storage.MockNvmeController(1), @@ -440,8 +443,7 @@ func TestDaosServer_Auto_confGen(t *testing.T) { }, }, expCfg: control.MockServerCfg("ofi+psm2", tmpfsEngineCfgs). - // 18+1 (extra MD-on-SSD sys-xstream) targets * 2 engines * 512 pages - WithNrHugepages(19 * 2 * 512). + WithNrHugepages(tmpfsNrHugepages). WithAccessPoints("localhost:10001"). WithControlLogFile("/tmp/daos_server.log"). WithControlMetadata(controlMetadata), diff --git a/src/control/cmd/dmg/auto_test.go b/src/control/cmd/dmg/auto_test.go index e19b082920e..d3406f4d61e 100644 --- a/src/control/cmd/dmg/auto_test.go +++ b/src/control/cmd/dmg/auto_test.go @@ -140,10 +140,11 @@ func TestAuto_confGen(t *testing.T) { Addr: "host1", Message: control.MockServerScanResp(t, "withSpaceUsage"), } - // Total mem to meet requirements 34GiB hugeMem, 1GiB per engine rsvd, 6GiB sys rsvd, - // 5GiB per engine for tmpfs. storRespHighMem := control.MockServerScanResp(t, "withSpaceUsage") - storRespHighMem.MemInfo.MemTotalKb = (humanize.GiByte * (34 + 2 + 6 + 10)) / humanize.KiByte + // Total mem to meet requirements 34GiB hugeMem, 2GiB per engine rsvd, 16GiB sys rsvd, + // 5GiB per engine for tmpfs. + storRespHighMem.MemInfo.MemTotalKb = (humanize.GiByte * (34 + 4 + 16 + 10)) / humanize.KiByte + mockRamdiskSize := 5 storHostRespHighMem := &control.HostResponse{ Addr: "host1", Message: storRespHighMem, @@ -151,7 +152,6 @@ func TestAuto_confGen(t *testing.T) { e0 := control.MockEngineCfg(0, 2, 4, 6, 8).WithHelperStreamCount(4) e1 := control.MockEngineCfg(1, 1, 3, 5, 7).WithHelperStreamCount(4) exmplEngineCfgs := []*engine.Config{e0, e1} - mockRamdiskSize := 5 // RoundDownGiB(16*0.75/2) metadataMountPath := "/mnt/daos_md" controlMetadata := storage.ControlMetadata{ Path: metadataMountPath, @@ -406,7 +406,7 @@ disable_vfio: false disable_vmd: false enable_hotplug: false nr_hugepages: 6144 -system_ram_reserved: 6 +system_ram_reserved: 16 disable_hugepages: false control_log_mask: INFO control_log_file: /tmp/daos_server.log diff --git a/src/control/lib/control/auto_test.go b/src/control/lib/control/auto_test.go index 0dea21db770..93a1d8ccd2a 100644 --- a/src/control/lib/control/auto_test.go +++ b/src/control/lib/control/auto_test.go @@ -1573,7 +1573,7 @@ func TestControl_AutoConfig_genConfig(t *testing.T) { MockEngineCfgTmpfs(1, 0, mockBdevTier(1, 3), mockBdevTier(1, 4, 5)), }, hpSize: defHpSizeKb, - memTotal: (52 * humanize.GiByte) / humanize.KiByte, + memTotal: (64 * humanize.GiByte) / humanize.KiByte, expCfg: MockServerCfg(exmplEngineCfg0.Fabric.Provider, []*engine.Config{ MockEngineCfgTmpfs(0, 5, /* tmpfs size in gib */ diff --git a/src/control/server/config/server.go b/src/control/server/config/server.go index e446e6c3c80..66330585b62 100644 --- a/src/control/server/config/server.go +++ b/src/control/server/config/server.go @@ -521,8 +521,12 @@ func (cfg *Server) CalcRamdiskSize(log logging.Logger, hpSizeKiB, memKiB int) (u // Calculate reserved system memory in bytes. memSys := uint64(cfg.SystemRamReserved * humanize.GiByte) - return storage.CalcRamdiskSize(log, memTotal, memHuge, memSys, - storage.DefaultEngineMemRsvd, len(cfg.Engines)) + if len(cfg.Engines) == 0 { + return 0, errors.New("no engines in config") + } + + return storage.CalcRamdiskSize(log, memTotal, memHuge, memSys, cfg.Engines[0].TargetCount, + len(cfg.Engines)) } // CalcMemForRamdiskSize calculates minimum memory needed for a given RAM-disk size. @@ -533,8 +537,12 @@ func (cfg *Server) CalcMemForRamdiskSize(log logging.Logger, hpSizeKiB int, ramd // Calculate reserved system memory in bytes. memSys := uint64(cfg.SystemRamReserved * humanize.GiByte) + if len(cfg.Engines) == 0 { + return 0, errors.New("no engines in config") + } + return storage.CalcMemForRamdiskSize(log, ramdiskSize, memHuge, memSys, - storage.DefaultEngineMemRsvd, len(cfg.Engines)) + cfg.Engines[0].TargetCount, len(cfg.Engines)) } // SetRamdiskSize calculates maximum RAM-disk size using total memory as reported by /proc/meminfo. diff --git a/src/control/server/config/server_test.go b/src/control/server/config/server_test.go index c9c647a9707..827a2632e41 100644 --- a/src/control/server/config/server_test.go +++ b/src/control/server/config/server_test.go @@ -1009,7 +1009,7 @@ func TestServerConfig_SetRamdiskSize(t *testing.T) { c.Engines[0].Storage.Tiers.ScmConfigs()[0].Scm.RamdiskSize = 11 return c.WithNrHugepages(16896) }, - expErr: FaultConfigRamdiskOverMaxMem(humanize.GiByte*11, humanize.GiByte*10, 0), + expErr: FaultConfigRamdiskOverMaxMem(humanize.GiByte*11, humanize.GiByte*9, 0), }, "low mem": { // 46 total - 40 reserved = 6 for tmpfs (3 gib per engine - too low) @@ -1019,7 +1019,7 @@ func TestServerConfig_SetRamdiskSize(t *testing.T) { }, // error indicates min RAM needed = 40 + 4 gib per engine expErr: storage.FaultRamdiskLowMem("Total", storage.MinRamdiskMem, - humanize.GiByte*48, humanize.GiByte*46), + humanize.GiByte*50, humanize.GiByte*46), }, "custom value set": { memTotBytes: humanize.GiByte * 60, @@ -1036,7 +1036,7 @@ func TestServerConfig_SetRamdiskSize(t *testing.T) { extraConfig: func(c *Server) *Server { return c.WithNrHugepages(16896) }, - expRamdiskSize: 10, + expRamdiskSize: 9, }, "custom system_ram_reserved value set": { // 33 huge mem + 2 sys rsv + 2 engine rsv = 37 gib reserved mem @@ -1046,7 +1046,7 @@ func TestServerConfig_SetRamdiskSize(t *testing.T) { c.SystemRamReserved = 2 return c.WithNrHugepages(16896) }, - expRamdiskSize: 11, + expRamdiskSize: 10, }, "no scm configured on second engine": { memTotBytes: humanize.GiByte * 80, diff --git a/src/control/server/storage/scm.go b/src/control/server/storage/scm.go index 056503abe24..3296575d54b 100644 --- a/src/control/server/storage/scm.go +++ b/src/control/server/storage/scm.go @@ -45,16 +45,17 @@ const ( // ScmUnknownMode indicates a pMem AppDirect region is in an unsupported memory mode. ScmUnknownMode - // DefaultSysMemRsvd is the default amount of memory reserved for system when calculating - // RAM-disk size for DAOS I/O engine. - DefaultSysMemRsvd = humanize.GiByte * 6 - // DefaultEngineMemRsvd is the default amount of memory reserved per-engine when - // calculating RAM-disk size for DAOS I/O engine. - DefaultEngineMemRsvd = humanize.GiByte * 1 // MinRamdiskMem is the minimum amount of memory needed for each engine's tmpfs RAM-disk. MinRamdiskMem = humanize.GiByte * 4 ) +// Memory reservation constant defaults to be used when calculating RAM-disk size for DAOS I/O engine. +const ( + DefaultSysMemRsvd = humanize.GiByte * 16 // per-system + DefaultTgtMemRsvd = humanize.MiByte * 128 // per-engine-target + DefaultEngineMemRsvd = humanize.GiByte * 1 // per-engine +) + func (ss ScmState) String() string { if val, exists := map[ScmState]string{ ScmStateUnknown: "Unknown", @@ -561,20 +562,28 @@ func (f *ScmFwForwarder) UpdateFirmware(req ScmFirmwareUpdateRequest) (*ScmFirmw } // CalcRamdiskSize returns recommended tmpfs RAM-disk size calculated as -// (total mem - hugepage mem - sys rsvd mem - (engine rsvd mem * nr engines)) / nr engines. +// (total mem - hugepage mem - sys rsvd mem - engine rsvd mem) / nr engines. // All values in units of bytes and return value is for a single RAM-disk/engine. -func CalcRamdiskSize(log logging.Logger, memTotal, memHuge, memSys, memEng uint64, engCount int) (uint64, error) { +func CalcRamdiskSize(log logging.Logger, memTotal, memHuge, memSys uint64, tgtCount, engCount int) (uint64, error) { if memTotal == 0 { return 0, errors.New("requires nonzero total mem") } - if engCount == 0 { - return 0, errors.New("requires nonzero nr engines") + if tgtCount <= 0 { + return 0, errors.New("requires positive nonzero nr engine targets") + } + if engCount <= 0 { + return 0, errors.New("requires positive nonzero nr engines") + } + + memEng := uint64(tgtCount) * DefaultTgtMemRsvd + if memEng < DefaultEngineMemRsvd { + memEng = DefaultEngineMemRsvd } msgStats := fmt.Sprintf("mem stats: total %s (%d) - (hugepages %s + sys rsvd %s + "+ - "(engine rsvd %s * nr engines %d))", humanize.IBytes(memTotal), memTotal, - humanize.IBytes(memHuge), humanize.IBytes(memSys), humanize.IBytes(memEng), - engCount) + "(engine rsvd %s * nr engines %d). %d tgts-per-engine)", humanize.IBytes(memTotal), + memTotal, humanize.IBytes(memHuge), humanize.IBytes(memSys), + humanize.IBytes(memEng), engCount, tgtCount) memRsvd := memHuge + memSys + (memEng * uint64(engCount)) if memTotal < memRsvd { @@ -590,18 +599,26 @@ func CalcRamdiskSize(log logging.Logger, memTotal, memHuge, memSys, memEng uint6 } // CalcMemForRamdiskSize returns the minimum RAM required for the input requested RAM-disk size. -func CalcMemForRamdiskSize(log logging.Logger, ramdiskSize, memHuge, memSys, memEng uint64, engCount int) (uint64, error) { +func CalcMemForRamdiskSize(log logging.Logger, ramdiskSize, memHuge, memSys uint64, tgtCount, engCount int) (uint64, error) { if ramdiskSize == 0 { return 0, errors.New("requires nonzero ram-disk size") } + if tgtCount <= 0 { + return 0, errors.New("requires positive nonzero nr engine targets") + } if engCount == 0 { return 0, errors.New("requires nonzero nr engines") } + memEng := uint64(tgtCount) * DefaultTgtMemRsvd + if memEng < DefaultEngineMemRsvd { + memEng = DefaultEngineMemRsvd + } + msgStats := fmt.Sprintf("required ram-disk size %s (%d). mem hugepage: %s, nr engines: %d, "+ - "sys mem rsvd: %s, engine mem rsvd: %s", humanize.IBytes(ramdiskSize), ramdiskSize, - humanize.IBytes(memHuge), engCount, humanize.IBytes(memSys), - humanize.IBytes(memEng)) + "sys mem rsvd: %s, engine mem rsvd: %s, %d tgts-per-engine", + humanize.IBytes(ramdiskSize), ramdiskSize, humanize.IBytes(memHuge), engCount, + humanize.IBytes(memSys), humanize.IBytes(memEng), tgtCount) memRsvd := memHuge + memSys + (memEng * uint64(engCount)) memReqd := memRsvd + (ramdiskSize * uint64(engCount)) diff --git a/src/control/server/storage/scm_test.go b/src/control/server/storage/scm_test.go index 20d3a641532..08e8638f8f4 100644 --- a/src/control/server/storage/scm_test.go +++ b/src/control/server/storage/scm_test.go @@ -21,7 +21,7 @@ func Test_CalcRamdiskSize(t *testing.T) { memTotal uint64 memHuge uint64 memSys uint64 - memEng uint64 + tgtCount int engCount int expSize uint64 expErr error @@ -29,40 +29,54 @@ func Test_CalcRamdiskSize(t *testing.T) { "no mem": { expErr: errors.New("requires nonzero total mem"), }, + "no targets": { + memTotal: humanize.GiByte, + expErr: errors.New("requires positive nonzero nr engine targets"), + }, "no engines": { memTotal: humanize.GiByte, - expErr: errors.New("requires nonzero nr engines"), + tgtCount: 8, + expErr: errors.New("requires positive nonzero nr engines"), }, "default values; low mem": { - memTotal: humanize.GiByte * 18, - memHuge: humanize.GiByte * 12, + memTotal: humanize.GiByte * 30, + memHuge: humanize.GiByte * 14, memSys: DefaultSysMemRsvd, - memEng: DefaultEngineMemRsvd, + tgtCount: 8, engCount: 1, - expErr: errors.New("insufficient ram"), + expErr: errors.New("insufficient ram"), // 30 - (14+16+1) = -1 }, "default values; high mem": { - memTotal: humanize.GiByte * 23, - memHuge: humanize.GiByte * 12, + memTotal: humanize.GiByte * 60, + memHuge: humanize.GiByte * 30, memSys: DefaultSysMemRsvd, - memEng: DefaultEngineMemRsvd, - engCount: 1, - expSize: humanize.GiByte * 4, + tgtCount: 16, + engCount: 2, + expSize: humanize.GiByte * 5, // (60 - (30+16+4)) / 2 + }, + "default values; low nr targets": { + memTotal: humanize.GiByte * 60, + memHuge: humanize.GiByte * 30, + memSys: DefaultSysMemRsvd, + tgtCount: 1, + engCount: 2, + expSize: humanize.GiByte * 6, // (60 - (30+16+2)) / 2 }, "custom values; low sys reservation": { + memTotal: humanize.GiByte * 60, + memHuge: humanize.GiByte * 30, memSys: humanize.GiByte * 4, - memEng: DefaultEngineMemRsvd, - memTotal: humanize.GiByte * 18, - memHuge: humanize.GiByte * 12, + tgtCount: 16, engCount: 2, + expSize: humanize.GiByte * 11, // (60 - (30+4+4)) / 2 }, - "custom values; high eng reservation": { - memSys: DefaultSysMemRsvd, - memEng: humanize.GiByte * 3, - memTotal: humanize.GiByte * 23, - memHuge: humanize.GiByte * 12, + "custom values; high sys reservation": { + memTotal: humanize.GiByte * 60, + memHuge: humanize.GiByte * 30, + memSys: humanize.GiByte * 27, + tgtCount: 16, engCount: 2, - expErr: errors.New("insufficient ram"), + expErr: errors.New("insufficient ram"), // 60 - (30+27+4) = -1 }, } { t.Run(name, func(t *testing.T) { @@ -70,14 +84,15 @@ func Test_CalcRamdiskSize(t *testing.T) { defer test.ShowBufferOnFailure(t, buf) gotSize, gotErr := CalcRamdiskSize(log, tc.memTotal, tc.memHuge, tc.memSys, - tc.memEng, tc.engCount) + tc.tgtCount, tc.engCount) test.CmpErr(t, tc.expErr, gotErr) if tc.expErr != nil { return } if gotSize != tc.expSize { - t.Fatalf("expected %d, got %d", tc.expSize, gotSize) + t.Fatalf("expected %s, got %s", + humanize.IBytes(tc.expSize), humanize.IBytes(gotSize)) } }) } diff --git a/src/tests/ftest/container/snapshot_aggregation.yaml b/src/tests/ftest/container/snapshot_aggregation.yaml index 1b7031404a8..5e94dceec65 100644 --- a/src/tests/ftest/container/snapshot_aggregation.yaml +++ b/src/tests/ftest/container/snapshot_aggregation.yaml @@ -20,6 +20,7 @@ server_config: fabric_iface_port: 31417 log_file: daos_server1.log storage: auto + system_ram_reserved: 8 pool: control_method: dmg scm_size: 80G diff --git a/src/tests/ftest/control/dmg_server_set_logmasks.yaml b/src/tests/ftest/control/dmg_server_set_logmasks.yaml index b4e3a1ddfeb..29b8e5326e1 100644 --- a/src/tests/ftest/control/dmg_server_set_logmasks.yaml +++ b/src/tests/ftest/control/dmg_server_set_logmasks.yaml @@ -4,6 +4,7 @@ timeout: 120 server_config: name: daos_server engines_per_host: 1 + system_ram_reserved: 6 engines: 0: storage: diff --git a/src/tests/ftest/pool/create_all_vm.yaml b/src/tests/ftest/pool/create_all_vm.yaml index 0e030ee8cbc..4daa7d3ea46 100644 --- a/src/tests/ftest/pool/create_all_vm.yaml +++ b/src/tests/ftest/pool/create_all_vm.yaml @@ -30,6 +30,7 @@ test_two_pools: server_config: name: daos_server engines_per_host: 1 + system_ram_reserved: 6 engines: 0: targets: 5 diff --git a/src/tests/ftest/security/cont_overwrite_acl.yaml b/src/tests/ftest/security/cont_overwrite_acl.yaml index dd5f4cabd79..83974c98fd6 100644 --- a/src/tests/ftest/security/cont_overwrite_acl.yaml +++ b/src/tests/ftest/security/cont_overwrite_acl.yaml @@ -7,6 +7,7 @@ timeout: 120 server_config: name: daos_server engines_per_host: 1 + system_ram_reserved: 6 engines: 0: targets: 4 diff --git a/src/tests/ftest/security/cont_update_acl.yaml b/src/tests/ftest/security/cont_update_acl.yaml index 65091e2b06a..78c114e7e27 100644 --- a/src/tests/ftest/security/cont_update_acl.yaml +++ b/src/tests/ftest/security/cont_update_acl.yaml @@ -7,6 +7,7 @@ timeout: 120 server_config: name: daos_server engines_per_host: 1 + system_ram_reserved: 6 engines: 0: targets: 4 diff --git a/src/tests/ftest/telemetry/pool_space_metrics.yaml b/src/tests/ftest/telemetry/pool_space_metrics.yaml index d041e936c54..459e2a1954d 100644 --- a/src/tests/ftest/telemetry/pool_space_metrics.yaml +++ b/src/tests/ftest/telemetry/pool_space_metrics.yaml @@ -1,7 +1,7 @@ hosts: test_servers: 2 test_clients: 1 -timeout: 120 +timeout: 180 server_config: name: daos_server engines_per_host: 2 diff --git a/utils/config/daos_server.yml b/utils/config/daos_server.yml index 3fba33dda67..73772a44ed4 100644 --- a/utils/config/daos_server.yml +++ b/utils/config/daos_server.yml @@ -235,9 +235,10 @@ ## of RAM resulting in MemAvailable value being too low to support the calculated RAM-disk size ## increasing the value will reduce the calculate size. Alternatively in situations where total ## RAM is low, reducing the value may prevent problems where RAM-disk size calculated is below the -## minimum of 4gib. +## minimum of 4gib. Increasing the value may help avoid the potential of OOM killer terminating +## engine processes but could also result in stopping DAOS from using available memory resources. # -## default: 6 +## default: 16 #system_ram_reserved: 5 # #