DAOS-13672 control: Calculate engine memory reservation on nr targets (…

…#12597) (#12755)

During the evaluation of an optimum RAM-disk size, update per-engine
memory reservation per-engine calculation to take into account the
number of targets. Reserve 128mib of RAM per target.

control: Bump system_ram_reserved to reduce OOM occurrences (#12430)

Attempt to reduce the chance of OOM killer terminating an engine
process when maximum pool space is allocated by slightly increasing
the system_ram_reserved default value from 6->16gib.

Some test yaml system_ram_reserved values have been reduced to 6 to
prevent the increase in the default from causing an available memory
check failure on engine start-up in memory constrained (VM)
environments. Increasing the default value should also resolve
DAOS-13918 by providing a larger memory buffer to reduce the chance
of intermittent failures related to this check.

test: Reduce system_ram_reserved for GHA ARM build (#12746)

Signed-off-by: Tom Nabarro <tom.nabarro@intel.com>

.github/workflows/landing-builds.yml

@@ -383,4 +383,4 @@ jobs:
       - name: Run NLT
         run: docker run --mount type=tmpfs,destination=/mnt/daos_0,tmpfs-mode=1777 --user root:root
                  build-image ./daos/utils/node_local_test.py --no-root
-                 --memcheck no --test cont_copy
+                 --memcheck no --test cont_copy --system-ram-reserved 6

src/control/cmd/daos_server/auto_test.go

@@ -151,17 +151,27 @@ func TestDaosServer_Auto_confGen(t *testing.T) {
 	controlMetadata := storage.ControlMetadata{
 		Path: metadataMountPath,
 	}
-	// SCM tmpfs 5GiB size calculated after subtracting reservations from MemTotalKiB.
+	// Nr hugepages expected with 18+1 (extra MD-on-SSD sys-xstream) targets * 2 engines * 512
+	// pages-per-target.
+	tmpfsNrHugepages := 19 * 2 * 512
+	tmpfsHugeMemGiB := (humanize.MiByte * 2 * tmpfsNrHugepages) / humanize.GiByte
+	tmpfsEngRsvdGiB := 2 /* calculated based on 18 targets */
+	tmpfsSysRsvdGiB := storage.DefaultSysMemRsvd / humanize.GiByte
+	tmpfsRamdiskGiB := storage.MinRamdiskMem / humanize.GiByte
+	// Total mem to meet requirements 39GiB hugeMem, 2GiB per engine rsvd, 6GiB sys rsvd, 4GiB
+	// per engine RAM-disk.
+	tmpfsMemTotalGiB := humanize.GiByte * (tmpfsHugeMemGiB + (2 * tmpfsEngRsvdGiB) + tmpfsSysRsvdGiB +
+		(2 * tmpfsRamdiskGiB) + 1 /* add 1GiB buffer */)
 	tmpfsEngineCfgs := []*engine.Config{
-		control.MockEngineCfgTmpfs(0, 5,
+		control.MockEngineCfgTmpfs(0, tmpfsRamdiskGiB,
 			control.MockBdevTierWithRole(0, storage.BdevRoleWAL, 2),
 			control.MockBdevTierWithRole(0, storage.BdevRoleMeta|storage.BdevRoleData, 4)).
 			WithStorageControlMetadataPath(metadataMountPath).
 			WithStorageConfigOutputPath(
 				filepath.Join(controlMetadata.EngineDirectory(0), storage.BdevOutConfName),
 			).
 			WithTargetCount(18).WithHelperStreamCount(4),
-		control.MockEngineCfgTmpfs(1, 5,
+		control.MockEngineCfgTmpfs(1, tmpfsRamdiskGiB,
 			control.MockBdevTierWithRole(1, storage.BdevRoleWAL, 1),
 			control.MockBdevTierWithRole(1, storage.BdevRoleMeta|storage.BdevRoleData, 3)).
 			WithStorageControlMetadataPath(metadataMountPath).
@@ -368,12 +378,7 @@ func TestDaosServer_Auto_confGen(t *testing.T) {
 					storage.MockScmNamespace(0),
 					storage.MockScmNamespace(1),
 				},
-				MemInfo: &common.MemInfo{
-					HugepageSizeKiB: 2048,
-					// Total mem to meet requirements 39GiB hugeMem, 1GiB per
-					// engine rsvd, 6GiB sys rsvd, 5GiB per engine for tmpfs.
-					MemTotalKiB: (humanize.GiByte * (39 + 2 + 6 + 10)) / humanize.KiByte,
-				},
+				MemInfo: &defMemInfo,
 				NvmeDevices: storage.NvmeControllers{
 					storage.MockNvmeController(1),
 					storage.MockNvmeController(2),
@@ -428,9 +433,7 @@ func TestDaosServer_Auto_confGen(t *testing.T) {
 				},
 				MemInfo: &common.MemInfo{
 					HugepageSizeKiB: 2048,
-					// Total mem to meet requirements 39GiB hugeMem, 1GiB per
-					// engine rsvd, 6GiB sys rsvd, 5GiB per engine for tmpfs.
-					MemTotalKiB: (humanize.GiByte * (39 + 2 + 6 + 10)) / humanize.KiByte,
+					MemTotalKiB:     tmpfsMemTotalGiB / humanize.KiByte,
 				},
 				NvmeDevices: storage.NvmeControllers{
 					storage.MockNvmeController(1),
@@ -440,8 +443,7 @@ func TestDaosServer_Auto_confGen(t *testing.T) {
 				},
 			},
 			expCfg: control.MockServerCfg("ofi+psm2", tmpfsEngineCfgs).
-				// 18+1 (extra MD-on-SSD sys-xstream) targets * 2 engines * 512 pages
-				WithNrHugepages(19 * 2 * 512).
+				WithNrHugepages(tmpfsNrHugepages).
 				WithAccessPoints("localhost:10001").
 				WithControlLogFile("/tmp/daos_server.log").
 				WithControlMetadata(controlMetadata),

src/control/cmd/dmg/auto_test.go

@@ -140,18 +140,18 @@ func TestAuto_confGen(t *testing.T) {
 		Addr:    "host1",
 		Message: control.MockServerScanResp(t, "withSpaceUsage"),
 	}
-	// Total mem to meet requirements 34GiB hugeMem, 1GiB per engine rsvd, 6GiB sys rsvd,
-	// 5GiB per engine for tmpfs.
 	storRespHighMem := control.MockServerScanResp(t, "withSpaceUsage")
-	storRespHighMem.MemInfo.MemTotalKb = (humanize.GiByte * (34 + 2 + 6 + 10)) / humanize.KiByte
+	// Total mem to meet requirements 34GiB hugeMem, 2GiB per engine rsvd, 16GiB sys rsvd,
+	// 5GiB per engine for tmpfs.
+	storRespHighMem.MemInfo.MemTotalKb = (humanize.GiByte * (34 + 4 + 16 + 10)) / humanize.KiByte
+	mockRamdiskSize := 5
 	storHostRespHighMem := &control.HostResponse{
 		Addr:    "host1",
 		Message: storRespHighMem,
 	}
 	e0 := control.MockEngineCfg(0, 2, 4, 6, 8).WithHelperStreamCount(4)
 	e1 := control.MockEngineCfg(1, 1, 3, 5, 7).WithHelperStreamCount(4)
 	exmplEngineCfgs := []*engine.Config{e0, e1}
-	mockRamdiskSize := 5 // RoundDownGiB(16*0.75/2)
 	metadataMountPath := "/mnt/daos_md"
 	controlMetadata := storage.ControlMetadata{
 		Path: metadataMountPath,
@@ -406,7 +406,7 @@ disable_vfio: false
 disable_vmd: false
 enable_hotplug: false
 nr_hugepages: 6144
-system_ram_reserved: 6
+system_ram_reserved: 16
 disable_hugepages: false
 control_log_mask: INFO
 control_log_file: /tmp/daos_server.log

src/control/lib/control/auto_test.go

@@ -1573,7 +1573,7 @@ func TestControl_AutoConfig_genConfig(t *testing.T) {
 				MockEngineCfgTmpfs(1, 0, mockBdevTier(1, 3), mockBdevTier(1, 4, 5)),
 			},
 			hpSize:   defHpSizeKb,
-			memTotal: (52 * humanize.GiByte) / humanize.KiByte,
+			memTotal: (64 * humanize.GiByte) / humanize.KiByte,
 			expCfg: MockServerCfg(exmplEngineCfg0.Fabric.Provider,
 				[]*engine.Config{
 					MockEngineCfgTmpfs(0, 5, /* tmpfs size in gib */

src/control/server/config/server.go

@@ -521,8 +521,12 @@ func (cfg *Server) CalcRamdiskSize(log logging.Logger, hpSizeKiB, memKiB int) (u
 	// Calculate reserved system memory in bytes.
 	memSys := uint64(cfg.SystemRamReserved * humanize.GiByte)
 
-	return storage.CalcRamdiskSize(log, memTotal, memHuge, memSys,
-		storage.DefaultEngineMemRsvd, len(cfg.Engines))
+	if len(cfg.Engines) == 0 {
+		return 0, errors.New("no engines in config")
+	}
+
+	return storage.CalcRamdiskSize(log, memTotal, memHuge, memSys, cfg.Engines[0].TargetCount,
+		len(cfg.Engines))
 }
 
 // CalcMemForRamdiskSize calculates minimum memory needed for a given RAM-disk size.
@@ -533,8 +537,12 @@ func (cfg *Server) CalcMemForRamdiskSize(log logging.Logger, hpSizeKiB int, ramd
 	// Calculate reserved system memory in bytes.
 	memSys := uint64(cfg.SystemRamReserved * humanize.GiByte)
 
+	if len(cfg.Engines) == 0 {
+		return 0, errors.New("no engines in config")
+	}
+
 	return storage.CalcMemForRamdiskSize(log, ramdiskSize, memHuge, memSys,
-		storage.DefaultEngineMemRsvd, len(cfg.Engines))
+		cfg.Engines[0].TargetCount, len(cfg.Engines))
 }
 
 // SetRamdiskSize calculates maximum RAM-disk size using total memory as reported by /proc/meminfo.

src/control/server/config/server_test.go

@@ -1009,7 +1009,7 @@ func TestServerConfig_SetRamdiskSize(t *testing.T) {
 				c.Engines[0].Storage.Tiers.ScmConfigs()[0].Scm.RamdiskSize = 11
 				return c.WithNrHugepages(16896)
 			},
-			expErr: FaultConfigRamdiskOverMaxMem(humanize.GiByte*11, humanize.GiByte*10, 0),
+			expErr: FaultConfigRamdiskOverMaxMem(humanize.GiByte*11, humanize.GiByte*9, 0),
 		},
 		"low mem": {
 			// 46 total - 40 reserved = 6 for tmpfs (3 gib per engine - too low)
@@ -1019,7 +1019,7 @@ func TestServerConfig_SetRamdiskSize(t *testing.T) {
 			},
 			// error indicates min RAM needed = 40 + 4 gib per engine
 			expErr: storage.FaultRamdiskLowMem("Total", storage.MinRamdiskMem,
-				humanize.GiByte*48, humanize.GiByte*46),
+				humanize.GiByte*50, humanize.GiByte*46),
 		},
 		"custom value set": {
 			memTotBytes: humanize.GiByte * 60,
@@ -1036,7 +1036,7 @@ func TestServerConfig_SetRamdiskSize(t *testing.T) {
 			extraConfig: func(c *Server) *Server {
 				return c.WithNrHugepages(16896)
 			},
-			expRamdiskSize: 10,
+			expRamdiskSize: 9,
 		},
 		"custom system_ram_reserved value set": {
 			// 33 huge mem + 2 sys rsv + 2 engine rsv = 37 gib reserved mem
@@ -1046,7 +1046,7 @@ func TestServerConfig_SetRamdiskSize(t *testing.T) {
 				c.SystemRamReserved = 2
 				return c.WithNrHugepages(16896)
 			},
-			expRamdiskSize: 11,
+			expRamdiskSize: 10,
 		},
 		"no scm configured on second engine": {
 			memTotBytes: humanize.GiByte * 80,

src/control/server/storage/scm.go

@@ -45,16 +45,17 @@ const (
 	// ScmUnknownMode indicates a pMem AppDirect region is in an unsupported memory mode.
 	ScmUnknownMode
 
-	// DefaultSysMemRsvd is the default amount of memory reserved for system when calculating
-	// RAM-disk size for DAOS I/O engine.
-	DefaultSysMemRsvd = humanize.GiByte * 6
-	// DefaultEngineMemRsvd is the default amount of memory reserved per-engine when
-	// calculating RAM-disk size for DAOS I/O engine.
-	DefaultEngineMemRsvd = humanize.GiByte * 1
 	// MinRamdiskMem is the minimum amount of memory needed for each engine's tmpfs RAM-disk.
 	MinRamdiskMem = humanize.GiByte * 4
 )
 
+// Memory reservation constant defaults to be used when calculating RAM-disk size for DAOS I/O engine.
+const (
+	DefaultSysMemRsvd    = humanize.GiByte * 16  // per-system
+	DefaultTgtMemRsvd    = humanize.MiByte * 128 // per-engine-target
+	DefaultEngineMemRsvd = humanize.GiByte * 1   // per-engine
+)
+
 func (ss ScmState) String() string {
 	if val, exists := map[ScmState]string{
 		ScmStateUnknown:   "Unknown",
@@ -561,20 +562,28 @@ func (f *ScmFwForwarder) UpdateFirmware(req ScmFirmwareUpdateRequest) (*ScmFirmw
 }
 
 // CalcRamdiskSize returns recommended tmpfs RAM-disk size calculated as
-// (total mem - hugepage mem - sys rsvd mem - (engine rsvd mem * nr engines)) / nr engines.
+// (total mem - hugepage mem - sys rsvd mem - engine rsvd mem) / nr engines.
 // All values in units of bytes and return value is for a single RAM-disk/engine.
-func CalcRamdiskSize(log logging.Logger, memTotal, memHuge, memSys, memEng uint64, engCount int) (uint64, error) {
+func CalcRamdiskSize(log logging.Logger, memTotal, memHuge, memSys uint64, tgtCount, engCount int) (uint64, error) {
 	if memTotal == 0 {
 		return 0, errors.New("requires nonzero total mem")
 	}
-	if engCount == 0 {
-		return 0, errors.New("requires nonzero nr engines")
+	if tgtCount <= 0 {
+		return 0, errors.New("requires positive nonzero nr engine targets")
+	}
+	if engCount <= 0 {
+		return 0, errors.New("requires positive nonzero nr engines")
+	}
+
+	memEng := uint64(tgtCount) * DefaultTgtMemRsvd
+	if memEng < DefaultEngineMemRsvd {
+		memEng = DefaultEngineMemRsvd
 	}
 
 	msgStats := fmt.Sprintf("mem stats: total %s (%d) - (hugepages %s + sys rsvd %s + "+
-		"(engine rsvd %s * nr engines %d))", humanize.IBytes(memTotal), memTotal,
-		humanize.IBytes(memHuge), humanize.IBytes(memSys), humanize.IBytes(memEng),
-		engCount)
+		"(engine rsvd %s * nr engines %d). %d tgts-per-engine)", humanize.IBytes(memTotal),
+		memTotal, humanize.IBytes(memHuge), humanize.IBytes(memSys),
+		humanize.IBytes(memEng), engCount, tgtCount)
 
 	memRsvd := memHuge + memSys + (memEng * uint64(engCount))
 	if memTotal < memRsvd {
@@ -590,18 +599,26 @@ func CalcRamdiskSize(log logging.Logger, memTotal, memHuge, memSys, memEng uint6
 }
 
 // CalcMemForRamdiskSize returns the minimum RAM required for the input requested RAM-disk size.
-func CalcMemForRamdiskSize(log logging.Logger, ramdiskSize, memHuge, memSys, memEng uint64, engCount int) (uint64, error) {
+func CalcMemForRamdiskSize(log logging.Logger, ramdiskSize, memHuge, memSys uint64, tgtCount, engCount int) (uint64, error) {
 	if ramdiskSize == 0 {
 		return 0, errors.New("requires nonzero ram-disk size")
 	}
+	if tgtCount <= 0 {
+		return 0, errors.New("requires positive nonzero nr engine targets")
+	}
 	if engCount == 0 {
 		return 0, errors.New("requires nonzero nr engines")
 	}
 
+	memEng := uint64(tgtCount) * DefaultTgtMemRsvd
+	if memEng < DefaultEngineMemRsvd {
+		memEng = DefaultEngineMemRsvd
+	}
+
 	msgStats := fmt.Sprintf("required ram-disk size %s (%d). mem hugepage: %s, nr engines: %d, "+
-		"sys mem rsvd: %s, engine mem rsvd: %s", humanize.IBytes(ramdiskSize), ramdiskSize,
-		humanize.IBytes(memHuge), engCount, humanize.IBytes(memSys),
-		humanize.IBytes(memEng))
+		"sys mem rsvd: %s, engine mem rsvd: %s, %d tgts-per-engine",
+		humanize.IBytes(ramdiskSize), ramdiskSize, humanize.IBytes(memHuge), engCount,
+		humanize.IBytes(memSys), humanize.IBytes(memEng), tgtCount)
 
 	memRsvd := memHuge + memSys + (memEng * uint64(engCount))
 	memReqd := memRsvd + (ramdiskSize * uint64(engCount))

src/control/server/storage/scm_test.go

@@ -21,63 +21,78 @@ func Test_CalcRamdiskSize(t *testing.T) {
 		memTotal uint64
 		memHuge  uint64
 		memSys   uint64
-		memEng   uint64
+		tgtCount int
 		engCount int
 		expSize  uint64
 		expErr   error
 	}{
 		"no mem": {
 			expErr: errors.New("requires nonzero total mem"),
 		},
+		"no targets": {
+			memTotal: humanize.GiByte,
+			expErr:   errors.New("requires positive nonzero nr engine targets"),
+		},
 		"no engines": {
 			memTotal: humanize.GiByte,
-			expErr:   errors.New("requires nonzero nr engines"),
+			tgtCount: 8,
+			expErr:   errors.New("requires positive nonzero nr engines"),
 		},
 		"default values; low mem": {
-			memTotal: humanize.GiByte * 18,
-			memHuge:  humanize.GiByte * 12,
+			memTotal: humanize.GiByte * 30,
+			memHuge:  humanize.GiByte * 14,
 			memSys:   DefaultSysMemRsvd,
-			memEng:   DefaultEngineMemRsvd,
+			tgtCount: 8,
 			engCount: 1,
-			expErr:   errors.New("insufficient ram"),
+			expErr:   errors.New("insufficient ram"), // 30 - (14+16+1) = -1
 		},
 		"default values; high mem": {
-			memTotal: humanize.GiByte * 23,
-			memHuge:  humanize.GiByte * 12,
+			memTotal: humanize.GiByte * 60,
+			memHuge:  humanize.GiByte * 30,
 			memSys:   DefaultSysMemRsvd,
-			memEng:   DefaultEngineMemRsvd,
-			engCount: 1,
-			expSize:  humanize.GiByte * 4,
+			tgtCount: 16,
+			engCount: 2,
+			expSize:  humanize.GiByte * 5, // (60 - (30+16+4)) / 2
+		},
+		"default values; low nr targets": {
+			memTotal: humanize.GiByte * 60,
+			memHuge:  humanize.GiByte * 30,
+			memSys:   DefaultSysMemRsvd,
+			tgtCount: 1,
+			engCount: 2,
+			expSize:  humanize.GiByte * 6, // (60 - (30+16+2)) / 2
 		},
 		"custom values; low sys reservation": {
+			memTotal: humanize.GiByte * 60,
+			memHuge:  humanize.GiByte * 30,
 			memSys:   humanize.GiByte * 4,
-			memEng:   DefaultEngineMemRsvd,
-			memTotal: humanize.GiByte * 18,
-			memHuge:  humanize.GiByte * 12,
+			tgtCount: 16,
 			engCount: 2,
+			expSize:  humanize.GiByte * 11, // (60 - (30+4+4)) / 2
 		},
-		"custom values; high eng reservation": {
-			memSys:   DefaultSysMemRsvd,
-			memEng:   humanize.GiByte * 3,
-			memTotal: humanize.GiByte * 23,
-			memHuge:  humanize.GiByte * 12,
+		"custom values; high sys reservation": {
+			memTotal: humanize.GiByte * 60,
+			memHuge:  humanize.GiByte * 30,
+			memSys:   humanize.GiByte * 27,
+			tgtCount: 16,
 			engCount: 2,
-			expErr:   errors.New("insufficient ram"),
+			expErr:   errors.New("insufficient ram"), // 60 - (30+27+4) = -1
 		},
 	} {
 		t.Run(name, func(t *testing.T) {
 			log, buf := logging.NewTestLogger(name)
 			defer test.ShowBufferOnFailure(t, buf)
 
 			gotSize, gotErr := CalcRamdiskSize(log, tc.memTotal, tc.memHuge, tc.memSys,
-				tc.memEng, tc.engCount)
+				tc.tgtCount, tc.engCount)
 			test.CmpErr(t, tc.expErr, gotErr)
 			if tc.expErr != nil {
 				return
 			}
 
 			if gotSize != tc.expSize {
-				t.Fatalf("expected %d, got %d", tc.expSize, gotSize)
+				t.Fatalf("expected %s, got %s",
+					humanize.IBytes(tc.expSize), humanize.IBytes(gotSize))
 			}
 		})
 	}

src/tests/ftest/container/snapshot_aggregation.yaml

@@ -20,6 +20,7 @@ server_config:
       fabric_iface_port: 31417
       log_file: daos_server1.log
       storage: auto
+  system_ram_reserved: 8
 pool:
   control_method: dmg
   scm_size: 80G

src/tests/ftest/control/dmg_server_set_logmasks.yaml

@@ -4,6 +4,7 @@ timeout: 120
 server_config:
   name: daos_server
   engines_per_host: 1
+  system_ram_reserved: 6
   engines:
     0:
       storage:

src/tests/ftest/pool/create_all_vm.yaml

@@ -30,6 +30,7 @@ test_two_pools:
 server_config:
   name: daos_server
   engines_per_host: 1
+  system_ram_reserved: 6
   engines:
     0:
       targets: 5