Skip to content

Commit

Permalink
Memory metrics (#526)
Browse files Browse the repository at this point in the history 
* feat: added new memory metrics into SystemSample - memoryCachedBytes, memorySharedBytes and memorySlabBytes (#501)

* test: added harvest tests for new memory metrics (#512)

* feat: added new memory metrics into SystemSample - memoryCachedBytes, memorySharedBytes and memorySlabBytes

Co-authored-by: Cristian Ciutea <cristi.ciutea@gmail.com>
  • Loading branch information
@brushknight @cristianciutea
brushknight and cristianciutea authored Jun 9, 2021
1 parent 53e1c11 commit 2efc208
Show file tree
Hide file tree
Showing 4 changed files with 255 additions and 17 deletions.
7 changes: 7 additions & 0 deletions pkg/metrics/memory.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,9 @@ type MemorySample struct {
MemoryUsed float64 `json:"memoryUsedBytes"`
MemoryFreePercent float64 `json:"memoryFreePercent"`
MemoryUsedPercent float64 `json:"memoryUsedPercent"`
MemoryCachedBytes float64 `json:"memoryCachedBytes"`
MemorySlabBytes float64 `json:"memorySlabBytes"`
MemorySharedBytes float64 `json:"memorySharedBytes"`

SwapTotal float64 `json:"swapTotalBytes"`
SwapFree float64 `json:"swapFreeBytes"`
Expand Down Expand Up @@ -53,6 +56,10 @@ func (mm *MemoryMonitor) Sample() (result *MemorySample, err error) {
MemoryTotal: float64(memory.Total),
MemoryFree: float64(memory.Available),
MemoryUsed: float64(memory.Used),
MemoryCachedBytes: float64(memory.Cached),
MemorySlabBytes: float64(memory.Slab),
MemorySharedBytes: float64(memory.Shared),

MemoryFreePercent: memoryFreePercent,
MemoryUsedPercent: memoryUsedPercent,

Expand Down
42 changes: 31 additions & 11 deletions pkg/metrics/memory_linux.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -35,6 +35,10 @@ func reclaimableAsFree() (*mem.VirtualMemoryStat, error) {
filename := helpers.HostProc("meminfo")
lines, _ := acquire.ReadLines(filename)

return reclaimableAsFreeParseMemInfo(lines)
}

func reclaimableAsFreeParseMemInfo(lines []string) (*mem.VirtualMemoryStat, error) {
ret := &mem.VirtualMemoryStat{}
readFields := 0
for _, line := range lines {
Expand Down Expand Up @@ -63,16 +67,23 @@ func reclaimableAsFree() (*mem.VirtualMemoryStat, error) {
case "Cached":
ret.Cached = t * 1024
readFields++
case "Shmem":
ret.Shared = t * 1024
readFields++
case "Slab":
ret.Slab = t * 1024
readFields++
case "SReclaimable":
ret.SReclaimable = t * 1024
readFields++
}
if readFields >= 5 { // stop reading the file when we have read all the fields we require
if readFields >= 7 { // stop reading the file when we have read all the fields we require
break
}
}

ret.Available = ret.Free + ret.Buffers + ret.Cached + ret.SReclaimable
ret.Cached += ret.SReclaimable
ret.Available = ret.Free + ret.Buffers + ret.Cached
ret.Used = ret.Total - ret.Available

return ret, nil
Expand All @@ -86,12 +97,14 @@ func reclaimableAsFree() (*mem.VirtualMemoryStat, error) {
func reclaimableAsUsed() (*mem.VirtualMemoryStat, error) {
filename := helpers.HostProc("meminfo")
lines, _ := acquire.ReadLines(filename)
return reclaimableAsUsedParseMemInfo(lines)
}

func reclaimableAsUsedParseMemInfo(lines []string) (*mem.VirtualMemoryStat, error) {
memAvailable := false
memTotal := false

ret := &mem.VirtualMemoryStat{}
parse:
readFields := 0
for _, line := range lines {
fields := strings.Split(line, ":")
if len(fields) != 2 {
Expand All @@ -109,23 +122,30 @@ parse:
case "MemAvailable":
ret.Available = t * 1024
memAvailable = true
if memTotal {
break parse // stop parsing if we have enough fields
}
case "MemTotal":
ret.Total = t * 1024
memTotal = true
if memAvailable {
break parse // stop parsing if we have enough fields
}
readFields++
case "MemFree":
ret.Free = t * 1024
readFields++
case "Buffers":
ret.Buffers = t * 1024
readFields++
case "Cached":
ret.Cached = t * 1024
readFields++
case "Shmem":
ret.Shared = t * 1024
readFields++
case "Slab":
ret.Slab = t * 1024
readFields++
case "SReclaimable":
ret.SReclaimable = t * 1024
readFields++
}
if readFields >= 7 && memAvailable { // stop reading the file when we have read all the fields we require
break
}
}
if !memAvailable {
Expand Down
110 changes: 110 additions & 0 deletions pkg/metrics/memory_linux_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,8 @@
package metrics

import (
"github.com/shirou/gopsutil/mem"
"strings"
"testing"

"github.com/stretchr/testify/assert"
Expand Down Expand Up @@ -47,3 +49,111 @@ func TestMemoryMonitor_ReclaimableValues(t *testing.T) {
"%v (MemoryFree without reclaimable) should be > %v (MemoryFree with reclaimable)", sf.MemoryFree, su.MemoryFree)

}

var memInfoWithMemAvailable = `MemTotal: 2040788 kB
MemFree: 1344288 kB
MemAvailable: 1595120 kB
Buffers: 59388 kB
Cached: 292896 kB
SwapCached: 0 kB
Active: 388176 kB
Inactive: 177040 kB
Active(anon): 203872 kB
Inactive(anon): 144 kB
Active(file): 184304 kB
Inactive(file): 176896 kB
Unevictable: 18476 kB
Mlocked: 18476 kB
SwapTotal: 0 kB
SwapFree: 0 kB
Dirty: 248 kB
Writeback: 184 kB
AnonPages: 231428 kB
Mapped: 163992 kB
Shmem: 1044 kB
Slab: 79668 kB
SReclaimable: 42636 kB
SUnreclaim: 37032 kB
KernelStack: 3984 kB
PageTables: 7012 kB
NFS_Unstable: 0 kB
Bounce: 0 kB
WritebackTmp: 0 kB
CommitLimit: 1020392 kB
Committed_AS: 1794120 kB
VmallocTotal: 34359738367 kB
VmallocUsed: 0 kB
VmallocChunk: 0 kB
HardwareCorrupted: 0 kB
AnonHugePages: 0 kB
ShmemHugePages: 0 kB
ShmemPmdMapped: 0 kB
CmaTotal: 0 kB
CmaFree: 0 kB
HugePages_Total: 0
HugePages_Free: 0
HugePages_Rsvd: 0
HugePages_Surp: 0
Hugepagesize: 2048 kB
DirectMap4k: 122816 kB
DirectMap2M: 1974272 kB
`

func TestMemoryMonitor_reclaimableAsUsedParseMemInfo(t *testing.T) {
actual, err := reclaimableAsUsedParseMemInfo(strings.Split(memInfoWithMemAvailable, "\n"))
assert.NoError(t, err)
expected := &mem.VirtualMemoryStat{
Available: 1595120 * 1024,
Total: 2040788 * 1024,
Free: 1344288 * 1024,
Buffers: 59388 * 1024,
Cached: 292896 * 1024,
Shared: 1044 * 1024,
Slab: 79668 * 1024,
SReclaimable: 42636 * 1024,
Used: (2040788 - 1595120) * 1024, // Total - Available
}
assert.Equal(t, expected.String(), actual.String())
}

func TestMemoryMonitor_reclaimableAsUsedParseMemInfoWithoutMemAvailable(t *testing.T) {
lines := strings.Split(memInfoWithMemAvailable, "\n")
// Remove MemAvailable line
lines = append(lines[:2], lines[3:]...)

memAvailable := uint64(1344288 + 59388 + 292896) // Free + Buffers + Cached

actual, err := reclaimableAsUsedParseMemInfo(lines)
assert.NoError(t, err)
expected := &mem.VirtualMemoryStat{
Available: memAvailable * 1024,
Total: 2040788 * 1024,
Free: 1344288 * 1024,
Buffers: 59388 * 1024,
Cached: 292896 * 1024,
Shared: 1044 * 1024,
Slab: 79668 * 1024,
SReclaimable: 42636 * 1024,
Used: (2040788 - memAvailable) * 1024, // Total - Available
}
assert.Equal(t, expected.String(), actual.String())
}

func TestMemoryMonitor_reclaimableAsFreeParseMemInfo(t *testing.T) {
actual, err := reclaimableAsFreeParseMemInfo(strings.Split(memInfoWithMemAvailable, "\n"))
assert.NoError(t, err)

memAvailable := uint64(1344288 + 59388 + 292896 + 42636) // Free + Buffers + Cached + SReclaimable
expected := &mem.VirtualMemoryStat{
Available: memAvailable * 1024,
Total: 2040788 * 1024,
Free: 1344288 * 1024,
Buffers: 59388 * 1024,
Cached: (292896 + 42636) * 1024, // Cached + SReclaimable
Shared: 1044 * 1024,
Slab: 79668 * 1024,
SReclaimable: 42636 * 1024,
Used: (2040788 - memAvailable) * 1024, // Total - Available
}
assert.Equal(t, expected.String(), actual.String())
}
113 changes: 107 additions & 6 deletions test/harvest/hostmetrics_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,19 +7,19 @@ package harvest

import (
"fmt"
"io/ioutil"
"os"
"runtime"
"testing"
"time"

"github.com/newrelic/infrastructure-agent/internal/agent/mocks"
"github.com/newrelic/infrastructure-agent/internal/testhelpers"
"github.com/newrelic/infrastructure-agent/pkg/config"
"github.com/newrelic/infrastructure-agent/pkg/metrics"
"github.com/newrelic/infrastructure-agent/pkg/metrics/storage"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"io/ioutil"
"os"
"os/exec"
"runtime"
"testing"
"time"
)

const timeout = 5 * time.Second
Expand Down Expand Up @@ -63,6 +63,75 @@ func TestHostCPU(t *testing.T) {
t.Logf("CPUPercents: %f, %f", beforeSample.CPUPercent, afterSample.CPUPercent)
})
}
func TestHostSharedMemory(t *testing.T) {
ctx := new(mocks.AgentContext)
ctx.On("Config").Return(&config.Config{
MetricsNetworkSampleRate: 1,
})
storageSampler := storage.NewSampler(ctx)

systemSampler := metrics.NewSystemSampler(ctx, storageSampler)

sampleB, _ := systemSampler.Sample()
beforeSample := sampleB[0].(*metrics.SystemSample)

f, err := os.Create("/dev/shm/test")
require.NoError(t, err)

for i := 0; i < 1024*1024; i++ {
f.Write([]byte("0"))
}

f.Sync()
defer func() {
require.NoError(t, f.Close())
os.Remove(f.Name())
}()

testhelpers.Eventually(t, timeout, func(st require.TestingT) {
sampleB, _ = systemSampler.Sample()
afterSample := sampleB[0].(*metrics.SystemSample)

assert.True(st, beforeSample.MemorySharedBytes+(1024*1024) == afterSample.MemorySharedBytes, "Shared Memory used did not increase enough, SharedMemoryBefore: %f SharedMemoryAfter %f ", beforeSample.MemorySharedBytes, afterSample.MemorySharedBytes)
})
}

func TestHostCachedMemory(t *testing.T) {
ctx := new(mocks.AgentContext)
ctx.On("Config").Return(&config.Config{
MetricsNetworkSampleRate: 1,
})
storageSampler := storage.NewSampler(ctx)

systemSampler := metrics.NewSystemSampler(ctx, storageSampler)

sampleB, _ := systemSampler.Sample()
beforeSample := sampleB[0].(*metrics.SystemSample)

f, err := ioutil.TempFile("/tmp", "")
assert.NoError(t, err)
defer os.Remove(f.Name())

// Force memory spike
for i := 0; i < 1e5; i++ {
f.Write([]byte("00000000000000000000"))
}

f.Sync()
f.Close()

_, err = ioutil.ReadFile(f.Name())

assert.NoError(t, err)

testhelpers.Eventually(t, timeout, func(st require.TestingT) {
sampleB, _ = systemSampler.Sample()
afterSample := sampleB[0].(*metrics.SystemSample)

expectedIncreaseBytes := 500000.0
assert.True(st, beforeSample.MemoryCachedBytes+expectedIncreaseBytes <= afterSample.MemoryCachedBytes, "CachedMemory used did not increase enough, expected an increase by %f CachedMemoryBefore: %f CachedMemoryAfter %f ", expectedIncreaseBytes, beforeSample.MemoryCachedBytes, afterSample.MemoryCachedBytes)
})
}

func TestHostMemory(t *testing.T) {
ctx := new(mocks.AgentContext)
Expand Down Expand Up @@ -108,6 +177,38 @@ func TestHostMemory(t *testing.T) {
})
}

func TestHostSlabMemory(t *testing.T) {
ctx := new(mocks.AgentContext)
ctx.On("Config").Return(&config.Config{
MetricsNetworkSampleRate: 1,
})
storageSampler := storage.NewSampler(ctx)

systemSampler := metrics.NewSystemSampler(ctx, storageSampler)

sampleB, _ := systemSampler.Sample()
beforeSample := sampleB[0].(*metrics.SystemSample)

for i := 0; i < 1000; i++ {
cmd := exec.Command("/bin/bash", "-c", "echo x")
err := cmd.Start()
require.NoError(t, err)
defer func() {
if cmd.Process != nil {
cmd.Process.Kill()
}
}()
}

testhelpers.Eventually(t, timeout, func(st require.TestingT) {
sampleB, _ = systemSampler.Sample()
afterSample := sampleB[0].(*metrics.SystemSample)

expectedIncreaseBytes := 500000.0
assert.True(t, beforeSample.MemorySlabBytes+expectedIncreaseBytes <= afterSample.MemorySlabBytes, "Slab memory used did not increase enough, expected %f increase, SlabMemoryBefore: %f SlabMemoryAfter %f ", expectedIncreaseBytes, beforeSample.MemorySlabBytes, afterSample.MemorySlabBytes)
})
}

func TestHostSwap(t *testing.T) {
if runtime.GOOS != "linux" {
t.Skip("not implemented on darwin or windows")
Expand Down

0 comments on commit 2efc208

Please sign in to comment.