Skip to content

Commit

Permalink
Revert "NR-72398 - Do not let Infra Agent's Fluent Bit config files f…
Browse files Browse the repository at this point in the history 
…ill up the disk (#1570)" (#1572)

This reverts commit 531ac5d.
  • Loading branch information
@rubenruizdegauna
rubenruizdegauna authored Mar 2, 2023
1 parent 531ac5d commit 5f1d8b8
Show file tree
Hide file tree
Showing 2 changed files with 2 additions and 216 deletions.
126 changes: 2 additions & 124 deletions pkg/integrations/v4/supervisor_fb.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,18 +5,14 @@ package v4
import (
ctx2 "context"
"fmt"
"io/fs"
"io/ioutil"
"os"
"path/filepath"
"regexp"
"sort"
"strings"
"time"

"github.com/newrelic/infrastructure-agent/pkg/sysinfo/hostname"
"github.com/pkg/errors"

"github.com/newrelic/infrastructure-agent/pkg/sysinfo/hostname"

"github.com/newrelic/infrastructure-agent/pkg/entity"
"github.com/newrelic/infrastructure-agent/pkg/sample"

Expand All @@ -27,7 +23,6 @@ import (
)

var sFBLogger = log.WithComponent("integrations.Supervisor").WithField("process", "log-forwarder")
var luaFilterTempFileRegex = regexp.MustCompile("nr_fb_lua_filter\\d+")

type FBSupervisorConfig struct {
FluentBitExePath string
Expand All @@ -36,35 +31,6 @@ type FBSupervisorConfig struct {
FluentBitVerbose bool
}

const (
MaxNumberOfFbConfigTempFiles int = 50
)

// listError error representing a list of errors.
type listError struct {
Errors []error
}

func (s *listError) Error() (err string) {
err = "List of errors:"
for _, e := range s.Errors {
err += fmt.Sprintf(" - %s", e.Error())
}

return err
}

func (s *listError) Add(e error) { s.Errors = append(s.Errors, e) }

// ErrorOrNil returns an error interface if the Error slice is not empty or nil otherwise.
func (s *listError) ErrorOrNil() error {
if s == nil || len(s.Errors) == 0 {
return nil
}

return s
}

// IsLogForwarderAvailable checks whether all the required files for FluentBit execution are available
func (c *FBSupervisorConfig) IsLogForwarderAvailable() bool {
if _, err := os.Stat(c.FluentBitExePath); err != nil {
Expand Down Expand Up @@ -132,15 +98,6 @@ func buildFbExecutor(fbIntCfg FBSupervisorConfig, cfgLoader *logs.CfgLoader) fun
return nil, errors.Wrap(err, "failed to create temporary fb sFBLogger config file")
}

removedFbConfigTempFiles, err := removeFbConfigTempFiles(MaxNumberOfFbConfigTempFiles)
if err != nil {
log.WithError(err).Warn("Failed removing config temp files.")
}

for _, file := range removedFbConfigTempFiles {
log.Debugf("Removed %s config temp file.", file)
}

args := []string{
fbIntCfg.FluentBitExePath,
"-c",
Expand Down Expand Up @@ -187,85 +144,6 @@ func saveToTempFile(config []byte) (string, error) {
return file.Name(), nil
}

// keeps the most recent config files, up to maxNumberOfFbConfigTempFiles, and removes the rest.
func removeFbConfigTempFiles(maxNumberOfFbConfigTempFiles int) ([]string, error) {
fbConfigTempFiles, err := readFbConfigTempFilesFromTempDirectory()
if err != nil {
return nil, fmt.Errorf("failed reading config temp files: %w", err)
}

if len(fbConfigTempFiles) <= maxNumberOfFbConfigTempFiles {
return nil, nil
}

// sort fbConfigTempFiles by ascending modification date
sort.Slice(fbConfigTempFiles, func(i, j int) bool {
fileInfo1, _ := fbConfigTempFiles[i].Info()
fileInfo2, _ := fbConfigTempFiles[j].Info()

return fileInfo1.ModTime().Before(fileInfo2.ModTime())
})

var configTempFilesToRemove []string
var removedConfigTempFiles []string
var listErrors listError

// create list of fbConfigTempFiles to remove
for i := 0; i < len(fbConfigTempFiles)-maxNumberOfFbConfigTempFiles; i++ {
configTempFilesToRemove = append(configTempFilesToRemove, fbConfigTempFiles[i].Name())
}

// extract lua filter filenames from config temp files to remove
for _, configTempFileToRemove := range configTempFilesToRemove {
if fbLuaFilterTempFilenames, err := extractLuaFilterFilenames(configTempFileToRemove); err != nil {
listErrors.Add(err)
} else {
configTempFilesToRemove = append(configTempFilesToRemove, fbLuaFilterTempFilenames...)
}
}

// remove all config and lua filter temp files from temporary directory
for _, configTempFileToRemove := range configTempFilesToRemove {
if err := os.Remove(filepath.Join(os.TempDir(), configTempFileToRemove)); err != nil {
listErrors.Add(err)
} else {
removedConfigTempFiles = append(removedConfigTempFiles, configTempFileToRemove)
}
}

return removedConfigTempFiles, listErrors.ErrorOrNil()
}

// return the list of temp config files from temporary directory.
func readFbConfigTempFilesFromTempDirectory() ([]fs.DirEntry, error) {
files, err := os.ReadDir(os.TempDir())
if err != nil {
return nil, fmt.Errorf("failed reading temp directory: %w", err)
}

var fbConfigTempFiles []fs.DirEntry

for _, file := range files {
if !file.IsDir() && strings.HasPrefix(file.Name(), "nr_fb_config") {
fbConfigTempFiles = append(fbConfigTempFiles, file)
}
}

return fbConfigTempFiles, nil
}

// extract lua filter temp filenames referenced by fbConfigTempFilename.
func extractLuaFilterFilenames(fbConfigTempFilename string) ([]string, error) {
fbConfigTempFileContent, err := os.ReadFile(filepath.Join(os.TempDir(), fbConfigTempFilename))
if err != nil {
return nil, fmt.Errorf("failed reading config temp file: %s error: %w", fbConfigTempFilename, err)
}

luaFilterTempFilenames := luaFilterTempFileRegex.FindAllString(string(fbConfigTempFileContent), -1)

return luaFilterTempFilenames, nil
}

// SupervisorEvent will be used to create an InfrastructureEvent when fb start/stop.
type SupervisorEvent struct {
sample.BaseEvent
Expand Down
92 changes: 0 additions & 92 deletions pkg/integrations/v4/supervisor_fb_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,6 @@ package v4
import (
"io/ioutil"
"os"
"path/filepath"
"testing"

executor2 "github.com/newrelic/infrastructure-agent/internal/integrations/v4/executor"
Expand Down Expand Up @@ -106,94 +105,3 @@ func TestFBSupervisorConfig_LicenseKeyShouldBePassedAsEnvVar(t *testing.T) {
assert.Contains(t, exec.(*executor2.Executor).Cfg.Environment, "NR_LICENSE_KEY_ENV_VAR") // nolint:forcetypeassert
assert.Equal(t, exec.(*executor2.Executor).Cfg.Environment["NR_LICENSE_KEY_ENV_VAR"], license) //nolint:forcetypeassert
}

// nolint:paralleltest
func TestRemoveFbConfigTempFiles(t *testing.T) {
configFiles := []struct {
name string
content string
}{
{"nr_fb_config1", "nr_fb_lua_filter1,nr_fb_lua_filter2"},
{"nr_fb_config2", ""},
{"nr_fb_config3", "nr_fb_lua_filter3"},
{"nr_fb_config4", "nr_fb_lua_filter0"},
{"nr_fb_config5", ""},
{"nr_fb_config6", ""},
{"nr_fb_lua_filter1", ""},
{"nr_fb_lua_filter2", ""},
{"nr_fb_lua_filter3", ""},
{"nr_fb_lua_filter4", ""},
}

tests := []struct {
name string
maxNumConfFiles int
expectedRemovedConfFiles []string
expectedKeptConfFiles []string
wantErr bool
}{
{
name: "No config files are removed",
maxNumConfFiles: 10,
expectedRemovedConfFiles: []string{},
expectedKeptConfFiles: []string{"nr_fb_config1", "nr_fb_config2", "nr_fb_config3", "nr_fb_config4", "nr_fb_config5", "nr_fb_config6", "nr_fb_lua_filter1", "nr_fb_lua_filter2", "nr_fb_lua_filter3", "nr_fb_lua_filter4"},
wantErr: false,
},
{
name: "Config file 1 and config lua files 1 and 2 are removed",
maxNumConfFiles: 5,
expectedRemovedConfFiles: []string{"nr_fb_config1", "nr_fb_lua_filter1", "nr_fb_lua_filter2"},
expectedKeptConfFiles: []string{"nr_fb_config2", "nr_fb_config3", "nr_fb_config4", "nr_fb_config5", "nr_fb_config6", "nr_fb_lua_filter3", "nr_fb_lua_filter4"},
wantErr: false,
},
{
name: "Config files 1, 2 and 3 and config lua files 1, 2 and 3 are removed",
maxNumConfFiles: 3,
expectedRemovedConfFiles: []string{"nr_fb_config1", "nr_fb_config2", "nr_fb_config3", "nr_fb_lua_filter1", "nr_fb_lua_filter2", "nr_fb_lua_filter3"},
expectedKeptConfFiles: []string{"nr_fb_config4", "nr_fb_config5", "nr_fb_config6", "nr_fb_lua_filter4"},
wantErr: false,
},
{
name: "Config files 1, 2, 3, 4 and 5 and config lua files 1, 2 and 3 are removed. Error removing non-existing lua file 0 referenced by config file 4.",
maxNumConfFiles: 1,
expectedRemovedConfFiles: []string{"nr_fb_config1", "nr_fb_config2", "nr_fb_config3", "nr_fb_config4", "nr_fb_config5", "nr_fb_lua_filter1", "nr_fb_lua_filter2", "nr_fb_lua_filter3"},
expectedKeptConfFiles: []string{"nr_fb_config6", "nr_fb_lua_filter4"},
wantErr: true,
},
}

for _, test := range tests {
// create temp directory and set it as default directory to use for temporary files
tmpDir := t.TempDir()
t.Setenv("TMPDIR", tmpDir)

t.Run(test.name, func(t *testing.T) {
// create config files in temp directory
for _, file := range configFiles {
addFile(t, tmpDir, file.name, file.content)
}

got, err := removeFbConfigTempFiles(test.maxNumConfFiles)
if (err != nil) != test.wantErr {
t.Errorf("removeFbConfigTempFiles() error = %v, wantErr %v", err, test.wantErr)

return
}

// read the remaining config file names from the temp directory
files, err := os.Open(tmpDir)
require.NoError(t, err)
keptConfTempFilenames, err := files.Readdirnames(0)
require.NoError(t, err)

assert.ElementsMatchf(t, test.expectedRemovedConfFiles, got, "Config files removed do not match")
assert.ElementsMatchf(t, test.expectedKeptConfFiles, keptConfTempFilenames, "Config files kept do not match")
})
}
}

func addFile(t *testing.T, dir, name, contents string) {
t.Helper()
filePath := filepath.Join(dir, name)
require.NoError(t, os.WriteFile(filePath, []byte(contents), 0o0600))
}

0 comments on commit 5f1d8b8

Please sign in to comment.