From a19bf2faf96efaa6fb93ec06a20eab91e9d7bce5 Mon Sep 17 00:00:00 2001 From: Cristian Ciutea Date: Wed, 12 Apr 2023 15:08:48 +0200 Subject: [PATCH] Large inventory (#1625) * NR-94488 * NR-95226 added ff for bulk inventory (#1581) * NR-95226 added ff for bulk inventory * NR-95231 : add inventory archive config option (#1608) * add inventory archive config option * add tests * Cciutea/inventory handler (#1602) added asyncinventoryhandler * fixed linter * fix linter --------- Co-authored-by: Grigorii Merkushev --- cmd/newrelic-infra/newrelic-infra.go | 3 + internal/agent/agent.go | 300 +++++++++++------- internal/agent/agent_test.go | 25 +- internal/agent/bulk_inventories.go | 4 +- internal/agent/bulk_inventories_test.go | 22 +- internal/agent/cmdchannel/fflag/ffhandler.go | 38 ++- .../agent/cmdchannel/fflag/ffhandler_test.go | 45 +++ internal/agent/delta/store.go | 28 +- internal/agent/delta/store_test.go | 87 +++-- internal/agent/inventory/entity_patcher.go | 192 +++++++++++ .../agent/inventory/entity_patcher_test.go | 105 ++++++ internal/agent/inventory/handler.go | 145 +++++++++ internal/agent/inventory/patcher.go | 123 +++++++ internal/agent/inventory/patcher_test.go | 141 ++++++++ internal/agent/mocks/AgentContext.go | 3 +- internal/agent/patch_reaper.go | 13 +- internal/agent/patch_reaper_test.go | 12 +- internal/agent/patch_sender.go | 21 +- internal/agent/patch_sender_test.go | 29 +- internal/agent/patch_sender_vortex.go | 3 +- internal/agent/patch_sender_vortex_test.go | 27 +- internal/agent/plugin.go | 37 +-- internal/agent/plugin_test.go | 11 +- internal/agent/types/types.go | 39 +++ internal/plugins/darwin/hostinfo.go | 5 +- .../darwin/processorinfo_test_amd64.go | 2 +- .../darwin/processorinfo_test_arm64.go | 2 +- .../plugins/linux/cloud_security_groups.go | 5 +- internal/plugins/linux/daemontools.go | 3 +- internal/plugins/linux/dpkg.go | 3 +- internal/plugins/linux/facter.go | 5 +- internal/plugins/linux/hostinfo.go | 5 +- internal/plugins/linux/kernel_modules.go | 5 +- internal/plugins/linux/rpm.go | 5 +- internal/plugins/linux/selinux.go | 13 +- internal/plugins/linux/sshd_config.go | 3 +- internal/plugins/linux/supervisor.go | 5 +- internal/plugins/linux/sysctl_polling.go | 7 +- internal/plugins/linux/sysctl_subscriber.go | 5 +- internal/plugins/linux/systemd.go | 5 +- internal/plugins/linux/sysvinit.go | 3 +- internal/plugins/linux/upstart.go | 5 +- internal/plugins/linux/users.go | 3 +- internal/plugins/testing/agent_mock.go | 11 +- internal/plugins/windows/hostinfo.go | 5 +- internal/plugins/windows/services.go | 3 +- internal/plugins/windows/services_test.go | 4 +- internal/plugins/windows/updates.go | 3 +- pkg/config/config.go | 12 + pkg/config/defaults.go | 1 + pkg/helpers/helpers.go | 16 +- pkg/integrations/legacy/runner_test.go | 15 +- pkg/integrations/legacy/utils.go | 6 +- pkg/integrations/v4/dm/emitter_bench_test.go | 2 +- pkg/integrations/v4/dm/emitter_test.go | 9 +- pkg/integrations/v4/emitter/emitter_test.go | 12 +- pkg/metrics/process/sampler_linux_test.go | 3 +- pkg/plugins/agent_config.go | 3 +- pkg/plugins/agent_config_test.go | 6 +- pkg/plugins/custom_attrs.go | 3 +- pkg/plugins/files_config.go | 3 +- pkg/plugins/host_aliases.go | 5 +- pkg/plugins/network_interface.go | 5 +- pkg/plugins/network_interface_test.go | 10 +- pkg/plugins/proxy/proxy_config.go | 5 +- test/cfgprotocol/agent/emulator.go | 1 + test/core/deltas_test.go | 137 +++++++- test/core/dummy_plugin.go | 37 ++- test/core/dummy_plugin_v4.go | 3 +- test/harvest/facter_test.go | 6 +- test/harvest/hostinfo_test.go | 10 +- test/infra/agent.go | 3 +- 72 files changed, 1516 insertions(+), 365 deletions(-) create mode 100644 internal/agent/inventory/entity_patcher.go create mode 100644 internal/agent/inventory/entity_patcher_test.go create mode 100644 internal/agent/inventory/handler.go create mode 100644 internal/agent/inventory/patcher.go create mode 100644 internal/agent/inventory/patcher_test.go create mode 100644 internal/agent/types/types.go diff --git a/cmd/newrelic-infra/newrelic-infra.go b/cmd/newrelic-infra/newrelic-infra.go index 864a10485..e1673d004 100644 --- a/cmd/newrelic-infra/newrelic-infra.go +++ b/cmd/newrelic-infra/newrelic-infra.go @@ -414,6 +414,9 @@ func initializeAgentAndRun(c *config.Config, logFwCfg config.LogForward) error { aslog.WithError(err).Warn("Commands initial fetch failed.") } + // Initialise the agent after fetching FF. + agt.Init() + if c.StatusServerEnabled || c.HTTPServerEnabled { rlog := wlog.WithComponent("status.Reporter") timeoutD, err := time.ParseDuration(c.StartupConnectionTimeout) diff --git a/internal/agent/agent.go b/internal/agent/agent.go index def22f167..dea92eefb 100644 --- a/internal/agent/agent.go +++ b/internal/agent/agent.go @@ -17,6 +17,8 @@ import ( "time" "github.com/newrelic/infrastructure-agent/internal/agent/instrumentation" + "github.com/newrelic/infrastructure-agent/internal/agent/inventory" + "github.com/newrelic/infrastructure-agent/internal/agent/types" "github.com/newrelic/infrastructure-agent/internal/feature_flags" "github.com/newrelic/infrastructure-agent/pkg/entity/host" @@ -51,8 +53,9 @@ import ( ) const ( - defaultRemoveEntitiesPeriod = 48 * time.Hour - activeEntitiesBufferLength = 32 + defaultRemoveEntitiesPeriod = 48 * time.Hour + activeEntitiesBufferLength = 32 + defaultBulkInventoryQueueLength = 1000 ) type registerableSender interface { @@ -60,16 +63,24 @@ type registerableSender interface { Stop() error } +type inventoryEntity struct { + reaper *PatchReaper + sender inventory.PatchSender + needsReaping bool + needsCleanup bool +} + type Agent struct { inv inventoryState - plugins []Plugin // Slice of registered plugins - oldPlugins []ids.PluginID // Deprecated plugins whose cached data must be removed, if existing - agentDir string // Base data directory for the agent - extDir string // Location of external data input - userAgent string // User-Agent making requests to warlock - inventories map[string]*inventory // Inventory reaper and sender instances (key: entity ID) - Context *context // Agent context data that is passed around the place + plugins []Plugin // Slice of registered plugins + oldPlugins []ids.PluginID // Deprecated plugins whose cached data must be removed, if existing + agentDir string // Base data directory for the agent + extDir string // Location of external data input + userAgent string // User-Agent making requests to warlock + inventories map[string]*inventoryEntity // Inventory reaper and sender instances (key: entity ID) + Context *context // Agent context data that is passed around the place metricsSender registerableSender + inventoryHandler *inventory.Handler store *delta.Store debugProvide debug.Provide httpClient backendhttp.Client // http client for both data submission types: events and inventory @@ -88,14 +99,6 @@ type inventoryState struct { sendErrorCount uint32 } -// inventory holds the reaper and sender for the inventories of a given entity (local or remote), as well as their status -type inventory struct { - reaper *patchReaper - sender patchSender - needsReaping bool - needsCleanup bool -} - var ( alog = log.WithComponent("Agent") aclog = log.WithComponent("AgentContext") @@ -104,7 +107,7 @@ var ( // AgentContext defines the interfaces between plugins and the agent type AgentContext interface { Context() context2.Context - SendData(PluginOutput) + SendData(types.PluginOutput) SendEvent(event sample.Event, entityKey entity.Key) Unregister(ids.PluginID) // Reconnecting tells the agent that this plugin must be re-executed when the agent reconnects after long time @@ -135,16 +138,19 @@ type AgentContext interface { // available to the various plugins and satisfies the // AgentContext interface type context struct { - Ctx context2.Context - CancelFn context2.CancelFunc - cfg *config.Config - id *id.Context - agentKey atomic.Value - reconnecting *sync.Map // Plugins that must be re-executed after a long disconnection - ch chan PluginOutput // Channel of inbound plugin data payloads - activeEntities chan string // Channel will be reported about the local/remote entities that are active - version string - eventSender eventSender + Ctx context2.Context + CancelFn context2.CancelFunc + cfg *config.Config + id *id.Context + agentKey atomic.Value + reconnecting *sync.Map // Plugins that must be re-executed after a long disconnection + ch chan types.PluginOutput // Channel of inbound plugin data payloads + + updateIDLookupTableFn func(hostAliases types.PluginInventoryDataset) (err error) + pluginOutputHandleFn func(types.PluginOutput) // Function to handle the PluginOutput (Inventory Data). When this is provided the ch would not be used (In future would be deprecared) + activeEntities chan string // Channel will be reported about the local/remote entities that are active + version string + eventSender eventSender servicePidLock *sync.RWMutex servicePids map[string]map[int]string // Map of plugin -> (map of pid -> service) @@ -262,8 +268,8 @@ func NewAgent( cfg *config.Config, buildVersion string, userAgent string, - ffRetriever feature_flags.Retriever) (a *Agent, err error) { - + ffRetriever feature_flags.Retriever, +) (a *Agent, err error) { hostnameResolver := hostname.CreateResolver( cfg.OverrideHostname, cfg.OverrideHostnameShort, cfg.DnsHostnameResolution) @@ -293,7 +299,7 @@ func NewAgent( maxInventorySize = delta.DisableInventorySplit } - s := delta.NewStore(dataDir, ctx.EntityKey(), maxInventorySize) + s := delta.NewStore(dataDir, ctx.EntityKey(), maxInventorySize, cfg.InventoryArchiveEnabled) transport := backendhttp.BuildTransport(cfg, backendhttp.ClientTimeout) @@ -329,7 +335,6 @@ func NewAgent( provideIDs := NewProvideIDs(registerClient, state.NewRegisterSM()) fpHarvester, err := fingerprint.NewHarvestor(cfg, hostnameResolver, cloudHarvester) - if err != nil { return nil, err } @@ -401,7 +406,7 @@ func New( notificationHandler.RegisterHandler(ipc.Shutdown, a.gracefulShutdown) // Instantiate reaper and sender - a.inventories = map[string]*inventory{} + a.inventories = map[string]*inventoryEntity{} // Make sure the network is working before continuing with identity if err := checkCollectorConnectivity(ctx.Ctx, cfg, backoff.NewRetrier(), a.userAgent, a.Context.getAgentKey(), transport); err != nil { @@ -416,14 +421,14 @@ func New( llog.Debug("Bootstrap Entity Key.") // Create the external directory for user-generated json - if err := disk.MkdirAll(a.extDir, 0755); err != nil { + if err := disk.MkdirAll(a.extDir, 0o755); err != nil { llog.WithField("path", a.extDir).WithError(err).Error("External json directory could not be initialized") return nil, err } // Create input channel for plugins to feed data back to the agent llog.WithField(config.TracesFieldName, config.FeatureTrace).Tracef("inventory parallelize queue: %v", a.Context.cfg.InventoryQueueLen) - a.Context.ch = make(chan PluginOutput, a.Context.cfg.InventoryQueueLen) + a.Context.ch = make(chan types.PluginOutput, a.Context.cfg.InventoryQueueLen) a.Context.activeEntities = make(chan string, activeEntitiesBufferLength) if cfg.RegisterEnabled { @@ -473,27 +478,35 @@ func (a *Agent) registerEntityInventory(entity entity.Entity) error { alog.WithField("entityKey", entityKey). WithField("entityID", entity.ID).Debug("Registering inventory for entity.") - var inv inventory + var patchSender inventory.PatchSender var err error if a.Context.cfg.RegisterEnabled { - inv.sender, err = newPatchSenderVortex(entityKey, a.Context.getAgentKey(), a.Context, a.store, a.userAgent, a.Context.Identity, a.provideIDs, a.entityMap, a.httpClient) + patchSender, err = newPatchSenderVortex(entityKey, a.Context.getAgentKey(), a.Context, a.store, a.userAgent, a.Context.Identity, a.provideIDs, a.entityMap, a.httpClient) } else { - fileName := a.store.EntityFolder(entity.Key.String()) - lastSubmission := delta.NewLastSubmissionStore(a.store.DataDir, fileName) - lastEntityID := delta.NewEntityIDFilePersist(a.store.DataDir, fileName) - inv.sender, err = newPatchSender(entity, a.Context, a.store, lastSubmission, lastEntityID, a.userAgent, a.Context.Identity, a.httpClient) + patchSender, err = a.newPatchSender(entity) } if err != nil { return err } - inv.reaper = newPatchReaper(entityKey, a.store) - a.inventories[entityKey] = &inv + reaper := newPatchReaper(entityKey, a.store) + a.inventories[entityKey] = &inventoryEntity{ + sender: patchSender, + reaper: reaper, + } return nil } +func (a *Agent) newPatchSender(entity entity.Entity) (inventory.PatchSender, error) { + fileName := a.store.EntityFolder(entity.Key.String()) + lastSubmission := delta.NewLastSubmissionStore(a.store.DataDir, fileName) + lastEntityID := delta.NewEntityIDFilePersist(a.store.DataDir, fileName) + + return newPatchSender(entity, a.Context, a.store, lastSubmission, lastEntityID, a.userAgent, a.Context.Identity, a.httpClient) +} + // removes the inventory object references to free the memory, and the respective directories func (a *Agent) unregisterEntityInventory(entityKey string) error { alog.WithField("entityKey", entityKey).Debug("Unregistering inventory for entity.") @@ -517,8 +530,8 @@ func (a *Agent) Terminate() { a.mtx.Lock() defer a.mtx.Unlock() alog.Debug("Terminating running plugins.") - for _, plugin := range a.plugins { - switch p := plugin.(type) { + for _, agentPlugin := range a.plugins { + switch p := agentPlugin.(type) { case Killable: p.Kill() } @@ -559,25 +572,24 @@ func (a *Agent) DeprecatePlugin(plugin ids.PluginID) { } // storePluginOutput will take a PluginOutput and persist it in the store -func (a *Agent) storePluginOutput(plugin PluginOutput) error { - - if plugin.Data == nil { - plugin.Data = make(PluginInventoryDataset, 0) +func (a *Agent) storePluginOutput(pluginOutput types.PluginOutput) error { + if pluginOutput.Data == nil { + pluginOutput.Data = make(types.PluginInventoryDataset, 0) } - sort.Sort(plugin.Data) + sort.Sort(pluginOutput.Data) // Filter out ignored inventory data before writing the file out var sortKey string ignore := a.Context.Config().IgnoredInventoryPathsMap simplifiedPluginData := make(map[string]interface{}) DataLoop: - for _, data := range plugin.Data { + for _, data := range pluginOutput.Data { if data == nil { continue } sortKey = data.SortKey() - pluginSource := fmt.Sprintf("%s/%s", plugin.Id, sortKey) + pluginSource := fmt.Sprintf("%s/%s", pluginOutput.Id, sortKey) if _, ok := ignore[strings.ToLower(pluginSource)]; ok { continue DataLoop } @@ -585,9 +597,9 @@ DataLoop: } return a.store.SavePluginSource( - plugin.Entity.Key.String(), - plugin.Id.Category, - plugin.Id.Term, + pluginOutput.Entity.Key.String(), + pluginOutput.Id.Category, + pluginOutput.Id.Term, simplifiedPluginData, ) } @@ -596,13 +608,13 @@ DataLoop: // we don't return until all plugins have been started func (a *Agent) startPlugins() { // iterate over and start each plugin - for _, plugin := range a.plugins { - plugin.LogInfo() + for _, agentPlugin := range a.plugins { + agentPlugin.LogInfo() go func(p Plugin) { _, trx := instrumentation.SelfInstrumentation.StartTransaction(context2.Background(), fmt.Sprintf("plugin. %s ", p.Id().String())) defer trx.End() p.Run() - }(plugin) + }(agentPlugin) } } @@ -618,7 +630,7 @@ func (a *Agent) LogExternalPluginsInfo() { var hostAliasesPluginID = ids.PluginID{Category: "metadata", Term: "host_aliases"} -func (a *Agent) updateIDLookupTable(hostAliases PluginInventoryDataset) (err error) { +func (a *Agent) updateIDLookupTable(hostAliases types.PluginInventoryDataset) (err error) { newIDLookupTable := make(map[string]string) for _, hAliases := range hostAliases { if alias, ok := hAliases.(sysinfo.HostAliases); ok { @@ -650,6 +662,41 @@ func (a *Agent) setAgentKey(idLookupTable host.IDLookup) error { return nil } +func (a *Agent) Init() { + cfg := a.Context.cfg + + // Configure AsyncInventoryHandler if FF is enabled. + if cfg.AsyncInventoryHandlerEnabled { + alog.Debug("Initialise async inventory handler") + + removeEntitiesPeriod, _ := time.ParseDuration(a.Context.Config().RemoveEntitiesPeriod) + + patcherConfig := inventory.PatcherConfig{ + IgnoredPaths: cfg.IgnoredInventoryPathsMap, + AgentEntity: entity.NewFromNameWithoutID(a.Context.EntityKey()), + RemoveEntitiesPeriod: removeEntitiesPeriod, + } + patcher := inventory.NewEntityPatcher(patcherConfig, a.store, a.newPatchSender) + + if cfg.InventoryQueueLen == 0 { + cfg.InventoryQueueLen = defaultBulkInventoryQueueLength + } + + inventoryHandlerCfg := inventory.HandlerConfig{ + SendInterval: cfg.SendInterval, + FirstReapInterval: cfg.FirstReapInterval, + ReapInterval: cfg.ReapInterval, + InventoryQueueLen: cfg.InventoryQueueLen, + } + a.inventoryHandler = inventory.NewInventoryHandler(a.Context.Ctx, inventoryHandlerCfg, patcher) + a.Context.pluginOutputHandleFn = a.inventoryHandler.Handle + a.Context.updateIDLookupTableFn = a.updateIDLookupTable + + // When AsyncInventoryHandlerEnabled is set disable inventory archiving. + a.store.SetArchiveEnabled(false) + } +} + // Run is the main event loop for the agent it starts up the plugins // kicks off a filesystem seed and watcher and listens for data from // the plugins @@ -672,6 +719,7 @@ func (a *Agent) Run() (err error) { // If the cloud provider was specified but we cannot get the instance ID, agent fails if err != nil { alog.WithError(err).Error("Couldn't detect the instance ID for the specified cloud") + return } } @@ -687,6 +735,33 @@ func (a *Agent) Run() (err error) { alog.WithError(err).Error("failed to start troubleshooting handler") } + // Start debugger routine. + go func() { + if a.Context.Config().DebugLogSec <= 0 { + return + } + + debugTimer := time.NewTicker(time.Duration(a.Context.Config().DebugLogSec) * time.Second) + + for { + select { + case <-debugTimer.C: + { + debugInfo, err := a.debugProvide() + if err != nil { + alog.WithError(err).Debug("failed to get debug stats") + } else if debugInfo != "" { + alog.Debug(debugInfo) + } + } + case <-a.Context.Ctx.Done(): + debugTimer.Stop() + + return + } + } + }() + if a.Context.eventSender != nil { if err := a.Context.eventSender.Start(); err != nil { alog.WithError(err).Error("failed to start event sender") @@ -699,14 +774,40 @@ func (a *Agent) Run() (err error) { } } + exit := make(chan struct{}) + + go func() { + <-a.Context.Ctx.Done() + + a.exitGracefully() + + close(exit) + }() + + if a.inventoryHandler != nil { + if a.shouldSendInventory() { + a.inventoryHandler.Start() + } + <-exit + + return nil + } + + a.handleInventory(exit) + + return nil +} + +func (a *Agent) handleInventory(exit chan struct{}) { + cfg := a.Context.cfg + // Timers reapInventoryTimer := time.NewTicker(cfg.FirstReapInterval) sendInventoryTimer := time.NewTimer(cfg.SendInterval) // Send any deltas every X seconds - debugTimer := time.Tick(time.Duration(a.Context.Config().DebugLogSec) * time.Second) - //Remove send timer + // Remove send timer if !a.shouldSendInventory() { - //If Stop returns false means that the timer has been already triggered + // If Stop returns false means that the timer has been already triggered if !sendInventoryTimer.Stop() { <-sendInventoryTimer.C } @@ -741,21 +842,6 @@ func (a *Agent) Run() (err error) { _ = a.registerEntityInventory(entity.NewFromNameWithoutID(a.Context.EntityKey())) } - exit := make(chan struct{}) - - go func() { - <-a.Context.Ctx.Done() - - a.exitGracefully(sendInventoryTimer, reapInventoryTimer, removeEntitiesTicker) - - close(exit) - - // Should not reach here, just a guard. - //<-time.After(service.GracefulExitTimeout) - //log.Warn("graceful stop time exceeded... forcing stop") - //os.Exit(0) - }() - // three states // -- reading data to write to json // -- reaping @@ -764,7 +850,16 @@ func (a *Agent) Run() (err error) { for { select { case <-exit: - return nil + if sendInventoryTimer != nil { + sendInventoryTimer.Stop() + } + if reapInventoryTimer != nil { + reapInventoryTimer.Stop() + } + if removeEntitiesTicker != nil { + removeEntitiesTicker.Stop() + } + return // agent gets notified about active entities case ent := <-a.Context.activeEntities: reportedEntities[ent] = true @@ -830,15 +925,6 @@ func (a *Agent) Run() (err error) { } case <-sendInventoryTimer.C: a.sendInventory(sendInventoryTimer) - case <-debugTimer: - { - debugInfo, err := a.debugProvide() - if err != nil { - alog.WithError(err).Debug("failed to get debug stats") - } else if debugInfo != "" { - alog.Debug(debugInfo) - } - } case <-removeEntitiesTicker.C: pastPeriodReportedEntities := reportedEntities reportedEntities = map[string]bool{} // reset the set of reporting entities the next period @@ -855,6 +941,7 @@ func (a *Agent) checkInstanceIDRetry(maxRetries, backoffTime int) error { if _, err = a.cloudHarvester.GetInstanceID(); err == nil { return nil } + if i >= maxRetries-1 { break } @@ -867,7 +954,6 @@ func (a *Agent) checkInstanceIDRetry(maxRetries, backoffTime int) error { } func (a *Agent) cpuProfileStart() *os.File { - // Start CPU profiling if a.Context.cfg.CPUProfile == "" { return nil @@ -895,7 +981,6 @@ func (a *Agent) cpuProfileStop(f *os.File) { } func (a *Agent) intervalMemoryProfile() { - cfg := a.Context.cfg if cfg.MemProfileInterval <= 0 { @@ -913,11 +998,9 @@ func (a *Agent) intervalMemoryProfile() { counter++ } } - } func (a *Agent) dumpMemoryProfile(agentRuntimeMark int) { - if a.Context.cfg.MemProfile == "" { return } @@ -936,21 +1019,11 @@ func (a *Agent) dumpMemoryProfile(agentRuntimeMark int) { mlog.WithError(err).Error("could not start memory profile") } } - } -func (a *Agent) exitGracefully(sendTimer *time.Timer, reapTimer, removeEntitiesTicker *time.Ticker) { +func (a *Agent) exitGracefully() { log.Info("Gracefully Exiting") - if sendTimer != nil { - sendTimer.Stop() - } - if reapTimer != nil { - reapTimer.Stop() - } - if removeEntitiesTicker != nil { - removeEntitiesTicker.Stop() - } if a.Context.eventSender != nil { if err := a.Context.eventSender.Stop(); err != nil { log.WithError(err).Error("failed to stop event sender") @@ -962,6 +1035,10 @@ func (a *Agent) exitGracefully(sendTimer *time.Timer, reapTimer, removeEntitiesT } } + if a.inventoryHandler != nil { + a.inventoryHandler.Stop() + } + if a.notificationHandler != nil { a.notificationHandler.Stop() } @@ -1035,7 +1112,14 @@ func (a *Agent) removeOutdatedEntities(reportedEntities map[string]bool) { alog.WithField("remaining", len(a.inventories)).Debug("Some entities may remain registered.") } -func (c *context) SendData(data PluginOutput) { +func (c *context) SendData(data types.PluginOutput) { + if c.pluginOutputHandleFn != nil { + if data.Id == hostAliasesPluginID && c.updateIDLookupTableFn != nil { + c.updateIDLookupTableFn(data.Data) + } + c.pluginOutputHandleFn(data) + return + } c.ch <- data } @@ -1087,7 +1171,7 @@ func (c *context) SendEvent(event sample.Event, entityKey entity.Key) { } func (c *context) Unregister(id ids.PluginID) { - c.ch <- NewNotApplicableOutput(id) + c.ch <- types.NewNotApplicableOutput(id) } func (c *context) Config() *config.Config { @@ -1142,11 +1226,11 @@ func (c *context) Reconnect() { // triggerAddReconnecting is used with sync.Map.Range to iterate through all plugins and reconnect them func triggerAddReconnecting(l log.Entry) func(pluginID interface{}, plugin interface{}) bool { - return func(pluginID, plugin interface{}) bool { + return func(pluginID, agentPlugin interface{}) bool { l.WithField("plugin", pluginID).Debug("Reconnecting plugin.") func(p Plugin) { go p.Run() - }(plugin.(Plugin)) + }(agentPlugin.(Plugin)) return true } } @@ -1166,6 +1250,10 @@ func (c *context) setAgentKey(agentKey string) { } func (c *context) getAgentKey() (agentKey string) { + loaded := c.agentKey.Load() + if loaded == nil { + return "" + } return c.agentKey.Load().(string) } diff --git a/internal/agent/agent_test.go b/internal/agent/agent_test.go index 1561f6739..513466ec6 100644 --- a/internal/agent/agent_test.go +++ b/internal/agent/agent_test.go @@ -8,6 +8,7 @@ import ( context2 "context" "encoding/json" "fmt" + "github.com/newrelic/infrastructure-agent/pkg/metrics/types" "io/ioutil" "net/http" "net/http/httptest" @@ -20,6 +21,7 @@ import ( "time" "github.com/newrelic/infrastructure-agent/internal/agent/delta" + agentTypes "github.com/newrelic/infrastructure-agent/internal/agent/types" "github.com/newrelic/infrastructure-agent/internal/feature_flags" "github.com/newrelic/infrastructure-agent/internal/feature_flags/test" "github.com/newrelic/infrastructure-agent/internal/testhelpers" @@ -35,7 +37,6 @@ import ( "github.com/newrelic/infrastructure-agent/pkg/helpers/fingerprint" "github.com/newrelic/infrastructure-agent/pkg/helpers/metric" "github.com/newrelic/infrastructure-agent/pkg/log" - "github.com/newrelic/infrastructure-agent/pkg/metrics/types" "github.com/newrelic/infrastructure-agent/pkg/plugins/ids" "github.com/newrelic/infrastructure-agent/pkg/sample" "github.com/newrelic/infrastructure-agent/pkg/sysinfo" @@ -63,7 +64,7 @@ func newTesting(cfg *config.Config) *Agent { ctx := NewContext(cfg, "1.2.3", testhelpers.NullHostnameResolver, lookups, matcher) - st := delta.NewStore(dataDir, "default", cfg.MaxInventorySize) + st := delta.NewStore(dataDir, "default", cfg.MaxInventorySize, true) fpHarvester, err := fingerprint.NewHarvestor(cfg, testhelpers.NullHostnameResolver, cloudDetector) if err != nil { @@ -93,6 +94,8 @@ func newTesting(cfg *config.Config) *Agent { panic(err) } + a.Init() + return a } @@ -116,10 +119,10 @@ func TestIgnoreInventory(t *testing.T) { _ = os.RemoveAll(a.store.DataDir) }() - assert.NoError(t, a.storePluginOutput(PluginOutput{ + assert.NoError(t, a.storePluginOutput(agentTypes.PluginOutput{ Id: ids.PluginID{"test", "plugin"}, Entity: entity.NewFromNameWithoutID("someEntity"), - Data: PluginInventoryDataset{ + Data: agentTypes.PluginInventoryDataset{ &TestAgentData{"yum", "value1"}, &TestAgentData{"myService", "value2"}, }, @@ -207,7 +210,7 @@ func TestUpdateIDLookupTable(t *testing.T) { a := newTesting(nil) defer os.RemoveAll(a.store.DataDir) - dataset := PluginInventoryDataset{} + dataset := agentTypes.PluginInventoryDataset{} dataset = append(dataset, sysinfo.HostAliases{ Alias: "hostName.com", Source: sysinfo.HOST_SOURCE_HOSTNAME, @@ -315,7 +318,7 @@ func TestRemoveOutdatedEntities(t *testing.T) { // Given an agent agent := newTesting(nil) defer os.RemoveAll(agent.store.DataDir) - agent.inventories = map[string]*inventory{} + agent.inventories = map[string]*inventoryEntity{} dataDir := agent.store.DataDir @@ -588,7 +591,7 @@ func TestAgent_Run_DontSendInventoryIfFwdOnly(t *testing.T) { //Inventory recording calls snd := &patchSenderCallRecorder{} - a.inventories = map[string]*inventory{"test": {sender: snd}} + a.inventories = map[string]*inventoryEntity{"test": {sender: snd}} go func() { assert.NoError(t, a.Run()) @@ -718,10 +721,10 @@ func TestStorePluginOutput(t *testing.T) { aV := "aValue" bV := "bValue" cV := "cValue" - err := a.storePluginOutput(PluginOutput{ + err := a.storePluginOutput(agentTypes.PluginOutput{ Id: ids.PluginID{"test", "plugin"}, Entity: entity.NewFromNameWithoutID("someEntity"), - Data: PluginInventoryDataset{ + Data: agentTypes.PluginInventoryDataset{ &testAgentNullableData{"cMyService", &cV}, &testAgentNullableData{"aMyService", &aV}, &testAgentNullableData{"NilService", nil}, @@ -777,7 +780,7 @@ func BenchmarkStorePluginOutput(b *testing.B) { for _, bm := range benchmarks { b.Run(bm.name, func(b *testing.B) { - var dataset PluginInventoryDataset + var dataset agentTypes.PluginInventoryDataset for i := 0; i < 6; i++ { mHostInfo := &mockHostinfoData{ System: fmt.Sprintf("system-%v", i), @@ -798,7 +801,7 @@ func BenchmarkStorePluginOutput(b *testing.B) { dataset = append(dataset, mHostInfo) } - output := PluginOutput{ + output := agentTypes.PluginOutput{ Id: ids.PluginID{"test", "plugin"}, Entity: entity.NewFromNameWithoutID("someEntity"), Data: dataset, diff --git a/internal/agent/bulk_inventories.go b/internal/agent/bulk_inventories.go index 1ecb01505..a40244918 100644 --- a/internal/agent/bulk_inventories.go +++ b/internal/agent/bulk_inventories.go @@ -27,14 +27,14 @@ type Inventories struct { ctx AgentContext // reference to the inventories from agent - inventories *map[string]*inventory + inventories *map[string]*inventoryEntity } // backendPost defines the prototype for a function that submits the bulk of PostDeltaBody objects to the backend type backendPost func(reqs []inventoryapi.PostDeltaBody) ([]inventoryapi.BulkDeltaResponse, error) // NewInventories instantiates and returns a new Inventories object given the configuration passed as arguments -func NewInventories(store *delta.Store, ctx AgentContext, client *inventoryapi.IngestClient, inventories *map[string]*inventory, +func NewInventories(store *delta.Store, ctx AgentContext, client *inventoryapi.IngestClient, inventories *map[string]*inventoryEntity, agentIdentifier string, compactEnabled bool, compactThreshold uint64, maxDataSize int) Inventories { b := bulk.NewBuffer(maxDataSize) diff --git a/internal/agent/bulk_inventories_test.go b/internal/agent/bulk_inventories_test.go index e5485fc92..d0173a7e4 100644 --- a/internal/agent/bulk_inventories_test.go +++ b/internal/agent/bulk_inventories_test.go @@ -16,8 +16,6 @@ import ( "github.com/stretchr/testify/assert" ) -const maxInventoryDataSize = 3 * 1000 * 1000 - var plugin = &delta.PluginInfo{ Source: "metadata/plugin", Plugin: "metadata", @@ -101,10 +99,10 @@ func TestInventories(t *testing.T) { // Given a delta store dataDir, err := ioutil.TempDir("", "test_inventories") assert.Nil(t, err) - store := delta.NewStore(dataDir, "default", maxInventoryDataSize) + store := delta.NewStore(dataDir, "default", maxInventoryDataSize, true) // And a set of registered entities - inv := map[string]*inventory{ + inv := map[string]*inventoryEntity{ "agent_id": nil, "some_other_cool_entity": nil, "some_other_cool_entity2": nil, @@ -157,10 +155,10 @@ func TestInventories_BulkPatchProcess(t *testing.T) { // Given a delta store dataDir, err := ioutil.TempDir("", "test_inventories") assert.Nil(t, err) - store := delta.NewStore(dataDir, "default", maxInventoryDataSize) + store := delta.NewStore(dataDir, "default", maxInventoryDataSize, true) // And a set of registered entities - inv := map[string]*inventory{ + inv := map[string]*inventoryEntity{ "agent_id": nil, "some_other_cool_entity": nil, "some_other_cool_entity2": nil, @@ -217,10 +215,10 @@ func TestInventories_NoDeltas(t *testing.T) { // Given a delta store dataDir, err := ioutil.TempDir("", "no_deltas") assert.Nil(t, err) - store := delta.NewStore(dataDir, "default", maxInventoryDataSize) + store := delta.NewStore(dataDir, "default", maxInventoryDataSize, true) // And a set of registered entities - inv := map[string]*inventory{ + inv := map[string]*inventoryEntity{ "agent_id": nil, "some_other_cool_entity": nil, "some_other_cool_entity2": nil, @@ -246,10 +244,10 @@ func TestInventories_ResetAll(t *testing.T) { // Given a delta store dataDir, err := ioutil.TempDir("", "reset_all") assert.Nil(t, err) - store := delta.NewStore(dataDir, "default", maxInventoryDataSize) + store := delta.NewStore(dataDir, "default", maxInventoryDataSize, true) // And a set of registered entities - inv := map[string]*inventory{ + inv := map[string]*inventoryEntity{ "agent_id": nil, "some_other_cool_entity": nil, "some_other_cool_entity2": nil, @@ -288,10 +286,10 @@ func TestInventories_Compacting(t *testing.T) { // Given a delta store dataDir, err := ioutil.TempDir("", "reset_all") assert.Nil(t, err) - store := delta.NewStore(dataDir, "default", maxInventoryDataSize) + store := delta.NewStore(dataDir, "default", maxInventoryDataSize, true) // And a set of registered entities - inv := map[string]*inventory{ + inv := map[string]*inventoryEntity{ "agent_id": nil, "some_other_cool_entity": nil, "some_other_cool_entity2": nil, diff --git a/internal/agent/cmdchannel/fflag/ffhandler.go b/internal/agent/cmdchannel/fflag/ffhandler.go index 81dc08bb9..faf93fcca 100644 --- a/internal/agent/cmdchannel/fflag/ffhandler.go +++ b/internal/agent/cmdchannel/fflag/ffhandler.go @@ -17,17 +17,20 @@ import ( const ( // FFs - FlagCategory = "Infra_Agent" - FlagNameRegister = "register_enabled" - FlagParallelizeInventory = "parallelize_inventory_enabled" + FlagCategory = "Infra_Agent" + FlagNameRegister = "register_enabled" + FlagParallelizeInventory = "parallelize_inventory_enabled" + FlagAsyncInventoryHandler = "async_inventory_handler_enabled" + FlagProtocolV4 = "protocol_v4_enabled" FlagFullProcess = "full_process_sampling" FlagDmRegisterDeprecated = "dm_register_deprecated" FlagFluentBit19 = "fluent_bit_19" // Config - CfgYmlRegisterEnabled = "register_enabled" - CfgYmlParallelizeInventory = "inventory_queue_len" - CfgValueParallelizeInventory = int64(100) // default value when no config provided by user and FF enabled + CfgYmlRegisterEnabled = "register_enabled" + CfgYmlParallelizeInventory = "inventory_queue_len" + CfgYmlAsyncInventoryHandlerEnabled = "async_inventory_handler_enabled" + CfgValueParallelizeInventory = int64(100) // default value when no config provided by user and FF enabled ) //nolint:gochecknoglobals @@ -141,6 +144,11 @@ func (h *handler) Handle(ctx context.Context, c commandapi.Command, isInitialFet return } + if ffArgs.Flag == FlagAsyncInventoryHandler { + handleAsyncInventoryHandlerEnabled(ffArgs, h.cfg, isInitialFetch) + return + } + // this is where we handle normal feature flags that are not related to OHIs. These are meant to just enable/disable // the falue of the feature flag if isBasicFeatureFlag(ffArgs.Flag) { @@ -257,3 +265,21 @@ func handleRegister(ffArgs args, c *config.Config, isInitialFetch bool) { Warn("unable to update config value") } } + +func handleAsyncInventoryHandlerEnabled(ffArgs args, c *config.Config, isInitialFetch bool) { + // feature already in desired state. + if ffArgs.Enabled == c.AsyncInventoryHandlerEnabled { + return + } + + if !isInitialFetch { + os.Exit(api.ExitCodeRestart) + } + + if err := c.SetBoolValueByYamlAttribute(CfgYmlAsyncInventoryHandlerEnabled, ffArgs.Enabled); err != nil { + ffLogger. + WithError(err). + WithField("field", CfgYmlAsyncInventoryHandlerEnabled). + Warn("unable to update config value") + } +} diff --git a/internal/agent/cmdchannel/fflag/ffhandler_test.go b/internal/agent/cmdchannel/fflag/ffhandler_test.go index 8153fa793..dae5bd425 100644 --- a/internal/agent/cmdchannel/fflag/ffhandler_test.go +++ b/internal/agent/cmdchannel/fflag/ffhandler_test.go @@ -112,6 +112,51 @@ func TestFFHandlerHandle_EnabledFFParallelizeInventoryDoesNotModifyProvidedConfi assert.Equal(t, 123, c.InventoryQueueLen) } +func TestFFHandlerHandle_AsyncInventoryHandlerEnabledInitialFetch(t *testing.T) { + c := config.Config{ + AsyncInventoryHandlerEnabled: false, + } + cmd := commandapi.Command{ + Args: []byte(`{ + "category": "Infra_Agent", + "flag": "async_inventory_handler_enabled", + "enabled": true }`), + } + NewHandler(&c, feature_flags.NewManager(nil), l).Handle(context.Background(), cmd, true) + + assert.True(t, c.AsyncInventoryHandlerEnabled) +} + +func TestFFHandlerHandle_AsyncInventoryHandlerEnabled(t *testing.T) { + c := config.Config{ + AsyncInventoryHandlerEnabled: true, + } + cmd := commandapi.Command{ + Args: []byte(`{ + "category": "Infra_Agent", + "flag": "async_inventory_handler_enabled", + "enabled": true }`), + } + NewHandler(&c, feature_flags.NewManager(nil), l).Handle(context.Background(), cmd, false) + + assert.True(t, c.AsyncInventoryHandlerEnabled) +} + +func TestFFHandlerHandle_AsyncInventoryHandler_Disabled(t *testing.T) { + c := config.Config{ + AsyncInventoryHandlerEnabled: true, + } + cmd := commandapi.Command{ + Args: []byte(`{ + "category": "Infra_Agent", + "flag": "async_inventory_handler_enabled", + "enabled": false }`), + } + NewHandler(&c, feature_flags.NewManager(nil), l).Handle(context.Background(), cmd, true) + + assert.False(t, c.AsyncInventoryHandlerEnabled) +} + func TestFFHandlerHandle_ExitsOnDiffValueAndNotInitialFetch(t *testing.T) { type testCase struct { name string diff --git a/internal/agent/delta/store.go b/internal/agent/delta/store.go index d957356ea..cb77d6e96 100644 --- a/internal/agent/delta/store.go +++ b/internal/agent/delta/store.go @@ -12,6 +12,7 @@ import ( "path/filepath" "regexp" "strings" + "sync" "time" "github.com/sirupsen/logrus" @@ -85,10 +86,12 @@ type Storage interface { ResetAllDeltas(entityKey string) UpdateState(entityKey string, deltas []*inventoryapi.RawDelta, deltaStateResults *inventoryapi.DeltaStateMap) SaveState() (err error) + IsArchiveEnabled() bool } // Store handles information about the storage of Deltas. type Store struct { + l sync.Mutex // DataDir holds the agent data directory DataDir string // CacheDir holds the agent cache directory @@ -101,10 +104,12 @@ type Store struct { plugins pluginSource2Info // stores time of last success submission of inventory to backend lastSuccessSubmission time.Time + // if enabled, will save archive deltas in .sent files + archiveEnabled bool } // NewStore creates a new Store and returns a pointer to it. If maxInventorySize <= 0, the inventory splitting is disabled -func NewStore(dataDir string, defaultEntityKey string, maxInventorySize int) *Store { +func NewStore(dataDir string, defaultEntityKey string, maxInventorySize int, archiveEnabled bool) *Store { if defaultEntityKey == "" { slog.Error("creating delta store: default entity ID can't be empty") panic("default entity ID can't be empty") @@ -116,6 +121,7 @@ func NewStore(dataDir string, defaultEntityKey string, maxInventorySize int) *St maxInventorySize: maxInventorySize, defaultEntityKey: defaultEntityKey, plugins: make(pluginSource2Info), + archiveEnabled: archiveEnabled, } // Nice2Have: remove side effects from constructor @@ -307,9 +313,11 @@ func (s *Store) archivePlugin(pluginItem *PluginInfo, entityKey string) (err err } } - err = s.rewriteDeltas(s.archiveFilePath(pluginItem, entityKey), os.O_CREATE|os.O_APPEND|os.O_WRONLY, archiveDeltas) - if err != nil { - return + if s.archiveEnabled { + err = s.rewriteDeltas(s.archiveFilePath(pluginItem, entityKey), os.O_CREATE|os.O_APPEND|os.O_WRONLY, archiveDeltas) + if err != nil { + return + } } return s.rewriteDeltas(s.DeltaFilePath(pluginItem, entityKey), os.O_CREATE|os.O_TRUNC|os.O_WRONLY, keepDeltas) @@ -737,12 +745,16 @@ func (s *Store) ReadDeltas(entityKey string) ([]inventoryapi.RawDeltaBlock, erro } func (s *Store) ChangeDefaultEntity(newEntityKey string) { + s.l.Lock() + defer s.l.Unlock() s.defaultEntityKey = newEntityKey } // EntityFolder provides the folder name for a given entity ID, or for the agent default entity in case entityKey is an // empty string func (s *Store) EntityFolder(entityKey string) string { + s.l.Lock() + defer s.l.Unlock() if entityKey == "" || entityKey == s.defaultEntityKey { return localEntityFolder } @@ -997,3 +1009,11 @@ func (s *Store) SavePluginSource(entityKey, category, term string, source map[st err = disk.WriteFile(outputFile, sourceB, DATA_FILE_MODE) return } + +func (s *Store) IsArchiveEnabled() bool { + return s.archiveEnabled +} + +func (s *Store) SetArchiveEnabled(archiveEnabled bool) { + s.archiveEnabled = archiveEnabled +} diff --git a/internal/agent/delta/store_test.go b/internal/agent/delta/store_test.go index 8d618b724..56a47b754 100644 --- a/internal/agent/delta/store_test.go +++ b/internal/agent/delta/store_test.go @@ -74,7 +74,7 @@ func TestNewDeltaStoreGolden(t *testing.T) { defer os.RemoveAll(dataDir) repoDir := filepath.Join(dataDir, "delta") - ds := NewStore(repoDir, "default", maxInventorySize) + ds := NewStore(repoDir, "default", maxInventorySize, true) assert.NotNil(t, ds) } @@ -84,7 +84,7 @@ func TestStorageSize(t *testing.T) { defer os.RemoveAll(dataDir) repoDir := filepath.Join(dataDir, "delta") - ds := NewStore(repoDir, "default", maxInventorySize) + ds := NewStore(repoDir, "default", maxInventorySize, true) size, _ := ds.StorageSize(ds.CacheDir) assert.Equal(t, uint64(0), size) @@ -135,7 +135,7 @@ func assertSuffix(t *testing.T, expected string, actual string) bool { func TestArchiveFilePath(t *testing.T) { s := SetUpTest(t) defer s.TearDownTest() - ds := NewStore(s.repoDir, "default", maxInventorySize) + ds := NewStore(s.repoDir, "default", maxInventorySize, true) actual := ds.archiveFilePath(s.plugin, "entity:id") expected := filepath.Join("delta", ".delta_repo", "metadata", "entityid", "plugin.sent") @@ -145,7 +145,7 @@ func TestArchiveFilePath(t *testing.T) { func TestDeltaFilePath(t *testing.T) { s := SetUpTest(t) defer s.TearDownTest() - ds := NewStore(s.repoDir, "default", maxInventorySize) + ds := NewStore(s.repoDir, "default", maxInventorySize, true) actual := ds.DeltaFilePath(s.plugin, "entity:id:2") expected := filepath.Join("delta", ".delta_repo", "metadata", "entityid2", "plugin.pending") @@ -155,7 +155,7 @@ func TestDeltaFilePath(t *testing.T) { func TestCachedFilePath(t *testing.T) { s := SetUpTest(t) defer s.TearDownTest() - ds := NewStore(s.repoDir, "default", maxInventorySize) + ds := NewStore(s.repoDir, "default", maxInventorySize, true) actual := ds.cachedFilePath(s.plugin, "hello!!everybody") expected := filepath.Join("delta", ".delta_repo", "metadata", "hello!!everybody", "plugin.json") @@ -165,7 +165,7 @@ func TestCachedFilePath(t *testing.T) { func TestSourceFilePath(t *testing.T) { s := SetUpTest(t) defer s.TearDownTest() - ds := NewStore(s.repoDir, "default", maxInventorySize) + ds := NewStore(s.repoDir, "default", maxInventorySize, true) actual := ds.SourceFilePath(s.plugin, "xxxx") expected := filepath.Join("delta", "metadata", "xxxx", "plugin.json") @@ -175,7 +175,7 @@ func TestSourceFilePath(t *testing.T) { func TestArchiveFilePath_localEntity(t *testing.T) { s := SetUpTest(t) defer s.TearDownTest() - ds := NewStore(s.repoDir, "my-hostname", maxInventorySize) + ds := NewStore(s.repoDir, "my-hostname", maxInventorySize, true) actual := ds.archiveFilePath(s.plugin, "") expected := filepath.Join("delta", ".delta_repo", "metadata", localEntityFolder, "plugin.sent") @@ -185,7 +185,7 @@ func TestArchiveFilePath_localEntity(t *testing.T) { func TestDeltaFilePath_localEntity(t *testing.T) { s := SetUpTest(t) defer s.TearDownTest() - ds := NewStore(s.repoDir, "my-hostname", maxInventorySize) + ds := NewStore(s.repoDir, "my-hostname", maxInventorySize, true) actual := ds.DeltaFilePath(s.plugin, "") expected := filepath.Join("delta", ".delta_repo", "metadata", localEntityFolder, "plugin.pending") @@ -195,7 +195,7 @@ func TestDeltaFilePath_localEntity(t *testing.T) { func TestCachedFilePath_localEntity(t *testing.T) { s := SetUpTest(t) defer s.TearDownTest() - ds := NewStore(s.repoDir, "my-hostname", maxInventorySize) + ds := NewStore(s.repoDir, "my-hostname", maxInventorySize, true) actual := ds.cachedFilePath(s.plugin, "") expected := filepath.Join("delta", ".delta_repo", "metadata", localEntityFolder, "plugin.json") @@ -205,7 +205,7 @@ func TestCachedFilePath_localEntity(t *testing.T) { func TestSourceFilePath_localEntity(t *testing.T) { s := SetUpTest(t) defer s.TearDownTest() - ds := NewStore(s.repoDir, "my-hostname", maxInventorySize) + ds := NewStore(s.repoDir, "my-hostname", maxInventorySize, true) actual := ds.SourceFilePath(s.plugin, "") expected := filepath.Join("delta", "metadata", localEntityFolder, "plugin.json") @@ -215,7 +215,7 @@ func TestSourceFilePath_localEntity(t *testing.T) { func TestBaseDirectories(t *testing.T) { s := SetUpTest(t) defer s.TearDownTest() - ds := NewStore(s.repoDir, "default", maxInventorySize) + ds := NewStore(s.repoDir, "default", maxInventorySize, true) assert.Equal(t, s.repoDir, ds.DataDir) assert.Equal(t, filepath.Join(s.repoDir, CACHE_DIR), ds.CacheDir) @@ -225,7 +225,7 @@ func TestResetAllSentDeltas(t *testing.T) { s := SetUpTest(t) defer s.TearDownTest() const eKey = "entityKey" - ds := NewStore(s.repoDir, "default", maxInventorySize) + ds := NewStore(s.repoDir, "default", maxInventorySize, true) srcFile := ds.SourceFilePath(s.plugin, eKey) err := os.MkdirAll(filepath.Dir(srcFile), 0755) require.NoError(t, err) @@ -251,7 +251,7 @@ func TestResetAllSentDeltas(t *testing.T) { func TestUpdateLastDeltaSentNoHint(t *testing.T) { s := SetUpTest(t) defer s.TearDownTest() - ds := NewStore(s.repoDir, "default", maxInventorySize) + ds := NewStore(s.repoDir, "default", maxInventorySize, true) s.plugin.setDeltaID("entityKey", 1) ds.plugins["metadata/plugin"] = s.plugin diff := make(map[string]interface{}) @@ -274,7 +274,7 @@ func TestUpdateLastDeltaSentNoHint(t *testing.T) { func TestUpdateLastDeltaSentNewDelta(t *testing.T) { s := SetUpTest(t) defer s.TearDownTest() - ds := NewStore(s.repoDir, "default", maxInventorySize) + ds := NewStore(s.repoDir, "default", maxInventorySize, true) s.plugin.setDeltaID("entityKey", 1) ds.plugins["metadata/plugin"] = s.plugin diff := make(map[string]interface{}) @@ -301,7 +301,7 @@ func TestUpdateLastDeltaSentNewDelta(t *testing.T) { func TestUpdateLastDeltaSentHintResend(t *testing.T) { s := SetUpTest(t) defer s.TearDownTest() - ds := NewStore(s.repoDir, "default", maxInventorySize) + ds := NewStore(s.repoDir, "default", maxInventorySize, true) s.plugin.setDeltaID("entityKey", 1) ds.plugins["metadata/plugin"] = s.plugin diff := make(map[string]interface{}) @@ -328,7 +328,7 @@ func TestUpdateLastDeltaSentHintResend(t *testing.T) { func TestUpdateLastDeltaSentHintRequestOlder(t *testing.T) { s := SetUpTest(t) defer s.TearDownTest() - ds := NewStore(s.repoDir, "default", maxInventorySize) + ds := NewStore(s.repoDir, "default", maxInventorySize, true) s.plugin.setDeltaID("entityKey", 1) ds.plugins["metadata/plugin"] = s.plugin diff := make(map[string]interface{}) @@ -356,7 +356,7 @@ func TestUpdateLastDeltaSentHintRequestOlder(t *testing.T) { func TestUpdateLastDeltaSentHintIsSameAsDelta(t *testing.T) { s := SetUpTest(t) defer s.TearDownTest() - ds := NewStore(s.repoDir, "default", maxInventorySize) + ds := NewStore(s.repoDir, "default", maxInventorySize, true) s.plugin.setDeltaID("entityKey", 1) ds.plugins["metadata/plugin"] = s.plugin diff := make(map[string]interface{}) @@ -386,7 +386,7 @@ func TestUpdatePluginInventoryCacheFirstRunGP(t *testing.T) { defer s.TearDownTest() const eKey = "entity:ID" - ds := NewStore(s.repoDir, "default", maxInventorySize) + ds := NewStore(s.repoDir, "default", maxInventorySize, true) srcFile := ds.SourceFilePath(s.plugin, eKey) err := os.MkdirAll(filepath.Dir(srcFile), 0755) require.NoError(t, err) @@ -412,7 +412,7 @@ func TestUpdatePluginInventoryCacheThreeChanges(t *testing.T) { defer s.TearDownTest() const eKey = "entity:ID" - ds := NewStore(s.repoDir, "default", maxInventorySize) + ds := NewStore(s.repoDir, "default", maxInventorySize, true) srcFile := ds.SourceFilePath(s.plugin, eKey) err := os.MkdirAll(filepath.Dir(srcFile), 0755) require.NoError(t, err) @@ -449,7 +449,7 @@ func TestSaveState(t *testing.T) { defer s.TearDownTest() const eKey = "entity:ID" - ds := NewStore(s.repoDir, "default", maxInventorySize) + ds := NewStore(s.repoDir, "default", maxInventorySize, true) srcFile := ds.SourceFilePath(s.plugin, eKey) err := os.MkdirAll(filepath.Dir(srcFile), 0755) require.NoError(t, err) @@ -497,7 +497,7 @@ func TestReadPluginIDMapNoContent(t *testing.T) { defer s.TearDownTest() const eKey = "entity:ID" - ds := NewStore(s.repoDir, "default", maxInventorySize) + ds := NewStore(s.repoDir, "default", maxInventorySize, true) srcFile := ds.SourceFilePath(s.plugin, eKey) err := os.MkdirAll(filepath.Dir(srcFile), 0755) require.NoError(t, err) @@ -539,7 +539,7 @@ func TestReadDeltas(t *testing.T) { s := SetUpTest(t) defer s.TearDownTest() // Given a delta file store - ds := NewStore(s.repoDir, "default", maxInventorySize) + ds := NewStore(s.repoDir, "default", maxInventorySize, true) // When a delta source file is created for an entity const eKey = "entity:ID" @@ -571,7 +571,7 @@ func TestReadDeltas_SamePluginWithMultipleEntitiesIncreaseIDIndependently(t *tes s := SetUpTest(t) defer s.TearDownTest() // Given a delta file store - ds := NewStore(s.repoDir, "default", maxInventorySize) + ds := NewStore(s.repoDir, "default", maxInventorySize, true) // When a delta source file is created for an entity const e1 = "entity:ID1" @@ -632,7 +632,7 @@ func TestReadDeltas_Divided(t *testing.T) { } // And a storer whose max inventory size is lower than the sum of the 3 (each one occupies ~150 bytes) - ds := NewStore(s.repoDir, "default", 350) + ds := NewStore(s.repoDir, "default", 350, true) var updated bool // And the deltas have been correctly stored @@ -686,7 +686,7 @@ func TestReadDeltas_Undivided(t *testing.T) { } // And a storer whose max inventory size higher than the sum of the 3 - ds := NewStore(s.repoDir, "default", maxInventorySize) + ds := NewStore(s.repoDir, "default", maxInventorySize, true) var updated bool // And the deltas have been correctly stored @@ -728,7 +728,7 @@ func TestReadDeltas_Undivided(t *testing.T) { func (d *DeltaUtilsCoreSuite) SetupSavedState(t *testing.T) (ds *Store) { const eKey = "entity:ID" - ds = NewStore(d.repoDir, "default", maxInventorySize) + ds = NewStore(d.repoDir, "default", maxInventorySize, true) srcFile := ds.SourceFilePath(d.plugin, eKey) err := os.MkdirAll(filepath.Dir(srcFile), 0755) require.NoError(t, err) @@ -841,11 +841,34 @@ func TestCompactStoreRemoveUnusedPlugin(t *testing.T) { assert.Error(t, err) } +func TestStoreNotArchiving(t *testing.T) { + const eKey = "entity:ID" + s := SetUpTest(t) + defer s.TearDownTest() + + ds := s.SetupSavedState(t) + ds.archiveEnabled = false + ds.plugins["metadata/plugin"].setLastSentID(eKey, 2) + origSize, err := ds.StorageSize(ds.CacheDir) + require.NoError(t, err) + + err = ds.archivePlugin(ds.plugins["metadata/plugin"], eKey) + require.NoError(t, err) + + require.NoError(t, err) + newSize, err := ds.StorageSize(ds.CacheDir) + require.NoError(t, err) + assert.Less(t, newSize, origSize) + + exists := exists(filepath.Join(ds.CacheDir, "metadata/entityID/plugin.sent")) + assert.False(t, exists, "expected .sent file to not exist") +} + func TestDeltaFileCorrupt(t *testing.T) { s := SetUpTest(t) defer s.TearDownTest() const eKey = "entity:ID" - ds := NewStore(s.repoDir, "default", maxInventorySize) + ds := NewStore(s.repoDir, "default", maxInventorySize, true) srcFile := ds.SourceFilePath(s.plugin, eKey) err := os.MkdirAll(filepath.Dir(srcFile), 0755) require.NoError(t, err) @@ -930,7 +953,7 @@ func TestRemoveEntity(t *testing.T) { for _, dir := range directories { assert.NoError(t, os.MkdirAll(dir.path, 0755)) } - store := NewStore(baseDir, "default", maxInventorySize) + store := NewStore(baseDir, "default", maxInventorySize, true) // When removing data from a given entity: _ = store.RemoveEntityFolders(entityToRemove) @@ -972,7 +995,7 @@ func TestScanEntityFolders(t *testing.T) { for _, dir := range directories { assert.NoError(t, os.MkdirAll(dir.path, 0755)) } - store := NewStore(baseDir, "default", maxInventorySize) + store := NewStore(baseDir, "default", maxInventorySize, true) // When fetching all the entities entities, err := store.ScanEntityFolders() @@ -1011,7 +1034,7 @@ func TestCollectPluginFiles(t *testing.T) { require.NoError(t, err) file.Close() } - store := NewStore(baseDir, "default", maxInventorySize) + store := NewStore(baseDir, "default", maxInventorySize, true) // When collecting all the plugins of a given entity plugins, err := store.collectPluginFiles(store.DataDir, anEntity, helpers.JsonFilesRegexp) @@ -1053,7 +1076,7 @@ func TestUpdatePluginInventoryCacheDeltaFileCorrupted(t *testing.T) { cacheJSON := filepath.Join(cacheDir, "corrupted.json") // And a delta storage - ds := NewStore(dataDir, "default", maxInventorySize) + ds := NewStore(dataDir, "default", maxInventorySize, true) require.NoError(t, os.MkdirAll(sourceDir, 0755)) require.NoError(t, os.MkdirAll(cacheDir, 0755)) require.NoError(t, ioutil.WriteFile(sourceJSON, testCase["source"], 0644)) @@ -1094,7 +1117,7 @@ func TestDeltaRoot_WithCorruptedFile_StartFresh(t *testing.T) { assert.NoError(t, err) // WHEN the data store is create - ds := NewStore(dataPath, "default", maxInventorySize) + ds := NewStore(dataPath, "default", maxInventorySize, true) assert.NotNil(t, ds) // THEN check that the corrupted json file has been deleted diff --git a/internal/agent/inventory/entity_patcher.go b/internal/agent/inventory/entity_patcher.go new file mode 100644 index 000000000..177f954a1 --- /dev/null +++ b/internal/agent/inventory/entity_patcher.go @@ -0,0 +1,192 @@ +// Copyright 2020 New Relic Corporation. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +package inventory + +import ( + "fmt" + "github.com/newrelic/infrastructure-agent/internal/agent/delta" + "github.com/newrelic/infrastructure-agent/internal/agent/types" + "github.com/newrelic/infrastructure-agent/pkg/entity" + "github.com/newrelic/infrastructure-agent/pkg/helpers" + "sync" + "time" +) + +type PatchSenderProviderFunc func(entity.Entity) (PatchSender, error) + +type EntityPatcher struct { + m sync.Mutex + + BasePatcher + entities map[entity.Key]struct { + sender PatchSender + needsReaping bool + } + seenEntities map[entity.Key]struct{} + + patchSenderProviderFn PatchSenderProviderFunc +} + +func NewEntityPatcher(cfg PatcherConfig, deltaStore *delta.Store, patchSenderProviderFn PatchSenderProviderFunc) Patcher { + ep := &EntityPatcher{ + BasePatcher: BasePatcher{ + deltaStore: deltaStore, + cfg: cfg, + lastClean: time.Now(), + }, + seenEntities: make(map[entity.Key]struct{}), + + entities: map[entity.Key]struct { + sender PatchSender + needsReaping bool + }{}, + patchSenderProviderFn: patchSenderProviderFn, + } + err := ep.registerEntity(cfg.AgentEntity) + if err != nil { + ilog.WithError(err).Error("Failed to register agent inventory entity") + } + return ep +} + +func (ep *EntityPatcher) Send() error { + if ep.needsCleanup() { + ep.m.Lock() + ep.seenEntities = make(map[entity.Key]struct{}) + ep.cleanOutdatedEntities() + ep.m.Unlock() + } + + ep.m.Lock() + senders := make([]PatchSender, len(ep.entities)) + + i := 0 + for _, inventory := range ep.entities { + senders[i] = inventory.sender + i++ + } + ep.m.Unlock() + + for _, sender := range senders { + if err := sender.Process(); err != nil { + return err + } + } + return nil +} + +func (ep *EntityPatcher) Reap() { + ep.m.Lock() + defer ep.m.Unlock() + + for key, inventory := range ep.entities { + if !inventory.needsReaping { + continue + } + ep.reapEntity(key) + inventory.needsReaping = false + } +} + +func (ep *EntityPatcher) Save(data types.PluginOutput) error { + ep.m.Lock() + defer ep.m.Unlock() + + if data.NotApplicable { + return nil + } + + if err := ep.registerEntity(data.Entity); err != nil { + return fmt.Errorf("failed to save plugin inventory data, error: %w", err) + } + + if err := ep.BasePatcher.save(data); err != nil { + return fmt.Errorf("failed to save plugin inventory data, error: %w", err) + } + + ep.seenEntities[data.Entity.Key] = struct{}{} + + e := ep.entities[data.Entity.Key] + e.needsReaping = true + return nil +} + +func (ep *EntityPatcher) registerEntity(entity entity.Entity) error { + if _, found := ep.entities[entity.Key]; found { + return nil + } + + ilog.WithField("entityKey", entity.Key.String()). + WithField("entityID", entity.ID).Debug("Registering inventory for entity.") + + sender, err := ep.patchSenderProviderFn(entity) + if err != nil { + return fmt.Errorf("failed to register inventory for entity: %s, %v", entity.Key, err) + } + + ep.entities[entity.Key] = struct { + sender PatchSender + needsReaping bool + }{sender: sender, needsReaping: true} + + return nil +} + +// removes the inventory object references to free the memory, and the respective directories +func (ep *EntityPatcher) unregisterEntity(entity entity.Key) error { + entityKey := entity.String() + ilog.WithField("entityKey", entityKey).Debug("Unregistering inventory for entity.") + + _, ok := ep.entities[entity] + if ok { + delete(ep.entities, entity) + } + + return ep.deltaStore.RemoveEntity(entityKey) +} + +func (ep *EntityPatcher) cleanOutdatedEntities() { + ilog.Debug("Triggered periodic removal of outdated entities.") + // The entities to remove are those entities that haven't reported activity in the last period and + // are registered in the system + entitiesToRemove := map[entity.Key]struct{}{} + + for entityKey := range ep.entities { + entitiesToRemove[entityKey] = struct{}{} + } + + delete(entitiesToRemove, ep.cfg.AgentEntity.Key) // never delete local entity + + for entityKey := range ep.seenEntities { + delete(entitiesToRemove, entityKey) + } + + for entityKey := range entitiesToRemove { + ilog.WithField("entityKey", entityKey.String()).Debug("Removing inventory for entity.") + if err := ep.unregisterEntity(entityKey); err != nil { + ilog.WithError(err).Warn("unregistering inventory for entity") + } + } + // Remove folders from unregistered entities that still have folders in the data directory (e.g. from + // previous agent executions) + foldersToRemove, err := ep.deltaStore.ScanEntityFolders() + if err != nil { + ilog.WithError(err).Warn("error scanning outdated entity folders") + // Continuing, because some entities may have been fetched despite the error + } + + if foldersToRemove != nil { + // We don't remove those entities that are registered + for entityKey := range ep.entities { + delete(foldersToRemove, helpers.SanitizeFileName(entityKey.String())) + } + for folder := range foldersToRemove { + if err := ep.deltaStore.RemoveEntityFolders(folder); err != nil { + ilog.WithField("folder", folder).WithError(err).Warn("error removing entity folder") + } + } + } + + ilog.WithField("remaining", len(ep.entities)).Debug("Some entities may remain registered.") +} diff --git a/internal/agent/inventory/entity_patcher_test.go b/internal/agent/inventory/entity_patcher_test.go new file mode 100644 index 000000000..53c96b92a --- /dev/null +++ b/internal/agent/inventory/entity_patcher_test.go @@ -0,0 +1,105 @@ +// Copyright 2020 New Relic Corporation. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +package inventory + +import ( + "github.com/newrelic/infrastructure-agent/internal/agent/delta" + agentTypes "github.com/newrelic/infrastructure-agent/internal/agent/types" + "github.com/newrelic/infrastructure-agent/pkg/entity" + "github.com/newrelic/infrastructure-agent/pkg/helpers" + "github.com/newrelic/infrastructure-agent/pkg/plugins/ids" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "io/ioutil" + "os" + "path/filepath" + "testing" +) + +func TestCleanOutdatedEntities(t *testing.T) { + dataDir, err := ioutil.TempDir("", "prefix") + require.NoError(t, err) + + deltaStore := delta.NewStore(dataDir, "default", 1024, false) + + defer os.RemoveAll(deltaStore.DataDir) + + const aPlugin = "aPlugin" + const anotherPlugin = "anotherPlugin" + + // GIVEN en entity patcher + entityPatcher := &EntityPatcher{ + BasePatcher: BasePatcher{ + cfg: PatcherConfig{}, + deltaStore: deltaStore, + }, + patchSenderProviderFn: func(e entity.Entity) (PatchSender, error) { + return nil, nil + }, + entities: map[entity.Key]struct { + sender PatchSender + needsReaping bool + }{}, + seenEntities: map[entity.Key]struct{}{}, + } + + dataDirPath := entityPatcher.deltaStore.DataDir + + defer os.RemoveAll(dataDirPath) + + // With a set of registered entities + for _, id := range []string{"entity:1", "entity:2", "entity:3"} { + entityPatcher.registerEntity(entity.NewFromNameWithoutID(id)) + + assert.NoError(t, os.MkdirAll(filepath.Join(dataDir, aPlugin, helpers.SanitizeFileName(id)), 0755)) + assert.NoError(t, os.MkdirAll(filepath.Join(dataDir, anotherPlugin, helpers.SanitizeFileName(id)), 0755)) + } + + // With some entity inventory folders from previous executions + assert.NoError(t, os.MkdirAll(filepath.Join(dataDir, aPlugin, "entity4"), 0755)) + assert.NoError(t, os.MkdirAll(filepath.Join(dataDir, aPlugin, "entity5"), 0755)) + assert.NoError(t, os.MkdirAll(filepath.Join(dataDir, aPlugin, "entity6"), 0755)) + assert.NoError(t, os.MkdirAll(filepath.Join(dataDir, anotherPlugin, "entity4"), 0755)) + assert.NoError(t, os.MkdirAll(filepath.Join(dataDir, anotherPlugin, "entity5"), 0755)) + assert.NoError(t, os.MkdirAll(filepath.Join(dataDir, anotherPlugin, "entity6"), 0755)) + + // WHEN not all the entities reported information during the last period + entityPatcher.Save(agentTypes.PluginOutput{ + Id: ids.PluginID{ + Category: "test", + Term: "plugin", + }, + + Entity: entity.NewFromNameWithoutID("entity:2"), + Data: agentTypes.PluginInventoryDataset{}, + }) + + // AND the "remove outdated entities" is triggered + entityPatcher.cleanOutdatedEntities() + + // THEN the entities that didn't reported information have been unregistered + // and only their folders are kept + entities := []struct { + ID string + Folder string + StillReporting bool + }{ + {"entity:1", "entity1", false}, + {"entity:2", "entity2", true}, + {"entity:3", "entity3", false}, + {"dontCare", "entity4", false}, + {"doesntMatter", "entity5", false}, + } + for _, e := range entities { + _, err1 := os.Stat(filepath.Join(dataDir, aPlugin, e.Folder)) + _, err2 := os.Stat(filepath.Join(dataDir, anotherPlugin, e.Folder)) + if e.StillReporting { + assert.NoError(t, err1) + assert.NoError(t, err2) + } else { + assert.True(t, os.IsNotExist(err1)) + assert.True(t, os.IsNotExist(err2)) + } + } +} diff --git a/internal/agent/inventory/handler.go b/internal/agent/inventory/handler.go new file mode 100644 index 000000000..bc0c2b8a6 --- /dev/null +++ b/internal/agent/inventory/handler.go @@ -0,0 +1,145 @@ +// Copyright 2020 New Relic Corporation. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +package inventory + +import ( + context2 "context" + "github.com/newrelic/infrastructure-agent/internal/agent/types" + "github.com/newrelic/infrastructure-agent/pkg/backend/inventoryapi" + "github.com/newrelic/infrastructure-agent/pkg/config" + "github.com/newrelic/infrastructure-agent/pkg/helpers" + "github.com/newrelic/infrastructure-agent/pkg/log" + "net/http" + "time" +) + +var ( + ilog = log.WithComponent("Inventory") +) + +type HandlerConfig struct { + FirstReapInterval time.Duration + ReapInterval time.Duration + SendInterval time.Duration + InventoryQueueLen int +} + +// Handler maintains the infrastructure inventory in an updated state. +// It will receive inventory data from integrations/plugins for processing deltas (differences between versions) +type Handler struct { + cfg HandlerConfig + + ctx context2.Context + cancelFn context2.CancelFunc + + patcher Patcher + + initialReap bool + + dataCh chan types.PluginOutput + + sendTimer *time.Timer + + sendErrorCount uint32 +} + +// NewInventoryHandler returns a new instances of an inventory.Handler. +func NewInventoryHandler(ctx context2.Context, cfg HandlerConfig, patcher Patcher) *Handler { + ctx2, cancelFn := context2.WithCancel(ctx) + + return &Handler{ + cfg: cfg, + dataCh: make(chan types.PluginOutput, cfg.InventoryQueueLen), + ctx: ctx2, + cancelFn: cancelFn, + patcher: patcher, + initialReap: true, + } +} + +// Handle the inventory data from a plugin/integration. +func (h *Handler) Handle(data types.PluginOutput) { + h.dataCh <- data +} + +// Start will run the routines that periodically checks for deltas and submit them. +func (h *Handler) Start() { + go h.listenForData() + h.doProcess() +} + +// Stop will gracefully stop the inventory.Handler. +func (h *Handler) Stop() { + h.cancelFn() +} + +// listenForData from plugins/integrations. +func (h *Handler) listenForData() { + for { + select { + case <-h.ctx.Done(): + return + case data := <-h.dataCh: + err := h.patcher.Save(data) + if err != nil { + ilog.WithError(err).Error("problem storing plugin output") + } + } + } +} + +// doProcess does the inventory processing. +func (h *Handler) doProcess() { + h.sendTimer = time.NewTimer(h.cfg.SendInterval) + reapTimer := time.NewTicker(h.cfg.FirstReapInterval) + + defer func() { + h.sendTimer.Stop() + reapTimer.Stop() + }() + + for { + select { + case <-h.ctx.Done(): + return + case <-reapTimer.C: + if h.initialReap { + h.initialReap = false + reapTimer.Reset(h.cfg.ReapInterval) + } + h.patcher.Reap() + case <-h.sendTimer.C: + h.send() + } + } +} + +// send will submit the deltas. +func (h *Handler) send() { + backoffMax := config.MAX_BACKOFF + + err := h.patcher.Send() + if err != nil { + if ingestError, ok := err.(*inventoryapi.IngestError); ok && + ingestError.StatusCode == http.StatusTooManyRequests { + + ilog.Warn("server is rate limiting inventory submission") + + backoffMax = config.RATE_LIMITED_BACKOFF + h.sendErrorCount = helpers.MaxBackoffErrorCount + } else { + h.sendErrorCount++ + } + + ilog.WithError(err).WithField("errorCount", h.sendErrorCount). + Debug("Inventory sender can't process data.") + } else { + h.sendErrorCount = 0 + } + + sendTimerVal := helpers.ExpBackoff(h.cfg.SendInterval, + time.Duration(backoffMax)*time.Second, + h.sendErrorCount) + h.sendTimer.Reset(sendTimerVal) +} diff --git a/internal/agent/inventory/patcher.go b/internal/agent/inventory/patcher.go new file mode 100644 index 000000000..7b18f8bfa --- /dev/null +++ b/internal/agent/inventory/patcher.go @@ -0,0 +1,123 @@ +// Copyright 2020 New Relic Corporation. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +package inventory + +import ( + "fmt" + "github.com/newrelic/infrastructure-agent/internal/agent/delta" + "github.com/newrelic/infrastructure-agent/internal/agent/types" + "github.com/newrelic/infrastructure-agent/pkg/entity" + "github.com/sirupsen/logrus" + "sort" + "strings" + "time" +) + +const ( + defaultRemoveEntitiesPeriod = 48 * time.Hour +) + +type PatchSender interface { + Process() error +} + +// Patcher performs the actions required to update the state of the inventory stored +// by the backend. +type Patcher interface { + // Save will store the new data received from a plugin for future processing. + Save(output types.PluginOutput) error + + // Reap will compare latest saved data with latest submitted and generate + // deltas in storage if required. + Reap() + + // Send will look for deltas in the storage and submit them to the backend. + Send() error +} + +type PatcherConfig struct { + IgnoredPaths map[string]struct{} + AgentEntity entity.Entity + RemoveEntitiesPeriod time.Duration +} + +// BasePatcher will keep the common functionality of a patcher. +type BasePatcher struct { + deltaStore *delta.Store + cfg PatcherConfig + lastClean time.Time +} + +func (b *BasePatcher) needsCleanup() bool { + if b.lastClean.Equal(time.Time{}) { + b.lastClean = time.Now() + return false + } + + removePeriod := b.cfg.RemoveEntitiesPeriod + if removePeriod <= 0 { + removePeriod = defaultRemoveEntitiesPeriod + } + + needsCleanup := b.lastClean.Add(removePeriod).Before(time.Now()) + if needsCleanup { + b.lastClean = time.Now() + } + return needsCleanup +} + +// save will take a PluginOutput and persist it in the store. +func (b *BasePatcher) save(pluginOutput types.PluginOutput) error { + + if pluginOutput.Data == nil { + pluginOutput.Data = make(types.PluginInventoryDataset, 0) + } + + sort.Sort(pluginOutput.Data) + + simplifiedPluginData := make(map[string]interface{}) + + for _, data := range pluginOutput.Data { + if data == nil { + continue + } + sortKey := data.SortKey() + + // Filter out ignored inventory data before writing the file out + pluginSource := fmt.Sprintf("%s/%s", pluginOutput.Id, sortKey) + if b.isIgnored(pluginSource) { + continue + } + simplifiedPluginData[sortKey] = data + } + + return b.deltaStore.SavePluginSource( + pluginOutput.Entity.Key.String(), + pluginOutput.Id.Category, + pluginOutput.Id.Term, + simplifiedPluginData, + ) +} + +// isIgnored will check if a specific plugin output should be ignored according to the configuration. +func (b *BasePatcher) isIgnored(pluginSource string) bool { + if b.cfg.IgnoredPaths == nil { + return false + } + _, ignored := b.cfg.IgnoredPaths[strings.ToLower(pluginSource)] + return ignored +} + +// reapEntity will tell storage to generate deltas for a specific entity. +func (b *BasePatcher) reapEntity(entityKey entity.Key) { + // Reap generates deltas from last iteration and persist. + entityKeyStr := entityKey.String() + err := b.deltaStore.UpdatePluginsInventoryCache(entityKeyStr) + if err != nil { + ilog.WithFields(logrus.Fields{ + "entityKey": entityKeyStr, + }).WithError(err).Error("failed to update inventory cache") + return + } +} diff --git a/internal/agent/inventory/patcher_test.go b/internal/agent/inventory/patcher_test.go new file mode 100644 index 000000000..15954607b --- /dev/null +++ b/internal/agent/inventory/patcher_test.go @@ -0,0 +1,141 @@ +// Copyright 2020 New Relic Corporation. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +package inventory + +import ( + "encoding/json" + "github.com/newrelic/infrastructure-agent/internal/agent/delta" + agentTypes "github.com/newrelic/infrastructure-agent/internal/agent/types" + "github.com/newrelic/infrastructure-agent/pkg/entity" + "github.com/newrelic/infrastructure-agent/pkg/plugins/ids" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "io/ioutil" + "os" + "path/filepath" + "testing" + "time" +) + +func TestPatcher_NeedsCleanup_NeverCleaned(t *testing.T) { + b := BasePatcher{} + assert.False(t, b.needsCleanup()) + assert.False(t, b.needsCleanup()) +} + +func TestPatcher_NeedsCleanup_DefaultRemoveEntitiesPeriodExceeded(t *testing.T) { + b := BasePatcher{ + lastClean: time.Now().Add(-(defaultRemoveEntitiesPeriod + 1)), + } + assert.True(t, b.needsCleanup()) + assert.False(t, b.needsCleanup()) +} + +func TestPatcher_NeedsCleanup_ConfigRemoveEntitiesPeriodExceeded(t *testing.T) { + b := BasePatcher{ + cfg: PatcherConfig{ + RemoveEntitiesPeriod: 1 * time.Hour, + }, + lastClean: time.Now().Add(-30 * time.Minute), + } + assert.False(t, b.needsCleanup()) + + b.lastClean = b.lastClean.Add(-(30*time.Minute + 1)) + assert.True(t, b.needsCleanup()) + assert.False(t, b.needsCleanup()) +} + +type testInventoryData struct { + Name string + Value *string +} + +func (t *testInventoryData) SortKey() string { + return t.Name +} + +func TestPatcher_Save(t *testing.T) { + dataDir, err := ioutil.TempDir("", "prefix") + require.NoError(t, err) + + deltaStore := delta.NewStore(dataDir, "default", 1024, false) + + b := BasePatcher{ + cfg: PatcherConfig{}, + deltaStore: deltaStore, + } + defer os.RemoveAll(deltaStore.DataDir) + + aV := "aValue" + bV := "bValue" + cV := "cValue" + + err = b.save(agentTypes.PluginOutput{ + Id: ids.PluginID{ + Category: "test", + Term: "plugin", + }, + + Entity: entity.NewFromNameWithoutID("someEntity"), + Data: agentTypes.PluginInventoryDataset{ + &testInventoryData{"cMyService", &cV}, + &testInventoryData{"aMyService", &aV}, + &testInventoryData{"NilService", nil}, + &testInventoryData{"bMyService", &bV}, + }, + }) + + assert.NoError(t, err) + + sourceFile := filepath.Join(deltaStore.DataDir, "test", "someEntity", "plugin.json") + sourceB, err := ioutil.ReadFile(sourceFile) + require.NoError(t, err) + + expected := []byte(`{"NilService":{"Name":"NilService"},"aMyService":{"Name":"aMyService","Value":"aValue"},"bMyService":{"Name":"bMyService","Value":"bValue"},"cMyService":{"Name":"cMyService","Value":"cValue"}}`) + + assert.Equal(t, string(expected), string(sourceB)) +} + +func TestPatcher_SaveIgnored(t *testing.T) { + dataDir, err := ioutil.TempDir("", "prefix") + require.NoError(t, err) + + deltaStore := delta.NewStore(dataDir, "default", 1024, false) + + b := BasePatcher{ + cfg: PatcherConfig{ + IgnoredPaths: map[string]struct{}{"test/plugin/yum": {}}, + }, + deltaStore: deltaStore, + } + defer os.RemoveAll(deltaStore.DataDir) + + aV := "aValue" + bV := "bValue" + + assert.NoError(t, b.save(agentTypes.PluginOutput{ + Id: ids.PluginID{ + Category: "test", + Term: "plugin", + }, + Entity: entity.NewFromNameWithoutID("someEntity"), + Data: agentTypes.PluginInventoryDataset{ + &testInventoryData{"yum", &aV}, + &testInventoryData{"myService", &bV}, + }, + })) + + restoredDataBytes, err := ioutil.ReadFile(filepath.Join(deltaStore.DataDir, "test", "someEntity", "plugin.json")) + require.NoError(t, err) + + var restoredData map[string]interface{} + require.NoError(t, json.Unmarshal(restoredDataBytes, &restoredData)) + + assert.Equal(t, restoredData, map[string]interface{}{ + "myService": map[string]interface{}{ + "Name": "myService", + "Value": "bValue", + }, + }) +} diff --git a/internal/agent/mocks/AgentContext.go b/internal/agent/mocks/AgentContext.go index b399ed8ee..c2488cbc1 100644 --- a/internal/agent/mocks/AgentContext.go +++ b/internal/agent/mocks/AgentContext.go @@ -5,6 +5,7 @@ package mocks import ( "context" + "github.com/newrelic/infrastructure-agent/internal/agent/types" "sync" agent "github.com/newrelic/infrastructure-agent/internal/agent" @@ -165,7 +166,7 @@ func (_m *AgentContext) Reconnect() { } // SendData provides a mock function with given fields: _a0 -func (_m *AgentContext) SendData(_a0 agent.PluginOutput) { +func (_m *AgentContext) SendData(_a0 types.PluginOutput) { _m.SendDataWg.Done() _m.Called(_a0) } diff --git a/internal/agent/patch_reaper.go b/internal/agent/patch_reaper.go index 1a9a1c8fb..2535c234f 100644 --- a/internal/agent/patch_reaper.go +++ b/internal/agent/patch_reaper.go @@ -14,21 +14,21 @@ import ( "github.com/sirupsen/logrus" ) -// patchReaper gets the inventory data that has changed since the last reap and commits it into storage -type patchReaper struct { +// PatchReaper gets the inventory data that has changed since the last reap and commits it into storage +type PatchReaper struct { entityKey string store *delta.Store } var prlog = log.WithComponent("PatchReaper") -func newPatchReaper(entityKey string, store *delta.Store) *patchReaper { +func newPatchReaper(entityKey string, store *delta.Store) *PatchReaper { if store == nil { prlog.WithField("entityKey", entityKey).Error("creating patch reaper: delta store can't be nil") panic("creating patch reaper: delta store can't be nil") } - return &patchReaper{ + return &PatchReaper{ entityKey: entityKey, store: store, } @@ -36,13 +36,14 @@ func newPatchReaper(entityKey string, store *delta.Store) *patchReaper { // CleanupOldPlugins deletes old json from plugins that have been // deprecated or are no longer used -func (p *patchReaper) CleanupOldPlugins(plugins []ids.PluginID) { +func (p *PatchReaper) CleanupOldPlugins(plugins []ids.PluginID) { for _, plugin := range plugins { // first check if file exists filename := filepath.Join(p.store.DataDir, plugin.Category, fmt.Sprintf("%s.json", plugin.Term)) if _, err := os.Stat(filename); err != nil { continue } + // next, remove the source file first, which will show // as a deleted delta if err := os.Remove(filename); err != nil { @@ -55,7 +56,7 @@ func (p *patchReaper) CleanupOldPlugins(plugins []ids.PluginID) { } // Reap generates deltas from last iteration and persist. -func (p *patchReaper) Reap() { +func (p *PatchReaper) Reap() { err := p.store.UpdatePluginsInventoryCache(p.entityKey) if err != nil { prlog.WithFields(logrus.Fields{ diff --git a/internal/agent/patch_reaper_test.go b/internal/agent/patch_reaper_test.go index 6ba0c6852..ae2c0e723 100644 --- a/internal/agent/patch_reaper_test.go +++ b/internal/agent/patch_reaper_test.go @@ -14,6 +14,8 @@ import ( "github.com/stretchr/testify/require" ) +const maxInventoryDataSize = 3 * 1000 * 1000 + func TestPatchReapNoCacheExists(t *testing.T) { fixture, err := ioutil.ReadFile(filepath.Join("fixtures", "packages_rpm_delta.json")) if err != nil { @@ -34,7 +36,7 @@ func TestPatchReapNoCacheExists(t *testing.T) { } defer dir.Clear() - store := delta.NewStore(dir.Path, "default", maxInventoryDataSize) + store := delta.NewStore(dir.Path, "default", maxInventoryDataSize, true) pr := newPatchReaper("", store) pr.Reap() cacheFilePath := filepath.Join( @@ -82,7 +84,7 @@ func TestPatchReapCacheUpdate(t *testing.T) { } defer dir.Clear() - store := delta.NewStore(dir.Path, "default", maxInventoryDataSize) + store := delta.NewStore(dir.Path, "default", maxInventoryDataSize, true) pr := newPatchReaper("", store) pr.Reap() cacheFilePath := filepath.Join( @@ -133,7 +135,7 @@ func TestPatchReapEmptyDelta(t *testing.T) { ) s1, err := os.Stat(cacheFilePath) - store := delta.NewStore(dir.Path, "default", maxInventoryDataSize) + store := delta.NewStore(dir.Path, "default", maxInventoryDataSize, true) pr := newPatchReaper("", store) pr.Reap() @@ -173,7 +175,7 @@ func BenchmarkPatchReapNoDiff(b *testing.B) { } defer dir.Clear() - store := delta.NewStore(dir.Path, "default", maxInventoryDataSize) + store := delta.NewStore(dir.Path, "default", maxInventoryDataSize, true) pr := newPatchReaper("", store) b.ResetTimer() for i := 0; i < b.N; i++ { @@ -213,7 +215,7 @@ func BenchmarkPatchReapWithDiff(b *testing.B) { } defer dir.Clear() - store := delta.NewStore(dir.Path, "default", maxInventoryDataSize) + store := delta.NewStore(dir.Path, "default", maxInventoryDataSize, true) pr := newPatchReaper("", store) var mockSource testhelpers.MockFile for i := 0; i < b.N; i++ { diff --git a/internal/agent/patch_sender.go b/internal/agent/patch_sender.go index 6a4ade6eb..9049cef1c 100644 --- a/internal/agent/patch_sender.go +++ b/internal/agent/patch_sender.go @@ -5,6 +5,7 @@ package agent import ( "encoding/json" "fmt" + "github.com/newrelic/infrastructure-agent/internal/agent/inventory" "os" "strings" "time" @@ -39,10 +40,6 @@ type patchSenderIngest struct { currentAgentID entity.ID } -type patchSender interface { - Process() error -} - // Reference to the `time.Now()` function that can be stubbed for unit testing var timeNow = time.Now @@ -51,7 +48,7 @@ var pslog = log.WithComponent("PatchSender") // Reference to post delta function that can be stubbed for unit testing type postDeltas func(entityKeys []string, entityID entity.ID, isAgent bool, deltas ...*inventoryapi.RawDelta) (*inventoryapi.PostDeltaResponse, error) -func newPatchSender(entityInfo entity.Entity, context AgentContext, store delta.Storage, lastSubmission delta.LastSubmissionStore, lastEntityID delta.EntityIDPersist, userAgent string, agentIDProvide id.Provide, httpClient http2.Client) (patchSender, error) { +func newPatchSender(entityInfo entity.Entity, context AgentContext, store delta.Storage, lastSubmission delta.LastSubmissionStore, lastEntityID delta.EntityIDPersist, userAgent string, agentIDProvide id.Provide, httpClient http2.Client) (inventory.PatchSender, error) { if store == nil { return nil, fmt.Errorf("creating patch sender: delta store can't be nil") } @@ -135,8 +132,12 @@ func (p *patchSenderIngest) Process() (err error) { if err := p.store.RemoveEntity(entityKey); err != nil { llog.WithError(err).Warn("Could not remove inventory cache") } - // Relaunching one-time harvesters to avoid losing the inventories after reset - p.context.Reconnect() + + if p.context.EntityKey() == p.entityInfo.Key.String() { + // Relaunching one-time harvesters to avoid losing the inventories after reset + p.context.Reconnect() + } + p.lastDeltaRemoval = now if agentEntityIDChanged { @@ -161,7 +162,7 @@ func (p *patchSenderIngest) Process() (err error) { llog.WithField("numberOfDeltas", len(deltas)).Info("suppressed PostDeltas") } - if p.compactEnabled { + if p.store.IsArchiveEnabled() && p.compactEnabled { if cerr := p.store.CompactStorage(entityKey, p.compactThreshold); cerr != nil { llog.WithError(cerr).WithField("compactThreshold", p.compactThreshold). Error("compaction error") @@ -188,7 +189,9 @@ func (p *patchSenderIngest) sendAllDeltas(allDeltas []inventoryapi.RawDeltaBlock if reset { llog.Debug("Full Plugin Inventory Reset Requested.") p.store.ResetAllDeltas(entityKey) - p.context.Reconnect() // Relaunching one-time harvesters to avoid losing the inventories after reset + if entityKey == p.context.EntityKey() { + p.context.Reconnect() // Relaunching one-time harvesters to avoid losing the inventories after reset + } err := p.store.SaveState() if err != nil { llog.WithError(err).Error("error after resetting deltas while flushing inventory to cache") diff --git a/internal/agent/patch_sender_test.go b/internal/agent/patch_sender_test.go index ef2c7d5e9..e98953fc5 100644 --- a/internal/agent/patch_sender_test.go +++ b/internal/agent/patch_sender_test.go @@ -5,6 +5,7 @@ package agent import ( ctx "context" "fmt" + "github.com/newrelic/infrastructure-agent/internal/agent/inventory" "github.com/stretchr/testify/mock" "io/ioutil" "math" @@ -55,7 +56,7 @@ func ResetPostDelta(_ []string, _ entity.ID, _ bool, _ ...*inventoryapi.RawDelta } func TestNewPatchSender(t *testing.T) { - assert.Implements(t, (*patchSender)(nil), newTestPatchSender(t, "", &delta.Store{}, delta.NewLastSubmissionInMemory(), nil)) + assert.Implements(t, (*inventory.PatchSender)(nil), newTestPatchSender(t, "", &delta.Store{}, delta.NewLastSubmissionInMemory(), nil)) } func cachePluginData(t *testing.T, store *delta.Store, entityKey string) { @@ -83,7 +84,7 @@ func TestPatchSender_Process_LongTermOffline(t *testing.T) { // Given a delta Store dataDir, err := TempDeltaStoreDir() assert.NoError(t, err) - store := delta.NewStore(dataDir, "default", maxInventoryDataSize) + store := delta.NewStore(dataDir, "default", maxInventoryDataSize, true) ls := delta.NewLastSubmissionInMemory() // With some cached plugin data @@ -113,7 +114,7 @@ func TestPatchSender_Process_LongTermOffline_ReconnectPlugins(t *testing.T) { // Given a delta Store dataDir, err := TempDeltaStoreDir() assert.NoError(t, err) - store := delta.NewStore(dataDir, "default", maxInventoryDataSize) + store := delta.NewStore(dataDir, "default", maxInventoryDataSize, true) ls := delta.NewLastSubmissionInMemory() // With some cached plugin data @@ -124,7 +125,7 @@ func TestPatchSender_Process_LongTermOffline_ReconnectPlugins(t *testing.T) { ps.postDeltas = FakePostDelta ps.lastDeltaRemoval = endOf18.Truncate(48 * time.Hour) var agentKey atomic.Value - agentKey.Store("test") + agentKey.Store(entityKey) ps.context = &context{ reconnecting: new(sync.Map), agentKey: agentKey, @@ -152,7 +153,7 @@ func TestPatchSender_Process_LongTermOffline_NoDeltasToPost_UpdateLastDeltaRemov // Given a delta Store dataDir, err := TempDeltaStoreDir() assert.NoError(t, err) - store := delta.NewStore(dataDir, "default", maxInventoryDataSize) + store := delta.NewStore(dataDir, "default", maxInventoryDataSize, true) ls := delta.NewLastSubmissionInMemory() // When it has successfully submitted some deltas require.NoError(t, ls.UpdateTime(time.Now())) @@ -181,7 +182,7 @@ func TestPatchSender_Process_LongTermOffline_AlreadyRemoved(t *testing.T) { // Given a delta Store dataDir, err := TempDeltaStoreDir() assert.NoError(t, err) - store := delta.NewStore(dataDir, "default", maxInventoryDataSize) + store := delta.NewStore(dataDir, "default", maxInventoryDataSize, true) ls := delta.NewLastSubmissionInMemory() // With some cached plugin data @@ -263,7 +264,7 @@ func TestPatchSender_Process_ShortTermOffline(t *testing.T) { // Given a delta Store dataDir, err := TempDeltaStoreDir() assert.NoError(t, err) - store := delta.NewStore(dataDir, "default", maxInventoryDataSize) + store := delta.NewStore(dataDir, "default", maxInventoryDataSize, true) ls := delta.NewLastSubmissionInMemory() // With some cached plugin data @@ -297,7 +298,7 @@ func TestPatchSender_Process_DividedDeltas(t *testing.T) { nowIsEndOf18() - store := delta.NewStore(dataDir, "localhost", maxInventoryDataSize) + store := delta.NewStore(dataDir, "localhost", maxInventoryDataSize, true) ls := delta.NewLastSubmissionInMemory() require.NoError(t, ls.UpdateTime(timeNow())) ps := newTestPatchSender(t, dataDir, store, ls, getLastEntityIDMock()) @@ -333,7 +334,7 @@ func TestPatchSender_Process_DisabledDeltaSplit(t *testing.T) { // Given a patch sender with disabled delta split dataDir, err := TempDeltaStoreDir() assert.NoError(t, err) - store := delta.NewStore(dataDir, "localhost", delta.DisableInventorySplit) + store := delta.NewStore(dataDir, "localhost", delta.DisableInventorySplit, true) ps := newTestPatchSender(t, dataDir, store, delta.NewLastSubmissionInMemory(), getLastEntityIDMock()) pdt := testhelpers.NewPostDeltaTracer(math.MaxInt32) ps.postDeltas = pdt.PostDeltas @@ -362,7 +363,7 @@ func TestPatchSender_Process_SingleRequestDeltas(t *testing.T) { // Given a patch sender dataDir, err := TempDeltaStoreDir() assert.NoError(t, err) - store := delta.NewStore(dataDir, "localhost", maxInventoryDataSize) + store := delta.NewStore(dataDir, "localhost", maxInventoryDataSize, true) ls := delta.NewLastSubmissionInMemory() pdt := testhelpers.NewPostDeltaTracer(maxInventoryDataSize) @@ -399,7 +400,7 @@ func TestPatchSender_Process_CompactEnabled(t *testing.T) { // Given a patch sender with compaction enabled dataDir, err := TempDeltaStoreDir() assert.NoError(t, err) - store := delta.NewStore(dataDir, "localhost", maxInventoryDataSize) + store := delta.NewStore(dataDir, "localhost", maxInventoryDataSize, true) ls := delta.NewLastSubmissionInMemory() resetTime, _ := time.ParseDuration("24h") @@ -433,7 +434,7 @@ func TestPatchSender_Process_Reset(t *testing.T) { // Given a patch sender dataDir, err := TempDeltaStoreDir() assert.NoError(t, err) - store := delta.NewStore(dataDir, "localhost", maxInventoryDataSize) + store := delta.NewStore(dataDir, "localhost", maxInventoryDataSize, true) ls := delta.NewLastSubmissionInMemory() resetTime, _ := time.ParseDuration("24h") @@ -554,6 +555,10 @@ func (m *mockStorage) ReadDeltas(entityKey string) ([]inventoryapi.RawDeltaBlock func (m *mockStorage) UpdateState(entityKey string, deltas []*inventoryapi.RawDelta, deltaStateResults *inventoryapi.DeltaStateMap) { } +func (m *mockStorage) IsArchiveEnabled() bool { + return true +} + type mockEntityIDPersist struct { mock.Mock } diff --git a/internal/agent/patch_sender_vortex.go b/internal/agent/patch_sender_vortex.go index d740ccf0b..8e25be9bb 100644 --- a/internal/agent/patch_sender_vortex.go +++ b/internal/agent/patch_sender_vortex.go @@ -5,6 +5,7 @@ package agent import ( "encoding/json" "fmt" + "github.com/newrelic/infrastructure-agent/internal/agent/inventory" "os" "strings" "time" @@ -48,7 +49,7 @@ var psvlog = wlog.WithComponent("PatchSenderVortex") // Reference to post delta function that can be stubbed for unit testing type postDeltasVortex func(entityID entity.ID, entityKeys []string, isAgent bool, deltas ...*inventoryapi.RawDelta) (*inventoryapi.PostDeltaResponse, error) -func newPatchSenderVortex(entityKey, agentKey string, context AgentContext, store *delta.Store, userAgent string, agentIDProvide id.Provide, provideIDs ProvideIDs, entityMap entity.KnownIDs, httpClient http2.Client) (patchSender, error) { +func newPatchSenderVortex(entityKey, agentKey string, context AgentContext, store *delta.Store, userAgent string, agentIDProvide id.Provide, provideIDs ProvideIDs, entityMap entity.KnownIDs, httpClient http2.Client) (inventory.PatchSender, error) { if store == nil { psvlog.WithField("entityKey", entityKey).Error("creating patch sender: delta store can't be nil") panic("creating patch sender: delta store can't be nil") diff --git a/internal/agent/patch_sender_vortex_test.go b/internal/agent/patch_sender_vortex_test.go index 6a8697be8..da33d50c7 100644 --- a/internal/agent/patch_sender_vortex_test.go +++ b/internal/agent/patch_sender_vortex_test.go @@ -4,6 +4,7 @@ package agent import ( "fmt" + "github.com/newrelic/infrastructure-agent/internal/agent/inventory" "math" http2 "net/http" "os" @@ -48,7 +49,7 @@ func TestPatchSenderVortex_Process_LongTermOffline(t *testing.T) { // Given a delta Store dataDir, err := TempDeltaStoreDir() assert.NoError(t, err) - store := delta.NewStore(dataDir, "default", maxInventoryDataSize) + store := delta.NewStore(dataDir, "default", maxInventoryDataSize, true) // With some cached plugin data cachePluginData(t, store, "entityKey") @@ -73,7 +74,7 @@ func TestPatchSenderVortex_Process_LongTermOffline_ReconnectPlugins(t *testing.T // Given a delta Store dataDir, err := TempDeltaStoreDir() assert.NoError(t, err) - store := delta.NewStore(dataDir, "default", maxInventoryDataSize) + store := delta.NewStore(dataDir, "default", maxInventoryDataSize, true) // With some cached plugin data cachePluginData(t, store, "entityKey") @@ -117,7 +118,7 @@ func TestPatchSenderVortex_Process_LongTermOffline_NoDeltasToPost_UpdateLastConn // Given a delta Store dataDir, err := TempDeltaStoreDir() assert.NoError(t, err) - store := delta.NewStore(dataDir, "default", maxInventoryDataSize) + store := delta.NewStore(dataDir, "default", maxInventoryDataSize, true) // And a patch sender that has been disconnected for less than 24 hours resetTime, _ := time.ParseDuration("24h") @@ -146,7 +147,7 @@ func TestPatchSenderVortex_Process_LongTermOffline_AlreadyRemoved(t *testing.T) // Given a delta Store dataDir, err := TempDeltaStoreDir() assert.NoError(t, err) - store := delta.NewStore(dataDir, "default", maxInventoryDataSize) + store := delta.NewStore(dataDir, "default", maxInventoryDataSize, true) // With some cached plugin data cachePluginData(t, store, "entityKey") @@ -179,7 +180,7 @@ func TestPatchSenderVortex_Process_ShortTermOffline(t *testing.T) { // Given a delta Store dataDir, err := TempDeltaStoreDir() assert.NoError(t, err) - store := delta.NewStore(dataDir, "default", maxInventoryDataSize) + store := delta.NewStore(dataDir, "default", maxInventoryDataSize, true) // With some cached plugin data cachePluginData(t, store, "entityKey") @@ -214,7 +215,7 @@ func TestPatchSenderVortex_Process_ShortTermOffline(t *testing.T) { func TestPatchSenderVortex_Process(t *testing.T) { dataDir, err := TempDeltaStoreDir() assert.NoError(t, err) - store := delta.NewStore(dataDir, "localhost", maxInventoryDataSize) + store := delta.NewStore(dataDir, "localhost", maxInventoryDataSize, true) // set of deltas from different plugins, whose total size is smaller than the max inventory data size testhelpers.PopulateDeltas(dataDir, "entityKey", []testhelpers.FakeDeltaEntry{ {Source: "plugin1/plugin1", DeltasSize: maxInventoryDataSize / 10, BodySize: 100}, @@ -242,7 +243,7 @@ func TestPatchSenderVortex_Process_WaitsForAgentID(t *testing.T) { agentIdentity := entity.Identity{ID: 123} dataDir, err := TempDeltaStoreDir() assert.NoError(t, err) - store := delta.NewStore(dataDir, "localhost", maxInventoryDataSize) + store := delta.NewStore(dataDir, "localhost", maxInventoryDataSize, true) testhelpers.PopulateDeltas(dataDir, "entityKey", []testhelpers.FakeDeltaEntry{ {Source: "test/dummy", DeltasSize: maxInventoryDataSize, BodySize: 100}, }) @@ -281,7 +282,7 @@ func TestPatchSenderVortex_Process_DividedDeltas(t *testing.T) { // Given a patch sender dataDir, err := TempDeltaStoreDir() assert.NoError(t, err) - store := delta.NewStore(dataDir, "localhost", maxInventoryDataSize) + store := delta.NewStore(dataDir, "localhost", maxInventoryDataSize, true) pdt := testhelpers.NewPostDeltaTracer(maxInventoryDataSize) ctx := newContextWithVortex() registerClient := &identityapi.RegisterClientMock{} @@ -315,7 +316,7 @@ func TestPatchSenderVortex_Process_DividedDeltas(t *testing.T) { func TestPatchSenderVortex_Process_DisabledDeltaSplit(t *testing.T) { dataDir, err := TempDeltaStoreDir() assert.NoError(t, err) - store := delta.NewStore(dataDir, "localhost", delta.DisableInventorySplit) + store := delta.NewStore(dataDir, "localhost", delta.DisableInventorySplit, true) // Given a patch sender with disabled delta split pdt := testhelpers.NewPostDeltaTracer(math.MaxInt32) @@ -349,7 +350,7 @@ func TestPatchSenderVortex_Process_DisabledDeltaSplit(t *testing.T) { func TestPatchSenderVortex_Process_SingleRequestDeltas(t *testing.T) { dataDir, err := TempDeltaStoreDir() assert.NoError(t, err) - store := delta.NewStore(dataDir, "localhost", maxInventoryDataSize) + store := delta.NewStore(dataDir, "localhost", maxInventoryDataSize, true) // Given a patch sender pdt := testhelpers.NewPostDeltaTracer(maxInventoryDataSize) @@ -385,7 +386,7 @@ func TestPatchSenderVortex_Process_CompactEnabled(t *testing.T) { // Given a patch sender with compaction enabled dataDir, err := TempDeltaStoreDir() assert.NoError(t, err) - store := delta.NewStore(dataDir, "localhost", maxInventoryDataSize) + store := delta.NewStore(dataDir, "localhost", maxInventoryDataSize, true) ctx := newContextWithVortex() registerClient := &identityapi.RegisterClientMock{} @@ -418,7 +419,7 @@ func TestPatchSenderVortex_Process_Reset(t *testing.T) { // Given a patch sender dataDir, err := TempDeltaStoreDir() assert.NoError(t, err) - store := delta.NewStore(dataDir, "localhost", maxInventoryDataSize) + store := delta.NewStore(dataDir, "localhost", maxInventoryDataSize, true) resetTime, _ := time.ParseDuration("24h") lastConnection := time.Date(2018, 12, 12, 0, 12, 12, 12, &time.Location{}) @@ -457,7 +458,7 @@ func TestPatchSenderVortex_Process_Reset(t *testing.T) { assert.True(t, storageSize < 10, "%v not smaller than 10", storageSize) } -func newSender(t *testing.T, ctx *context, store *delta.Store, client http.Client, registerClient identityapi.RegisterClient) patchSender { // nolint:ireturn +func newSender(t *testing.T, ctx *context, store *delta.Store, client http.Client, registerClient identityapi.RegisterClient) inventory.PatchSender { // nolint:ireturn t.Helper() pSender, err := newPatchSenderVortex("entityKey", agentKey, ctx, store, "user-agent", ctx.Identity, NewProvideIDs(registerClient, state.NewRegisterSM()), entity.NewKnownIDs(), client) require.NoError(t, err) diff --git a/internal/agent/plugin.go b/internal/agent/plugin.go index 9282344a3..f75e51547 100644 --- a/internal/agent/plugin.go +++ b/internal/agent/plugin.go @@ -6,6 +6,7 @@ import ( goContext "context" "fmt" "github.com/newrelic/infrastructure-agent/internal/agent/instrumentation" + "github.com/newrelic/infrastructure-agent/internal/agent/types" "sync" "time" @@ -59,36 +60,6 @@ func NewExternalPluginCommon(id ids.PluginID, ctx AgentContext, name string) Plu } } -// Anything implementing the sortable interface must implement a -// method to return a string Sort key -type Sortable interface { - SortKey() string -} - -type PluginInventoryDataset []Sortable // PluginInventoryDataset is a slice of sortable things - -// PluginInventoryDataset also implements the sort.Sort interface -func (pd PluginInventoryDataset) Len() int { return len(pd) } -func (pd PluginInventoryDataset) Swap(i, j int) { pd[i], pd[j] = pd[j], pd[i] } -func (pd PluginInventoryDataset) Less(i, j int) bool { return pd[i].SortKey() < pd[j].SortKey() } - -// PluginOutput contains metadata about the inventory provided by Plugins, which will be used for its later addition -// to the delta store -type PluginOutput struct { - Id ids.PluginID - Entity entity.Entity - Data PluginInventoryDataset - NotApplicable bool -} - -func NewPluginOutput(id ids.PluginID, entity entity.Entity, data PluginInventoryDataset) PluginOutput { - return PluginOutput{Id: id, Entity: entity, Data: data} -} - -func NewNotApplicableOutput(id ids.PluginID) PluginOutput { - return PluginOutput{Id: id, NotApplicable: true} -} - // Id is the accessor for the id field func (pc *PluginCommon) Id() ids.PluginID { return pc.ID @@ -115,16 +86,16 @@ func (pc *PluginCommon) GetExternalPluginName() string { } type PluginEmitter interface { - EmitInventory(data PluginInventoryDataset, entity entity.Entity) + EmitInventory(data types.PluginInventoryDataset, entity entity.Entity) EmitEvent(eventData map[string]interface{}, entityKey entity.Key) } // EmitInventory sends data collected by the plugin to the agent -func (pc *PluginCommon) EmitInventory(data PluginInventoryDataset, entity entity.Entity) { +func (pc *PluginCommon) EmitInventory(data types.PluginInventoryDataset, entity entity.Entity) { _, txn := instrumentation.SelfInstrumentation.StartTransaction(goContext.Background(), "plugin.emit_inventory") txn.AddAttribute("plugin_id", fmt.Sprintf("%s:%s", pc.ID.Category, pc.ID.Term)) defer txn.End() - pc.Context.SendData(NewPluginOutput(pc.ID, entity, data)) + pc.Context.SendData(types.NewPluginOutput(pc.ID, entity, data)) } func (pc *PluginCommon) EmitEvent(eventData map[string]interface{}, entityKey entity.Key) { diff --git a/internal/agent/plugin_test.go b/internal/agent/plugin_test.go index dc9462e95..dacaa0d04 100644 --- a/internal/agent/plugin_test.go +++ b/internal/agent/plugin_test.go @@ -3,6 +3,7 @@ package agent import ( + "github.com/newrelic/infrastructure-agent/internal/agent/types" "testing" "github.com/newrelic/infrastructure-agent/pkg/config" @@ -14,11 +15,11 @@ import ( ) func TestPluginOutput(t *testing.T) { - pluginOutput := NewPluginOutput(ids.PluginID{}, entity.NewFromNameWithoutID(""), nil) + pluginOutput := types.NewPluginOutput(ids.PluginID{}, entity.NewFromNameWithoutID(""), nil) assert.False(t, pluginOutput.NotApplicable) assert.NotNil(t, pluginOutput) - pluginOutput = NewNotApplicableOutput(ids.PluginID{"a", "b"}) + pluginOutput = types.NewNotApplicableOutput(ids.PluginID{"a", "b"}) assert.Equal(t, ids.PluginID{"a", "b"}, pluginOutput.Id) assert.True(t, pluginOutput.NotApplicable) } @@ -49,14 +50,14 @@ func TestPluginIDSortable(t *testing.T) { func newFakeContext(resolver hostname.Resolver) *fakeContext { return &fakeContext{ resolver: resolver, - data: make(chan PluginOutput), + data: make(chan types.PluginOutput), ev: make(chan sample.Event), } } type fakeContext struct { resolver hostname.Resolver - data chan PluginOutput + data chan types.PluginOutput ev chan sample.Event } @@ -82,7 +83,7 @@ func (c *fakeContext) GetServiceForPid(pid int) (service string, ok bool) { return "", false } -func (c *fakeContext) SendData(data PluginOutput) { +func (c *fakeContext) SendData(data types.PluginOutput) { c.data <- data } diff --git a/internal/agent/types/types.go b/internal/agent/types/types.go new file mode 100644 index 000000000..09f6e325c --- /dev/null +++ b/internal/agent/types/types.go @@ -0,0 +1,39 @@ +// Copyright 2020 New Relic Corporation. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +package types + +import ( + "github.com/newrelic/infrastructure-agent/pkg/entity" + "github.com/newrelic/infrastructure-agent/pkg/plugins/ids" +) + +// Anything implementing the sortable interface must implement a +// method to return a string Sort key +type Sortable interface { + SortKey() string +} + +type PluginInventoryDataset []Sortable // PluginInventoryDataset is a slice of sortable things + +// PluginInventoryDataset also implements the sort.Sort interface +func (pd PluginInventoryDataset) Len() int { return len(pd) } +func (pd PluginInventoryDataset) Swap(i, j int) { pd[i], pd[j] = pd[j], pd[i] } +func (pd PluginInventoryDataset) Less(i, j int) bool { return pd[i].SortKey() < pd[j].SortKey() } + +// PluginOutput contains metadata about the inventory provided by Plugins, which will be used for its later addition +// to the delta store +type PluginOutput struct { + Id ids.PluginID + Entity entity.Entity + Data PluginInventoryDataset + NotApplicable bool +} + +func NewPluginOutput(id ids.PluginID, entity entity.Entity, data PluginInventoryDataset) PluginOutput { + return PluginOutput{Id: id, Entity: entity, Data: data} +} + +func NewNotApplicableOutput(id ids.PluginID) PluginOutput { + return PluginOutput{Id: id, NotApplicable: true} +} diff --git a/internal/plugins/darwin/hostinfo.go b/internal/plugins/darwin/hostinfo.go index 747257ffe..b7ca1a3f6 100644 --- a/internal/plugins/darwin/hostinfo.go +++ b/internal/plugins/darwin/hostinfo.go @@ -8,6 +8,7 @@ package darwin import ( "errors" "fmt" + "github.com/newrelic/infrastructure-agent/internal/agent/types" "regexp" "strconv" "strings" @@ -72,8 +73,8 @@ func (hip *HostinfoPlugin) Run() { } // Data collects the hostinfo. -func (hip *HostinfoPlugin) Data() agent.PluginInventoryDataset { - return agent.PluginInventoryDataset{hip.gatherHostinfo(hip.Context)} +func (hip *HostinfoPlugin) Data() types.PluginInventoryDataset { + return types.PluginInventoryDataset{hip.gatherHostinfo(hip.Context)} } func (hip *HostinfoPlugin) gatherHostinfo(context agent.AgentContext) *HostInfoDarwin { diff --git a/internal/plugins/darwin/processorinfo_test_amd64.go b/internal/plugins/darwin/processorinfo_test_amd64.go index 6f9d2775d..c5271138d 100644 --- a/internal/plugins/darwin/processorinfo_test_amd64.go +++ b/internal/plugins/darwin/processorinfo_test_amd64.go @@ -7,9 +7,9 @@ package darwin import ( "errors" + "github.com/newrelic/infrastructure-agent/internal/agent" "testing" - "github.com/newrelic/infrastructure-agent/internal/agent" "github.com/newrelic/infrastructure-agent/internal/plugins/common" testing2 "github.com/newrelic/infrastructure-agent/internal/plugins/testing" "github.com/newrelic/infrastructure-agent/pkg/plugins/ids" diff --git a/internal/plugins/darwin/processorinfo_test_arm64.go b/internal/plugins/darwin/processorinfo_test_arm64.go index b315b3d6b..9be054fdc 100644 --- a/internal/plugins/darwin/processorinfo_test_arm64.go +++ b/internal/plugins/darwin/processorinfo_test_arm64.go @@ -7,9 +7,9 @@ package darwin import ( "errors" + "github.com/newrelic/infrastructure-agent/internal/agent" "testing" - "github.com/newrelic/infrastructure-agent/internal/agent" "github.com/newrelic/infrastructure-agent/internal/plugins/common" testing2 "github.com/newrelic/infrastructure-agent/internal/plugins/testing" "github.com/newrelic/infrastructure-agent/pkg/plugins/ids" diff --git a/internal/plugins/linux/cloud_security_groups.go b/internal/plugins/linux/cloud_security_groups.go index be413944f..2ac8e4be7 100644 --- a/internal/plugins/linux/cloud_security_groups.go +++ b/internal/plugins/linux/cloud_security_groups.go @@ -7,6 +7,7 @@ package linux import ( "errors" + "github.com/newrelic/infrastructure-agent/internal/agent/types" "github.com/newrelic/infrastructure-agent/pkg/entity" "strings" "time" @@ -52,7 +53,7 @@ func NewCloudSecurityGroupsPlugin(id ids.PluginID, ctx agent.AgentContext, harve } } -func (p *CloudSecurityGroupsPlugin) getCloudSecurityGroupsDataset() (dataset agent.PluginInventoryDataset, err error) { +func (p *CloudSecurityGroupsPlugin) getCloudSecurityGroupsDataset() (dataset types.PluginInventoryDataset, err error) { var h cloud.Harvester h, err = p.harvester.GetHarvester() if err != nil { @@ -96,7 +97,7 @@ func (p *CloudSecurityGroupsPlugin) Run() { refreshTimer.Stop() refreshTimer = time.NewTicker(p.frequency) { - var dataset agent.PluginInventoryDataset + var dataset types.PluginInventoryDataset var err error if dataset, err = p.getCloudSecurityGroupsDataset(); err != nil { // Silence errors here, they are only advisory and the function returns an diff --git a/internal/plugins/linux/daemontools.go b/internal/plugins/linux/daemontools.go index 1db2b7ff7..684d4457f 100644 --- a/internal/plugins/linux/daemontools.go +++ b/internal/plugins/linux/daemontools.go @@ -9,6 +9,7 @@ import ( "bufio" "bytes" "fmt" + "github.com/newrelic/infrastructure-agent/internal/agent/types" "github.com/newrelic/infrastructure-agent/pkg/entity" "os" "os/exec" @@ -63,7 +64,7 @@ func daemonToolsPresent() bool { return err == nil } -func getDaemontoolsServiceStatus() (data agent.PluginInventoryDataset, pidMap map[int]string, err error) { +func getDaemontoolsServiceStatus() (data types.PluginInventoryDataset, pidMap map[int]string, err error) { pidMap = make(map[int]string) var ( psOut []byte diff --git a/internal/plugins/linux/dpkg.go b/internal/plugins/linux/dpkg.go index 04cc1385d..03d0c2c80 100644 --- a/internal/plugins/linux/dpkg.go +++ b/internal/plugins/linux/dpkg.go @@ -11,6 +11,7 @@ import ( "bufio" "fmt" "github.com/fsnotify/fsnotify" + "github.com/newrelic/infrastructure-agent/internal/agent/types" "github.com/newrelic/infrastructure-agent/pkg/entity" "github.com/newrelic/infrastructure-agent/pkg/log" "github.com/newrelic/infrastructure-agent/pkg/plugins/ids" @@ -94,7 +95,7 @@ func (self *DpkgPlugin) guessInstallTime(packageName string, arch string) string return fmt.Sprintf("%d", ctime.Unix()) } -func (self *DpkgPlugin) fetchPackageInfo() (packages agent.PluginInventoryDataset, err error) { +func (self *DpkgPlugin) fetchPackageInfo() (packages types.PluginInventoryDataset, err error) { output, err := helpers.RunCommand("/usr/bin/dpkg-query", "", "-W", "-f=${Package} ${Status} ${Architecture} ${Version} ${Essential} ${Priority}\n") if err != nil { return nil, err diff --git a/internal/plugins/linux/facter.go b/internal/plugins/linux/facter.go index c22625ffc..8dec235aa 100644 --- a/internal/plugins/linux/facter.go +++ b/internal/plugins/linux/facter.go @@ -8,6 +8,7 @@ package linux import ( "encoding/json" "fmt" + "github.com/newrelic/infrastructure-agent/internal/agent/types" "github.com/newrelic/infrastructure-agent/pkg/entity" "os/exec" "strings" @@ -71,12 +72,12 @@ func (self *FacterPlugin) CanRun() bool { return err == nil } -func (self *FacterPlugin) Data() (agent.PluginInventoryDataset, error) { +func (self *FacterPlugin) Data() (types.PluginInventoryDataset, error) { services, err := self.facter.Facts() if err != nil { return nil, err } - a := agent.PluginInventoryDataset{} + a := types.PluginInventoryDataset{} for _, svc := range services { a = append(a, svc) } diff --git a/internal/plugins/linux/hostinfo.go b/internal/plugins/linux/hostinfo.go index a41dd945c..ac92169ec 100644 --- a/internal/plugins/linux/hostinfo.go +++ b/internal/plugins/linux/hostinfo.go @@ -9,6 +9,7 @@ import ( "bufio" "errors" "fmt" + "github.com/newrelic/infrastructure-agent/internal/agent/types" "github.com/newrelic/infrastructure-agent/internal/plugins/common" "io/ioutil" "os" @@ -147,8 +148,8 @@ func NewHostinfoPlugin(ctx agent.AgentContext, hostInfo common.HostInfo) agent.P } } -func (self *HostinfoPlugin) Data() agent.PluginInventoryDataset { - return agent.PluginInventoryDataset{self.gatherHostinfo(self.Context)} +func (self *HostinfoPlugin) Data() types.PluginInventoryDataset { + return types.PluginInventoryDataset{self.gatherHostinfo(self.Context)} } func (self *HostinfoPlugin) Run() { diff --git a/internal/plugins/linux/kernel_modules.go b/internal/plugins/linux/kernel_modules.go index 81e52b6ee..79a8df9fa 100644 --- a/internal/plugins/linux/kernel_modules.go +++ b/internal/plugins/linux/kernel_modules.go @@ -8,6 +8,7 @@ package linux import ( "bufio" "fmt" + "github.com/newrelic/infrastructure-agent/internal/agent/types" "github.com/newrelic/infrastructure-agent/pkg/entity" "os" "regexp" @@ -57,8 +58,8 @@ func NewKernelModulesPlugin(id ids.PluginID, ctx agent.AgentContext) agent.Plugi } } -func (self KernelModulesPlugin) getKernelModulesDataset() agent.PluginInventoryDataset { - var dataset agent.PluginInventoryDataset +func (self KernelModulesPlugin) getKernelModulesDataset() types.PluginInventoryDataset { + var dataset types.PluginInventoryDataset for _, v := range self.loadedModules { dataset = append(dataset, v) diff --git a/internal/plugins/linux/rpm.go b/internal/plugins/linux/rpm.go index 1f897946b..11f8a04d7 100644 --- a/internal/plugins/linux/rpm.go +++ b/internal/plugins/linux/rpm.go @@ -11,6 +11,7 @@ import ( "bufio" "fmt" "github.com/fsnotify/fsnotify" + "github.com/newrelic/infrastructure-agent/internal/agent/types" "github.com/newrelic/infrastructure-agent/pkg/entity" "github.com/newrelic/infrastructure-agent/pkg/log" "github.com/newrelic/infrastructure-agent/pkg/plugins/ids" @@ -66,7 +67,7 @@ func NewRpmPlugin(ctx agent.AgentContext) agent.Plugin { } } -func (p *rpmPlugin) fetchPackageInfo() (packages agent.PluginInventoryDataset, err error) { +func (p *rpmPlugin) fetchPackageInfo() (packages types.PluginInventoryDataset, err error) { output, err := helpers.RunCommand(RpmPath, "", "-qa", "--queryformat=%{NAME} %{VERSION} %{RELEASE} %{ARCH} %{INSTALLTIME} %{EPOCH}\n") if err != nil { return nil, err @@ -74,7 +75,7 @@ func (p *rpmPlugin) fetchPackageInfo() (packages agent.PluginInventoryDataset, e return p.parsePackageInfo(output) } -func (p *rpmPlugin) parsePackageInfo(output string) (packages agent.PluginInventoryDataset, err error) { +func (p *rpmPlugin) parsePackageInfo(output string) (packages types.PluginInventoryDataset, err error) { // Get output and sort it alphabetically to ensure consistent ordering var outputLines []string scanner := bufio.NewScanner(strings.NewReader(output)) diff --git a/internal/plugins/linux/selinux.go b/internal/plugins/linux/selinux.go index d6bf4777f..e9844e8ff 100644 --- a/internal/plugins/linux/selinux.go +++ b/internal/plugins/linux/selinux.go @@ -9,6 +9,7 @@ import ( "bufio" "errors" "fmt" + "github.com/newrelic/infrastructure-agent/internal/agent/types" "github.com/newrelic/infrastructure-agent/pkg/entity" "regexp" "strings" @@ -75,7 +76,7 @@ func NewSELinuxPlugin(id ids.PluginID, ctx agent.AgentContext) agent.Plugin { // basicData: Overall SELinux status - whether it's running, what mode it's in, etc. // policyData: Individual SELinux policy flags - a high-level overview of SELinux configuration // policyModules: Listing of policy modules in use and which version of modules are active -func (self *SELinuxPlugin) getDataset() (basicData agent.PluginInventoryDataset, policyData agent.PluginInventoryDataset, policyModules agent.PluginInventoryDataset, err error) { +func (self *SELinuxPlugin) getDataset() (basicData types.PluginInventoryDataset, policyData types.PluginInventoryDataset, policyModules types.PluginInventoryDataset, err error) { // Get basic selinux status data using sestatus. If selinux isn't enabled or installed, this will fail. output, err := helpers.RunCommand("sestatus", "", "-b") if err != nil { @@ -97,7 +98,7 @@ func (self *SELinuxPlugin) getDataset() (basicData agent.PluginInventoryDataset, return } -func (self *SELinuxPlugin) parseSestatusOutput(output string) (basicResult agent.PluginInventoryDataset, policyResult agent.PluginInventoryDataset, err error) { +func (self *SELinuxPlugin) parseSestatusOutput(output string) (basicResult types.PluginInventoryDataset, policyResult types.PluginInventoryDataset, err error) { labelRegex, err := regexp.Compile(`([^:]*):\s+(.*)`) if err != nil { return @@ -155,7 +156,7 @@ func (self *SELinuxPlugin) sELinuxActive() bool { return err == nil } -func (self *SELinuxPlugin) parseSemoduleOutput(output string) (result agent.PluginInventoryDataset, err error) { +func (self *SELinuxPlugin) parseSemoduleOutput(output string) (result types.PluginInventoryDataset, err error) { moduleRegex, err := regexp.Compile(`(\S+)\s+(\S+)`) if err != nil { return @@ -196,10 +197,10 @@ func (self *SELinuxPlugin) Run() { } entity := entity.NewFromNameWithoutID(self.Context.EntityKey()) - self.Context.SendData(agent.NewPluginOutput(self.Id(), entity, basicData)) - self.Context.SendData(agent.NewPluginOutput(ids.PluginID{self.ID.Category, fmt.Sprintf("%s-policies", self.ID.Term)}, entity, policyData)) + self.Context.SendData(types.NewPluginOutput(self.Id(), entity, basicData)) + self.Context.SendData(types.NewPluginOutput(ids.PluginID{self.ID.Category, fmt.Sprintf("%s-policies", self.ID.Term)}, entity, policyData)) if self.enableSemodule { - self.Context.SendData(agent.NewPluginOutput(ids.PluginID{self.ID.Category, fmt.Sprintf("%s-modules", self.ID.Term)}, entity, policyModules)) + self.Context.SendData(types.NewPluginOutput(ids.PluginID{self.ID.Category, fmt.Sprintf("%s-modules", self.ID.Term)}, entity, policyModules)) } <-refreshTimer.C diff --git a/internal/plugins/linux/sshd_config.go b/internal/plugins/linux/sshd_config.go index 1747b0570..8f1301ded 100644 --- a/internal/plugins/linux/sshd_config.go +++ b/internal/plugins/linux/sshd_config.go @@ -7,6 +7,7 @@ package linux import ( "fmt" + "github.com/newrelic/infrastructure-agent/internal/agent/types" "github.com/newrelic/infrastructure-agent/pkg/entity" "io/ioutil" "regexp" @@ -94,7 +95,7 @@ func parseSshdConfig(configText string) (values map[string]string, err error) { return } -func convertSshValuesToPluginData(configValues map[string]string) (dataset agent.PluginInventoryDataset) { +func convertSshValuesToPluginData(configValues map[string]string) (dataset types.PluginInventoryDataset) { for key, value := range configValues { dataset = append(dataset, SshdConfigValue{key, value}) } diff --git a/internal/plugins/linux/supervisor.go b/internal/plugins/linux/supervisor.go index 1f4d44959..02692374f 100644 --- a/internal/plugins/linux/supervisor.go +++ b/internal/plugins/linux/supervisor.go @@ -7,6 +7,7 @@ package linux import ( "fmt" + "github.com/newrelic/infrastructure-agent/internal/agent/types" "github.com/newrelic/infrastructure-agent/pkg/entity" "net/http" "strconv" @@ -96,7 +97,7 @@ func (self *SupervisorPlugin) CanRun() bool { return err == nil } -func (self *SupervisorPlugin) Data() (agent.PluginInventoryDataset, map[int]string, error) { +func (self *SupervisorPlugin) Data() (types.PluginInventoryDataset, map[int]string, error) { client, err := self.GetClient() if err != nil { return nil, nil, err @@ -105,7 +106,7 @@ func (self *SupervisorPlugin) Data() (agent.PluginInventoryDataset, map[int]stri if err != nil { return nil, nil, err } - a := agent.PluginInventoryDataset{} + a := types.PluginInventoryDataset{} pidMap := make(map[int]string) for _, proc := range procs { if proc.State != 10 && proc.State != 20 { diff --git a/internal/plugins/linux/sysctl_polling.go b/internal/plugins/linux/sysctl_polling.go index 78243a549..a49ddffcf 100644 --- a/internal/plugins/linux/sysctl_polling.go +++ b/internal/plugins/linux/sysctl_polling.go @@ -7,6 +7,7 @@ package linux import ( "fmt" + "github.com/newrelic/infrastructure-agent/internal/agent/types" "github.com/newrelic/infrastructure-agent/pkg/entity" "io/ioutil" "os" @@ -24,7 +25,7 @@ import ( type SysctlPlugin struct { agent.PluginCommon - sysctls agent.PluginInventoryDataset + sysctls types.PluginInventoryDataset errorsLogged map[string]bool frequency time.Duration procSysDir string @@ -114,9 +115,9 @@ func (sp *SysctlPlugin) newSysctlItem(filePath string, output []byte) SysctlItem return SysctlItem{keyPath, strings.TrimSpace(string(output))} } -func (sp *SysctlPlugin) Sysctls() (dataset agent.PluginInventoryDataset, err error) { +func (sp *SysctlPlugin) Sysctls() (dataset types.PluginInventoryDataset, err error) { // Clear out the list, since we're going to be repopulating it completely anyway and we want to drop any entries we don't find anymore. - sp.sysctls = make([]agent.Sortable, 0) + sp.sysctls = make([]types.Sortable, 0) if err := sp.fileService.walk(sp.procSysDir, sp.walkSysctl); err != nil { return nil, err diff --git a/internal/plugins/linux/sysctl_subscriber.go b/internal/plugins/linux/sysctl_subscriber.go index 7af44eeb0..a0747591b 100644 --- a/internal/plugins/linux/sysctl_subscriber.go +++ b/internal/plugins/linux/sysctl_subscriber.go @@ -6,6 +6,7 @@ package linux import ( + "github.com/newrelic/infrastructure-agent/internal/agent/types" "github.com/newrelic/infrastructure-agent/pkg/entity" "io/ioutil" "time" @@ -53,7 +54,7 @@ func (p *SysctlSubscriberPlugin) Run() { var initialSubmitOk bool var needsFlush bool - var deltas agent.PluginInventoryDataset + var deltas types.PluginInventoryDataset for { select { case <-ticker.C: @@ -68,7 +69,7 @@ func (p *SysctlSubscriberPlugin) Run() { } else if needsFlush { p.EmitInventory(deltas, entity.NewFromNameWithoutID(p.Context.EntityKey())) needsFlush = false - deltas = agent.PluginInventoryDataset{} + deltas = types.PluginInventoryDataset{} } ticker.Stop() ticker = time.NewTicker(p.frequency) diff --git a/internal/plugins/linux/systemd.go b/internal/plugins/linux/systemd.go index 7cf7cd957..0054c2a88 100644 --- a/internal/plugins/linux/systemd.go +++ b/internal/plugins/linux/systemd.go @@ -7,6 +7,7 @@ package linux import ( "bufio" + "github.com/newrelic/infrastructure-agent/internal/agent/types" "github.com/newrelic/infrastructure-agent/pkg/entity" "os/exec" "regexp" @@ -44,8 +45,8 @@ func (self SystemdService) SortKey() string { return self.Name } -func (self SystemdPlugin) getSystemdDataset() agent.PluginInventoryDataset { - var dataset agent.PluginInventoryDataset +func (self SystemdPlugin) getSystemdDataset() types.PluginInventoryDataset { + var dataset types.PluginInventoryDataset for _, v := range self.runningServices { dataset = append(dataset, v) diff --git a/internal/plugins/linux/sysvinit.go b/internal/plugins/linux/sysvinit.go index 625f7c0b3..650276a45 100644 --- a/internal/plugins/linux/sysvinit.go +++ b/internal/plugins/linux/sysvinit.go @@ -9,6 +9,7 @@ import ( "bufio" "bytes" "fmt" + "github.com/newrelic/infrastructure-agent/internal/agent/types" "github.com/newrelic/infrastructure-agent/pkg/entity" "io/ioutil" "os" @@ -76,7 +77,7 @@ func (self *SysvInitPlugin) Run() { time.Sleep(self.frequency) } - dataset := agent.PluginInventoryDataset{} + dataset := types.PluginInventoryDataset{} pidMap := make(map[int]string) a, err := self.services(SYSV_INIT_DIR) diff --git a/internal/plugins/linux/upstart.go b/internal/plugins/linux/upstart.go index f9464fbc1..319d890b7 100644 --- a/internal/plugins/linux/upstart.go +++ b/internal/plugins/linux/upstart.go @@ -7,6 +7,7 @@ package linux import ( "bufio" + "github.com/newrelic/infrastructure-agent/internal/agent/types" "github.com/newrelic/infrastructure-agent/pkg/entity" "os/exec" "regexp" @@ -42,8 +43,8 @@ func (us UpstartService) SortKey() string { return us.Name } -func (up UpstartPlugin) getUpstartDataset() agent.PluginInventoryDataset { - var dataset agent.PluginInventoryDataset +func (up UpstartPlugin) getUpstartDataset() types.PluginInventoryDataset { + var dataset types.PluginInventoryDataset for _, v := range up.runningServices { dataset = append(dataset, v) diff --git a/internal/plugins/linux/users.go b/internal/plugins/linux/users.go index 0b29e77c0..ae709d09c 100644 --- a/internal/plugins/linux/users.go +++ b/internal/plugins/linux/users.go @@ -8,6 +8,7 @@ package linux import ( "bufio" "github.com/fsnotify/fsnotify" + "github.com/newrelic/infrastructure-agent/internal/agent/types" "github.com/newrelic/infrastructure-agent/pkg/entity" "github.com/newrelic/infrastructure-agent/pkg/log" "strings" @@ -52,7 +53,7 @@ func NewUsersPlugin(ctx agent.AgentContext) agent.Plugin { // getUserDetails runs the who command, parses it's output and returns // a dataset of users. -func (self UsersPlugin) getUserDetails() (dataset agent.PluginInventoryDataset) { +func (self UsersPlugin) getUserDetails() (dataset types.PluginInventoryDataset) { output, err := helpers.RunCommand("/usr/bin/env", "", "who") if err != nil { usrlog.WithError(err).Error("failed to fetch user information") diff --git a/internal/plugins/testing/agent_mock.go b/internal/plugins/testing/agent_mock.go index 88e004b9e..04b3f5b0c 100644 --- a/internal/plugins/testing/agent_mock.go +++ b/internal/plugins/testing/agent_mock.go @@ -4,6 +4,7 @@ package testing import ( "context" + "github.com/newrelic/infrastructure-agent/internal/agent/types" "time" "github.com/newrelic/infrastructure-agent/pkg/entity/host" @@ -21,7 +22,7 @@ import ( ) type MockAgent struct { - ch chan agent.PluginOutput + ch chan types.PluginOutput registered bool cfg *config.Config entities chan string @@ -42,7 +43,7 @@ func (m *MockAgent) Identity() entity.Identity { func NewMockAgent() *MockAgent { return &MockAgent{ registered: true, - ch: make(chan agent.PluginOutput, 1), + ch: make(chan types.PluginOutput, 1), cfg: &config.Config{ SupervisorRefreshSec: 1, SupervisorRpcSocket: "/tmp/supervisor.sock.test", @@ -65,7 +66,7 @@ func (self *MockAgent) WithConfig(cfg *config.Config) *MockAgent { return self } -func (self *MockAgent) GetData(c *C) (output agent.PluginOutput) { +func (self *MockAgent) GetData(c *C) (output types.PluginOutput) { select { case output = <-self.ch: case <-time.After(50 * time.Millisecond): @@ -74,7 +75,7 @@ func (self *MockAgent) GetData(c *C) (output agent.PluginOutput) { return } -func (self *MockAgent) SendData(data agent.PluginOutput) { +func (self *MockAgent) SendData(data types.PluginOutput) { self.ch <- data } @@ -84,7 +85,7 @@ func (self *MockAgent) SendEvent(event sample.Event, entityKey entity.Key) { func (self *MockAgent) Unregister(id ids.PluginID) { self.registered = false - self.ch <- agent.NewNotApplicableOutput(id) + self.ch <- types.NewNotApplicableOutput(id) } func (self *MockAgent) Config() *config.Config { diff --git a/internal/plugins/windows/hostinfo.go b/internal/plugins/windows/hostinfo.go index 98e5a8068..5d7014e83 100644 --- a/internal/plugins/windows/hostinfo.go +++ b/internal/plugins/windows/hostinfo.go @@ -9,6 +9,7 @@ import ( "context" "errors" "fmt" + "github.com/newrelic/infrastructure-agent/internal/agent/types" "runtime" "strconv" "strings" @@ -66,9 +67,9 @@ func NewHostinfoPlugin(id ids.PluginID, ctx agent.AgentContext, hostInfo common. } } -func (self *HostinfoPlugin) Data() agent.PluginInventoryDataset { +func (self *HostinfoPlugin) Data() types.PluginInventoryDataset { info := getHostInfo() - return agent.PluginInventoryDataset{self.gatherHostinfo(self.Context, info)} + return types.PluginInventoryDataset{self.gatherHostinfo(self.Context, info)} } func (self *HostinfoPlugin) Run() { diff --git a/internal/plugins/windows/services.go b/internal/plugins/windows/services.go index a69c902ec..c635d4904 100644 --- a/internal/plugins/windows/services.go +++ b/internal/plugins/windows/services.go @@ -7,6 +7,7 @@ package windows import ( "fmt" + "github.com/newrelic/infrastructure-agent/internal/agent/types" "github.com/newrelic/infrastructure-agent/pkg/entity" "time" @@ -97,7 +98,7 @@ func (self *ServicesPlugin) getServicePID(mgr windows.Handle, serviceName string return status.ProcessId, status.CurrentState, nil } -func (self *ServicesPlugin) getDataset() (result agent.PluginInventoryDataset, err error) { +func (self *ServicesPlugin) getDataset() (result types.PluginInventoryDataset, err error) { // Windows registry path that contains all the services on the local machine key, err := registry.OpenKey(registry.LOCAL_MACHINE, `System\CurrentControlSet\Services\`, registry.QUERY_VALUE|registry.ENUMERATE_SUB_KEYS) if err != nil { diff --git a/internal/plugins/windows/services_test.go b/internal/plugins/windows/services_test.go index 3d5b0be35..33015ff56 100644 --- a/internal/plugins/windows/services_test.go +++ b/internal/plugins/windows/services_test.go @@ -7,11 +7,11 @@ package windows import ( "fmt" + "github.com/newrelic/infrastructure-agent/internal/agent/types" "sort" "testing" "github.com/StackExchange/wmi" - "github.com/newrelic/infrastructure-agent/internal/agent" "github.com/newrelic/infrastructure-agent/pkg/config" "github.com/stretchr/testify/assert" ) @@ -24,7 +24,7 @@ type Win32_Service struct { ProcessId uint32 } -func getDatasetWMI() (result agent.PluginInventoryDataset, err error) { +func getDatasetWMI() (result types.PluginInventoryDataset, err error) { var wmiResults []Win32_Service // Only get running services which are set to start automatically. diff --git a/internal/plugins/windows/updates.go b/internal/plugins/windows/updates.go index 8c27e490d..14ae54b9a 100644 --- a/internal/plugins/windows/updates.go +++ b/internal/plugins/windows/updates.go @@ -7,6 +7,7 @@ package windows import ( "fmt" + "github.com/newrelic/infrastructure-agent/internal/agent/types" "github.com/newrelic/infrastructure-agent/pkg/entity" "time" @@ -50,7 +51,7 @@ func NewUpdatesPlugin(id ids.PluginID, ctx agent.AgentContext) agent.Plugin { } } -func (self *UpdatesPlugin) getDataset() (result agent.PluginInventoryDataset, err error) { +func (self *UpdatesPlugin) getDataset() (result types.PluginInventoryDataset, err error) { var wmiResults []Win32_QuickFixEngineering wmiQuery := wmi.CreateQuery(&wmiResults, "") if err = wmi.QueryNamespace(wmiQuery, &wmiResults, config.DefaultWMINamespace); err != nil { diff --git a/pkg/config/config.go b/pkg/config/config.go index abce9cf32..1a641e31e 100644 --- a/pkg/config/config.go +++ b/pkg/config/config.go @@ -674,12 +674,23 @@ type Config struct { // Public: Yes InventoryQueueLen int `yaml:"inventory_queue_len" envconfig:"inventory_queue_len" public:"true"` + // AsyncInventoryHandlerEnabled when set to true, enables the inventory handler that parallelize processing that allows handling larger inventory payloads. + // Default: false + // Public: no + AsyncInventoryHandlerEnabled bool `yaml:"async_inventory_handler_enabled" envconfig:"async_inventory_handler_enabled" public:"false"` + // EnableWinUpdatePlugin enables the windows updates plugin which retrieves the lists of hotfix that are installed // on the host. // Default: False // Public: Yes EnableWinUpdatePlugin bool `yaml:"enable_win_update_plugin" envconfig:"enable_win_update_plugin" os:"windows"` + // InventoryArchiveEnabled When enabled, the delta storage will save each successful deltas submission into + // .sent files in the delta store. + // Default: True + // Public: True + InventoryArchiveEnabled bool `yaml:"inventory_archive_enabled" envconfig:"inventory_archive_enabled" public:"true"` + // CompactEnabled When enabled, the delta storage will be compacted after its storage directory surpasses a // certain threshold set by the CompactTreshold options. Compaction works by removing the data of inactive plugins // and the archived deltas of the active plugins; archive deltas are deltas that have already been sent to the @@ -1687,6 +1698,7 @@ func NewConfig() *Config { StartupConnectionRetries: defaultStartupConnectionRetries, DisableZeroRSSFilter: defaultDisableZeroRSSFilter, DisableWinSharedWMI: defaultDisableWinSharedWMI, + InventoryArchiveEnabled: defaultInventoryArchiveEnabled, CompactEnabled: defaultCompactEnabled, StripCommandLine: DefaultStripCommandLine, NetworkInterfaceFilters: defaultNetworkInterfaceFilters, diff --git a/pkg/config/defaults.go b/pkg/config/defaults.go index dee8bfaa5..3c60f87fd 100644 --- a/pkg/config/defaults.go +++ b/pkg/config/defaults.go @@ -57,6 +57,7 @@ var ( defaultAppDataDir = "" defaultCmdChannelEndpoint = "/agent_commands/v1/commands" defaultCmdChannelIntervalSec = 60 + defaultInventoryArchiveEnabled = true defaultCompactEnabled = true defaultCompactThreshold = 20 * 1024 * 1024 // (in bytes) compact repo when it hits 20MB defaultIgnoreReclaimable = false diff --git a/pkg/helpers/helpers.go b/pkg/helpers/helpers.go index 484a22fe4..6036f0654 100644 --- a/pkg/helpers/helpers.go +++ b/pkg/helpers/helpers.go @@ -16,6 +16,7 @@ import ( "regexp" "runtime" "strings" + "sync" "time" "github.com/newrelic/infrastructure-agent/pkg/log" @@ -24,10 +25,17 @@ import ( "github.com/newrelic/infrastructure-agent/pkg/helpers/lru" ) +type fileNameCache struct { + *lru.Cache + m sync.Mutex +} + var ( JsonFilesRegexp = regexp.MustCompile("^[^~]+.json$") - sanitizeFileNameCache = lru.New() - HiddenField = "" + sanitizeFileNameCache = &fileNameCache{ + Cache: lru.New(), + } + HiddenField = "" ) var quotations = map[uint8]bool{ @@ -220,6 +228,10 @@ func SanitizeFileName(fileName string) string { if fileName == "" { return fileName } + + sanitizeFileNameCache.m.Lock() + defer sanitizeFileNameCache.m.Unlock() + value, found := sanitizeFileNameCache.Get(fileName) if found { if stringValue, isString := value.(string); isString { diff --git a/pkg/integrations/legacy/runner_test.go b/pkg/integrations/legacy/runner_test.go index 94bcbb7b0..03f432694 100644 --- a/pkg/integrations/legacy/runner_test.go +++ b/pkg/integrations/legacy/runner_test.go @@ -8,6 +8,7 @@ import ( "encoding/json" "errors" "fmt" + "github.com/newrelic/infrastructure-agent/internal/agent/types" "io" "io/ioutil" "os" @@ -408,7 +409,7 @@ func (rs *RunnerSuite) TestRegisterInstances(c *C) { // customContext implements the AgentContext interface for testing purposes // It only has two channels to read/write events and inventory data from plugins type customContext struct { - ch chan agent.PluginOutput + ch chan types.PluginOutput ev chan sample.Event cfg *config.Config } @@ -439,7 +440,7 @@ func (cc customContext) GetServiceForPid(pid int) (service string, ok bool) { return "", false } -func (cc customContext) SendData(data agent.PluginOutput) { +func (cc customContext) SendData(data types.PluginOutput) { cc.ch <- data } @@ -486,7 +487,7 @@ custom_attributes: cfg, _ := config.LoadConfig(f.Name()) return customContext{ - ch: make(chan agent.PluginOutput), + ch: make(chan types.PluginOutput), ev: make(chan sample.Event), cfg: cfg, } @@ -611,8 +612,8 @@ func readFromChannel(ch chan interface{}) (interface{}, error) { } } -func readData(ch chan agent.PluginOutput) (agent.PluginOutput, error) { - var output agent.PluginOutput +func readData(ch chan types.PluginOutput) (types.PluginOutput, error) { + var output types.PluginOutput ticker := time.NewTicker(time.Second) defer ticker.Stop() @@ -1542,10 +1543,10 @@ type fakeEmitter struct { lastEntityKey string } -func (f *fakeEmitter) EmitInventoryWithPluginId(data agent.PluginInventoryDataset, entityKey string, pluginId ids.PluginID) { +func (f *fakeEmitter) EmitInventoryWithPluginId(data types.PluginInventoryDataset, entityKey string, pluginId ids.PluginID) { } -func (f *fakeEmitter) EmitInventory(data agent.PluginInventoryDataset, entity entity.Entity) {} +func (f *fakeEmitter) EmitInventory(data types.PluginInventoryDataset, entity entity.Entity) {} func (f *fakeEmitter) EmitEvent(eventData map[string]interface{}, entityKey entity.Key) { f.lastEventData = eventData diff --git a/pkg/integrations/legacy/utils.go b/pkg/integrations/legacy/utils.go index 7981a6ed8..512b61b88 100644 --- a/pkg/integrations/legacy/utils.go +++ b/pkg/integrations/legacy/utils.go @@ -4,8 +4,8 @@ package legacy import ( "fmt" + "github.com/newrelic/infrastructure-agent/internal/agent/types" - "github.com/newrelic/infrastructure-agent/internal/agent" event2 "github.com/newrelic/infrastructure-agent/pkg/event" "github.com/newrelic/infrastructure-agent/pkg/integrations/v4/protocol" "github.com/newrelic/infrastructure-agent/pkg/log" @@ -18,8 +18,8 @@ func BuildInventoryDataSet( labels map[string]string, integrationUser string, pluginName string, - entityKey string) agent.PluginInventoryDataset { - var inventoryDataSet agent.PluginInventoryDataset + entityKey string) types.PluginInventoryDataset { + var inventoryDataSet types.PluginInventoryDataset for key, item := range inventoryData { item["id"] = key diff --git a/pkg/integrations/v4/dm/emitter_bench_test.go b/pkg/integrations/v4/dm/emitter_bench_test.go index f25b9689c..c73a4e709 100644 --- a/pkg/integrations/v4/dm/emitter_bench_test.go +++ b/pkg/integrations/v4/dm/emitter_bench_test.go @@ -177,7 +177,7 @@ package dm // return context.TODO() //} // -//func (n noopAgentContext) SendData(output agent.PluginOutput) { +//func (n noopAgentContext) SendData(output types.PluginOutput) { // panic("implement me") //} // diff --git a/pkg/integrations/v4/dm/emitter_test.go b/pkg/integrations/v4/dm/emitter_test.go index 1ad53171d..d006dbbe2 100644 --- a/pkg/integrations/v4/dm/emitter_test.go +++ b/pkg/integrations/v4/dm/emitter_test.go @@ -7,6 +7,7 @@ import ( "encoding/json" "errors" "fmt" + "github.com/newrelic/infrastructure-agent/internal/agent/types" "io/ioutil" "os" "sync" @@ -170,8 +171,8 @@ func TestEmitter_Send_usingIDCache(t *testing.T) { aCtx := getAgentContext("TestEmitter_Send_usingIDCache") aCtx.On("SendEvent", mock.Anything, mock.Anything) - aCtx.On("SendData", agent.PluginOutput{Id: ids.PluginID{Category: "integration", Term: "Sample"}, Entity: firstEntity, Data: agent.PluginInventoryDataset{protocol.InventoryData{"id": "inventory_payload_one", "value": "foo-one"}}, NotApplicable: false}) - aCtx.On("SendData", agent.PluginOutput{Id: ids.PluginID{Category: "integration", Term: "Sample"}, Entity: secondEntity, Data: agent.PluginInventoryDataset{protocol.InventoryData{"id": "inventory_payload_two", "value": "bar-two"}}, NotApplicable: false}) + aCtx.On("SendData", types.PluginOutput{Id: ids.PluginID{Category: "integration", Term: "Sample"}, Entity: firstEntity, Data: types.PluginInventoryDataset{protocol.InventoryData{"id": "inventory_payload_one", "value": "foo-one"}}, NotApplicable: false}) + aCtx.On("SendData", types.PluginOutput{Id: ids.PluginID{Category: "integration", Term: "Sample"}, Entity: secondEntity, Data: types.PluginInventoryDataset{protocol.InventoryData{"id": "inventory_payload_two", "value": "bar-two"}}, NotApplicable: false}) dmSender := &mockedMetricsSender{ wg: sync.WaitGroup{}, @@ -288,10 +289,10 @@ func TestEmitter_Send(t *testing.T) { if testCase.register { aCtx = getAgentContext("TestEmitter_Send") aCtx.On("SendData", - agent.PluginOutput{ + types.PluginOutput{ Id: ids.PluginID{Category: "integration", Term: "integration name"}, Entity: entity.New("unique name", eID), - Data: agent.PluginInventoryDataset{ + Data: types.PluginInventoryDataset{ protocol.InventoryData{"id": "inventory_foo", "value": "bar"}, protocol.InventoryData{"entityKey": "unique name", "id": "integrationUser", "value": "root"}, }, diff --git a/pkg/integrations/v4/emitter/emitter_test.go b/pkg/integrations/v4/emitter/emitter_test.go index 411d78ff6..b8261750a 100644 --- a/pkg/integrations/v4/emitter/emitter_test.go +++ b/pkg/integrations/v4/emitter/emitter_test.go @@ -5,6 +5,7 @@ package emitter import ( "encoding/json" "fmt" + "github.com/newrelic/infrastructure-agent/internal/agent/types" "strings" "testing" @@ -14,7 +15,6 @@ import ( "github.com/newrelic/infrastructure-agent/internal/agent/mocks" - "github.com/newrelic/infrastructure-agent/internal/agent" "github.com/newrelic/infrastructure-agent/internal/agent/cmdchannel/fflag" "github.com/newrelic/infrastructure-agent/internal/feature_flags" "github.com/newrelic/infrastructure-agent/internal/integrations/v4/integration" @@ -498,7 +498,7 @@ func TestLegacy_Emit(t *testing.T) { called := tc.ma.Calls[c] if called.Method == "SendData" { //t.Log(called) - po := called.Arguments[0].(agent.PluginOutput) + po := called.Arguments[0].(types.PluginOutput) assert.Equal(t, tc.expectedId, po.Id) } } @@ -582,7 +582,7 @@ func TestProtocolV4_Emit(t *testing.T) { if called.Method == "SendData" { //t.Log(called) - pluginOutput := called.Arguments[0].(agent.PluginOutput) + pluginOutput := called.Arguments[0].(types.PluginOutput) assert.Equal(t, "unique name", pluginOutput.Entity.Key) assert.Equal(t, "labels/foo", pluginOutput.Data[1].(protocol.InventoryData)["id"]) assert.Equal(t, "bar", pluginOutput.Data[1].(protocol.InventoryData)["value"]) @@ -837,7 +837,7 @@ func TestEmit_SendCustomAttributes_SendCAInSecureForwardMode(t *testing.T) { func mockAgent2Payloads() *mocks.AgentContext { ma := mockAgent() - ma.On("SendData", mock.AnythingOfType("agent.PluginOutput")).Twice() + ma.On("SendData", mock.AnythingOfType("types.PluginOutput")).Twice() ma.SendDataWg.Add(1) return ma @@ -857,7 +857,7 @@ func mockAgent() *mocks.AgentContext { ma := &mocks.AgentContext{} ma.On("EntityKey").Return("bob") ma.On("IDLookup").Return(aID) - ma.On("SendData", mock.AnythingOfType("agent.PluginOutput")).Once() + ma.On("SendData", mock.AnythingOfType("types.PluginOutput")).Once() ma.SendDataWg.Add(1) ma.On("SendEvent", mock.AnythingOfType("agent.mapEvent"), mock.AnythingOfType("entity.Key")).Once() ma.On("Config").Return(cfg) @@ -882,7 +882,7 @@ func mockForwardAgent(isForwardOnly bool, customAttributes config.CustomAttribut ma := &mocks.AgentContext{} ma.On("EntityKey").Return("bob") ma.On("IDLookup").Return(aID) - ma.On("SendData", mock.AnythingOfType("agent.PluginOutput")).Once() + ma.On("SendData", mock.AnythingOfType("types.PluginOutput")).Once() ma.SendDataWg.Add(1) ma.On("SendEvent", mock.AnythingOfType("agent.mapEvent"), mock.AnythingOfType("entity.Key")).Once() ma.On("Config").Return(cfg) diff --git a/pkg/metrics/process/sampler_linux_test.go b/pkg/metrics/process/sampler_linux_test.go index f83747223..5f3e823db 100644 --- a/pkg/metrics/process/sampler_linux_test.go +++ b/pkg/metrics/process/sampler_linux_test.go @@ -14,6 +14,7 @@ import ( "github.com/newrelic/infrastructure-agent/internal/agent" "github.com/newrelic/infrastructure-agent/internal/agent/mocks" + agentTypes "github.com/newrelic/infrastructure-agent/internal/agent/types" "github.com/newrelic/infrastructure-agent/pkg/config" "github.com/newrelic/infrastructure-agent/pkg/entity" "github.com/newrelic/infrastructure-agent/pkg/entity/host" @@ -137,7 +138,7 @@ func (*dummyAgentContext) HostnameResolver() hostname.Resolver { func (*dummyAgentContext) Reconnect() {} -func (*dummyAgentContext) SendData(agent.PluginOutput) {} +func (*dummyAgentContext) SendData(agentTypes.PluginOutput) {} func (*dummyAgentContext) SendEvent(event sample.Event, entityKey entity.Key) {} diff --git a/pkg/plugins/agent_config.go b/pkg/plugins/agent_config.go index 7a67cf2d0..6cf233347 100644 --- a/pkg/plugins/agent_config.go +++ b/pkg/plugins/agent_config.go @@ -3,6 +3,7 @@ package plugins import ( + "github.com/newrelic/infrastructure-agent/internal/agent/types" "github.com/newrelic/infrastructure-agent/pkg/entity" "github.com/newrelic/infrastructure-agent/pkg/log" "github.com/sirupsen/logrus" @@ -57,5 +58,5 @@ func (ac *AgentConfigPlugin) Run() { helpers.LogStructureDetails(aclog, inventoryItems, "config", "raw", logrus.Fields{}) - ac.EmitInventory(agent.PluginInventoryDataset{ConfigAttrs(inventoryItems)}, entity.NewFromNameWithoutID(ac.Context.EntityKey())) + ac.EmitInventory(types.PluginInventoryDataset{ConfigAttrs(inventoryItems)}, entity.NewFromNameWithoutID(ac.Context.EntityKey())) } diff --git a/pkg/plugins/agent_config_test.go b/pkg/plugins/agent_config_test.go index 330816431..3c37c4cbd 100644 --- a/pkg/plugins/agent_config_test.go +++ b/pkg/plugins/agent_config_test.go @@ -3,13 +3,13 @@ package plugins import ( + "github.com/newrelic/infrastructure-agent/internal/agent/types" "github.com/newrelic/infrastructure-agent/pkg/entity" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "testing" - "github.com/newrelic/infrastructure-agent/internal/agent" "github.com/newrelic/infrastructure-agent/internal/agent/mocks" "github.com/newrelic/infrastructure-agent/pkg/config" "github.com/newrelic/infrastructure-agent/pkg/plugins/ids" @@ -35,7 +35,7 @@ func TestConfig(t *testing.T) { args := <-ch - _, ok := args[0].(agent.PluginOutput) + _, ok := args[0].(types.PluginOutput) assert.True(t, ok) gotPluginOutput := args[0] @@ -48,7 +48,7 @@ func TestConfig(t *testing.T) { "value": value, } } - expectedPluginOutput := agent.NewPluginOutput(*pluginId, entity.NewFromNameWithoutID(agentId), agent.PluginInventoryDataset{ConfigAttrs(expectedInvItems)}) + expectedPluginOutput := types.NewPluginOutput(*pluginId, entity.NewFromNameWithoutID(agentId), types.PluginInventoryDataset{ConfigAttrs(expectedInvItems)}) assert.Equal(t, gotPluginOutput, expectedPluginOutput) ctx.AssertExpectations(t) diff --git a/pkg/plugins/custom_attrs.go b/pkg/plugins/custom_attrs.go index 22abe1d12..7102d9c0b 100644 --- a/pkg/plugins/custom_attrs.go +++ b/pkg/plugins/custom_attrs.go @@ -4,6 +4,7 @@ package plugins import ( "github.com/newrelic/infrastructure-agent/internal/agent" + "github.com/newrelic/infrastructure-agent/internal/agent/types" "github.com/newrelic/infrastructure-agent/pkg/config" "github.com/newrelic/infrastructure-agent/pkg/entity" "github.com/newrelic/infrastructure-agent/pkg/plugins/ids" @@ -34,7 +35,7 @@ func NewCustomAttrsPlugin(ctx agent.AgentContext) agent.Plugin { func (self *CustomAttrsPlugin) Run() { self.Context.AddReconnecting(self) - data := agent.PluginInventoryDataset{CustomAttrs(self.customAttributes)} + data := types.PluginInventoryDataset{CustomAttrs(self.customAttributes)} entityKey := self.Context.EntityKey() aclog. diff --git a/pkg/plugins/files_config.go b/pkg/plugins/files_config.go index 5c78f8c1e..4aeb85b2f 100644 --- a/pkg/plugins/files_config.go +++ b/pkg/plugins/files_config.go @@ -6,6 +6,7 @@ package plugins import ( "encoding/json" "fmt" + "github.com/newrelic/infrastructure-agent/internal/agent/types" "github.com/newrelic/infrastructure-agent/pkg/entity" "io/ioutil" "os" @@ -147,7 +148,7 @@ func fileTypeString(fi os.FileInfo) (fileType string) { return fileType } -func getPluginDataset() (dataset agent.PluginInventoryDataset, err error) { +func getPluginDataset() (dataset types.PluginInventoryDataset, err error) { for filename := range monitoredFiles { var d FileData if d, err = getFileData(filename); err != nil { diff --git a/pkg/plugins/host_aliases.go b/pkg/plugins/host_aliases.go index 8177c5bff..6fe969c26 100644 --- a/pkg/plugins/host_aliases.go +++ b/pkg/plugins/host_aliases.go @@ -4,6 +4,7 @@ package plugins import ( "fmt" + "github.com/newrelic/infrastructure-agent/internal/agent/types" "github.com/newrelic/infrastructure-agent/pkg/entity" "time" @@ -41,7 +42,7 @@ func NewHostAliasesPlugin(ctx agent.AgentContext, cloudHarvester cloud.Harvester } } -func (self *HostAliasesPlugin) getHostAliasesDataset() (dataset agent.PluginInventoryDataset, err error) { +func (self *HostAliasesPlugin) getHostAliasesDataset() (dataset types.PluginInventoryDataset, err error) { fullHostname, shortHostname, err := self.resolver.Query() if err != nil { return nil, fmt.Errorf("error resolving hostname: %s", err) @@ -116,7 +117,7 @@ func (self *HostAliasesPlugin) Run() { refreshTimer.Stop() refreshTimer = time.NewTicker(config.FREQ_PLUGIN_HOST_ALIASES * time.Second) { - var dataset agent.PluginInventoryDataset + var dataset types.PluginInventoryDataset var err error self.logger.Debug("Starting harvest.") if dataset, err = self.getHostAliasesDataset(); err != nil { diff --git a/pkg/plugins/network_interface.go b/pkg/plugins/network_interface.go index 67084204a..59443ce13 100644 --- a/pkg/plugins/network_interface.go +++ b/pkg/plugins/network_interface.go @@ -3,6 +3,7 @@ package plugins import ( + "github.com/newrelic/infrastructure-agent/internal/agent/types" "github.com/newrelic/infrastructure-agent/pkg/entity" "time" @@ -61,8 +62,8 @@ func (self *NetworkInterfacePlugin) WithInterfacesProvider(p network_helpers.Int return self } -func (self *NetworkInterfacePlugin) getNetworkInterfaceData() (agent.PluginInventoryDataset, error) { - var dataset agent.PluginInventoryDataset +func (self *NetworkInterfacePlugin) getNetworkInterfaceData() (types.PluginInventoryDataset, error) { + var dataset types.PluginInventoryDataset interfaces, err := self.getInterfaces() if err != nil { diff --git a/pkg/plugins/network_interface_test.go b/pkg/plugins/network_interface_test.go index 5bc0149fb..56c2c3e33 100644 --- a/pkg/plugins/network_interface_test.go +++ b/pkg/plugins/network_interface_test.go @@ -3,13 +3,13 @@ package plugins import ( + "github.com/newrelic/infrastructure-agent/internal/agent/types" "github.com/newrelic/infrastructure-agent/pkg/entity" "github.com/stretchr/testify/mock" "strings" "testing" "time" - "github.com/newrelic/infrastructure-agent/internal/agent" "github.com/newrelic/infrastructure-agent/internal/agent/mocks" "github.com/newrelic/infrastructure-agent/pkg/config" "github.com/newrelic/infrastructure-agent/pkg/helpers/network" @@ -136,7 +136,7 @@ func TestGetNetworkInterfaceData(t *testing.T) { ni := interfaceStatAsNetworkInterfaceData(&getTestInterfaces()[0]) assert.NotNil(t, ni) - assert.Equal(t, data, agent.PluginInventoryDataset{ni}) + assert.Equal(t, data, types.PluginInventoryDataset{ni}) } func TestNetworkPlugin(t *testing.T) { @@ -144,12 +144,12 @@ func TestNetworkPlugin(t *testing.T) { assert.NoError(t, err) assert.NotNil(t, interfaces) - pluginInventory := agent.PluginInventoryDataset{} + pluginInventory := types.PluginInventoryDataset{} for _, ni := range interfaces { pluginInventory = append(pluginInventory, interfaceStatAsNetworkInterfaceData(&ni)) } - expectedInventory := agent.NewPluginOutput(getPluginId(), entity.NewFromNameWithoutID(agentId), pluginInventory) + expectedInventory := types.NewPluginOutput(getPluginId(), entity.NewFromNameWithoutID(agentId), pluginInventory) assert.NotNil(t, expectedInventory) ctx := &mocks.AgentContext{} @@ -168,7 +168,7 @@ func TestNetworkPlugin(t *testing.T) { go plugin.Run() args := <-ch - _, ok := args[0].(agent.PluginOutput) + _, ok := args[0].(types.PluginOutput) assert.True(t, ok) actualInventory := args[0] diff --git a/pkg/plugins/proxy/proxy_config.go b/pkg/plugins/proxy/proxy_config.go index a42e3d24b..d10a7ba69 100644 --- a/pkg/plugins/proxy/proxy_config.go +++ b/pkg/plugins/proxy/proxy_config.go @@ -3,6 +3,7 @@ package proxy import ( + "github.com/newrelic/infrastructure-agent/internal/agent/types" "github.com/newrelic/infrastructure-agent/pkg/entity" "net/url" "os" @@ -51,7 +52,7 @@ func (e *entry) SortKey() string { // ProxyConfigPlugins reports the ProxyConfig as inventory type configPlugin struct { agent.PluginCommon - config []agent.Sortable + config []types.Sortable } func ConfigPlugin(ctx agent.AgentContext) agent.Plugin { @@ -60,7 +61,7 @@ func ConfigPlugin(ctx agent.AgentContext) agent.Plugin { cfg = &config.Config{} // empty config to avoid null pointers } - proxyConfig := make([]agent.Sortable, 0) + proxyConfig := make([]types.Sortable, 0) if e := urlEntry(os.Getenv("HTTPS_PROXY")); e != nil { e.Id = "HTTPS_PROXY" diff --git a/test/cfgprotocol/agent/emulator.go b/test/cfgprotocol/agent/emulator.go index 003f72357..1657c8969 100644 --- a/test/cfgprotocol/agent/emulator.go +++ b/test/cfgprotocol/agent/emulator.go @@ -136,6 +136,7 @@ func (ae *Emulator) RunAgent() error { } go integrationManager.Start(ae.agent.Context.Ctx) go func() { + ae.agent.Init() if err := ae.agent.Run(); err != nil { panic(err) } diff --git a/test/core/deltas_test.go b/test/core/deltas_test.go index 4ce15b702..217e1d4b5 100644 --- a/test/core/deltas_test.go +++ b/test/core/deltas_test.go @@ -4,6 +4,11 @@ package core import ( "bytes" + context2 "context" + agentTypes "github.com/newrelic/infrastructure-agent/internal/agent/types" + "github.com/newrelic/infrastructure-agent/pkg/plugins/ids" + "github.com/newrelic/infrastructure-agent/pkg/sysinfo" + "github.com/stretchr/testify/suite" "io/ioutil" "net/http" "testing" @@ -20,14 +25,33 @@ import ( "github.com/stretchr/testify/assert" ) -func TestDeltas_nestedObjectsV4(t *testing.T) { +type InventoryTestSuite struct { + suite.Suite + AsyncInventoryHandlerEnabled bool +} + +func TestInventorySuite_AsyncInventoryHandlerEnabled(t *testing.T) { + suite.Run(t, &InventoryTestSuite{ + AsyncInventoryHandlerEnabled: true, + }) +} + +func TestInventorySuite_AsyncInventoryHandlerDisabled(t *testing.T) { + suite.Run(t, &InventoryTestSuite{}) +} + +func (s *InventoryTestSuite) TestDeltas_nestedObjectsV4() { + t := s.T() const timeout = 5 * time.Second // Given an agent testClient := ihttp.NewRequestRecorderClient( ihttp.AcceptedResponse("test/dummy", 1), ihttp.AcceptedResponse("test/dummy", 2)) - a := infra.NewAgent(testClient.Client) + a := infra.NewAgent(testClient.Client, func(config *config.Config) { + config.AsyncInventoryHandlerEnabled = s.AsyncInventoryHandlerEnabled + }) + a.Context.SetAgentIdentity(entity.Identity{10, "abcdef"}) // That runs a v4 plugin with nested inventory @@ -87,14 +111,19 @@ func TestDeltas_nestedObjectsV4(t *testing.T) { }) } -func TestDeltas_BasicWorkflow(t *testing.T) { +func (s *InventoryTestSuite) TestDeltas_BasicWorkflow() { + t := s.T() + const timeout = 5 * time.Second // Given an agent testClient := ihttp.NewRequestRecorderClient( ihttp.AcceptedResponse("test/dummy", 1), ihttp.AcceptedResponse("test/dummy", 2)) - a := infra.NewAgent(testClient.Client) + a := infra.NewAgent(testClient.Client, func(config *config.Config) { + config.AsyncInventoryHandlerEnabled = s.AsyncInventoryHandlerEnabled + }) + a.Context.SetAgentIdentity(entity.Identity{10, "abcdef"}) // That runs a plugin @@ -171,7 +200,51 @@ func TestDeltas_BasicWorkflow(t *testing.T) { }) } -func TestDeltas_ResendIfFailure(t *testing.T) { +func (s *InventoryTestSuite) TestDeltas_ForwardOnly() { + t := s.T() + + const timeout = 5 * time.Second + + // Given an agent + testClient := ihttp.NewRequestRecorderClient( + ihttp.AcceptedResponse("test/dummy", 1), + ihttp.AcceptedResponse("test/dummy", 2)) + a := infra.NewAgent(testClient.Client, func(config *config.Config) { + config.AsyncInventoryHandlerEnabled = s.AsyncInventoryHandlerEnabled + config.IsForwardOnly = true + config.FirstReapInterval = time.Nanosecond + config.SendInterval = time.Nanosecond + }) + + a.Context.SetAgentIdentity(entity.Identity{10, "abcdef"}) + + //Give time to at least send one request + ctxTimeout, _ := context2.WithTimeout(a.Context.Ctx, time.Millisecond*10) + a.Context.Ctx = ctxTimeout + + // That runs a plugin + plugin := newDummyPlugin("hello", a.Context) + a.RegisterPlugin(plugin) + + go a.Run() + + // When the plugin harvests inventory data + plugin.harvest() + + select { + case <-testClient.RequestCh: + a.Terminate() + assert.FailNow(t, "Agent must not send data yet") + case <-ctxTimeout.Done(): + // Success + return + case <-time.After(timeout): + a.Terminate() + } +} + +func (s *InventoryTestSuite) TestDeltas_ResendIfFailure() { + t := s.T() const timeout = 5 * time.Second // Given an agent that fails submitting the deltas in the second invocation @@ -180,7 +253,10 @@ func TestDeltas_ResendIfFailure(t *testing.T) { ihttp.ErrorResponse, ihttp.AcceptedResponse("test/dummy", 2)) - a := infra.NewAgent(testClient.Client) + a := infra.NewAgent(testClient.Client, func(config *config.Config) { + config.AsyncInventoryHandlerEnabled = s.AsyncInventoryHandlerEnabled + }) + a.Context.SetAgentIdentity(entity.Identity{10, "abcdef"}) // That runs a plugin @@ -266,7 +342,9 @@ func TestDeltas_ResendIfFailure(t *testing.T) { } -func TestDeltas_ResendAfterReset(t *testing.T) { +func (s *InventoryTestSuite) TestDeltas_ResendAfterReset() { + t := s.T() + const timeout = 10 * time.Second agentDir, err := ioutil.TempDir("", "prefix") @@ -279,6 +357,7 @@ func TestDeltas_ResendAfterReset(t *testing.T) { a := infra.NewAgent(testClient.Client, func(config *config.Config) { config.SendInterval = time.Hour config.AgentDir = agentDir + config.AsyncInventoryHandlerEnabled = s.AsyncInventoryHandlerEnabled }) a.Context.SetAgentIdentity(entity.Identity{10, "abcdef"}) @@ -303,6 +382,7 @@ func TestDeltas_ResendAfterReset(t *testing.T) { // When another agent process starts again a = infra.NewAgent(testClient.Client, func(config *config.Config) { config.AgentDir = agentDir + config.AsyncInventoryHandlerEnabled = s.AsyncInventoryHandlerEnabled }) a.Context.SetAgentIdentity(entity.Identity{10, "abcdef"}) a.RegisterPlugin(plugin1) @@ -330,7 +410,9 @@ func TestDeltas_ResendAfterReset(t *testing.T) { a.Terminate() } -func TestDeltas_HarvestAfterStoreCleanup(t *testing.T) { +func (s *InventoryTestSuite) TestDeltas_HarvestAfterStoreCleanup() { + t := s.T() + const timeout = 5 * time.Second // Given an agent @@ -344,6 +426,7 @@ func TestDeltas_HarvestAfterStoreCleanup(t *testing.T) { "someother": "other_attr", } cfg.Log.Level = config.LogLevelDebug + cfg.AsyncInventoryHandlerEnabled = s.AsyncInventoryHandlerEnabled }) a.Context.SetAgentIdentity(entity.Identity{10, "abcdef"}) @@ -409,6 +492,44 @@ func TestDeltas_HarvestAfterStoreCleanup(t *testing.T) { }) } +func (s *InventoryTestSuite) TestDeltas_UpdateIDLookupTable() { + t := s.T() + + // Given an agent + testClient := ihttp.NewRequestRecorderClient( + ihttp.AcceptedResponse("metadata/attributes", 1)) + + a := infra.NewAgent(testClient.Client, func(cfg *config.Config) { + cfg.CustomAttributes = config.CustomAttributeMap{ + "some": "attr", + "someother": "other_attr", + } + cfg.Log.Level = config.LogLevelDebug + cfg.AsyncInventoryHandlerEnabled = s.AsyncInventoryHandlerEnabled + }) + + go a.Run() + defer a.Terminate() + assert.Equal(t, "display-name", a.Context.EntityKey()) + + dataset := agentTypes.PluginInventoryDataset{} + dataset = append(dataset, sysinfo.HostAliases{ + Alias: "hostName.com", + Source: sysinfo.HOST_SOURCE_HOSTNAME, + }) + dataset = append(dataset, sysinfo.HostAliases{ + Alias: "instanceId", + Source: sysinfo.HOST_SOURCE_INSTANCE_ID, + }) + dataset = append(dataset, sysinfo.HostAliases{ + Alias: "hostName", + Source: sysinfo.HOST_SOURCE_HOSTNAME_SHORT, + }) + a.Context.SendData(agentTypes.NewPluginOutput(ids.PluginID{Category: "metadata", Term: "host_aliases"}, entity.NewWithoutID("test"), dataset)) + + assert.Equal(t, "instanceId", a.Context.EntityKey()) +} + func BenchmarkInventoryProcessingPipeline(b *testing.B) { const timeout = 5 * time.Second diff --git a/test/core/dummy_plugin.go b/test/core/dummy_plugin.go index 80e326903..84f8ff821 100644 --- a/test/core/dummy_plugin.go +++ b/test/core/dummy_plugin.go @@ -5,6 +5,7 @@ package core import ( "github.com/newrelic/infrastructure-agent/internal/agent" + "github.com/newrelic/infrastructure-agent/internal/agent/types" "github.com/newrelic/infrastructure-agent/pkg/entity" "github.com/newrelic/infrastructure-agent/pkg/plugins/ids" ) @@ -39,7 +40,7 @@ func (cp *dummyPlugin) Run() { for { select { case <-cp.ticker: - dataset := agent.PluginInventoryDataset{ + dataset := types.PluginInventoryDataset{ &valueEntry{Id: "dummy", Value: cp.value}, } cp.EmitInventory(dataset, entity.NewFromNameWithoutID(cp.Context.EntityKey())) @@ -54,3 +55,37 @@ func (cp *dummyPlugin) Id() ids.PluginID { func (cp *dummyPlugin) harvest() { cp.ticker <- 1 } + +type dummyPluginWithOutput struct { + agent.PluginCommon + ticker chan interface{} + outputData types.PluginOutput +} + +func newDummyPluginWithOutput(context agent.AgentContext, pluginId ids.PluginID, output types.PluginOutput) *dummyPluginWithOutput { + return &dummyPluginWithOutput{ + PluginCommon: agent.PluginCommon{ + ID: pluginId, + Context: context, + }, + ticker: make(chan interface{}), + outputData: output, + } +} + +func (cp *dummyPluginWithOutput) Run() { + for { + select { + case <-cp.ticker: + cp.Context.SendData(cp.outputData) + } + } +} + +func (cp *dummyPluginWithOutput) Id() ids.PluginID { + return cp.ID +} + +func (cp *dummyPluginWithOutput) harvest() { + cp.ticker <- 1 +} diff --git a/test/core/dummy_plugin_v4.go b/test/core/dummy_plugin_v4.go index f3e7a1844..288ad5604 100644 --- a/test/core/dummy_plugin_v4.go +++ b/test/core/dummy_plugin_v4.go @@ -4,6 +4,7 @@ package core import ( + "github.com/newrelic/infrastructure-agent/internal/agent/types" "testing" "github.com/newrelic/infrastructure-agent/internal/agent" @@ -60,7 +61,7 @@ func (cp *dummyV4Plugin) harvest() { cp.ticker <- 1 } -func InventoryDatasetsForPayload(t *testing.T, payload []byte) (dss []agent.PluginInventoryDataset) { +func InventoryDatasetsForPayload(t *testing.T, payload []byte) (dss []types.PluginInventoryDataset) { dataV4, err := dm.ParsePayloadV4(payload, test.NewFFRetrieverReturning(true, true)) require.NoError(t, err) diff --git a/test/harvest/facter_test.go b/test/harvest/facter_test.go index 96224c011..6f4262515 100644 --- a/test/harvest/facter_test.go +++ b/test/harvest/facter_test.go @@ -6,11 +6,11 @@ package harvest import ( + "github.com/newrelic/infrastructure-agent/internal/agent/types" "os/exec" "os/user" "testing" - "github.com/newrelic/infrastructure-agent/internal/agent" "github.com/newrelic/infrastructure-agent/internal/agent/mocks" pluginsLinux "github.com/newrelic/infrastructure-agent/internal/plugins/linux" "github.com/newrelic/infrastructure-agent/pkg/config" @@ -35,9 +35,9 @@ func TestFacter(t *testing.T) { FacterHomeDir: usr.HomeDir, }) ctx.On("EntityKey").Return(agentIdentifier) - ch := make(chan agent.PluginOutput) + ch := make(chan types.PluginOutput) ctx.On("SendData", mock.Anything).Return().Run(func(args mock.Arguments) { - ch <- args[0].(agent.PluginOutput) + ch <- args[0].(types.PluginOutput) }) ctx.SendDataWg.Add(1) diff --git a/test/harvest/hostinfo_test.go b/test/harvest/hostinfo_test.go index b33af2a23..c8bf81954 100644 --- a/test/harvest/hostinfo_test.go +++ b/test/harvest/hostinfo_test.go @@ -7,6 +7,7 @@ package harvest import ( "fmt" + "github.com/newrelic/infrastructure-agent/internal/agent/types" "github.com/newrelic/infrastructure-agent/pkg/entity" "os" "regexp" @@ -15,7 +16,6 @@ import ( "github.com/newrelic/infrastructure-agent/pkg/sysinfo/cloud" - "github.com/newrelic/infrastructure-agent/internal/agent" "github.com/newrelic/infrastructure-agent/internal/agent/mocks" "github.com/newrelic/infrastructure-agent/internal/plugins/common" pluginsLinux "github.com/newrelic/infrastructure-agent/internal/plugins/linux" @@ -116,10 +116,10 @@ func TestHostInfo(t *testing.T) { ctx.AssertExpectations(t) // Retrieve the PluginOutput from the mock - var actual agent.PluginOutput + var actual types.PluginOutput for _, call := range ctx.Calls { if call.Method == "SendData" { - actual = call.Arguments[0].(agent.PluginOutput) + actual = call.Arguments[0].(types.PluginOutput) break } } @@ -131,13 +131,13 @@ func TestHostInfo(t *testing.T) { uptimeRegex := regexp.MustCompile("^[0-9]{4}-[0-9]{2}-[0-9]{2} [0-9]{2}:[0-9]{2}:[0-9]{2}$|^$|unknown") assert.Regexp(t, uptimeRegex, actualUpSince) - expectedPluginOutput := agent.PluginOutput{ + expectedPluginOutput := types.PluginOutput{ Id: ids.PluginID{ Category: "metadata", Term: "system", }, Entity: entity.NewFromNameWithoutID(agentIdentifier), - Data: agent.PluginInventoryDataset{ + Data: types.PluginInventoryDataset{ &pluginsLinux.HostInfoLinux{ HostInfoData: common.HostInfoData{ System: "system", diff --git a/test/infra/agent.go b/test/infra/agent.go index 57c0cea08..403c48bbc 100644 --- a/test/infra/agent.go +++ b/test/infra/agent.go @@ -68,7 +68,7 @@ func NewAgentWithConnectClientAndConfig(connectClient *http.Client, dataClient b } } dataDir := filepath.Join(cfg.AgentDir, "data") - st := delta.NewStore(dataDir, "default", cfg.MaxInventorySize) + st := delta.NewStore(dataDir, "default", cfg.MaxInventorySize, cfg.InventoryArchiveEnabled) cloudDetector := cloud.NewDetector(true, 0, 0, 0, false) @@ -107,5 +107,6 @@ func NewAgentWithConnectClientAndConfig(connectClient *http.Client, dataClient b panic(err) } + a.Init() return a }