From c1ebd1e4585be8b934e5430eea38bc3460489963 Mon Sep 17 00:00:00 2001 From: Paul Cacheux Date: Fri, 15 Dec 2023 16:09:47 +0100 Subject: [PATCH 01/17] fix missing `pidToCleanup` when handling exit event in windows probe (#21585) fix missing `pidToCleanup` when handling exit event in windows probe --- pkg/security/probe/probe_windows.go | 1 + 1 file changed, 1 insertion(+) diff --git a/pkg/security/probe/probe_windows.go b/pkg/security/probe/probe_windows.go index 6e119b6fd85d5..7bc239c793cbb 100644 --- a/pkg/security/probe/probe_windows.go +++ b/pkg/security/probe/probe_windows.go @@ -115,6 +115,7 @@ func (p *WindowsProbe) Start() error { log.Infof("Received stop %v", stop) pce = p.Resolvers.ProcessResolver.GetEntry(pid) + pidToCleanup = pid ev.Type = uint32(model.ExitEventType) if pce == nil { From 1001fbce8bf8d284699750a64c5c0725dd69ae9b Mon Sep 17 00:00:00 2001 From: Pedro Lambert Date: Fri, 15 Dec 2023 10:20:37 -0500 Subject: [PATCH 02/17] [usm] Handle invalid events (#21550) --- pkg/network/protocols/events/consumer.go | 72 +++++++++++++++++++----- 1 file changed, 57 insertions(+), 15 deletions(-) diff --git a/pkg/network/protocols/events/consumer.go b/pkg/network/protocols/events/consumer.go index 08c96d75d01b7..0e8771a3a4abe 100644 --- a/pkg/network/protocols/events/consumer.go +++ b/pkg/network/protocols/events/consumer.go @@ -8,6 +8,7 @@ package events import ( + "errors" "fmt" "sync" "unsafe" @@ -23,8 +24,11 @@ import ( const ( batchMapSuffix = "_batches" eventsMapSuffix = "_batch_events" + sizeOfBatch = int(unsafe.Sizeof(batch{})) ) +var errInvalidPerfEvent = errors.New("invalid perf event") + // Consumer provides a standardized abstraction for consuming (batched) events from eBPF type Consumer[V any] struct { mux sync.Mutex @@ -40,11 +44,12 @@ type Consumer[V any] struct { stopped bool // telemetry - metricGroup *telemetry.MetricGroup - eventsCount *telemetry.Counter - missesCount *telemetry.Counter - kernelDropsCount *telemetry.Counter - batchSize *atomic.Int64 + metricGroup *telemetry.MetricGroup + eventsCount *telemetry.Counter + missesCount *telemetry.Counter + kernelDropsCount *telemetry.Counter + invalidEventsCount *telemetry.Counter + batchSize *atomic.Int64 } // NewConsumer instantiates a new event Consumer @@ -85,6 +90,7 @@ func NewConsumer[V any](proto string, ebpf *manager.Manager, callback func([]V)) eventsCount := metricGroup.NewCounter("events_captured") missesCount := metricGroup.NewCounter("events_missed") kernelDropsCount := metricGroup.NewCounter("kernel_dropped_events") + invalidEventsCount := metricGroup.NewCounter("invalid_events") return &Consumer[V]{ proto: proto, @@ -95,11 +101,12 @@ func NewConsumer[V any](proto string, ebpf *manager.Manager, callback func([]V)) batchReader: batchReader, // telemetry - metricGroup: metricGroup, - eventsCount: eventsCount, - missesCount: missesCount, - kernelDropsCount: kernelDropsCount, - batchSize: atomic.NewInt64(0), + metricGroup: metricGroup, + eventsCount: eventsCount, + missesCount: missesCount, + kernelDropsCount: kernelDropsCount, + invalidEventsCount: invalidEventsCount, + batchSize: atomic.NewInt64(0), }, nil } @@ -115,8 +122,12 @@ func (c *Consumer[V]) Start() { return } - b := batchFromEventData(dataEvent.Data) - c.process(dataEvent.CPU, b, false) + b, err := batchFromEventData(dataEvent.Data) + if err == nil { + c.process(dataEvent.CPU, b, false) + } else { + c.invalidEventsCount.Add(1) + } dataEvent.Done() case _, ok := <-c.handler.LostChannel: if !ok { @@ -174,7 +185,26 @@ func (c *Consumer[V]) Stop() { } func (c *Consumer[V]) process(cpu int, b *batch, syncing bool) { + // Determine the subset of data we're interested in as we might have read + // part of this batch before during a Sync() call begin, end := c.offsets.Get(cpu, b, syncing) + length := end - begin + + // This can happen in the context of a low-traffic host + // (that is, when no events are enqueued in a batch between two consecutive + // calls to `Sync()`) + if length == 0 { + return + } + + // Sanity check. Ideally none of these conditions should evaluate to + // true. In case they do we bail out and increment the counter tracking + // invalid events + // TODO: investigate why we're sometimes getting invalid offsets + if length < 0 || length > int(b.Cap) { + c.invalidEventsCount.Add(1) + return + } // telemetry stuff c.batchSize.Store(int64(b.Cap)) @@ -182,15 +212,27 @@ func (c *Consumer[V]) process(cpu int, b *batch, syncing bool) { c.kernelDropsCount.Add(int64(b.Dropped_events)) // generate a slice of type []V from the batch - length := end - begin ptr := pointerToElement[V](b, begin) events := unsafe.Slice(ptr, length) c.callback(events) } -func batchFromEventData(data []byte) *batch { - return (*batch)(unsafe.Pointer(&data[0])) +func batchFromEventData(data []byte) (*batch, error) { + if len(data) < sizeOfBatch { + // For some reason the eBPF program sent us a perf event with a size + // different from what we're expecting. + // + // TODO: we're not ensuring that len(data) == sizeOfBatch, because we're + // consistently getting events that have a few bytes more than + // `sizeof(batch_event_t)`. I haven't determined yet where these extra + // bytes are coming from, but I already validated that is not padding + // coming from the clang/LLVM toolchain for alignment purposes, so it's + // something happening *after* the call to bpf_perf_event_output. + return nil, errInvalidPerfEvent + } + + return (*batch)(unsafe.Pointer(&data[0])), nil } func pointerToElement[V any](b *batch, elementIdx int) *V { From 81dd765245af0efda477baab7bf7ef6edb0e0cd1 Mon Sep 17 00:00:00 2001 From: Olivier G <52180542+ogaca-dd@users.noreply.github.com> Date: Fri, 15 Dec 2023 16:56:32 +0100 Subject: [PATCH 03/17] Migrate demultiplexer to new file hierarchy (#21478) Migrate demultiplexer to new file hierarchy --- cmd/agent/subcommands/run/command.go | 7 ++++--- .../internal/settings/runtime_settings_test.go | 3 ++- .../subcommands/run/command.go | 7 ++++--- cmd/cluster-agent/subcommands/start/command.go | 7 ++++--- cmd/dogstatsd/subcommands/start/command.go | 7 ++++--- cmd/otel-agent/main.go | 8 ++++---- cmd/security-agent/main_windows.go | 7 ++++--- cmd/security-agent/subcommands/start/command.go | 7 ++++--- comp/aggregator/bundle.go | 4 ++-- comp/aggregator/bundle_test.go | 4 ++-- comp/aggregator/demultiplexer/component.go | 9 --------- comp/aggregator/demultiplexer/component_mock.go | 9 --------- .../{ => demultiplexerimpl}/demultiplexer.go | 15 ++++++++++++--- .../{ => demultiplexerimpl}/demultiplexer_mock.go | 14 +++++++++++--- .../demultiplexer_mock_test.go | 7 ++++--- .../{ => demultiplexerimpl}/params.go | 2 +- comp/ndmtmp/bundle_test.go | 6 +++--- comp/netflow/server/server_test.go | 4 ++-- pkg/cli/subcommands/check/command.go | 7 ++++--- pkg/collector/corechecks/snmp/snmp_test.go | 3 ++- pkg/metadata/scheduler_test.go | 3 ++- 21 files changed, 75 insertions(+), 65 deletions(-) rename comp/aggregator/demultiplexer/{ => demultiplexerimpl}/demultiplexer.go (82%) rename comp/aggregator/demultiplexer/{ => demultiplexerimpl}/demultiplexer_mock.go (76%) rename comp/aggregator/demultiplexer/{ => demultiplexerimpl}/demultiplexer_mock_test.go (82%) rename comp/aggregator/demultiplexer/{ => demultiplexerimpl}/params.go (94%) diff --git a/cmd/agent/subcommands/run/command.go b/cmd/agent/subcommands/run/command.go index 14fd7b756c05b..07b4eadb3a7c7 100644 --- a/cmd/agent/subcommands/run/command.go +++ b/cmd/agent/subcommands/run/command.go @@ -33,6 +33,7 @@ import ( "github.com/DataDog/datadog-agent/cmd/agent/subcommands/run/internal/clcrunnerapi" "github.com/DataDog/datadog-agent/cmd/manager" "github.com/DataDog/datadog-agent/comp/aggregator/demultiplexer" + "github.com/DataDog/datadog-agent/comp/aggregator/demultiplexer/demultiplexerimpl" netflowServer "github.com/DataDog/datadog-agent/comp/netflow/server" // checks implemented as components @@ -346,12 +347,12 @@ func getSharedFxOption() fx.Option { metadata.Bundle(), // injecting the aggregator demultiplexer to FX until we migrate it to a proper component. This allows // other already migrated components to request it. - fx.Provide(func(config config.Component) demultiplexer.Params { + fx.Provide(func(config config.Component) demultiplexerimpl.Params { opts := aggregator.DefaultAgentDemultiplexerOptions() opts.EnableNoAggregationPipeline = config.GetBool("dogstatsd_no_aggregation_pipeline") - return demultiplexer.Params{Options: opts} + return demultiplexerimpl.Params{Options: opts} }), - demultiplexer.Module(), + demultiplexerimpl.Module(), orchestratorForwarderImpl.Module(), fx.Supply(orchestratorForwarderImpl.NewDefaultParams()), // injecting the shared Serializer to FX until we migrate it to a prpoper component. This allows other diff --git a/cmd/agent/subcommands/run/internal/settings/runtime_settings_test.go b/cmd/agent/subcommands/run/internal/settings/runtime_settings_test.go index 7c96c38cd0562..0c6aaba6806ce 100644 --- a/cmd/agent/subcommands/run/internal/settings/runtime_settings_test.go +++ b/cmd/agent/subcommands/run/internal/settings/runtime_settings_test.go @@ -10,6 +10,7 @@ import ( global "github.com/DataDog/datadog-agent/cmd/agent/dogstatsd" "github.com/DataDog/datadog-agent/comp/aggregator/demultiplexer" + "github.com/DataDog/datadog-agent/comp/aggregator/demultiplexer/demultiplexerimpl" "github.com/DataDog/datadog-agent/comp/core" "github.com/DataDog/datadog-agent/comp/dogstatsd" "github.com/DataDog/datadog-agent/comp/dogstatsd/server" @@ -45,7 +46,7 @@ func TestDogstatsdMetricsStats(t *testing.T) { }), dogstatsd.Bundle(), defaultforwarder.MockModule(), - demultiplexer.MockModule(), + demultiplexerimpl.MockModule(), )) demux := deps.Demultiplexer diff --git a/cmd/cluster-agent-cloudfoundry/subcommands/run/command.go b/cmd/cluster-agent-cloudfoundry/subcommands/run/command.go index d05160ca4f2a5..309a57b17a4ec 100644 --- a/cmd/cluster-agent-cloudfoundry/subcommands/run/command.go +++ b/cmd/cluster-agent-cloudfoundry/subcommands/run/command.go @@ -26,6 +26,7 @@ import ( "github.com/DataDog/datadog-agent/cmd/cluster-agent/api" dcav1 "github.com/DataDog/datadog-agent/cmd/cluster-agent/api/v1" "github.com/DataDog/datadog-agent/comp/aggregator/demultiplexer" + "github.com/DataDog/datadog-agent/comp/aggregator/demultiplexer/demultiplexerimpl" "github.com/DataDog/datadog-agent/comp/core" "github.com/DataDog/datadog-agent/comp/core/config" "github.com/DataDog/datadog-agent/comp/core/log" @@ -69,13 +70,13 @@ func Commands(globalParams *command.GlobalParams) []*cobra.Command { core.Bundle(), forwarder.Bundle(), fx.Provide(defaultforwarder.NewParamsWithResolvers), - demultiplexer.Module(), + demultiplexerimpl.Module(), orchestratorForwarderImpl.Module(), fx.Supply(orchestratorForwarderImpl.NewDisabledParams()), - fx.Provide(func() demultiplexer.Params { + fx.Provide(func() demultiplexerimpl.Params { opts := aggregator.DefaultAgentDemultiplexerOptions() opts.UseEventPlatformForwarder = false - return demultiplexer.Params{Options: opts} + return demultiplexerimpl.Params{Options: opts} }), // setup workloadmeta collectors.GetCatalog(), diff --git a/cmd/cluster-agent/subcommands/start/command.go b/cmd/cluster-agent/subcommands/start/command.go index 4dfcd291e12ae..fe38311c2e122 100644 --- a/cmd/cluster-agent/subcommands/start/command.go +++ b/cmd/cluster-agent/subcommands/start/command.go @@ -26,6 +26,7 @@ import ( "github.com/DataDog/datadog-agent/cmd/cluster-agent/command" "github.com/DataDog/datadog-agent/cmd/cluster-agent/custommetrics" "github.com/DataDog/datadog-agent/comp/aggregator/demultiplexer" + "github.com/DataDog/datadog-agent/comp/aggregator/demultiplexer/demultiplexerimpl" "github.com/DataDog/datadog-agent/comp/core" "github.com/DataDog/datadog-agent/comp/core/config" "github.com/DataDog/datadog-agent/comp/core/log" @@ -94,13 +95,13 @@ func Commands(globalParams *command.GlobalParams) []*cobra.Command { params.Options.DisableAPIKeyChecking = true return params }), - demultiplexer.Module(), + demultiplexerimpl.Module(), orchestratorForwarderImpl.Module(), fx.Supply(orchestratorForwarderImpl.NewDefaultParams()), - fx.Provide(func() demultiplexer.Params { + fx.Provide(func() demultiplexerimpl.Params { opts := aggregator.DefaultAgentDemultiplexerOptions() opts.UseEventPlatformForwarder = false - return demultiplexer.Params{Options: opts} + return demultiplexerimpl.Params{Options: opts} }), // setup workloadmeta collectors.GetCatalog(), diff --git a/cmd/dogstatsd/subcommands/start/command.go b/cmd/dogstatsd/subcommands/start/command.go index f0a64a3df0dd0..5887c779532dd 100644 --- a/cmd/dogstatsd/subcommands/start/command.go +++ b/cmd/dogstatsd/subcommands/start/command.go @@ -19,6 +19,7 @@ import ( "github.com/DataDog/datadog-agent/cmd/agent/common" "github.com/DataDog/datadog-agent/comp/aggregator/demultiplexer" + "github.com/DataDog/datadog-agent/comp/aggregator/demultiplexer/demultiplexerimpl" "github.com/DataDog/datadog-agent/comp/core/config" "github.com/DataDog/datadog-agent/comp/core/log" @@ -133,7 +134,7 @@ func RunDogstatsdFct(cliParams *CLIParams, defaultConfPath string, defaultLogFil } }), workloadmeta.OptionalModule(), - demultiplexer.Module(), + demultiplexerimpl.Module(), secretsimpl.Module(), orchestratorForwarderImpl.Module(), fx.Supply(orchestratorForwarderImpl.NewDisabledParams()), @@ -142,11 +143,11 @@ func RunDogstatsdFct(cliParams *CLIParams, defaultConfPath string, defaultLogFil fx.Provide(func(demuxInstance demultiplexer.Component) serializer.MetricSerializer { return demuxInstance.Serializer() }), - fx.Provide(func(config config.Component) demultiplexer.Params { + fx.Provide(func(config config.Component) demultiplexerimpl.Params { opts := aggregator.DefaultAgentDemultiplexerOptions() opts.UseEventPlatformForwarder = false opts.EnableNoAggregationPipeline = config.GetBool("dogstatsd_no_aggregation_pipeline") - return demultiplexer.Params{Options: opts, ContinueOnMissingHostname: true} + return demultiplexerimpl.Params{Options: opts, ContinueOnMissingHostname: true} }), fx.Supply(resourcesimpl.Disabled()), metadatarunnerimpl.Module(), diff --git a/cmd/otel-agent/main.go b/cmd/otel-agent/main.go index a585a8b5a3678..8b5a2ae121f7e 100644 --- a/cmd/otel-agent/main.go +++ b/cmd/otel-agent/main.go @@ -17,7 +17,7 @@ import ( "os" "os/signal" - "github.com/DataDog/datadog-agent/comp/aggregator/demultiplexer" + "github.com/DataDog/datadog-agent/comp/aggregator/demultiplexer/demultiplexerimpl" "github.com/DataDog/datadog-agent/comp/core" "github.com/DataDog/datadog-agent/comp/core/config" corelog "github.com/DataDog/datadog-agent/comp/core/log" @@ -73,14 +73,14 @@ func main() { }, ), fx.Provide(newForwarderParams), - demultiplexer.Module(), + demultiplexerimpl.Module(), orchestratorForwarderImpl.Module(), fx.Supply(orchestratorForwarderImpl.NewDisabledParams()), fx.Provide(newSerializer), - fx.Provide(func(cfg config.Component) demultiplexer.Params { + fx.Provide(func(cfg config.Component) demultiplexerimpl.Params { opts := aggregator.DefaultAgentDemultiplexerOptions() opts.EnableNoAggregationPipeline = cfg.GetBool("dogstatsd_no_aggregation_pipeline") - return demultiplexer.Params{Options: opts} + return demultiplexerimpl.Params{Options: opts} }), ) if err != nil { diff --git a/cmd/security-agent/main_windows.go b/cmd/security-agent/main_windows.go index 3f5744c24f40d..7fe177d79941b 100644 --- a/cmd/security-agent/main_windows.go +++ b/cmd/security-agent/main_windows.go @@ -22,6 +22,7 @@ import ( "github.com/DataDog/datadog-agent/cmd/security-agent/subcommands" "github.com/DataDog/datadog-agent/cmd/security-agent/subcommands/start" "github.com/DataDog/datadog-agent/comp/aggregator/demultiplexer" + "github.com/DataDog/datadog-agent/comp/aggregator/demultiplexer/demultiplexerimpl" "github.com/DataDog/datadog-agent/comp/core" "github.com/DataDog/datadog-agent/comp/core/config" "github.com/DataDog/datadog-agent/comp/core/log" @@ -103,13 +104,13 @@ func (s *service) Run(svcctx context.Context) error { dogstatsd.ClientBundle, forwarder.Bundle(), fx.Provide(defaultforwarder.NewParamsWithResolvers), - demultiplexer.Module(), + demultiplexerimpl.Module(), orchestratorForwarderImpl.Module(), fx.Supply(orchestratorForwarderImpl.NewDisabledParams()), - fx.Provide(func() demultiplexer.Params { + fx.Provide(func() demultiplexerimpl.Params { opts := aggregator.DefaultAgentDemultiplexerOptions() opts.UseEventPlatformForwarder = false - return demultiplexer.Params{Options: opts} + return demultiplexerimpl.Params{Options: opts} }), // workloadmeta setup diff --git a/cmd/security-agent/subcommands/start/command.go b/cmd/security-agent/subcommands/start/command.go index de7523b67fe32..23eca06e719fc 100644 --- a/cmd/security-agent/subcommands/start/command.go +++ b/cmd/security-agent/subcommands/start/command.go @@ -28,6 +28,7 @@ import ( "github.com/DataDog/datadog-agent/cmd/security-agent/subcommands/compliance" "github.com/DataDog/datadog-agent/cmd/security-agent/subcommands/runtime" "github.com/DataDog/datadog-agent/comp/aggregator/demultiplexer" + "github.com/DataDog/datadog-agent/comp/aggregator/demultiplexer/demultiplexerimpl" "github.com/DataDog/datadog-agent/comp/core" "github.com/DataDog/datadog-agent/comp/core/config" "github.com/DataDog/datadog-agent/comp/core/log" @@ -93,14 +94,14 @@ func Commands(globalParams *command.GlobalParams) []*cobra.Command { dogstatsd.ClientBundle, forwarder.Bundle(), fx.Provide(defaultforwarder.NewParamsWithResolvers), - demultiplexer.Module(), + demultiplexerimpl.Module(), orchestratorForwarderImpl.Module(), fx.Supply(orchestratorForwarderImpl.NewDisabledParams()), - fx.Provide(func() demultiplexer.Params { + fx.Provide(func() demultiplexerimpl.Params { opts := aggregator.DefaultAgentDemultiplexerOptions() opts.UseEventPlatformForwarder = false - return demultiplexer.Params{Options: opts} + return demultiplexerimpl.Params{Options: opts} }), // workloadmeta setup collectors.GetCatalog(), diff --git a/comp/aggregator/bundle.go b/comp/aggregator/bundle.go index f0203ec24b807..f806fe33fb778 100644 --- a/comp/aggregator/bundle.go +++ b/comp/aggregator/bundle.go @@ -7,7 +7,7 @@ package aggregator import ( - "github.com/DataDog/datadog-agent/comp/aggregator/demultiplexer" + "github.com/DataDog/datadog-agent/comp/aggregator/demultiplexer/demultiplexerimpl" "github.com/DataDog/datadog-agent/pkg/util/fxutil" ) @@ -16,5 +16,5 @@ import ( // Bundle defines the fx options for this bundle. func Bundle() fxutil.BundleOptions { return fxutil.Bundle( - demultiplexer.Module()) + demultiplexerimpl.Module()) } diff --git a/comp/aggregator/bundle_test.go b/comp/aggregator/bundle_test.go index f8f4794716dff..88822dc07d410 100644 --- a/comp/aggregator/bundle_test.go +++ b/comp/aggregator/bundle_test.go @@ -8,7 +8,7 @@ package aggregator import ( "testing" - "github.com/DataDog/datadog-agent/comp/aggregator/demultiplexer" + "github.com/DataDog/datadog-agent/comp/aggregator/demultiplexer/demultiplexerimpl" "github.com/DataDog/datadog-agent/comp/core" "github.com/DataDog/datadog-agent/comp/forwarder/defaultforwarder" orchestratorForwarderImpl "github.com/DataDog/datadog-agent/comp/forwarder/orchestrator/orchestratorimpl" @@ -21,6 +21,6 @@ func TestBundleDependencies(t *testing.T) { core.MockBundle(), defaultforwarder.MockModule(), orchestratorForwarderImpl.MockModule(), - fx.Supply(demultiplexer.Params{}), + fx.Supply(demultiplexerimpl.Params{}), ) } diff --git a/comp/aggregator/demultiplexer/component.go b/comp/aggregator/demultiplexer/component.go index 74a8c3163f810..93860e3ea060d 100644 --- a/comp/aggregator/demultiplexer/component.go +++ b/comp/aggregator/demultiplexer/component.go @@ -7,12 +7,9 @@ package demultiplexer import ( - "go.uber.org/fx" - "github.com/DataDog/datadog-agent/pkg/aggregator" "github.com/DataDog/datadog-agent/pkg/aggregator/sender" "github.com/DataDog/datadog-agent/pkg/serializer" - "github.com/DataDog/datadog-agent/pkg/util/fxutil" ) // team: agent-shared-components @@ -32,9 +29,3 @@ type Component interface { sender.DiagnoseSenderManager } - -// Module defines the fx options for this component. -func Module() fxutil.Module { - return fxutil.Component( - fx.Provide(newDemultiplexer)) -} diff --git a/comp/aggregator/demultiplexer/component_mock.go b/comp/aggregator/demultiplexer/component_mock.go index dc453df5f9bc2..6181190464adb 100644 --- a/comp/aggregator/demultiplexer/component_mock.go +++ b/comp/aggregator/demultiplexer/component_mock.go @@ -8,10 +8,7 @@ package demultiplexer import ( - "go.uber.org/fx" - "github.com/DataDog/datadog-agent/pkg/aggregator/sender" - "github.com/DataDog/datadog-agent/pkg/util/fxutil" ) // Mock implements mock-specific methods. @@ -19,9 +16,3 @@ type Mock interface { SetDefaultSender(sender.Sender) Component } - -// MockModule defines the fx options for this component. -func MockModule() fxutil.Module { - return fxutil.Component( - fx.Provide(newMock)) -} diff --git a/comp/aggregator/demultiplexer/demultiplexer.go b/comp/aggregator/demultiplexer/demultiplexerimpl/demultiplexer.go similarity index 82% rename from comp/aggregator/demultiplexer/demultiplexer.go rename to comp/aggregator/demultiplexer/demultiplexerimpl/demultiplexer.go index 215f929e48fe8..3cfa579c000ae 100644 --- a/comp/aggregator/demultiplexer/demultiplexer.go +++ b/comp/aggregator/demultiplexer/demultiplexerimpl/demultiplexer.go @@ -3,21 +3,30 @@ // This product includes software developed at Datadog (https://www.datadoghq.com/). // Copyright 2023-present Datadog, Inc. -package demultiplexer +// Package demultiplexerimpl defines the aggregator demultiplexer +package demultiplexerimpl import ( "context" + demultiplexerComp "github.com/DataDog/datadog-agent/comp/aggregator/demultiplexer" "github.com/DataDog/datadog-agent/comp/aggregator/diagnosesendermanager" "github.com/DataDog/datadog-agent/comp/core/log" "github.com/DataDog/datadog-agent/comp/forwarder/defaultforwarder" orchestratorforwarder "github.com/DataDog/datadog-agent/comp/forwarder/orchestrator" "github.com/DataDog/datadog-agent/pkg/aggregator" "github.com/DataDog/datadog-agent/pkg/aggregator/sender" + "github.com/DataDog/datadog-agent/pkg/util/fxutil" "github.com/DataDog/datadog-agent/pkg/util/hostname" "go.uber.org/fx" ) +// Module defines the fx options for this component. +func Module() fxutil.Module { + return fxutil.Component( + fx.Provide(newDemultiplexer)) +} + type dependencies struct { fx.In Log log.Component @@ -33,14 +42,14 @@ type demultiplexer struct { type provides struct { fx.Out - Comp Component + Comp demultiplexerComp.Component // Both demultiplexer.Component and diagnosesendermanager.Component expose a different instance of SenderManager. // It means that diagnosesendermanager.Component must not be used when there is demultiplexer.Component instance. // // newDemultiplexer returns both demultiplexer.Component and diagnosesendermanager.Component (Note: demultiplexer.Component // implements diagnosesendermanager.Component). This has the nice consequence of preventing having - // demultiplexer.Module and diagnosesendermanagerimpl.Module in the same fx.App because there would + // demultiplexerimpl.Module and diagnosesendermanagerimpl.Module in the same fx.App because there would // be two ways to create diagnosesendermanager.Component. SenderManager diagnosesendermanager.Component } diff --git a/comp/aggregator/demultiplexer/demultiplexer_mock.go b/comp/aggregator/demultiplexer/demultiplexerimpl/demultiplexer_mock.go similarity index 76% rename from comp/aggregator/demultiplexer/demultiplexer_mock.go rename to comp/aggregator/demultiplexer/demultiplexerimpl/demultiplexer_mock.go index 140635e91da44..66ef0c0443ae2 100644 --- a/comp/aggregator/demultiplexer/demultiplexer_mock.go +++ b/comp/aggregator/demultiplexer/demultiplexerimpl/demultiplexer_mock.go @@ -5,18 +5,26 @@ //go:build test -package demultiplexer +package demultiplexerimpl import ( + demultiplexerComp "github.com/DataDog/datadog-agent/comp/aggregator/demultiplexer" "github.com/DataDog/datadog-agent/comp/core/log" "github.com/DataDog/datadog-agent/comp/forwarder/defaultforwarder" "github.com/DataDog/datadog-agent/pkg/aggregator" "github.com/DataDog/datadog-agent/pkg/aggregator/sender" + "github.com/DataDog/datadog-agent/pkg/util/fxutil" "go.uber.org/fx" ) +// MockModule defines the fx options for this component. +func MockModule() fxutil.Module { + return fxutil.Component( + fx.Provide(newMock)) +} + type mock struct { - Component + demultiplexerComp.Component sender *sender.Sender } @@ -40,7 +48,7 @@ type mockDependencies struct { Log log.Component } -func newMock(deps mockDependencies) (Component, Mock) { +func newMock(deps mockDependencies) (demultiplexerComp.Component, demultiplexerComp.Mock) { opts := aggregator.DefaultAgentDemultiplexerOptions() opts.DontStartForwarders = true diff --git a/comp/aggregator/demultiplexer/demultiplexer_mock_test.go b/comp/aggregator/demultiplexer/demultiplexerimpl/demultiplexer_mock_test.go similarity index 82% rename from comp/aggregator/demultiplexer/demultiplexer_mock_test.go rename to comp/aggregator/demultiplexer/demultiplexerimpl/demultiplexer_mock_test.go index 8967030a7a12a..9988c8dbe87f5 100644 --- a/comp/aggregator/demultiplexer/demultiplexer_mock_test.go +++ b/comp/aggregator/demultiplexer/demultiplexerimpl/demultiplexer_mock_test.go @@ -5,11 +5,12 @@ //go:build test -package demultiplexer +package demultiplexerimpl import ( "testing" + demultiplexerComp "github.com/DataDog/datadog-agent/comp/aggregator/demultiplexer" "github.com/DataDog/datadog-agent/comp/core" "github.com/DataDog/datadog-agent/comp/forwarder/defaultforwarder" "github.com/DataDog/datadog-agent/pkg/aggregator/mocksender" @@ -18,14 +19,14 @@ import ( ) func TestSetDefaultSender(t *testing.T) { - mock := fxutil.Test[Mock](t, MockModule(), + mock := fxutil.Test[demultiplexerComp.Mock](t, MockModule(), core.MockBundle(), defaultforwarder.MockModule()) sender := &mocksender.MockSender{} mock.SetDefaultSender(sender) - var component Component = mock + var component demultiplexerComp.Component = mock lazySenderManager, err := component.LazyGetSenderManager() require.NoError(t, err) diff --git a/comp/aggregator/demultiplexer/params.go b/comp/aggregator/demultiplexer/demultiplexerimpl/params.go similarity index 94% rename from comp/aggregator/demultiplexer/params.go rename to comp/aggregator/demultiplexer/demultiplexerimpl/params.go index cd7b433a1c57e..3472622932dd4 100644 --- a/comp/aggregator/demultiplexer/params.go +++ b/comp/aggregator/demultiplexer/demultiplexerimpl/params.go @@ -3,7 +3,7 @@ // This product includes software developed at Datadog (https://www.datadoghq.com/). // Copyright 2023-present Datadog, Inc. -package demultiplexer +package demultiplexerimpl import "github.com/DataDog/datadog-agent/pkg/aggregator" diff --git a/comp/ndmtmp/bundle_test.go b/comp/ndmtmp/bundle_test.go index 1d9f851a159d5..0d7dd977e7f2f 100644 --- a/comp/ndmtmp/bundle_test.go +++ b/comp/ndmtmp/bundle_test.go @@ -8,7 +8,7 @@ package ndmtmp import ( "testing" - "github.com/DataDog/datadog-agent/comp/aggregator/demultiplexer" + "github.com/DataDog/datadog-agent/comp/aggregator/demultiplexer/demultiplexerimpl" "github.com/DataDog/datadog-agent/comp/core" "github.com/DataDog/datadog-agent/comp/forwarder/defaultforwarder" orchestratorForwarderImpl "github.com/DataDog/datadog-agent/comp/forwarder/orchestrator/orchestratorimpl" @@ -19,10 +19,10 @@ import ( func TestBundleDependencies(t *testing.T) { fxutil.TestBundle(t, Bundle(), - demultiplexer.Module(), + demultiplexerimpl.Module(), orchestratorForwarderImpl.MockModule(), defaultforwarder.Module(), - fx.Supply(demultiplexer.Params{}), + fx.Supply(demultiplexerimpl.Params{}), fx.Supply(defaultforwarder.Params{}), core.MockBundle(), ) diff --git a/comp/netflow/server/server_test.go b/comp/netflow/server/server_test.go index 47377e880446f..a3ad0b38d8d34 100644 --- a/comp/netflow/server/server_test.go +++ b/comp/netflow/server/server_test.go @@ -19,7 +19,7 @@ import ( "go.uber.org/fx" "go.uber.org/fx/fxtest" - "github.com/DataDog/datadog-agent/comp/aggregator/demultiplexer" + "github.com/DataDog/datadog-agent/comp/aggregator/demultiplexer/demultiplexerimpl" "github.com/DataDog/datadog-agent/comp/core/config" "github.com/DataDog/datadog-agent/comp/core/hostname/hostnameimpl" "github.com/DataDog/datadog-agent/comp/core/log/logimpl" @@ -68,7 +68,7 @@ var testOptions = fx.Options( forwarderimpl.MockModule(), hostnameimpl.MockModule(), logimpl.MockModule(), - demultiplexer.MockModule(), + demultiplexerimpl.MockModule(), defaultforwarder.MockModule(), config.MockModule(), fx.Invoke(func(lc fx.Lifecycle, c Component) { diff --git a/pkg/cli/subcommands/check/command.go b/pkg/cli/subcommands/check/command.go index 71d0dff4d2a98..2b1857a98aea2 100644 --- a/pkg/cli/subcommands/check/command.go +++ b/pkg/cli/subcommands/check/command.go @@ -27,6 +27,7 @@ import ( "github.com/DataDog/datadog-agent/cmd/agent/common" "github.com/DataDog/datadog-agent/cmd/agent/common/path" "github.com/DataDog/datadog-agent/comp/aggregator/demultiplexer" + "github.com/DataDog/datadog-agent/comp/aggregator/demultiplexer/demultiplexerimpl" internalAPI "github.com/DataDog/datadog-agent/comp/api/api" "github.com/DataDog/datadog-agent/comp/api/api/apiimpl" "github.com/DataDog/datadog-agent/comp/core" @@ -155,15 +156,15 @@ func MakeCommand(globalParamsGetter func() GlobalParams) *cobra.Command { }), fx.Provide(func() serializer.MetricSerializer { return nil }), fx.Supply(defaultforwarder.Params{UseNoopForwarder: true}), - demultiplexer.Module(), + demultiplexerimpl.Module(), orchestratorForwarderImpl.Module(), fx.Supply(orchestratorForwarderImpl.NewNoopParams()), - fx.Provide(func() demultiplexer.Params { + fx.Provide(func() demultiplexerimpl.Params { // Initializing the aggregator with a flush interval of 0 (to disable the flush goroutines) opts := aggregator.DefaultAgentDemultiplexerOptions() opts.FlushInterval = 0 opts.UseNoopEventPlatformForwarder = true - return demultiplexer.Params{Options: opts} + return demultiplexerimpl.Params{Options: opts} }), // TODO(components): this is a temporary hack as the StartServer() method of the API package was previously called with nil arguments diff --git a/pkg/collector/corechecks/snmp/snmp_test.go b/pkg/collector/corechecks/snmp/snmp_test.go index bff2ffe0b86ab..057d3833c04a4 100644 --- a/pkg/collector/corechecks/snmp/snmp_test.go +++ b/pkg/collector/corechecks/snmp/snmp_test.go @@ -21,6 +21,7 @@ import ( "go.uber.org/fx" "github.com/DataDog/datadog-agent/comp/aggregator/demultiplexer" + "github.com/DataDog/datadog-agent/comp/aggregator/demultiplexer/demultiplexerimpl" "github.com/DataDog/datadog-agent/comp/core/log/logimpl" "github.com/DataDog/datadog-agent/comp/forwarder/defaultforwarder" "github.com/DataDog/datadog-agent/comp/netflow/config" @@ -58,7 +59,7 @@ type deps struct { } func createDeps(t *testing.T) deps { - return fxutil.Test[deps](t, demultiplexer.MockModule(), defaultforwarder.MockModule(), config.MockModule(), logimpl.MockModule()) + return fxutil.Test[deps](t, demultiplexerimpl.MockModule(), defaultforwarder.MockModule(), config.MockModule(), logimpl.MockModule()) } func Test_Run_simpleCase(t *testing.T) { diff --git a/pkg/metadata/scheduler_test.go b/pkg/metadata/scheduler_test.go index a459fa9df952a..11e5e795b31eb 100644 --- a/pkg/metadata/scheduler_test.go +++ b/pkg/metadata/scheduler_test.go @@ -14,6 +14,7 @@ import ( "go.uber.org/fx" "github.com/DataDog/datadog-agent/comp/aggregator/demultiplexer" + "github.com/DataDog/datadog-agent/comp/aggregator/demultiplexer/demultiplexerimpl" "github.com/DataDog/datadog-agent/comp/core/config" "github.com/DataDog/datadog-agent/comp/core/log/logimpl" "github.com/DataDog/datadog-agent/comp/forwarder/defaultforwarder" @@ -234,5 +235,5 @@ type deps struct { func buildDeps(t *testing.T) deps { opts := aggregator.DefaultAgentDemultiplexerOptions() opts.DontStartForwarders = true - return fxutil.Test[deps](t, defaultforwarder.MockModule(), config.MockModule(), logimpl.MockModule(), demultiplexer.MockModule()) + return fxutil.Test[deps](t, defaultforwarder.MockModule(), config.MockModule(), logimpl.MockModule(), demultiplexerimpl.MockModule()) } From da32cbe9b20f3da628cd6da9f68815df4315b727 Mon Sep 17 00:00:00 2001 From: Sylvain Afchain Date: Fri, 15 Dec 2023 17:43:32 +0100 Subject: [PATCH 04/17] [CWS] handle process credentials in the ptracer (#21559) --- pkg/security/probe/probe_epbfless.go | 13 +++ pkg/security/proto/ebpfless/msg.go | 51 +++++++-- pkg/security/ptracer/cws.go | 100 +++++++++++++++++- pkg/security/ptracer/syscalls_amd64.go | 8 ++ pkg/security/ptracer/syscalls_arm64.go | 26 +++-- .../resolvers/process/resolver_ebpfless.go | 34 ++++++ 6 files changed, 210 insertions(+), 22 deletions(-) diff --git a/pkg/security/probe/probe_epbfless.go b/pkg/security/probe/probe_epbfless.go index ae4af546fbe2e..3bec17a96917e 100644 --- a/pkg/security/probe/probe_epbfless.go +++ b/pkg/security/probe/probe_epbfless.go @@ -82,6 +82,14 @@ func (p *EBPFLessProbe) handleClientMsg(msg *clientMsg) { case ebpfless.SyscallTypeExec: event.Type = uint32(model.ExecEventType) entry := p.Resolvers.ProcessResolver.AddExecEntry(process.CacheResolverKey{Pid: syscallMsg.PID, NSID: syscallMsg.NSID}, syscallMsg.Exec.Filename, syscallMsg.Exec.Args, syscallMsg.Exec.Envs, syscallMsg.ContainerContext.ID) + + if syscallMsg.Exec.Credentials != nil { + entry.Credentials.UID = syscallMsg.Exec.Credentials.UID + entry.Credentials.EUID = syscallMsg.Exec.Credentials.EUID + entry.Credentials.GID = syscallMsg.Exec.Credentials.GID + entry.Credentials.EGID = syscallMsg.Exec.Credentials.EGID + } + event.Exec.Process = &entry.Process case ebpfless.SyscallTypeFork: event.Type = uint32(model.ForkEventType) @@ -92,6 +100,11 @@ func (p *EBPFLessProbe) handleClientMsg(msg *clientMsg) { event.Open.File.BasenameStr = filepath.Base(syscallMsg.Open.Filename) event.Open.Flags = syscallMsg.Open.Flags event.Open.Mode = syscallMsg.Open.Mode + case ebpfless.SyscallTypeSetUID: + p.Resolvers.ProcessResolver.UpdateUID(process.CacheResolverKey{Pid: syscallMsg.PID, NSID: syscallMsg.NSID}, syscallMsg.SetUID.UID, syscallMsg.SetUID.EUID) + + case ebpfless.SyscallTypeSetGID: + p.Resolvers.ProcessResolver.UpdateGID(process.CacheResolverKey{Pid: syscallMsg.PID, NSID: syscallMsg.NSID}, syscallMsg.SetGID.GID, syscallMsg.SetGID.EGID) } // container context diff --git a/pkg/security/proto/ebpfless/msg.go b/pkg/security/proto/ebpfless/msg.go index c8a51ca0bc611..37f7c2c990135 100644 --- a/pkg/security/proto/ebpfless/msg.go +++ b/pkg/security/proto/ebpfless/msg.go @@ -13,17 +13,21 @@ type SyscallType int32 const ( // SyscallTypeUnknown unknown type - SyscallTypeUnknown SyscallType = 0 + SyscallTypeUnknown SyscallType = iota // SyscallTypeExec exec type - SyscallTypeExec SyscallType = 1 + SyscallTypeExec // SyscallTypeFork fork type - SyscallTypeFork SyscallType = 2 + SyscallTypeFork // SyscallTypeOpen open type - SyscallTypeOpen SyscallType = 3 + SyscallTypeOpen // SyscallTypeExit exit type - SyscallTypeExit SyscallType = 4 + SyscallTypeExit // SyscallTypeFcntl fcntl type - SyscallTypeFcntl SyscallType = 5 + SyscallTypeFcntl + // SyscallTypeSetUID setuid/setreuid type + SyscallTypeSetUID + // SyscallTypeSetGID setgid/setregid type + SyscallTypeSetGID ) // ContainerContext defines a container context @@ -40,11 +44,20 @@ type FcntlSyscallMsg struct { Cmd uint32 } +// Credentials defines process credentials +type Credentials struct { + UID uint32 + EUID uint32 + GID uint32 + EGID uint32 +} + // ExecSyscallMsg defines an exec message type ExecSyscallMsg struct { - Filename string - Args []string - Envs []string + Filename string + Args []string + Envs []string + Credentials *Credentials } // ForkSyscallMsg defines a fork message @@ -72,6 +85,18 @@ type ChdirSyscallFakeMsg struct { Path string } +// SetUIDSyscallMsg defines a setreuid message +type SetUIDSyscallMsg struct { + UID int32 + EUID int32 +} + +// SetGIDSyscallMsg defines a setregid message +type SetGIDSyscallMsg struct { + GID int32 + EGID int32 +} + // SyscallMsg defines a syscall message type SyscallMsg struct { SeqNum uint64 @@ -84,8 +109,12 @@ type SyscallMsg struct { Fork *ForkSyscallMsg Exit *ExitSyscallMsg Fcntl *FcntlSyscallMsg - Dup *DupSyscallFakeMsg - Chdir *ChdirSyscallFakeMsg + SetUID *SetUIDSyscallMsg + SetGID *SetGIDSyscallMsg + + // internals + Dup *DupSyscallFakeMsg + Chdir *ChdirSyscallFakeMsg } // String returns string representation diff --git a/pkg/security/ptracer/cws.go b/pkg/security/ptracer/cws.go index 23ea5421325a9..134314a8463fc 100644 --- a/pkg/security/ptracer/cws.go +++ b/pkg/security/ptracer/cws.go @@ -238,6 +238,41 @@ func handleFchdir(tracer *Tracer, process *Process, msg *ebpfless.SyscallMsg, re return nil } +func handleSetuid(tracer *Tracer, _ *Process, msg *ebpfless.SyscallMsg, regs syscall.PtraceRegs) error { + msg.Type = ebpfless.SyscallTypeSetUID + msg.SetUID = &ebpfless.SetUIDSyscallMsg{ + UID: tracer.ReadArgInt32(regs, 0), + EUID: -1, + } + return nil +} + +func handleSetgid(tracer *Tracer, _ *Process, msg *ebpfless.SyscallMsg, regs syscall.PtraceRegs) error { + msg.Type = ebpfless.SyscallTypeSetGID + msg.SetGID = &ebpfless.SetGIDSyscallMsg{ + GID: tracer.ReadArgInt32(regs, 0), + } + return nil +} + +func handleSetreuid(tracer *Tracer, _ *Process, msg *ebpfless.SyscallMsg, regs syscall.PtraceRegs) error { + msg.Type = ebpfless.SyscallTypeSetUID + msg.SetUID = &ebpfless.SetUIDSyscallMsg{ + UID: tracer.ReadArgInt32(regs, 0), + EUID: tracer.ReadArgInt32(regs, 1), + } + return nil +} + +func handleSetregid(tracer *Tracer, _ *Process, msg *ebpfless.SyscallMsg, regs syscall.PtraceRegs) error { + msg.Type = ebpfless.SyscallTypeSetGID + msg.SetGID = &ebpfless.SetGIDSyscallMsg{ + GID: tracer.ReadArgInt32(regs, 0), + EGID: tracer.ReadArgInt32(regs, 1), + } + return nil +} + // ECSMetadata defines ECS metadatas type ECSMetadata struct { DockerID string `json:"DockerId"` @@ -436,6 +471,14 @@ func StartCWSPtracer(args []string, probeAddr string, creds Creds, verbose bool) return err } + // first process + process := &Process{ + Pid: tracer.PID, + Nr: make(map[int]*ebpfless.SyscallMsg), + Fd: make(map[int32]string), + } + cache.Add(tracer.PID, process) + go func() { var seq uint64 @@ -517,6 +560,30 @@ func StartCWSPtracer(args []string, probeAddr string, creds Creds, verbose bool) logErrorf("unable to handle execve: %v", err) return } + + // Top level pid, add creds. For the other PIDs the creds will be propagated at the probe side + if process.Pid == tracer.PID { + var uid, gid uint32 + + if creds.UID != nil { + uid = *creds.UID + } else { + uid = uint32(os.Getuid()) + } + + if creds.GID != nil { + gid = *creds.GID + } else { + gid = uint32(os.Getgid()) + } + + msg.Exec.Credentials = &ebpfless.Credentials{ + UID: uid, + EUID: uid, + GID: gid, + EGID: gid, + } + } case ExecveatNr: if err = handleExecveAt(tracer, process, msg, regs); err != nil { logErrorf("unable to handle execveat: %v", err) @@ -539,7 +606,26 @@ func StartCWSPtracer(args []string, probeAddr string, creds Creds, verbose bool) logErrorf("unable to handle fchdir: %v", err) return } - + case SetuidNr: + if err = handleSetuid(tracer, process, msg, regs); err != nil { + logErrorf("unable to handle fchdir: %v", err) + return + } + case SetgidNr: + if err = handleSetgid(tracer, process, msg, regs); err != nil { + logErrorf("unable to handle fchdir: %v", err) + return + } + case SetreuidNr: + if err = handleSetreuid(tracer, process, msg, regs); err != nil { + logErrorf("unable to handle fchdir: %v", err) + return + } + case SetregidNr: + if err = handleSetregid(tracer, process, msg, regs); err != nil { + logErrorf("unable to handle fchdir: %v", err) + return + } } case CallbackPostType: switch nr { @@ -547,17 +633,25 @@ func StartCWSPtracer(args []string, probeAddr string, creds Creds, verbose bool) send(process.Nr[nr]) case OpenNr, OpenatNr: if ret := tracer.ReadRet(regs); ret >= 0 { - msg, exists := process.Nr[nr] if !exists { return } - send(process.Nr[nr]) + send(msg) // maintain fd/path mapping process.Fd[int32(ret)] = msg.Open.Filename } + case SetuidNr, SetgidNr, SetreuidNr, SetregidNr: + if ret := tracer.ReadRet(regs); ret >= 0 { + msg, exists := process.Nr[nr] + if !exists { + return + } + + send(msg) + } case ForkNr, VforkNr, CloneNr: msg := &ebpfless.SyscallMsg{ ContainerContext: &containerCtx, diff --git a/pkg/security/ptracer/syscalls_amd64.go b/pkg/security/ptracer/syscalls_amd64.go index 92cad5f4f8f8c..3f602a93b4e24 100644 --- a/pkg/security/ptracer/syscalls_amd64.go +++ b/pkg/security/ptracer/syscalls_amd64.go @@ -29,6 +29,10 @@ const ( Dup3Nr = 292 // Dup3Nr defines the syscall ID for amd64 ChdirNr = 80 // ChdirNr defines the syscall ID for amd64 FchdirNr = 81 // FchdirNr defines the syscall ID for amd64 + SetuidNr = 105 // SetuidNr defines the syscall ID for amd64 + SetgidNr = 106 // SetgidNr defines the syscall ID for amd64 + SetreuidNr = 113 // SetreuidNr defines the syscall ID for amd64 + SetregidNr = 114 // SetregidNr defines the syscall ID for amd64 ptraceFlags = 0 | syscall.PTRACE_O_TRACEVFORK | @@ -57,6 +61,10 @@ var ( "dup3", "chdir", "fchdir", + "setuid", + "setgid", + "setreuid", + "setregid", } ) diff --git a/pkg/security/ptracer/syscalls_arm64.go b/pkg/security/ptracer/syscalls_arm64.go index 35561f458ec20..86a8cff2d2ac6 100644 --- a/pkg/security/ptracer/syscalls_arm64.go +++ b/pkg/security/ptracer/syscalls_arm64.go @@ -21,15 +21,19 @@ const ( CloneNr = 220 // CloneNr defines the syscall ID for arm64 ExitNr = 93 // ExitNr defines the syscall ID for arm64 FcntlNr = 25 // FcntlNr defines the syscall ID for arm64 - DupNr = 23 // DupNr defines the syscall ID for amd64 - Dup3Nr = 24 // Dup3Nr defines the syscall ID for amd64 - ChdirNr = 49 // ChdirNr defines the syscall ID for amd64 - FchdirNr = 50 // FchdirNr defines the syscall ID for amd64 + DupNr = 23 // DupNr defines the syscall ID for arm64 + Dup3Nr = 24 // Dup3Nr defines the syscall ID for arm64 + ChdirNr = 49 // ChdirNr defines the syscall ID for arm64 + FchdirNr = 50 // FchdirNr defines the syscall ID for arm64 + SetuidNr = 146 // SetuidNr defines the syscall ID for arm64 + SetgidNr = 144 // SetgidNr defines the syscall ID for arm64 + SetreuidNr = 145 // SetreuidNr defines the syscall ID for arm64 + SetregidNr = 143 // SetregidNr defines the syscall ID for arm64 - OpenNr = 9990 // OpenNr not available on arm64 - ForkNr = 9991 // ForkNr not available on arm64 - VforkNr = 9992 // VforkNr not available on arm64 - Dup2Nr = 9993 // Dup2Nr not available on arm64 + OpenNr = -1 // OpenNr not available on arm64 + ForkNr = -2 // ForkNr not available on arm64 + VforkNr = -3 // VforkNr not available on arm64 + Dup2Nr = -4 // Dup2Nr not available on arm64 ptraceFlags = 0 | syscall.PTRACE_O_TRACECLONE | @@ -52,6 +56,12 @@ var ( "dup3", "chdir", "fchdir", + "setuid", + "setgid", + "setuid", + "setgid", + "setreuid", + "setregid", } ) diff --git a/pkg/security/resolvers/process/resolver_ebpfless.go b/pkg/security/resolvers/process/resolver_ebpfless.go index fe0da8f973563..4e79dcabf83d1 100644 --- a/pkg/security/resolvers/process/resolver_ebpfless.go +++ b/pkg/security/resolvers/process/resolver_ebpfless.go @@ -156,6 +156,8 @@ func (p *EBPFLessResolver) insertForkEntry(key CacheResolverKey, entry *model.Pr func (p *EBPFLessResolver) insertExecEntry(key CacheResolverKey, entry *model.ProcessCacheEntry) { prev := p.entryCache[key] if prev != nil { + // inheritate credentials as exec event, uid/gid can be update by setuid/setgid events + entry.Credentials = prev.Credentials prev.Exec(entry) } @@ -172,6 +174,38 @@ func (p *EBPFLessResolver) Resolve(key CacheResolverKey) *model.ProcessCacheEntr return nil } +// UpdateUID updates the credentials of the provided pid +func (p *EBPFLessResolver) UpdateUID(key CacheResolverKey, uid int32, euid int32) { + p.Lock() + defer p.Unlock() + + entry := p.entryCache[key] + if entry != nil { + if uid != -1 { + entry.Credentials.UID = uint32(uid) + } + if euid != -1 { + entry.Credentials.EUID = uint32(euid) + } + } +} + +// UpdateGID updates the credentials of the provided pid +func (p *EBPFLessResolver) UpdateGID(key CacheResolverKey, gid int32, egid int32) { + p.Lock() + defer p.Unlock() + + entry := p.entryCache[key] + if entry != nil { + if gid != -1 { + entry.Credentials.GID = uint32(gid) + } + if egid != -1 { + entry.Credentials.EGID = uint32(egid) + } + } +} + // getCacheSize returns the cache size of the process resolver func (p *EBPFLessResolver) getCacheSize() float64 { p.RLock() From bc10f388637f3eb22d26ae4b8c20f2f5a7f8bee2 Mon Sep 17 00:00:00 2001 From: Joshua Lineaweaver Date: Fri, 15 Dec 2023 12:02:32 -0500 Subject: [PATCH 05/17] [Orchestrator] Set HPA to stable in cluster agent (#21589) * Set HPA to stable in cluster agent * Add release note --- .../orchestrator/collectors/k8s/horizontalpodautoscaler.go | 2 +- .../notes/HorizontalPodAutoscalerEnable-353a13c703f9dc48.yaml | 4 ++++ 2 files changed, 5 insertions(+), 1 deletion(-) create mode 100644 releasenotes-dca/notes/HorizontalPodAutoscalerEnable-353a13c703f9dc48.yaml diff --git a/pkg/collector/corechecks/cluster/orchestrator/collectors/k8s/horizontalpodautoscaler.go b/pkg/collector/corechecks/cluster/orchestrator/collectors/k8s/horizontalpodautoscaler.go index c91450daeac68..8747a95c19f9a 100644 --- a/pkg/collector/corechecks/cluster/orchestrator/collectors/k8s/horizontalpodautoscaler.go +++ b/pkg/collector/corechecks/cluster/orchestrator/collectors/k8s/horizontalpodautoscaler.go @@ -41,7 +41,7 @@ func NewHorizontalPodAutoscalerCollector() *HorizontalPodAutoscalerCollector { return &HorizontalPodAutoscalerCollector{ metadata: &collectors.CollectorMetadata{ IsDefaultVersion: true, - IsStable: false, + IsStable: true, IsMetadataProducer: true, IsManifestProducer: true, SupportsManifestBuffering: true, diff --git a/releasenotes-dca/notes/HorizontalPodAutoscalerEnable-353a13c703f9dc48.yaml b/releasenotes-dca/notes/HorizontalPodAutoscalerEnable-353a13c703f9dc48.yaml new file mode 100644 index 0000000000000..ff645c6e9ae78 --- /dev/null +++ b/releasenotes-dca/notes/HorizontalPodAutoscalerEnable-353a13c703f9dc48.yaml @@ -0,0 +1,4 @@ +--- +features: + - | + Enable Horizontal Pod Autoscaler collection for the Orchestrator by default From 578dbd4e7eb4366bebc11d10d6260ab545ce2ac2 Mon Sep 17 00:00:00 2001 From: Yakov Shapiro Date: Fri, 15 Dec 2023 12:45:41 -0500 Subject: [PATCH 06/17] Allow the install type to be unset for the trace agent (#21578) --- pkg/trace/agent/agent.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/trace/agent/agent.go b/pkg/trace/agent/agent.go index 54649a968f686..56c0a622f93ec 100644 --- a/pkg/trace/agent/agent.go +++ b/pkg/trace/agent/agent.go @@ -234,7 +234,7 @@ func (a *Agent) setRootSpanTags(root *pb.Span) { // setFirstTraceTags sets additional tags on the first trace ever processed by the agent, // so that we can see that the customer has successfully onboarded onto APM. func (a *Agent) setFirstTraceTags(root *pb.Span) { - if a.conf == nil || a.conf.InstallSignature.InstallType == "" || root == nil { + if a.conf == nil || a.conf.InstallSignature.InstallID == "" || root == nil { return } a.firstSpanOnce.Do(func() { From d5a0f884e6ab2bef9f8f9325c27cd637039dbd62 Mon Sep 17 00:00:00 2001 From: Pierre Gimalac Date: Fri, 15 Dec 2023 19:44:21 +0100 Subject: [PATCH 07/17] Add used_by_otel field in modules.py and update update-go invoke task (#21588) Add used_by_otel field in modules.py and update update-go invoke task --- tasks/modules.py | 20 ++++++++++++-------- tasks/update_go.py | 14 ++++++++++---- 2 files changed, 22 insertions(+), 12 deletions(-) diff --git a/tasks/modules.py b/tasks/modules.py index 5837a86d59c34..a04b7ab378211 100644 --- a/tasks/modules.py +++ b/tasks/modules.py @@ -27,6 +27,7 @@ def __init__( importable=True, independent=False, lint_targets=None, + used_by_otel=False, ): self.path = path self.targets = targets if targets else ["."] @@ -39,6 +40,7 @@ def __init__( # at the cost of spending some time parsing the module. self.importable = importable self.independent = independent + self.used_by_otel = used_by_otel self._dependencies = None @@ -154,10 +156,10 @@ def dependency_path(self, agent_version): "test/fakeintake": GoModule("test/fakeintake", independent=True), "pkg/aggregator/ckey": GoModule("pkg/aggregator/ckey", independent=True), "pkg/errors": GoModule("pkg/errors", independent=True), - "pkg/obfuscate": GoModule("pkg/obfuscate", independent=True), + "pkg/obfuscate": GoModule("pkg/obfuscate", independent=True, used_by_otel=True), "pkg/gohai": GoModule("pkg/gohai", independent=True, importable=False), - "pkg/proto": GoModule("pkg/proto", independent=True), - "pkg/trace": GoModule("pkg/trace", independent=True), + "pkg/proto": GoModule("pkg/proto", independent=True, used_by_otel=True), + "pkg/trace": GoModule("pkg/trace", independent=True, used_by_otel=True), "pkg/tagset": GoModule("pkg/tagset", independent=True), "pkg/metrics": GoModule("pkg/metrics", independent=True), "pkg/telemetry": GoModule("pkg/telemetry", independent=True), @@ -170,12 +172,14 @@ def dependency_path(self, agent_version): "pkg/config/remote": GoModule("pkg/config/remote", independent=True), "pkg/security/secl": GoModule("pkg/security/secl", independent=True), "pkg/status/health": GoModule("pkg/status/health", independent=True), - "pkg/remoteconfig/state": GoModule("pkg/remoteconfig/state", independent=True), - "pkg/util/cgroups": GoModule("pkg/util/cgroups", independent=True, condition=lambda: sys.platform == "linux"), + "pkg/remoteconfig/state": GoModule("pkg/remoteconfig/state", independent=True, used_by_otel=True), + "pkg/util/cgroups": GoModule( + "pkg/util/cgroups", independent=True, condition=lambda: sys.platform == "linux", used_by_otel=True + ), "pkg/util/http": GoModule("pkg/util/http", independent=True), - "pkg/util/log": GoModule("pkg/util/log", independent=True), - "pkg/util/pointer": GoModule("pkg/util/pointer", independent=True), - "pkg/util/scrubber": GoModule("pkg/util/scrubber", independent=True), + "pkg/util/log": GoModule("pkg/util/log", independent=True, used_by_otel=True), + "pkg/util/pointer": GoModule("pkg/util/pointer", independent=True, used_by_otel=True), + "pkg/util/scrubber": GoModule("pkg/util/scrubber", independent=True, used_by_otel=True), "pkg/util/backoff": GoModule("pkg/util/backoff", independent=True), "pkg/util/cache": GoModule("pkg/util/cache", independent=True), "pkg/util/common": GoModule("pkg/util/common", independent=True), diff --git a/tasks/update_go.py b/tasks/update_go.py index 7b3d6e3463ab6..153979db0e3b0 100644 --- a/tasks/update_go.py +++ b/tasks/update_go.py @@ -24,6 +24,7 @@ def go_version(_): "test_version": "Whether the image is a test image or not", "warn": "Don't exit in case of matching error, just warn.", "release_note": "Whether to create a release note or not. The default behaviour is to create a release note", + "include_otel_modules": "Whether to update the version in go.mod files used by otel.", } ) def update_go( @@ -33,6 +34,7 @@ def update_go( test_version: Optional[bool] = False, warn: Optional[bool] = False, release_note: Optional[bool] = True, + include_otel_modules: Optional[bool] = False, ): """ Updates the version of Go and build images. @@ -68,7 +70,7 @@ def update_go( _update_root_readme(warn, new_major) _update_fakeintake_readme(warn, new_major) - _update_go_mods(warn, new_major) + _update_go_mods(warn, new_major, include_otel_modules) _update_process_agent_readme(warn, new_major) _update_windowsevent_readme(warn, new_major) _update_go_version_file(warn, version) @@ -217,9 +219,13 @@ def _update_windowsevent_readme(warn: bool, major: str): _update_file(warn, path, pattern, replace) -def _update_go_mods(warn: bool, major: str): - mod_files = [f"./{module}/go.mod" for module in DEFAULT_MODULES] - for mod_file in mod_files: +def _update_go_mods(warn: bool, major: str, include_otel_modules: bool): + for path, module in DEFAULT_MODULES.items(): + if not include_otel_modules and module.used_by_otel: + # only update the go directives in go.mod files not used by otel + # to allow them to keep using the modules + continue + mod_file = f"./{path}/go.mod" _update_file(warn, mod_file, "^go [.0-9]+$", f"go {major}") From d15268d030a6b06f163623c6940248ba81ec38df Mon Sep 17 00:00:00 2001 From: Bryce Kahle Date: Fri, 15 Dec 2023 11:26:02 -0800 Subject: [PATCH 08/17] fix KMT cleanup null check (#21576) fix KMT cleanup null check --- .gitlab/kernel_version_testing/system_probe.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab/kernel_version_testing/system_probe.yml b/.gitlab/kernel_version_testing/system_probe.yml index c7745d7a56d64..0d169eaf0d164 100644 --- a/.gitlab/kernel_version_testing/system_probe.yml +++ b/.gitlab/kernel_version_testing/system_probe.yml @@ -437,7 +437,7 @@ kernel_matrix_testing_run_tests_arm64: - INSTANCE_ID="$(jq -r '.[0][0]' < instance.json)" - echo ${INSTANCE_ID} - | - if [[ "${INSTANCE_ID}" != "" ]]; then + if [[ "${INSTANCE_ID}" != "" ]] && [[ "${INSTANCE_ID}" != "null" ]]; then aws ec2 terminate-instances --instance-ids "${INSTANCE_ID}" fi From 9c2a358144c1bec888f089d785a2555de1a2188a Mon Sep 17 00:00:00 2001 From: Pierre Gimalac Date: Fri, 15 Dec 2023 21:09:46 +0100 Subject: [PATCH 09/17] [ASCII-977] Go update auto-comment old versions (#21476) [ASCII-977] Go update auto-comment old versions --- .github/CODEOWNERS | 2 + .github/workflows/go-update-commenter.yml | 65 ++++++++++++++++ tools/go-update/detect-old-version.sh | 92 +++++++++++++++++++++++ 3 files changed, 159 insertions(+) create mode 100644 .github/workflows/go-update-commenter.yml create mode 100644 tools/go-update/detect-old-version.sh diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index 200c83e62ab97..bcf2998e991e3 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -47,6 +47,7 @@ /.github/workflows/windows-*.yml @DataDog/windows-agent /.github/workflows/cws-btfhub-sync.yml @DataDog/agent-security /.github/workflows/gohai.yml @DataDog/agent-shared-components +/.github/workflows/go-update-commenter.yml @DataDog/agent-shared-components # Gitlab files # Files containing job contents are owned by teams in charge of the jobs + agent-platform @@ -507,6 +508,7 @@ /tools/ @DataDog/agent-platform /tools/ebpf/ @DataDog/ebpf-platform /tools/gdb/ @DataDog/agent-shared-components +/tools/go-update/ @DataDog/agent-shared-components /tools/retry_file_dump/ @DataDog/agent-metrics-logs /tools/windows/ @DataDog/windows-agent /tools/windows/DatadogAgentInstaller/WixSetup/localization-en-us.wxl @DataDog/windows-agent @DataDog/documentation diff --git a/.github/workflows/go-update-commenter.yml b/.github/workflows/go-update-commenter.yml new file mode 100644 index 0000000000000..c25a907d753ec --- /dev/null +++ b/.github/workflows/go-update-commenter.yml @@ -0,0 +1,65 @@ +name: "Go update commenter" + +on: + pull_request: + # Only run on PR label events (in particular not on every commit) + types: [ labeled ] + +jobs: + old-versions-match: + # Only run if the PR is labeled with 'go-update' + if: ${{ github.event.label.name == 'go-update' }} + runs-on: ubuntu-latest + steps: + # get the Go version of the target branch + - uses: actions/checkout@v3 + with: + ref: ${{ github.base_ref }} + - name: Get former Go version + id: former_go_version + run: | + echo version="$(cat .go-version)" >> $GITHUB_OUTPUT + + # get the Go version of the PR branch + - uses: actions/checkout@v3 + - name: Get current Go version + id: new_go_version + run: | + echo version="$(cat .go-version)" >> $GITHUB_OUTPUT + + # build the comment + - name: Build full comment + id: old_versions + run: | + set -euxo pipefail + # build the base of the Github URL to the current commit + GITHUB_HEAD_URL='${{ github.server_url }}/${{ github.repository }}/blob/${{ github.sha }}' + { + echo "matches<> $GITHUB_OUTPUT + + # and display it + - uses: actions/github-script@v7 + env: + # We need to store the output in an environment variable and not use it directly in the createComment, + # as it will likely not be a valid JS string (eg. if it contains a quote character) + CONTENT: ${{ steps.old_versions.outputs.matches }} + with: + script: | + github.rest.issues.createComment({ + issue_number: context.issue.number, + owner: context.repo.owner, + repo: context.repo.repo, + body: process.env.CONTENT + }) diff --git a/tools/go-update/detect-old-version.sh b/tools/go-update/detect-old-version.sh new file mode 100644 index 0000000000000..d8cc714e1420f --- /dev/null +++ b/tools/go-update/detect-old-version.sh @@ -0,0 +1,92 @@ +#!/bin/bash + +set -euo pipefail + +# This script is used to display files which might contain a reference to an old Go version. +# +# It can be explicitely given a Go version as argument to look for. +# Otherwise, it assumes that the current branch is the one which contains the new Go version, +# and compares it to $GITHUB_BASE_REF if it is defined, or "main" otherwise + +if [ $# -gt 2 ]; then + echo "This script takes at most two arguments, the old Go version we want to look for and the new one" 1>&2 + echo "If no argument is given, they are fetched from the '.go-version' file, respectively of the branch from GITHUB_BASE_REF, or main if it is not defined." 1>&2 + echo "" 1>&2 + echo "If only one version is given, it is looked for without any particular checking." 1>&2 + echo "If zero or two versions are given, the script checks whether the minor changed or only the bugfix, and uses either accordingly." 1>&2 + exit 1 +fi + +if [ $# -eq 1 ]; then + # use the version given as argument + PATTERN_GO_VERSION="$1" +else + if [ $# -eq 0 ]; then + # use the version from the .go-version file, and compare it to the one from GITHUB_BASE_REF, or main + GO_VERSION_PREV_BUGFIX=$(git show "${GITHUB_BASE_REF:-main}":.go-version) + GO_VERSION_NEW_BUGFIX=$(cat .go-version) + else + GO_VERSION_PREV_BUGFIX="$1" + GO_VERSION_NEW_BUGFIX="$2" + fi + + GO_VERSION_PREV_MINOR="${GO_VERSION_PREV_BUGFIX%.*}" + echo "Former bugfix version: $GO_VERSION_PREV_BUGFIX" 1>&2 + echo "Former minor version: $GO_VERSION_PREV_MINOR" 1>&2 + + GO_VERSION_NEW_MINOR="${GO_VERSION_NEW_BUGFIX%.*}" + echo "New bugfix version: $GO_VERSION_NEW_BUGFIX" 1>&2 + echo "New minor version: $GO_VERSION_NEW_MINOR" 1>&2 + + # if the old bugfix is the same as the new one, return + if [ "$GO_VERSION_PREV_BUGFIX" == "$GO_VERSION_NEW_BUGFIX" ]; then + echo "This branch doesn't change the Go version" 1>&2 + exit 1 + fi + + if [ "$GO_VERSION_PREV_MINOR" != "$GO_VERSION_NEW_MINOR" ]; then + # minor update + PATTERN_GO_VERSION="$GO_VERSION_PREV_MINOR" + else + # bugfix update + PATTERN_GO_VERSION="$GO_VERSION_PREV_BUGFIX" + fi +fi + +echo "Looking for Go version: $PATTERN_GO_VERSION" 1>&2 + +# Go versions can be preceded by a 'g' (golang), 'o' (go), 'v', or a non-alphanumerical character +# Prevent matching when preceded by a dot, as it is likely not a Go version in that case +PATTERN_PREFIX='(^|[^.a-fh-np-uw-z0-9])' +# Go versions in go.dev/dl URLs are followed by a dot, so we need to allow it in the regex +PATTERN_SUFFIX='($|[^a-z0-9])' + +# Go versions contain dots, which are special characters in regexes, so we need to escape them +# Safely assume that the version only contains numbers and dots +PATTERN_GO_VERSION_ESCAPED="$(echo "$PATTERN_GO_VERSION" | sed 's/\./\\./g')" +# The regex is not perfect, but it should be good enough +PATTERN="${PATTERN_PREFIX}${PATTERN_GO_VERSION_ESCAPED}${PATTERN_SUFFIX}" +echo "Using pattern: $PATTERN" 1>&2 + +# grep returns 1 when no match is found, which would cause the script to fail, wrap it to return 0 in that case. +# It returns a non-zero value if grep returned >1. +function safegrep() { + grep "$@" || [ $? -eq 1 ] +} + +# -r: recursive +# -I: ignore binary files +# -i: ignore case +# -n: print line number +# -E: extended regexp pattern to match +# --exclude-dir: exclude directories +# --exclude: exclude file name patterns +safegrep -r -I -i -n -E "$PATTERN" . \ + --exclude-dir fixtures --exclude-dir .git --exclude-dir releasenotes \ + --exclude-dir omnibus --exclude-dir snmp --exclude-dir testdata \ + --exclude '*.rst' --exclude '*.sum' --exclude '*generated*.go' --exclude '*.svg' | \ +# -v: invert match +# exclude matches in go.mod files that are dependency versions (note the 'v' before the version) +safegrep -v -E "go\.mod:.*v${PATTERN_GO_VERSION_ESCAPED}" | \ +# grep outputs paths starting with ./, so we remove that +sed -e 's|^./||' From 5771179a3ea3a466b6660e2a26d256b8508fb090 Mon Sep 17 00:00:00 2001 From: Bryce Kahle Date: Fri, 15 Dec 2023 12:55:28 -0800 Subject: [PATCH 10/17] fix ebpf-platform revive issues (#21574) fix ebpf-platform revive issues --- cmd/system-probe/api/client.go | 2 +- cmd/system-probe/api/module/common.go | 2 +- cmd/system-probe/command/command.go | 2 +- cmd/system-probe/common/common.go | 2 +- cmd/system-probe/common/common_nix.go | 2 +- cmd/system-probe/common/common_unsupported.go | 2 +- cmd/system-probe/common/common_windows.go | 2 +- cmd/system-probe/config/adjust.go | 2 +- cmd/system-probe/config/config.go | 14 ++-- cmd/system-probe/config/config_test.go | 43 ++++++----- cmd/system-probe/main.go | 1 - cmd/system-probe/main_common.go | 2 +- cmd/system-probe/modules/all_linux.go | 5 +- cmd/system-probe/modules/all_unsupported.go | 2 +- cmd/system-probe/modules/all_windows.go | 2 +- .../modules/dynamic_instrumentation.go | 2 +- .../modules/language_detection.go | 2 +- cmd/system-probe/subcommands/debug/command.go | 2 +- .../subcommands/modrestart/command.go | 2 +- cmd/system-probe/subcommands/run/command.go | 12 ++- cmd/system-probe/subcommands/subcommands.go | 2 +- .../subcommands/version/command.go | 4 +- cmd/system-probe/utils/limiter.go | 2 +- .../corechecks/ebpf/probe/ebpfcheck/probe.go | 7 +- .../corechecks/ebpf/probe/ebpfcheck/prog.go | 49 ++++++------ .../corechecks/ebpf/probe/ebpfcheck/utils.go | 10 --- pkg/ebpf/bytecode/asset_reader.go | 2 +- pkg/ebpf/bytecode/runtime/all_helpers.go | 2 +- pkg/ebpf/bytecode/runtime/asset.go | 2 +- pkg/ebpf/bytecode/runtime/generated_asset.go | 4 +- pkg/ebpf/bytecode/runtime/helpers_test.go | 2 +- pkg/ebpf/bytecode/runtime/protected_file.go | 17 ++--- .../runtime/runtime_compilation_helpers.go | 2 +- pkg/ebpf/cgo/genpost.go | 2 +- pkg/ebpf/common.go | 2 +- pkg/ebpf/compiler/compiler.go | 6 +- pkg/ebpf/debugfs_stat_collector_linux.go | 7 +- pkg/ebpf/ebpftest/bpfdebug_linux.go | 2 +- pkg/ebpf/ebpftest/bpfdebug_unsupported.go | 2 +- pkg/ebpf/ebpftest/buildmode.go | 2 +- pkg/ebpf/ebpftest/buildmode_linux.go | 6 +- pkg/ebpf/ebpftest/buildmode_windows.go | 6 +- pkg/ebpf/ebpftest/log.go | 2 +- pkg/ebpf/perf.go | 8 +- pkg/network/telemetry/stat_counter_wrapper.go | 11 +-- pkg/network/telemetry/stat_gauge_wrapper.go | 12 +-- pkg/util/kernel/arch.go | 2 +- pkg/util/kernel/download_headers_test.go | 3 +- pkg/util/kernel/find_headers.go | 8 +- pkg/util/kernel/find_headers_test.go | 2 +- pkg/util/kernel/fs_nolinux.go | 4 +- test/new-e2e/scenarios/system-probe/main.go | 9 +-- .../system-probe/system-probe-test-env.go | 75 ++++++++----------- .../system-probe/test-json-review/main.go | 2 +- test/new-e2e/system-probe/test-runner/main.go | 9 +-- 55 files changed, 171 insertions(+), 221 deletions(-) diff --git a/cmd/system-probe/api/client.go b/cmd/system-probe/api/client.go index 58d7350e0dc8c..e6ffb8e341fd3 100644 --- a/cmd/system-probe/api/client.go +++ b/cmd/system-probe/api/client.go @@ -3,7 +3,7 @@ // This product includes software developed at Datadog (https://www.datadoghq.com/). // Copyright 2016-present Datadog, Inc. -//nolint:revive // TODO(EBPF) Fix revive linter +// Package api contains the API exposed by system-probe package api import ( diff --git a/cmd/system-probe/api/module/common.go b/cmd/system-probe/api/module/common.go index e50a10cbf9f81..7a303623cd95f 100644 --- a/cmd/system-probe/api/module/common.go +++ b/cmd/system-probe/api/module/common.go @@ -3,7 +3,7 @@ // This product includes software developed at Datadog (https://www.datadoghq.com/). // Copyright 2016-present Datadog, Inc. -//nolint:revive // TODO(EBPF) Fix revive linter +// Package module is the scaffolding for a system-probe module and the loader used upon start package module import ( diff --git a/cmd/system-probe/command/command.go b/cmd/system-probe/command/command.go index 6206e134f2e4a..2dec9a443b4af 100644 --- a/cmd/system-probe/command/command.go +++ b/cmd/system-probe/command/command.go @@ -3,7 +3,7 @@ // This product includes software developed at Datadog (https://www.datadoghq.com/). // Copyright 2016-present Datadog, Inc. -//nolint:revive // TODO(EBPF) Fix revive linter +// Package command contains utilities for creating system-probe commands package command import ( diff --git a/cmd/system-probe/common/common.go b/cmd/system-probe/common/common.go index ef66dd5bfb3da..60e55c9e71896 100644 --- a/cmd/system-probe/common/common.go +++ b/cmd/system-probe/common/common.go @@ -3,7 +3,7 @@ // This product includes software developed at Datadog (https://www.datadoghq.com/). // Copyright 2016-present Datadog, Inc. -//nolint:revive // TODO(EBPF) Fix revive linter +// Package common is global variables for the system-probe process package common import ( diff --git a/cmd/system-probe/common/common_nix.go b/cmd/system-probe/common/common_nix.go index 77aa9cae82c8b..bb50ddef3b86c 100644 --- a/cmd/system-probe/common/common_nix.go +++ b/cmd/system-probe/common/common_nix.go @@ -8,6 +8,6 @@ package common const ( - //nolint:revive // TODO(EBPF) Fix revive linter + // DefaultLogFile is the default path to the system-probe log file DefaultLogFile = "/var/log/datadog/system-probe.log" ) diff --git a/cmd/system-probe/common/common_unsupported.go b/cmd/system-probe/common/common_unsupported.go index 81f2ac7f2bfec..024ab6ccb3cfe 100644 --- a/cmd/system-probe/common/common_unsupported.go +++ b/cmd/system-probe/common/common_unsupported.go @@ -8,6 +8,6 @@ package common const ( - //nolint:revive // TODO(EBPF) Fix revive linter + // DefaultLogFile is the default path to the system-probe log file DefaultLogFile = "" ) diff --git a/cmd/system-probe/common/common_windows.go b/cmd/system-probe/common/common_windows.go index 7da3eeef06321..a88a8001241bc 100644 --- a/cmd/system-probe/common/common_windows.go +++ b/cmd/system-probe/common/common_windows.go @@ -6,6 +6,6 @@ package common const ( - //nolint:revive // TODO(EBPF) Fix revive linter + // DefaultLogFile is the default path to the system-probe log file DefaultLogFile = "c:\\programdata\\datadog\\logs\\system-probe.log" ) diff --git a/cmd/system-probe/config/adjust.go b/cmd/system-probe/config/adjust.go index d2b8e30a01a1b..1856aa69a791e 100644 --- a/cmd/system-probe/config/adjust.go +++ b/cmd/system-probe/config/adjust.go @@ -3,7 +3,7 @@ // This product includes software developed at Datadog (https://www.datadoghq.com/). // Copyright 2016-present Datadog, Inc. -//nolint:revive // TODO(EBPF) Fix revive linter +// Package config contains the general configuration for system-probe package config import ( diff --git a/cmd/system-probe/config/config.go b/cmd/system-probe/config/config.go index 1d8830400a72c..100bbd1ae31ad 100644 --- a/cmd/system-probe/config/config.go +++ b/cmd/system-probe/config/config.go @@ -95,18 +95,16 @@ func newSysprobeConfig(configPath string) (*Config, error) { // load the configuration _, err := aconfig.LoadCustom(aconfig.SystemProbe, "system-probe", optional.NewNoneOption[secrets.Component](), aconfig.Datadog.GetEnvVars()) if err != nil { - var e viper.ConfigFileNotFoundError - //nolint:revive // TODO(EBPF) Fix revive linter - if errors.As(err, &e) || errors.Is(err, os.ErrNotExist) { - // do nothing, we can ignore a missing system-probe.yaml config file - } else if errors.Is(err, fs.ErrPermission) { + if errors.Is(err, fs.ErrPermission) { // special-case permission-denied with a clearer error message if runtime.GOOS == "windows" { return nil, fmt.Errorf(`cannot access the system-probe config file (%w); try running the command in an Administrator shell"`, err) - } else { //nolint:revive // TODO(EBPF) Fix revive linter - return nil, fmt.Errorf("cannot access the system-probe config file (%w); try running the command under the same user as the Datadog Agent", err) } - } else { + return nil, fmt.Errorf("cannot access the system-probe config file (%w); try running the command under the same user as the Datadog Agent", err) + } + + var e viper.ConfigFileNotFoundError + if !errors.As(err, &e) && !errors.Is(err, os.ErrNotExist) { return nil, fmt.Errorf("unable to load system-probe config file: %w", err) } } diff --git a/cmd/system-probe/config/config_test.go b/cmd/system-probe/config/config_test.go index 051bb611c20a8..e50baaf38f9ea 100644 --- a/cmd/system-probe/config/config_test.go +++ b/cmd/system-probe/config/config_test.go @@ -32,34 +32,33 @@ func TestEventMonitor(t *testing.T) { newConfig(t) for i, tc := range []struct { - //nolint:revive // TODO(EBPF) Fix revive linter - cws, fim, process_events, network_events bool - enabled bool + cws, fim, processEvents, networkEvents bool + enabled bool }{ - {cws: false, fim: false, process_events: false, network_events: false, enabled: false}, - {cws: false, fim: false, process_events: true, network_events: false, enabled: true}, - {cws: false, fim: true, process_events: false, network_events: false, enabled: true}, - {cws: false, fim: true, process_events: true, network_events: false, enabled: true}, - {cws: true, fim: false, process_events: false, network_events: false, enabled: true}, - {cws: true, fim: false, process_events: true, network_events: false, enabled: true}, - {cws: true, fim: true, process_events: false, network_events: false, enabled: true}, - {cws: true, fim: true, process_events: true, network_events: false, enabled: true}, - {cws: false, fim: false, process_events: false, network_events: true, enabled: true}, - {cws: false, fim: false, process_events: true, network_events: true, enabled: true}, - {cws: false, fim: true, process_events: false, network_events: true, enabled: true}, - {cws: false, fim: true, process_events: true, network_events: true, enabled: true}, - {cws: true, fim: false, process_events: false, network_events: true, enabled: true}, - {cws: true, fim: false, process_events: true, network_events: true, enabled: true}, - {cws: true, fim: true, process_events: false, network_events: true, enabled: true}, - {cws: true, fim: true, process_events: true, network_events: true, enabled: true}, + {cws: false, fim: false, processEvents: false, networkEvents: false, enabled: false}, + {cws: false, fim: false, processEvents: true, networkEvents: false, enabled: true}, + {cws: false, fim: true, processEvents: false, networkEvents: false, enabled: true}, + {cws: false, fim: true, processEvents: true, networkEvents: false, enabled: true}, + {cws: true, fim: false, processEvents: false, networkEvents: false, enabled: true}, + {cws: true, fim: false, processEvents: true, networkEvents: false, enabled: true}, + {cws: true, fim: true, processEvents: false, networkEvents: false, enabled: true}, + {cws: true, fim: true, processEvents: true, networkEvents: false, enabled: true}, + {cws: false, fim: false, processEvents: false, networkEvents: true, enabled: true}, + {cws: false, fim: false, processEvents: true, networkEvents: true, enabled: true}, + {cws: false, fim: true, processEvents: false, networkEvents: true, enabled: true}, + {cws: false, fim: true, processEvents: true, networkEvents: true, enabled: true}, + {cws: true, fim: false, processEvents: false, networkEvents: true, enabled: true}, + {cws: true, fim: false, processEvents: true, networkEvents: true, enabled: true}, + {cws: true, fim: true, processEvents: false, networkEvents: true, enabled: true}, + {cws: true, fim: true, processEvents: true, networkEvents: true, enabled: true}, } { t.Run(fmt.Sprintf("%d", i), func(t *testing.T) { t.Logf("%+v\n", tc) t.Setenv("DD_RUNTIME_SECURITY_CONFIG_ENABLED", strconv.FormatBool(tc.cws)) t.Setenv("DD_RUNTIME_SECURITY_CONFIG_FIM_ENABLED", strconv.FormatBool(tc.fim)) - t.Setenv("DD_SYSTEM_PROBE_EVENT_MONITORING_PROCESS_ENABLED", strconv.FormatBool(tc.process_events)) - t.Setenv("DD_SYSTEM_PROBE_EVENT_MONITORING_NETWORK_PROCESS_ENABLED", strconv.FormatBool(tc.network_events)) - t.Setenv("DD_SYSTEM_PROBE_NETWORK_ENABLED", strconv.FormatBool(tc.network_events)) + t.Setenv("DD_SYSTEM_PROBE_EVENT_MONITORING_PROCESS_ENABLED", strconv.FormatBool(tc.processEvents)) + t.Setenv("DD_SYSTEM_PROBE_EVENT_MONITORING_NETWORK_PROCESS_ENABLED", strconv.FormatBool(tc.networkEvents)) + t.Setenv("DD_SYSTEM_PROBE_NETWORK_ENABLED", strconv.FormatBool(tc.networkEvents)) cfg, err := New("/doesnotexist") t.Logf("%+v\n", cfg) diff --git a/cmd/system-probe/main.go b/cmd/system-probe/main.go index 20084530887fc..5a40451d37a80 100644 --- a/cmd/system-probe/main.go +++ b/cmd/system-probe/main.go @@ -5,7 +5,6 @@ //go:build linux -//nolint:revive // TODO(EBPF) Fix revive linter package main import ( diff --git a/cmd/system-probe/main_common.go b/cmd/system-probe/main_common.go index 8fa26b5d90b08..b27a87970430c 100644 --- a/cmd/system-probe/main_common.go +++ b/cmd/system-probe/main_common.go @@ -5,7 +5,7 @@ //go:build linux || windows -//nolint:revive // TODO(EBPF) Fix revive linter +// Package main is the entrypoint for system-probe process package main import ( diff --git a/cmd/system-probe/modules/all_linux.go b/cmd/system-probe/modules/all_linux.go index de2f5429c9427..6b34b702dec73 100644 --- a/cmd/system-probe/modules/all_linux.go +++ b/cmd/system-probe/modules/all_linux.go @@ -5,7 +5,7 @@ //go:build linux -//nolint:revive // TODO(EBPF) Fix revive linter +// Package modules is all the module definitions for system-probe package modules import ( @@ -27,7 +27,6 @@ var All = []module.Factory{ ComplianceModule, } -//nolint:revive // TODO(EBPF) Fix revive linter -func inactivityEventLog(duration time.Duration) { +func inactivityEventLog(_ time.Duration) { } diff --git a/cmd/system-probe/modules/all_unsupported.go b/cmd/system-probe/modules/all_unsupported.go index 22eb6ca73f741..59e3fd37f5edd 100644 --- a/cmd/system-probe/modules/all_unsupported.go +++ b/cmd/system-probe/modules/all_unsupported.go @@ -5,7 +5,7 @@ //go:build !linux && !windows -//nolint:revive // TODO(EBPF) Fix revive linter +// Package modules is all the module definitions for system-probe package modules import "github.com/DataDog/datadog-agent/cmd/system-probe/api/module" diff --git a/cmd/system-probe/modules/all_windows.go b/cmd/system-probe/modules/all_windows.go index fdf9b4beba52d..554c4a3eda939 100644 --- a/cmd/system-probe/modules/all_windows.go +++ b/cmd/system-probe/modules/all_windows.go @@ -5,7 +5,7 @@ //go:build windows -//nolint:revive // TODO(EBPF) Fix revive linter +// Package modules is all the module definitions for system-probe package modules import ( diff --git a/cmd/system-probe/modules/dynamic_instrumentation.go b/cmd/system-probe/modules/dynamic_instrumentation.go index bdb089772ff99..343c89e269bf3 100644 --- a/cmd/system-probe/modules/dynamic_instrumentation.go +++ b/cmd/system-probe/modules/dynamic_instrumentation.go @@ -16,7 +16,7 @@ import ( "github.com/DataDog/datadog-agent/pkg/ebpf" ) -//nolint:revive // TODO(EBPF) Fix revive linter +// DynamicInstrumentation is the dynamic instrumentation module factory var DynamicInstrumentation = module.Factory{ Name: config.DynamicInstrumentationModule, ConfigNamespaces: []string{}, diff --git a/cmd/system-probe/modules/language_detection.go b/cmd/system-probe/modules/language_detection.go index 9e6f8ef33c85c..6bbc6ed7d96f5 100644 --- a/cmd/system-probe/modules/language_detection.go +++ b/cmd/system-probe/modules/language_detection.go @@ -23,7 +23,7 @@ import ( "github.com/DataDog/datadog-agent/pkg/util/log" ) -//nolint:revive // TODO(EBPF) Fix revive linter +// LanguageDetectionModule is the language detection module factory var LanguageDetectionModule = module.Factory{ Name: config.LanguageDetectionModule, ConfigNamespaces: []string{"language_detection"}, diff --git a/cmd/system-probe/subcommands/debug/command.go b/cmd/system-probe/subcommands/debug/command.go index 963801f8c3f52..763ad53334c87 100644 --- a/cmd/system-probe/subcommands/debug/command.go +++ b/cmd/system-probe/subcommands/debug/command.go @@ -3,7 +3,7 @@ // This product includes software developed at Datadog (https://www.datadoghq.com/). // Copyright 2016-present Datadog, Inc. -//nolint:revive // TODO(EBPF) Fix revive linter +// Package debug is the debug system-probe subcommand package debug import ( diff --git a/cmd/system-probe/subcommands/modrestart/command.go b/cmd/system-probe/subcommands/modrestart/command.go index b63730250365e..f94c7f3724567 100644 --- a/cmd/system-probe/subcommands/modrestart/command.go +++ b/cmd/system-probe/subcommands/modrestart/command.go @@ -3,7 +3,7 @@ // This product includes software developed at Datadog (https://www.datadoghq.com/). // Copyright 2016-present Datadog, Inc. -//nolint:revive // TODO(EBPF) Fix revive linter +// Package modrestart is the module-restart system-probe subcommand package modrestart import ( diff --git a/cmd/system-probe/subcommands/run/command.go b/cmd/system-probe/subcommands/run/command.go index 0e37ba0dce1c2..37588c8f4ba74 100644 --- a/cmd/system-probe/subcommands/run/command.go +++ b/cmd/system-probe/subcommands/run/command.go @@ -3,7 +3,7 @@ // This product includes software developed at Datadog (https://www.datadoghq.com/). // Copyright 2016-present Datadog, Inc. -//nolint:revive // TODO(EBPF) Fix revive linter +// Package run is the run system-probe subcommand package run import ( @@ -11,9 +11,7 @@ import ( "errors" "fmt" "net/http" - - //nolint:revive // TODO(EBPF) Fix revive linter - _ "net/http/pprof" + _ "net/http/pprof" // activate pprof profiling "os" "os/signal" "os/user" @@ -132,14 +130,14 @@ func run(log log.Component, _ config.Component, statsd compstatsd.Component, tel sigpipeCh := make(chan os.Signal, 1) signal.Notify(sigpipeCh, syscall.SIGPIPE) go func() { - //nolint:revive // TODO(EBPF) Fix revive linter + //nolint:revive for range sigpipeCh { - // do nothing + // intentionally drain channel } }() if err := startSystemProbe(cliParams, log, statsd, telemetry, sysprobeconfig, rcclient); err != nil { - if err == ErrNotEnabled { + if errors.Is(err, ErrNotEnabled) { // A sleep is necessary to ensure that supervisor registers this process as "STARTED" // If the exit is "too quick", we enter a BACKOFF->FATAL loop even though this is an expected exit // http://supervisord.org/subprocess.html#process-states diff --git a/cmd/system-probe/subcommands/subcommands.go b/cmd/system-probe/subcommands/subcommands.go index ae9b8662626d7..dd2cdaa3b27d1 100644 --- a/cmd/system-probe/subcommands/subcommands.go +++ b/cmd/system-probe/subcommands/subcommands.go @@ -3,7 +3,7 @@ // This product includes software developed at Datadog (https://www.datadoghq.com/). // Copyright 2016-present Datadog, Inc. -//nolint:revive // TODO(EBPF) Fix revive linter +// Package subcommands contains the subcommands for system-probe package subcommands import ( diff --git a/cmd/system-probe/subcommands/version/command.go b/cmd/system-probe/subcommands/version/command.go index 90508b45430c5..b2630d9aa383b 100644 --- a/cmd/system-probe/subcommands/version/command.go +++ b/cmd/system-probe/subcommands/version/command.go @@ -14,8 +14,6 @@ import ( ) // Commands returns a slice of subcommands for the 'agent' command. -// -//nolint:revive // TODO(EBPF) Fix revive linter -func Commands(globalParams *command.GlobalParams) []*cobra.Command { +func Commands(_ *command.GlobalParams) []*cobra.Command { return []*cobra.Command{version.MakeCommand("System Probe")} } diff --git a/cmd/system-probe/utils/limiter.go b/cmd/system-probe/utils/limiter.go index 49fc17c552ed2..6ec6aa71f6e9f 100644 --- a/cmd/system-probe/utils/limiter.go +++ b/cmd/system-probe/utils/limiter.go @@ -3,7 +3,7 @@ // This product includes software developed at Datadog (https://www.datadoghq.com/). // Copyright 2021-present Datadog, Inc. -//nolint:revive // TODO(EBPF) Fix revive linter +// Package utils are utilities for system-probe package utils import ( diff --git a/pkg/collector/corechecks/ebpf/probe/ebpfcheck/probe.go b/pkg/collector/corechecks/ebpf/probe/ebpfcheck/probe.go index 74ac1d26ecaa8..fe1398eb8e8e5 100644 --- a/pkg/collector/corechecks/ebpf/probe/ebpfcheck/probe.go +++ b/pkg/collector/corechecks/ebpf/probe/ebpfcheck/probe.go @@ -146,10 +146,9 @@ func (k *Probe) attach(collSpec *ebpf.CollectionSpec) (err error) { spec := collSpec.Programs[name] switch prog.Type() { case ebpf.Kprobe: - //nolint:revive // TODO(EBPF) Fix revive linter - const kProbePrefix, kretprobePrefix = "kprobe/", "kretprobe/" - if strings.HasPrefix(spec.SectionName, kProbePrefix) { - attachPoint := spec.SectionName[len(kProbePrefix):] + const kprobePrefix, kretprobePrefix = "kprobe/", "kretprobe/" + if strings.HasPrefix(spec.SectionName, kprobePrefix) { + attachPoint := spec.SectionName[len(kprobePrefix):] l, err := link.Kprobe(attachPoint, prog, &link.KprobeOptions{ TraceFSPrefix: "ddebpfc", }) diff --git a/pkg/collector/corechecks/ebpf/probe/ebpfcheck/prog.go b/pkg/collector/corechecks/ebpf/probe/ebpfcheck/prog.go index 981d99cef0d7e..26873926fe0b5 100644 --- a/pkg/collector/corechecks/ebpf/probe/ebpfcheck/prog.go +++ b/pkg/collector/corechecks/ebpf/probe/ebpfcheck/prog.go @@ -24,14 +24,14 @@ func ProgObjInfo(fd uint32, info *ProgInfo) error { return err } -//nolint:revive // TODO(EBPF) Fix revive linter +// ObjGetInfoByFdAttr is the attributes for the BPF_OBJ_GET_INFO_BY_FD mode of the bpf syscall type ObjGetInfoByFdAttr struct { BpfFd uint32 InfoLen uint32 Info Pointer } -//nolint:revive // TODO(EBPF) Fix revive linter +// ObjGetInfoByFd implements the BPF_OBJ_GET_INFO_BY_FD mode of the bpf syscall func ObjGetInfoByFd(attr *ObjGetInfoByFdAttr) error { _, _, errNo := unix.Syscall(unix.SYS_BPF, uintptr(unix.BPF_OBJ_GET_INFO_BY_FD), uintptr(unsafe.Pointer(attr)), unsafe.Sizeof(*attr)) if errNo != 0 { @@ -58,30 +58,27 @@ func ProgGetFdByID(attr *ProgGetFdByIDAttr) (uint32, error) { // ProgInfo corresponds to kernel C type `bpf_prog_info` type ProgInfo struct { - Type uint32 - //nolint:revive // TODO(EBPF) Fix revive linter - Id uint32 - Tag [8]uint8 - JitedProgLen uint32 - XlatedProgLen uint32 - JitedProgInsns uint64 - XlatedProgInsns Pointer - LoadTime uint64 - //nolint:revive // TODO(EBPF) Fix revive linter - CreatedByUid uint32 - NrMapIds uint32 - MapIds Pointer - Name ObjName - Ifindex uint32 - _ [4]byte /* unsupported bitfield */ - NetnsDev uint64 - NetnsIno uint64 - NrJitedKsyms uint32 - NrJitedFuncLens uint32 - JitedKsyms uint64 - JitedFuncLens uint64 - //nolint:revive // TODO(EBPF) Fix revive linter - BtfId BTFID + Type uint32 + ID uint32 + Tag [8]uint8 + JitedProgLen uint32 + XlatedProgLen uint32 + JitedProgInsns uint64 + XlatedProgInsns Pointer + LoadTime uint64 + CreatedByUID uint32 + NrMapIds uint32 + MapIds Pointer + Name ObjName + Ifindex uint32 + _ [4]byte /* unsupported bitfield */ + NetnsDev uint64 + NetnsIno uint64 + NrJitedKsyms uint32 + NrJitedFuncLens uint32 + JitedKsyms uint64 + JitedFuncLens uint64 + BtfID BTFID FuncInfoRecSize uint32 FuncInfo uint64 NrFuncInfo uint32 diff --git a/pkg/collector/corechecks/ebpf/probe/ebpfcheck/utils.go b/pkg/collector/corechecks/ebpf/probe/ebpfcheck/utils.go index 672354b408916..36b5a30bc264c 100644 --- a/pkg/collector/corechecks/ebpf/probe/ebpfcheck/utils.go +++ b/pkg/collector/corechecks/ebpf/probe/ebpfcheck/utils.go @@ -42,13 +42,3 @@ func pageAlign[T constraints.Integer](x T) T { func align[T constraints.Integer](x, a T) T { return (x + (a - 1)) & ^(a - 1) } - -// generic slice deletion at position i without concern for slice order -// -//nolint:unused // TODO(EBPF) Fix unused linter -func deleteAtNoOrder[S ~[]E, E any](s S, i int) S { - s[i] = s[len(s)-1] - // use zero value here to ensure no memory leaks - s[len(s)-1] = *new(E) - return s[:len(s)-1] -} diff --git a/pkg/ebpf/bytecode/asset_reader.go b/pkg/ebpf/bytecode/asset_reader.go index a0d8c2080358f..885f1a60d15ef 100644 --- a/pkg/ebpf/bytecode/asset_reader.go +++ b/pkg/ebpf/bytecode/asset_reader.go @@ -3,7 +3,7 @@ // This product includes software developed at Datadog (https://www.datadoghq.com/). // Copyright 2016-present Datadog, Inc. -//nolint:revive // TODO(EBPF) Fix revive linter +// Package bytecode contains types and functions for eBPF bytecode package bytecode import ( diff --git a/pkg/ebpf/bytecode/runtime/all_helpers.go b/pkg/ebpf/bytecode/runtime/all_helpers.go index 50fd1858239a1..485e07133b7b7 100644 --- a/pkg/ebpf/bytecode/runtime/all_helpers.go +++ b/pkg/ebpf/bytecode/runtime/all_helpers.go @@ -5,7 +5,7 @@ //go:build linux_bpf -//nolint:revive // TODO(EBPF) Fix revive linter +// Package runtime is for runtime compilation related types and functions package runtime // updated as of Linux v6.0 commit 4fe89d07dcc2804c8b562f6c7896a45643d34b2f diff --git a/pkg/ebpf/bytecode/runtime/asset.go b/pkg/ebpf/bytecode/runtime/asset.go index 998ac217dd416..2d0812368b5b2 100644 --- a/pkg/ebpf/bytecode/runtime/asset.go +++ b/pkg/ebpf/bytecode/runtime/asset.go @@ -51,7 +51,7 @@ func (a *asset) Compile(config *ebpf.Config, additionalFlags []string, client st } }() - opts := kernel.KernelHeaderOptions{ + opts := kernel.HeaderOptions{ DownloadEnabled: config.EnableKernelHeaderDownload, Dirs: config.KernelHeadersDirs, DownloadDir: config.KernelHeadersDownloadDir, diff --git a/pkg/ebpf/bytecode/runtime/generated_asset.go b/pkg/ebpf/bytecode/runtime/generated_asset.go index 622a6221fd738..59f4153741a7e 100644 --- a/pkg/ebpf/bytecode/runtime/generated_asset.go +++ b/pkg/ebpf/bytecode/runtime/generated_asset.go @@ -20,7 +20,7 @@ import ( "github.com/DataDog/datadog-agent/pkg/util/log" ) -//nolint:revive // TODO(EBPF) Fix revive linter +// ConstantFetcher is the generated asset for constant_fetcher.c var ConstantFetcher = newGeneratedAsset("constant_fetcher.c") // generatedAsset represents an asset whose contents will be dynamically generated at runtime @@ -50,7 +50,7 @@ func (a *generatedAsset) Compile(config *ebpf.Config, inputCode string, addition } }() - opts := kernel.KernelHeaderOptions{ + opts := kernel.HeaderOptions{ DownloadEnabled: config.EnableKernelHeaderDownload, Dirs: config.KernelHeadersDirs, DownloadDir: config.KernelHeadersDownloadDir, diff --git a/pkg/ebpf/bytecode/runtime/helpers_test.go b/pkg/ebpf/bytecode/runtime/helpers_test.go index 87bacedc21822..33f6fa044b9b5 100644 --- a/pkg/ebpf/bytecode/runtime/helpers_test.go +++ b/pkg/ebpf/bytecode/runtime/helpers_test.go @@ -30,7 +30,7 @@ func TestGetAvailableHelpers(t *testing.T) { } cfg := ebpf.NewConfig() - opts := kernel.KernelHeaderOptions{ + opts := kernel.HeaderOptions{ DownloadEnabled: cfg.EnableKernelHeaderDownload, Dirs: cfg.KernelHeadersDirs, DownloadDir: cfg.KernelHeadersDownloadDir, diff --git a/pkg/ebpf/bytecode/runtime/protected_file.go b/pkg/ebpf/bytecode/runtime/protected_file.go index 555aaf58b550d..a59224f09cf19 100644 --- a/pkg/ebpf/bytecode/runtime/protected_file.go +++ b/pkg/ebpf/bytecode/runtime/protected_file.go @@ -13,13 +13,12 @@ import ( "os" "path/filepath" - "github.com/DataDog/datadog-agent/pkg/util/log" "github.com/justincormack/go-memfd" + + "github.com/DataDog/datadog-agent/pkg/util/log" ) -// This represent a symlink to a sealed ram-backed file -// -//nolint:revive // TODO(EBPF) Fix revive linter +// ProtectedFile represents a symlink to a sealed ram-backed file type ProtectedFile interface { Close() error Reader() io.Reader @@ -31,9 +30,7 @@ type ramBackedFile struct { file *memfd.Memfd } -// This function returns a sealed ram backed file -// -//nolint:revive // TODO(EBPF) Fix revive linter +// NewProtectedFile returns a sealed ram backed file func NewProtectedFile(name, dir string, source io.Reader) (ProtectedFile, error) { var err error @@ -66,8 +63,7 @@ func NewProtectedFile(name, dir string, source io.Reader) (ProtectedFile, error) return nil, fmt.Errorf("failed to create symlink %s from target %s: %w", tmpFile, target, err) } - //nolint:staticcheck // TODO(EBPF) Fix staticcheck linter - if _, err := memfdFile.Seek(0, os.SEEK_SET); err != nil { + if _, err := memfdFile.Seek(0, io.SeekStart); err != nil { return nil, fmt.Errorf("failed to reset cursor: %w", err) } @@ -95,8 +91,7 @@ func setupSourceInfoFile(source io.Reader, path string) error { func (m *ramBackedFile) Close() error { os.Remove(m.symlink) - //nolint:staticcheck // TODO(EBPF) Fix staticcheck linter - if _, err := m.file.Seek(0, os.SEEK_SET); err != nil { + if _, err := m.file.Seek(0, io.SeekStart); err != nil { log.Debug(err) } if err := setupSourceInfoFile(m.file, m.symlink); err != nil { diff --git a/pkg/ebpf/bytecode/runtime/runtime_compilation_helpers.go b/pkg/ebpf/bytecode/runtime/runtime_compilation_helpers.go index 9f5b74f231c0a..58b4a961ec2ce 100644 --- a/pkg/ebpf/bytecode/runtime/runtime_compilation_helpers.go +++ b/pkg/ebpf/bytecode/runtime/runtime_compilation_helpers.go @@ -24,7 +24,7 @@ import ( "github.com/DataDog/datadog-agent/pkg/util/log" ) -//nolint:revive // TODO(EBPF) Fix revive linter +// CompiledOutput is the interface for a compiled output from runtime compilation type CompiledOutput interface { io.Reader io.ReaderAt diff --git a/pkg/ebpf/cgo/genpost.go b/pkg/ebpf/cgo/genpost.go index 6186df1e1760b..e21570299ab50 100644 --- a/pkg/ebpf/cgo/genpost.go +++ b/pkg/ebpf/cgo/genpost.go @@ -3,7 +3,7 @@ // This product includes software developed at Datadog (https://www.datadoghq.com/). // Copyright 2016-present Datadog, Inc. -//nolint:revive // TODO(EBPF) Fix revive linter +// Package main is the program to fixup cgo generated types package main import ( diff --git a/pkg/ebpf/common.go b/pkg/ebpf/common.go index 7a0f9071ac28a..cc6c913eaa83e 100644 --- a/pkg/ebpf/common.go +++ b/pkg/ebpf/common.go @@ -3,7 +3,7 @@ // This product includes software developed at Datadog (https://www.datadoghq.com/). // Copyright 2016-present Datadog, Inc. -//nolint:revive // TODO(EBPF) Fix revive linter +// Package ebpf contains general eBPF related types and functions package ebpf import ( diff --git a/pkg/ebpf/compiler/compiler.go b/pkg/ebpf/compiler/compiler.go index 46f2e5dec6ed3..55b88d342d0d4 100644 --- a/pkg/ebpf/compiler/compiler.go +++ b/pkg/ebpf/compiler/compiler.go @@ -5,7 +5,7 @@ //go:build linux_bpf -//nolint:revive // TODO(EBPF) Fix revive linter +// Package compiler is the runtime compiler for eBPF package compiler import ( @@ -92,14 +92,14 @@ func CompileToObjectFile(inFile, outputFile string, cflags []string, headerDirs return nil } -//nolint:revive // TODO(EBPF) Fix revive linter +// WithStdin assigns the provided io.Reader as Stdin for the command func WithStdin(in io.Reader) func(*exec.Cmd) { return func(c *exec.Cmd) { c.Stdin = in } } -//nolint:revive // TODO(EBPF) Fix revive linter +// WithStdout assigns the provided io.Writer as Stdout for the command func WithStdout(out io.Writer) func(*exec.Cmd) { return func(c *exec.Cmd) { c.Stdout = out diff --git a/pkg/ebpf/debugfs_stat_collector_linux.go b/pkg/ebpf/debugfs_stat_collector_linux.go index 02684e6029863..b611b44479540 100644 --- a/pkg/ebpf/debugfs_stat_collector_linux.go +++ b/pkg/ebpf/debugfs_stat_collector_linux.go @@ -19,8 +19,7 @@ import ( "github.com/DataDog/datadog-agent/pkg/util/log" ) -//nolint:revive // TODO(EBPF) Fix revive linter -const kProbeTelemetryName = "ebpf__probes" +const kprobeTelemetryName = "ebpf__probes" type profileType byte @@ -82,8 +81,8 @@ func NewDebugFsStatCollector() prometheus.Collector { return &NoopDebugFsStatCollector{} } return &DebugFsStatCollector{ - hits: prometheus.NewDesc(kProbeTelemetryName+"__hits", "Counter tracking number of probe hits", []string{"probe_name", "probe_type"}, nil), - misses: prometheus.NewDesc(kProbeTelemetryName+"__misses", "Counter tracking number of probe misses", []string{"probe_name", "probe_type"}, nil), + hits: prometheus.NewDesc(kprobeTelemetryName+"__hits", "Counter tracking number of probe hits", []string{"probe_name", "probe_type"}, nil), + misses: prometheus.NewDesc(kprobeTelemetryName+"__misses", "Counter tracking number of probe misses", []string{"probe_name", "probe_type"}, nil), lastProbeStats: make(map[eventKey]int), tracefsRoot: root, } diff --git a/pkg/ebpf/ebpftest/bpfdebug_linux.go b/pkg/ebpf/ebpftest/bpfdebug_linux.go index 877a870a180cf..9aa7674db0342 100644 --- a/pkg/ebpf/ebpftest/bpfdebug_linux.go +++ b/pkg/ebpf/ebpftest/bpfdebug_linux.go @@ -3,7 +3,7 @@ // This product includes software developed at Datadog (https://www.datadoghq.com/). // Copyright 2016-present Datadog, Inc. -//nolint:revive // TODO(EBPF) Fix revive linter +// Package ebpftest is utilities for tests against eBPF package ebpftest import ( diff --git a/pkg/ebpf/ebpftest/bpfdebug_unsupported.go b/pkg/ebpf/ebpftest/bpfdebug_unsupported.go index e8f77b46ed6e5..74097c8878c23 100644 --- a/pkg/ebpf/ebpftest/bpfdebug_unsupported.go +++ b/pkg/ebpf/ebpftest/bpfdebug_unsupported.go @@ -5,7 +5,7 @@ //go:build !linux -//nolint:revive // TODO(EBPF) Fix revive linter +// Package ebpftest is utilities for tests against eBPF package ebpftest import "testing" diff --git a/pkg/ebpf/ebpftest/buildmode.go b/pkg/ebpf/ebpftest/buildmode.go index f4b5c46ef30f8..4a2a990dfc255 100644 --- a/pkg/ebpf/ebpftest/buildmode.go +++ b/pkg/ebpf/ebpftest/buildmode.go @@ -24,7 +24,7 @@ func init() { Fentry = fentry{} } -//nolint:revive // TODO(EBPF) Fix revive linter +// BuildMode is an eBPF build mode type BuildMode interface { fmt.Stringer Env() map[string]string diff --git a/pkg/ebpf/ebpftest/buildmode_linux.go b/pkg/ebpf/ebpftest/buildmode_linux.go index 761c1eaddf8be..e68bcf4ac0192 100644 --- a/pkg/ebpf/ebpftest/buildmode_linux.go +++ b/pkg/ebpf/ebpftest/buildmode_linux.go @@ -22,7 +22,7 @@ func init() { hostinfo, _ = host.Info() } -//nolint:revive // TODO(EBPF) Fix revive linter +// SupportedBuildModes returns the build modes supported on the current host func SupportedBuildModes() []BuildMode { modes := []BuildMode{Prebuilt, RuntimeCompiled, CORE} if os.Getenv("TEST_FENTRY_OVERRIDE") == "true" || (runtime.GOARCH == "amd64" && (hostinfo.Platform == "amazon" || hostinfo.Platform == "amzn") && kv.Major() == 5 && kv.Minor() == 10) { @@ -31,14 +31,14 @@ func SupportedBuildModes() []BuildMode { return modes } -//nolint:revive // TODO(EBPF) Fix revive linter +// TestBuildModes runs the test under all the provided build modes func TestBuildModes(t *testing.T, modes []BuildMode, name string, fn func(t *testing.T)) { for _, mode := range modes { TestBuildMode(t, mode, name, fn) } } -//nolint:revive // TODO(EBPF) Fix revive linter +// TestBuildMode runs the test under the provided build mode func TestBuildMode(t *testing.T, mode BuildMode, name string, fn func(t *testing.T)) { t.Run(mode.String(), func(t *testing.T) { for k, v := range mode.Env() { diff --git a/pkg/ebpf/ebpftest/buildmode_windows.go b/pkg/ebpf/ebpftest/buildmode_windows.go index 770a28bcbd26a..7a0da5e343c16 100644 --- a/pkg/ebpf/ebpftest/buildmode_windows.go +++ b/pkg/ebpf/ebpftest/buildmode_windows.go @@ -7,18 +7,18 @@ package ebpftest import "testing" -//nolint:revive // TODO(EBPF) Fix revive linter +// SupportedBuildModes returns the build modes supported on the current host func SupportedBuildModes() []BuildMode { return []BuildMode{Prebuilt} } -//nolint:revive // TODO(EBPF) Fix revive linter +// TestBuildModes runs the test under all the provided build modes func TestBuildModes(t *testing.T, modes []BuildMode, name string, fn func(t *testing.T)) { //nolint:revive // TODO fix revive unused-parameter // ignore provided modes and only use prebuilt TestBuildMode(t, Prebuilt, name, fn) } -//nolint:revive // TODO(EBPF) Fix revive linter +// TestBuildMode runs the test under the provided build mode func TestBuildMode(t *testing.T, mode BuildMode, name string, fn func(t *testing.T)) { if mode != Prebuilt { t.Skipf("unsupported build mode %s", mode) diff --git a/pkg/ebpf/ebpftest/log.go b/pkg/ebpf/ebpftest/log.go index 7eb02542f3bc1..33ceef7aa3f9b 100644 --- a/pkg/ebpf/ebpftest/log.go +++ b/pkg/ebpf/ebpftest/log.go @@ -14,7 +14,7 @@ import ( "github.com/DataDog/datadog-agent/pkg/util/log" ) -//nolint:revive // TODO(EBPF) Fix revive linter +// LogLevel sets the logger level for this test only func LogLevel(t testing.TB, level string) { t.Cleanup(func() { log.SetupLogger(seelog.Default, "off") diff --git a/pkg/ebpf/perf.go b/pkg/ebpf/perf.go index 3254c33a0b344..cf362331a32cf 100644 --- a/pkg/ebpf/perf.go +++ b/pkg/ebpf/perf.go @@ -55,9 +55,7 @@ func NewPerfHandler(dataChannelSize int) *PerfHandler { } // LostHandler is the callback intended to be used when configuring PerfMapOptions -// -//nolint:revive // TODO(EBPF) Fix revive linter -func (c *PerfHandler) LostHandler(CPU int, lostCount uint64, perfMap *manager.PerfMap, manager *manager.Manager) { +func (c *PerfHandler) LostHandler(_ int, lostCount uint64, _ *manager.PerfMap, _ *manager.Manager) { if c.closed { return } @@ -65,9 +63,7 @@ func (c *PerfHandler) LostHandler(CPU int, lostCount uint64, perfMap *manager.Pe } // RecordHandler is the callback intended to be used when configuring PerfMapOptions -// -//nolint:revive // TODO(EBPF) Fix revive linter -func (c *PerfHandler) RecordHandler(record *perf.Record, perfMap *manager.PerfMap, manager *manager.Manager) { +func (c *PerfHandler) RecordHandler(record *perf.Record, _ *manager.PerfMap, _ *manager.Manager) { if c.closed { return } diff --git a/pkg/network/telemetry/stat_counter_wrapper.go b/pkg/network/telemetry/stat_counter_wrapper.go index 5a6be346e7691..de1e4f948dfee 100644 --- a/pkg/network/telemetry/stat_counter_wrapper.go +++ b/pkg/network/telemetry/stat_counter_wrapper.go @@ -3,7 +3,7 @@ // This product includes software developed at Datadog (https://www.datadoghq.com/). // Copyright 2016-present Datadog, Inc. -//nolint:revive // TODO(EBPF) Fix revive linter +// Package telemetry provides types and functions for internal telemetry package telemetry import ( @@ -19,29 +19,30 @@ type StatCounterWrapper struct { counter telemetry.Counter } -//nolint:revive // TODO(EBPF) Fix revive linter +// Inc increments the counter with the given tags value. func (sgw *StatCounterWrapper) Inc(tags ...string) { sgw.stat.Inc() sgw.counter.Inc(tags...) } -//nolint:revive // TODO(EBPF) Fix revive linter +// Delete deletes the value for the counter with the given tags value. func (sgw *StatCounterWrapper) Delete() { sgw.stat.Store(0) sgw.counter.Delete() } -//nolint:revive // TODO(EBPF) Fix revive linter +// Add adds the given value to the counter with the given tags value. func (sgw *StatCounterWrapper) Add(v int64, tags ...string) { sgw.stat.Add(v) sgw.counter.Add(float64(v), tags...) } -//nolint:revive // TODO(EBPF) Fix revive linter +// Load atomically loads the wrapped value. func (sgw *StatCounterWrapper) Load() int64 { return sgw.stat.Load() } +// NewStatCounterWrapper returns a new StatCounterWrapper func NewStatCounterWrapper(subsystem string, statName string, tags []string, description string) *StatCounterWrapper { return &StatCounterWrapper{ stat: atomic.NewInt64(0), diff --git a/pkg/network/telemetry/stat_gauge_wrapper.go b/pkg/network/telemetry/stat_gauge_wrapper.go index 746de999ec96f..3c0ae4db4254f 100644 --- a/pkg/network/telemetry/stat_gauge_wrapper.go +++ b/pkg/network/telemetry/stat_gauge_wrapper.go @@ -18,36 +18,36 @@ type StatGaugeWrapper struct { gauge telemetry.Gauge } -//nolint:revive // TODO(EBPF) Fix revive linter +// Inc increments the Gauge value. func (sgw *StatGaugeWrapper) Inc() { sgw.stat.Inc() sgw.gauge.Inc() } -//nolint:revive // TODO(EBPF) Fix revive linter +// Dec decrements the Gauge value. func (sgw *StatGaugeWrapper) Dec() { sgw.stat.Dec() sgw.gauge.Dec() } -//nolint:revive // TODO(EBPF) Fix revive linter +// Add adds the value to the Gauge value. func (sgw *StatGaugeWrapper) Add(v int64) { sgw.stat.Add(v) sgw.gauge.Add(float64(v)) } -//nolint:revive // TODO(EBPF) Fix revive linter +// Set stores the value for the given tags. func (sgw *StatGaugeWrapper) Set(v int64) { sgw.stat.Store(v) sgw.gauge.Set(float64(v)) } -//nolint:revive // TODO(EBPF) Fix revive linter +// Load atomically loads the wrapped value. func (sgw *StatGaugeWrapper) Load() int64 { return sgw.stat.Load() } -//nolint:revive // TODO(EBPF) Fix revive linter +// NewStatGaugeWrapper returns a new StatGaugeWrapper func NewStatGaugeWrapper(subsystem string, statName string, tags []string, description string) *StatGaugeWrapper { return &StatGaugeWrapper{ stat: atomic.NewInt64(0), diff --git a/pkg/util/kernel/arch.go b/pkg/util/kernel/arch.go index 9a9b724a79b8a..e7b6558accbe8 100644 --- a/pkg/util/kernel/arch.go +++ b/pkg/util/kernel/arch.go @@ -3,7 +3,7 @@ // This product includes software developed at Datadog (https://www.datadoghq.com/). // Copyright 2016-present Datadog, Inc. -//nolint:revive // TODO(EBPF) Fix revive linter +// Package kernel is utilities for the Linux kernel package kernel import "runtime" diff --git a/pkg/util/kernel/download_headers_test.go b/pkg/util/kernel/download_headers_test.go index 81ae433d88a29..239306544cf1a 100644 --- a/pkg/util/kernel/download_headers_test.go +++ b/pkg/util/kernel/download_headers_test.go @@ -102,8 +102,7 @@ var targets = map[string]TargetSetup{ }, } -//nolint:revive // TODO(EBPF) Fix revive linter -func setup(target types.Target, repos []string, dname string) error { +func setup(_ types.Target, repos []string, dname string) error { // Make source-list.d sources := fmt.Sprintf(reposSourceDir, dname) if err := os.MkdirAll(sources, 0744); err != nil { diff --git a/pkg/util/kernel/find_headers.go b/pkg/util/kernel/find_headers.go index 64ea8bb44e279..1937e4fadcb8d 100644 --- a/pkg/util/kernel/find_headers.go +++ b/pkg/util/kernel/find_headers.go @@ -78,8 +78,8 @@ type headerProvider struct { kernelHeaders []string } -//nolint:revive // TODO(EBPF) Fix revive linter -type KernelHeaderOptions struct { +// HeaderOptions are options for the kernel header download process +type HeaderOptions struct { DownloadEnabled bool Dirs []string DownloadDir string @@ -89,7 +89,7 @@ type KernelHeaderOptions struct { ZypperReposDir string } -func initProvider(opts KernelHeaderOptions) { +func initProvider(opts HeaderOptions) { HeaderProvider = &headerProvider{ downloadEnabled: opts.DownloadEnabled, headerDirs: opts.Dirs, @@ -114,7 +114,7 @@ func initProvider(opts KernelHeaderOptions) { // Any subsequent calls to GetKernelHeaders will return the result of the first call. This is because // kernel header downloading can be a resource intensive process, so we don't want to retry it an unlimited // number of times. -func GetKernelHeaders(opts KernelHeaderOptions, client statsd.ClientInterface) []string { +func GetKernelHeaders(opts HeaderOptions, client statsd.ClientInterface) []string { providerMu.Lock() defer providerMu.Unlock() diff --git a/pkg/util/kernel/find_headers_test.go b/pkg/util/kernel/find_headers_test.go index 9529bdad16679..9de3abd068445 100644 --- a/pkg/util/kernel/find_headers_test.go +++ b/pkg/util/kernel/find_headers_test.go @@ -20,7 +20,7 @@ func TestGetKernelHeaders(t *testing.T) { t.Skip("set INTEGRATION environment variable to run") } - opts := KernelHeaderOptions{} + opts := HeaderOptions{} dirs := GetKernelHeaders(opts, nil) assert.NotZero(t, len(dirs), "expected to find header directories") t.Log(dirs) diff --git a/pkg/util/kernel/fs_nolinux.go b/pkg/util/kernel/fs_nolinux.go index f2e710276ccd0..244924539ea14 100644 --- a/pkg/util/kernel/fs_nolinux.go +++ b/pkg/util/kernel/fs_nolinux.go @@ -9,12 +9,12 @@ package kernel import "github.com/DataDog/datadog-agent/pkg/util/funcs" -//nolint:revive // TODO(EBPF) Fix revive linter +// ProcFSRoot is the path to procfs var ProcFSRoot = funcs.MemoizeNoError(func() string { return "" }) -//nolint:revive // TODO(EBPF) Fix revive linter +// SysFSRoot is the path to sysfs var SysFSRoot = funcs.MemoizeNoError(func() string { return "" }) diff --git a/test/new-e2e/scenarios/system-probe/main.go b/test/new-e2e/scenarios/system-probe/main.go index 5d33c87a90111..2ac9154d456e1 100644 --- a/test/new-e2e/scenarios/system-probe/main.go +++ b/test/new-e2e/scenarios/system-probe/main.go @@ -5,7 +5,7 @@ //go:build !windows -//nolint:revive // TODO(EBPF) Fix revive linter +// Package main is the entrypoint for the system-probe e2e testing scenario package main import ( @@ -18,10 +18,9 @@ import ( systemProbe "github.com/DataDog/datadog-agent/test/new-e2e/system-probe" ) -var DD_AGENT_TESTING_DIR = os.Getenv("DD_AGENT_TESTING_DIR") var defaultVMConfigPath = filepath.Join(".", "system-probe", "config", "vmconfig.json") -func run(envName, x86InstanceType, armInstanceType string, destroy bool, opts *systemProbe.SystemProbeEnvOpts) error { +func run(envName, x86InstanceType, armInstanceType string, destroy bool, opts *systemProbe.EnvOpts) error { if destroy { return systemProbe.Destroy(envName) } @@ -49,7 +48,7 @@ func main() { sshKeyFile := flag.String("ssh-key-path", "", "path of private ssh key for ec2 instances") sshKeyName := flag.String("ssh-key-name", "", "name of ssh key pair to use for ec2 instances") infraEnv := flag.String("infra-env", "", "name of infra env to use") - dependenciesDirectoryPtr := flag.String("dependencies-dir", DD_AGENT_TESTING_DIR, "directory where dependencies package is present") + dependenciesDirectoryPtr := flag.String("dependencies-dir", os.Getenv("DD_AGENT_TESTING_DIR"), "directory where dependencies package is present") vmconfigPathPtr := flag.String("vmconfig", defaultVMConfigPath, "vmconfig path") local := flag.Bool("local", false, "is scenario running locally") runAgentPtr := flag.Bool("run-agent", false, "Run datadog agent on the metal instance") @@ -62,7 +61,7 @@ func main() { failOnMissing = true } - opts := systemProbe.SystemProbeEnvOpts{ + opts := systemProbe.EnvOpts{ X86AmiID: *x86AmiIDPtr, ArmAmiID: *armAmiIDPtr, ShutdownPeriod: *shutdownPtr, diff --git a/test/new-e2e/system-probe/system-probe-test-env.go b/test/new-e2e/system-probe/system-probe-test-env.go index ad6b9cff2b0c7..e19cbb749626e 100644 --- a/test/new-e2e/system-probe/system-probe-test-env.go +++ b/test/new-e2e/system-probe/system-probe-test-env.go @@ -10,8 +10,7 @@ package systemprobe import ( "context" - //nolint:revive // TODO(EBPF) Fix revive linter - _ "embed" + _ "embed" // embed files used in this scenario "fmt" "os" "path/filepath" @@ -31,35 +30,26 @@ import ( ) const ( - //nolint:revive // TODO(EBPF) Fix revive linter - AgentQAPrimaryAZ = "subnet-03061a1647c63c3c3" - //nolint:revive // TODO(EBPF) Fix revive linter - AgentQASecondaryAZ = "subnet-0f1ca3e929eb3fb8b" - //nolint:revive // TODO(EBPF) Fix revive linter - AgentQABackupAZ = "subnet-071213aedb0e1ae54" - - //nolint:revive // TODO(EBPF) Fix revive linter - SandboxPrimaryAz = "subnet-b89e00e2" - //nolint:revive // TODO(EBPF) Fix revive linter - SandboxSecondaryAz = "subnet-8ee8b1c6" - //nolint:revive // TODO(EBPF) Fix revive linter - SandboxBackupAz = "subnet-3f5db45b" - - //nolint:revive // TODO(EBPF) Fix revive linter - DatadogAgentQAEnv = "aws/agent-qa" - //nolint:revive // TODO(EBPF) Fix revive linter - SandboxEnv = "aws/sandbox" - //nolint:revive // TODO(EBPF) Fix revive linter - EC2TagsEnvVar = "RESOURCE_TAGS" + agentQAPrimaryAZ = "subnet-03061a1647c63c3c3" + agentQASecondaryAZ = "subnet-0f1ca3e929eb3fb8b" + agentQABackupAZ = "subnet-071213aedb0e1ae54" + + sandboxPrimaryAz = "subnet-b89e00e2" + sandboxSecondaryAz = "subnet-8ee8b1c6" + sandboxBackupAz = "subnet-3f5db45b" + + datadogAgentQAEnv = "aws/agent-qa" + sandboxEnv = "aws/sandbox" + ec2TagsEnvVar = "RESOURCE_TAGS" ) var availabilityZones = map[string][]string{ - DatadogAgentQAEnv: {AgentQAPrimaryAZ, AgentQASecondaryAZ, AgentQABackupAZ}, - SandboxEnv: {SandboxPrimaryAz, SandboxSecondaryAz, SandboxBackupAz}, + datadogAgentQAEnv: {agentQAPrimaryAZ, agentQASecondaryAZ, agentQABackupAZ}, + sandboxEnv: {sandboxPrimaryAz, sandboxSecondaryAz, sandboxBackupAz}, } -//nolint:revive // TODO(EBPF) Fix revive linter -type SystemProbeEnvOpts struct { +// EnvOpts are the options for the system-probe scenario +type EnvOpts struct { X86AmiID string ArmAmiID string SSHKeyPath string @@ -76,7 +66,7 @@ type SystemProbeEnvOpts struct { AgentVersion string } -//nolint:revive // TODO(EBPF) Fix revive linter +// TestEnv represents options for a particular test environment type TestEnv struct { context context.Context name string @@ -87,17 +77,13 @@ type TestEnv struct { } var ( - //nolint:revive // TODO(EBPF) Fix revive linter - MicroVMsDependenciesPath = filepath.Join("/", "opt", "kernel-version-testing", "dependencies-%s.tar.gz") - //nolint:revive // TODO(EBPF) Fix revive linter - CustomAMIWorkingDir = filepath.Join("/", "home", "kernel-version-testing") + customAMIWorkingDir = filepath.Join("/", "home", "kernel-version-testing") - //nolint:revive // TODO(EBPF) Fix revive linter - CI_PROJECT_DIR = GetEnv("CI_PROJECT_DIR", "/tmp") - sshKeyX86 = GetEnv("LibvirtSSHKeyX86", "/tmp/libvirt_rsa-x86_64") - sshKeyArm = GetEnv("LibvirtSSHKeyARM", "/tmp/libvirt_rsa-arm64") + ciProjectDir = getEnv("CI_PROJECT_DIR", "/tmp") + sshKeyX86 = getEnv("LibvirtSSHKeyX86", "/tmp/libvirt_rsa-x86_64") + sshKeyArm = getEnv("LibvirtSSHKeyARM", "/tmp/libvirt_rsa-arm64") - stackOutputs = filepath.Join(CI_PROJECT_DIR, "stack.output") + stackOutputs = filepath.Join(ciProjectDir, "stack.output") ) func outputsToFile(output auto.OutputMap) error { @@ -119,8 +105,7 @@ func outputsToFile(output auto.OutputMap) error { return f.Sync() } -//nolint:revive // TODO(EBPF) Fix revive linter -func GetEnv(key, fallback string) string { +func getEnv(key, fallback string) string { if value, ok := os.LookupEnv(key); ok { return value } @@ -157,8 +142,8 @@ func getAvailabilityZone(env string, azIndx int) string { return "" } -//nolint:revive // TODO(EBPF) Fix revive linter -func NewTestEnv(name, x86InstanceType, armInstanceType string, opts *SystemProbeEnvOpts) (*TestEnv, error) { +// NewTestEnv creates a new test environment +func NewTestEnv(name, x86InstanceType, armInstanceType string, opts *EnvOpts) (*TestEnv, error) { var err error var sudoPassword string @@ -178,7 +163,7 @@ func NewTestEnv(name, x86InstanceType, armInstanceType string, opts *SystemProbe sudoPassword = "" } - apiKey := GetEnv("DD_API_KEY", "") + apiKey := getEnv("DD_API_KEY", "") if opts.RunAgent && apiKey == "" { return nil, fmt.Errorf("No API Key for datadog-agent provided") } @@ -200,7 +185,7 @@ func NewTestEnv(name, x86InstanceType, armInstanceType string, opts *SystemProbe "microvm:provision": auto.ConfigValue{Value: strconv.FormatBool(opts.Provision)}, "microvm:x86AmiID": auto.ConfigValue{Value: opts.X86AmiID}, "microvm:arm64AmiID": auto.ConfigValue{Value: opts.ArmAmiID}, - "microvm:workingDir": auto.ConfigValue{Value: CustomAMIWorkingDir}, + "microvm:workingDir": auto.ConfigValue{Value: customAMIWorkingDir}, "ddagent:deploy": auto.ConfigValue{Value: strconv.FormatBool(opts.RunAgent)}, "ddagent:apiKey": auto.ConfigValue{Value: apiKey, Secret: true}, } @@ -221,7 +206,7 @@ func NewTestEnv(name, x86InstanceType, armInstanceType string, opts *SystemProbe config["ddagent:version"] = auto.ConfigValue{Value: opts.AgentVersion} } - if envVars := GetEnv(EC2TagsEnvVar, ""); envVars != "" { + if envVars := getEnv(ec2TagsEnvVar, ""); envVars != "" { config["ddinfra:extraResourcesTags"] = auto.ConfigValue{Value: envVars} } @@ -268,12 +253,12 @@ func NewTestEnv(name, x86InstanceType, armInstanceType string, opts *SystemProbe return systemProbeTestEnv, nil } -//nolint:revive // TODO(EBPF) Fix revive linter +// Destroy deletes the stack with the provided name func Destroy(name string) error { return infra.GetStackManager().DeleteStack(context.Background(), name) } -//nolint:revive // TODO(EBPF) Fix revive linter +// RemoveStack removes the stack configuration with the provided name func (env *TestEnv) RemoveStack() error { return infra.GetStackManager().ForceRemoveStackConfiguration(env.context, env.name) } diff --git a/test/new-e2e/system-probe/test-json-review/main.go b/test/new-e2e/system-probe/test-json-review/main.go index 4f884742443b8..afe80ded2aecf 100644 --- a/test/new-e2e/system-probe/test-json-review/main.go +++ b/test/new-e2e/system-probe/test-json-review/main.go @@ -5,7 +5,7 @@ //go:build linux -//nolint:revive // TODO(EBPF) Fix revive linter +// Package main is the test-json-review tool which reports all failed tests from the test JSON output package main import ( diff --git a/test/new-e2e/system-probe/test-runner/main.go b/test/new-e2e/system-probe/test-runner/main.go index 4b098dddc3031..6099758cb12e6 100644 --- a/test/new-e2e/system-probe/test-runner/main.go +++ b/test/new-e2e/system-probe/test-runner/main.go @@ -5,7 +5,7 @@ //go:build linux -//nolint:revive // TODO(EBPF) Fix revive linter +// Package main is the test-runner tool which runs the system-probe tests package main import ( @@ -136,16 +136,15 @@ func buildCommandArgs(pkg string, xmlpath string, jsonpath string, file string, // concatenateJsons combines all the test json output files into a single file. func concatenateJsons(indir, outdir string) error { - //nolint:revive // TODO(EBPF) Fix revive linter - testJsonFile := filepath.Join(outdir, "out.json") + testJSONFile := filepath.Join(outdir, "out.json") matches, err := glob(indir, `.*\.json`, func(path string) bool { return true }) if err != nil { return fmt.Errorf("json glob: %s", err) } - f, err := os.OpenFile(testJsonFile, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0o666) + f, err := os.OpenFile(testJSONFile, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0o666) if err != nil { - return fmt.Errorf("open %s: %s", testJsonFile, err) + return fmt.Errorf("open %s: %s", testJSONFile, err) } defer f.Close() From 8afcd59e794ef67e6c3309a10a9ef658a6b43ae9 Mon Sep 17 00:00:00 2001 From: Nenad Noveljic <18366081+nenadnoveljic@users.noreply.github.com> Date: Fri, 15 Dec 2023 22:31:48 +0100 Subject: [PATCH 11/17] obfuscation error false positive (#21594) --- pkg/collector/corechecks/oracle-dbm/statements.go | 5 +++-- ...racle-obuscation-error-false-207d782244887115.yaml | 11 +++++++++++ 2 files changed, 14 insertions(+), 2 deletions(-) create mode 100644 releasenotes/notes/oracle-obuscation-error-false-207d782244887115.yaml diff --git a/pkg/collector/corechecks/oracle-dbm/statements.go b/pkg/collector/corechecks/oracle-dbm/statements.go index 21b8678f08b71..cfc1cc6b7ad57 100644 --- a/pkg/collector/corechecks/oracle-dbm/statements.go +++ b/pkg/collector/corechecks/oracle-dbm/statements.go @@ -337,12 +337,13 @@ func (c *Check) copyToPreviousMap(newMap map[StatementMetricsKeyDB]StatementMetr } func handlePredicate(predicateType string, dbValue sql.NullString, payloadValue *string, statement StatementMetricsDB, c *Check, o *obfuscate.Obfuscator) { - if dbValue.Valid { + if dbValue.Valid && dbValue.String != "" { obfuscated, err := o.ObfuscateSQLString(dbValue.String) if err == nil { *payloadValue = obfuscated.Query } else { - *payloadValue = fmt.Sprintf("%s obfuscation error", predicateType) + *payloadValue = fmt.Sprintf("%s obfuscation error %d", predicateType, len(dbValue.String)) + //*payloadValue = dbValue.String logEntry := fmt.Sprintf("%s %s for sql_id: %s, plan_hash_value: %d", c.logPrompt, *payloadValue, statement.SQLID, statement.PlanHashValue) if c.config.ExecutionPlans.LogUnobfuscatedPlans { logEntry = fmt.Sprintf("%s unobfuscated filter: %s", logEntry, dbValue.String) diff --git a/releasenotes/notes/oracle-obuscation-error-false-207d782244887115.yaml b/releasenotes/notes/oracle-obuscation-error-false-207d782244887115.yaml new file mode 100644 index 0000000000000..ab9d5980b0b51 --- /dev/null +++ b/releasenotes/notes/oracle-obuscation-error-false-207d782244887115.yaml @@ -0,0 +1,11 @@ +# Each section from every release note are combined when the +# CHANGELOG.rst is rendered. So the text needs to be worded so that +# it does not depend on any information only available in another +# section. This may mean repeating some details, but each section +# must be readable independently of the other. +# +# Each section note must be formatted as reStructuredText. +--- +fixes: + - | + Fixed obfuscation error false positive when the access or filter predicates are empty. From f425dfa882dd9ca8533172c246ea047be1a40799 Mon Sep 17 00:00:00 2001 From: Gustavo Caso Date: Fri, 15 Dec 2023 22:41:10 +0100 Subject: [PATCH 12/17] core/status component implementation (#21536) [ASCII-976] Status component implementation --- comp/README.md | 4 + comp/core/status/component.go | 77 ++ comp/core/status/component_mock.go | 13 + comp/core/status/render_helpers.go | 286 ++++++++ .../statusimpl/common_header_provider.go | 106 +++ .../statusimpl/common_header_provider_test.go | 148 ++++ comp/core/status/statusimpl/status.go | 338 +++++++++ comp/core/status/statusimpl/status_mock.go | 38 + comp/core/status/statusimpl/status_test.go | 671 ++++++++++++++++++ .../status/statusimpl/templates/errors.tmpl | 6 + .../status/statusimpl/templates/html.tmpl | 32 + .../status/statusimpl/templates/text.tmpl | 13 + docs/components/README.md | 2 +- docs/components/defining-components.md | 2 +- pkg/util/flavor/flavor.go | 23 + pkg/util/flavor/flavor_test.go | 29 + 16 files changed, 1786 insertions(+), 2 deletions(-) create mode 100644 comp/core/status/component.go create mode 100644 comp/core/status/component_mock.go create mode 100644 comp/core/status/render_helpers.go create mode 100644 comp/core/status/statusimpl/common_header_provider.go create mode 100644 comp/core/status/statusimpl/common_header_provider_test.go create mode 100644 comp/core/status/statusimpl/status.go create mode 100644 comp/core/status/statusimpl/status_mock.go create mode 100644 comp/core/status/statusimpl/status_test.go create mode 100644 comp/core/status/statusimpl/templates/errors.tmpl create mode 100644 comp/core/status/statusimpl/templates/html.tmpl create mode 100644 comp/core/status/statusimpl/templates/text.tmpl create mode 100644 pkg/util/flavor/flavor_test.go diff --git a/comp/README.md b/comp/README.md index 2b6f5e7f1970e..7d642373c8e33 100644 --- a/comp/README.md +++ b/comp/README.md @@ -84,6 +84,10 @@ Package log implements a component to handle logging internal to the agent. Package secrets decodes secret values by invoking the configured executable command +### [comp/core/status](https://pkg.go.dev/github.com/DataDog/datadog-agent/comp/core/status) + +Package status displays information about the agent. + ### [comp/core/sysprobeconfig](https://pkg.go.dev/github.com/DataDog/datadog-agent/comp/core/sysprobeconfig) *Datadog Team*: ebpf-platform diff --git a/comp/core/status/component.go b/comp/core/status/component.go new file mode 100644 index 0000000000000..0a28f5ec4a424 --- /dev/null +++ b/comp/core/status/component.go @@ -0,0 +1,77 @@ +// Unless explicitly stated otherwise all files in this repository are licensed +// under the Apache License Version 2.0. +// This product includes software developed at Datadog (https://www.datadoghq.com/). +// Copyright 2023-present Datadog, Inc. + +// Package status displays information about the agent. +package status + +import ( + "io" + + "go.uber.org/fx" +) + +// team: agent-shared-components + +// CollectorSection stores the collector section name +const CollectorSection string = "collector" + +// Component interface to access the agent status. +type Component interface { + // Returns all the agent status information for the format type + GetStatus(format string, verbose bool) ([]byte, error) + // Returns only the agent status for the especify section and format type + GetStatusBySection(section string, format string, verbose bool) ([]byte, error) +} + +// Provider interface +type Provider interface { + // Name is used to sort the status providers alphabetically. + Name() string + // Section is used to group the status providers. + // When displaying the Text output the section is render as a header + Section() string + JSON(stats map[string]interface{}) error + Text(buffer io.Writer) error + HTML(buffer io.Writer) error +} + +// HeaderProvider interface +type HeaderProvider interface { + // Index is used to choose the order in which the header information is displayed. + Index() int + // When displaying the Text output the name is render as a header + Name() string + JSON(stats map[string]interface{}) error + Text(buffer io.Writer) error + HTML(buffer io.Writer) error +} + +// InformationProvider stores the Provider instance +type InformationProvider struct { + fx.Out + + Provider Provider `group:"status"` +} + +// HeaderInformationProvider stores the HeaderProvider instance +type HeaderInformationProvider struct { + fx.Out + + Provider HeaderProvider `group:"header_status"` +} + +// NewInformationProvider returns a InformationProvider to be called when generating the agent status +func NewInformationProvider(provider Provider) InformationProvider { + return InformationProvider{ + Provider: provider, + } +} + +// NewHeaderInformationProvider returns a new HeaderInformationProvider to be called when generating the agent status +func NewHeaderInformationProvider(provider HeaderProvider) HeaderInformationProvider { + return HeaderInformationProvider{ + Provider: provider, + } +} diff --git a/comp/core/status/component_mock.go b/comp/core/status/component_mock.go new file mode 100644 index 0000000000000..07fc21adf09f1 --- /dev/null +++ b/comp/core/status/component_mock.go @@ -0,0 +1,13 @@ +// Unless explicitly stated otherwise all files in this repository are licensed +// under the Apache License Version 2.0. +// This product includes software developed at Datadog (https://www.datadoghq.com/). +// Copyright 2023-present Datadog, Inc. + +//go:build test + +package status + +// Mock implements mock-specific methods. +type Mock interface { + Component +} diff --git a/comp/core/status/render_helpers.go b/comp/core/status/render_helpers.go new file mode 100644 index 0000000000000..ed398a0a62c65 --- /dev/null +++ b/comp/core/status/render_helpers.go @@ -0,0 +1,286 @@ +// Unless explicitly stated otherwise all files in this repository are licensed +// under the Apache License Version 2.0. +// This product includes software developed at Datadog (https://www.datadoghq.com/). +// Copyright 2016-present Datadog, Inc. + +package status + +import ( + "encoding/json" + "fmt" + htemplate "html/template" + "strconv" + "strings" + "sync" + ttemplate "text/template" + "time" + "unicode" + + "github.com/dustin/go-humanize" + "github.com/fatih/color" + + "golang.org/x/text/cases" + "golang.org/x/text/language" + "golang.org/x/text/unicode/norm" +) + +var ( + htmlFuncOnce sync.Once + htmlFuncMap htemplate.FuncMap + textFuncOnce sync.Once + textFuncMap ttemplate.FuncMap +) + +// HTMLFmap return a map of utility functions for HTML templating +func HTMLFmap() htemplate.FuncMap { + htmlFuncOnce.Do(func() { + htmlFuncMap = htemplate.FuncMap{ + "doNotEscape": doNotEscape, + "lastError": lastError, + "lastErrorTraceback": func(s string) htemplate.HTML { return doNotEscape(lastErrorTraceback(s)) }, + "lastErrorMessage": func(s string) htemplate.HTML { return doNotEscape(lastErrorMessage(s)) }, + "configError": configError, + "printDashes": PrintDashes, + "formatUnixTime": formatUnixTime, + "humanize": mkHuman, + "humanizeDuration": mkHumanDuration, + "toUnsortedList": toUnsortedList, + "formatTitle": formatTitle, + "add": add, + "status": status, + "redText": redText, + "yellowText": yellowText, + "greenText": greenText, + "ntpWarning": ntpWarning, + "version": getVersion, + "percent": func(v float64) string { return fmt.Sprintf("%02.1f", v*100) }, + "complianceResult": complianceResult, + } + }) + return htmlFuncMap +} + +// TextFmap map of utility functions for text templating +func TextFmap() ttemplate.FuncMap { + textFuncOnce.Do(func() { + textFuncMap = ttemplate.FuncMap{ + "lastErrorTraceback": lastErrorTraceback, + "lastErrorMessage": lastErrorMessage, + "printDashes": PrintDashes, + "formatUnixTime": formatUnixTime, + "humanize": mkHuman, + "humanizeDuration": mkHumanDuration, + "toUnsortedList": toUnsortedList, + "formatTitle": formatTitle, + "add": add, + "status": status, + "redText": redText, + "yellowText": yellowText, + "greenText": greenText, + "ntpWarning": ntpWarning, + "version": getVersion, + "percent": func(v float64) string { return fmt.Sprintf("%02.1f", v*100) }, + "complianceResult": complianceResult, + } + }) + + return textFuncMap +} + +const timeFormat = "2006-01-02 15:04:05.999 MST" + +func doNotEscape(value string) htemplate.HTML { + return htemplate.HTML(value) +} + +func configError(value string) htemplate.HTML { + return htemplate.HTML(value + "\n") +} + +func lastError(value string) htemplate.HTML { + return htemplate.HTML(value) +} + +func lastErrorTraceback(value string) string { + var lastErrorArray []map[string]string + + err := json.Unmarshal([]byte(value), &lastErrorArray) + if err != nil || len(lastErrorArray) == 0 { + return "No traceback" + } + lastErrorArray[0]["traceback"] = strings.Replace(lastErrorArray[0]["traceback"], "\n", "\n ", -1) + lastErrorArray[0]["traceback"] = strings.TrimRight(lastErrorArray[0]["traceback"], "\n\t ") + return lastErrorArray[0]["traceback"] +} + +// lastErrorMessage converts the last error message to html +func lastErrorMessage(value string) string { + var lastErrorArray []map[string]string + err := json.Unmarshal([]byte(value), &lastErrorArray) + if err == nil && len(lastErrorArray) > 0 { + if msg, ok := lastErrorArray[0]["message"]; ok { + return msg + } + } + return value +} + +// formatUnixTime formats the unix time to make it more readable +func formatUnixTime(unixTime int64) string { + // Initially treat given unixTime is in nanoseconds + t := time.Unix(0, int64(unixTime)) + // If year returned 1970, assume unixTime actually in seconds + if t.Year() == time.Unix(0, 0).Year() { + t = time.Unix(int64(unixTime), 0) + } + + _, tzoffset := t.Zone() + result := t.Format(timeFormat) + if tzoffset != 0 { + result += " / " + t.UTC().Format(timeFormat) + } + msec := t.UnixNano() / int64(time.Millisecond) + result += " (" + strconv.Itoa(int(msec)) + ")" + return result +} + +// PrintDashes repeats the pattern (dash) for the length of s +func PrintDashes(s string, dash string) string { + return strings.Repeat(dash, stringLength(s)) +} + +func toUnsortedList(s map[string]interface{}) string { + res := make([]string, 0, len(s)) + for key := range s { + res = append(res, key) + } + return fmt.Sprintf("%s", res) +} + +// mkHuman adds commas to large numbers to assist readability in status outputs +func mkHuman(f float64) string { + return humanize.Commaf(f) +} + +// mkHumanDuration makes time values more readable +func mkHumanDuration(f float64, unit string) string { + var duration time.Duration + if unit != "" { + duration, _ = time.ParseDuration(fmt.Sprintf("%f%s", f, unit)) + } else { + duration = time.Duration(int64(f)) * time.Second + } + + return duration.String() +} + +func stringLength(s string) int { + /* + len(string) is wrong if the string has unicode characters in it, + for example, something like 'Agent (v6.0.0+Χελωνη)' has len(s) == 27. + This is a better way of counting a string length + (credit goes to https://stackoverflow.com/a/12668840) + */ + var ia norm.Iter + ia.InitString(norm.NFKD, s) + nc := 0 + for !ia.Done() { + nc = nc + 1 + ia.Next() + } + return nc +} + +// add two integer together +func add(x, y int) int { + return x + y +} + +// formatTitle split a camel case string into space-separated words +func formatTitle(title string) string { + if title == "os" { + return "OS" + } + + // Split camel case words + var words []string + var l int + + for s := title; s != ""; s = s[l:] { + l = strings.IndexFunc(s[1:], unicode.IsUpper) + 1 + if l <= 0 { + l = len(s) + } + words = append(words, s[:l]) + } + title = strings.Join(words, " ") + + // Capitalize the first letter + return cases.Title(language.English, cases.NoLower).String(title) +} + +func status(check map[string]interface{}) string { + if check["LastError"].(string) != "" { + return fmt.Sprintf("[%s]", color.RedString("ERROR")) + } + if len(check["LastWarnings"].([]interface{})) != 0 { + return fmt.Sprintf("[%s]", color.YellowString("WARNING")) + } + return fmt.Sprintf("[%s]", color.GreenString("OK")) +} + +func complianceResult(result string) string { + switch result { + case "error": + return fmt.Sprintf("[%s]", color.RedString("ERROR")) + case "failed": + return fmt.Sprintf("[%s]", color.RedString("FAILED")) + case "passed": + return fmt.Sprintf("[%s]", color.GreenString("PASSED")) + default: + return fmt.Sprintf("[%s]", color.YellowString("UNKNOWN")) + } +} + +// Renders the message in a red color +func redText(message string) string { + return color.RedString(message) +} + +// Renders the message in a yellow color +func yellowText(message string) string { + return color.YellowString(message) +} + +// Renders the message in a green color +func greenText(message string) string { + return color.GreenString(message) +} + +// Tells if the ntp offset may be too large, resulting in metrics +// from the agent being dropped by metrics-intake +func ntpWarning(ntpOffset float64) bool { + // Negative offset => clock is in the future, 10 minutes (600s) allowed + // Positive offset => clock is in the past, 60 minutes (3600s) allowed + // According to https://docs.datadoghq.com/developers/metrics/#submitting-metrics + return ntpOffset <= -600 || ntpOffset >= 3600 +} + +func getVersion(instances map[string]interface{}) string { + if len(instances) == 0 { + return "" + } + for _, instance := range instances { + instanceMap := instance.(map[string]interface{}) + version, ok := instanceMap["CheckVersion"] + if !ok { + return "" + } + str, ok := version.(string) + if !ok { + return "" + } + return str + } + return "" +} diff --git a/comp/core/status/statusimpl/common_header_provider.go b/comp/core/status/statusimpl/common_header_provider.go new file mode 100644 index 0000000000000..5094476299d5f --- /dev/null +++ b/comp/core/status/statusimpl/common_header_provider.go @@ -0,0 +1,106 @@ +// Unless explicitly stated otherwise all files in this repository are licensed +// under the Apache License Version 2.0. +// This product includes software developed at Datadog (https://www.datadoghq.com/). +// Copyright 2023-present Datadog, Inc. + +package statusimpl + +import ( + "fmt" + htmlTemplate "html/template" + "io" + "os" + "path" + "runtime" + "strings" + textTemplate "text/template" + "time" + + "github.com/DataDog/datadog-agent/comp/core/config" + "github.com/DataDog/datadog-agent/comp/core/status" + "github.com/DataDog/datadog-agent/pkg/collector/python" + pkgConfig "github.com/DataDog/datadog-agent/pkg/config" + "github.com/DataDog/datadog-agent/pkg/util/flavor" + "github.com/DataDog/datadog-agent/pkg/version" +) + +var nowFunc = time.Now +var startTimeProvider = pkgConfig.StartTime + +type headerProvider struct { + data map[string]interface{} + name string + textTemplatesFunctions textTemplate.FuncMap + htmlTemplatesFunctions htmlTemplate.FuncMap +} + +func (h *headerProvider) Index() int { + return 0 +} + +func (h *headerProvider) Name() string { + return h.name +} + +func (h *headerProvider) JSON(stats map[string]interface{}) error { + for k, v := range h.data { + stats[k] = v + } + + return nil +} + +func (h *headerProvider) Text(buffer io.Writer) error { + tmpl, tmplErr := templatesFS.ReadFile(path.Join("templates", "text.tmpl")) + if tmplErr != nil { + return tmplErr + } + t := textTemplate.Must(textTemplate.New("header").Funcs(h.textTemplatesFunctions).Parse(string(tmpl))) + return t.Execute(buffer, h.data) +} + +func (h *headerProvider) HTML(buffer io.Writer) error { + tmpl, tmplErr := templatesFS.ReadFile(path.Join("templates", "html.tmpl")) + if tmplErr != nil { + return tmplErr + } + t := htmlTemplate.Must(htmlTemplate.New("header").Funcs(h.htmlTemplatesFunctions).Parse(string(tmpl))) + return t.Execute(buffer, h.data) +} + +func newCommonHeaderProvider(config config.Component) status.HeaderProvider { + + data := map[string]interface{}{} + data["version"] = version.AgentVersion + data["flavor"] = flavor.GetFlavor() + data["conf_file"] = config.ConfigFileUsed() + data["pid"] = os.Getpid() + data["go_version"] = runtime.Version() + data["agent_start_nano"] = startTimeProvider.UnixNano() + pythonVersion := python.GetPythonVersion() + data["python_version"] = strings.Split(pythonVersion, " ")[0] + data["build_arch"] = runtime.GOARCH + data["time_nano"] = nowFunc().UnixNano() + data["config"] = populateConfig(config) + + return &headerProvider{ + data: data, + name: fmt.Sprintf("%s (v%s)", flavor.GetHumanReadableFlavor(), data["version"]), + textTemplatesFunctions: status.TextFmap(), + htmlTemplatesFunctions: status.HTMLFmap(), + } +} + +func populateConfig(config config.Component) map[string]string { + conf := make(map[string]string) + conf["log_file"] = config.GetString("log_file") + conf["log_level"] = config.GetString("log_level") + conf["confd_path"] = config.GetString("confd_path") + conf["additional_checksd"] = config.GetString("additional_checksd") + + conf["fips_enabled"] = config.GetString("fips.enabled") + conf["fips_local_address"] = config.GetString("fips.local_address") + conf["fips_port_range_start"] = config.GetString("fips.port_range_start") + + return conf +} diff --git a/comp/core/status/statusimpl/common_header_provider_test.go b/comp/core/status/statusimpl/common_header_provider_test.go new file mode 100644 index 0000000000000..dfd7d616047f0 --- /dev/null +++ b/comp/core/status/statusimpl/common_header_provider_test.go @@ -0,0 +1,148 @@ +// Unless explicitly stated otherwise all files in this repository are licensed +// under the Apache License Version 2.0. +// This product includes software developed at Datadog (https://www.datadoghq.com/). +// Copyright 2016-present Datadog, Inc. + +package statusimpl + +import ( + "bytes" + "fmt" + "os" + "strings" + "testing" + "time" + + "github.com/DataDog/datadog-agent/comp/core/config" + pkgConfig "github.com/DataDog/datadog-agent/pkg/config" + "github.com/DataDog/datadog-agent/pkg/util/fxutil" + "github.com/DataDog/datadog-agent/pkg/version" + "github.com/stretchr/testify/assert" +) + +func TestCommonHeaderProviderIndex(t *testing.T) { + config := fxutil.Test[config.Component](t, config.MockModule()) + + provider := newCommonHeaderProvider(config) + + assert.Equal(t, 0, provider.Index()) +} + +func TestCommonHeaderProviderJSON(t *testing.T) { + nowFunc = func() time.Time { return time.Unix(1515151515, 0) } + startTimeProvider = time.Unix(1515151515, 0) + originalTZ := os.Getenv("TZ") + os.Setenv("TZ", "UTC") + + defer func() { + nowFunc = time.Now + startTimeProvider = pkgConfig.StartTime + os.Setenv("TZ", originalTZ) + }() + + config := fxutil.Test[config.Component](t, config.MockModule()) + + provider := newCommonHeaderProvider(config) + stats := map[string]interface{}{} + provider.JSON(stats) + + assert.Equal(t, version.AgentVersion, stats["version"]) + assert.Equal(t, agentFlavor, stats["flavor"]) + assert.Equal(t, config.ConfigFileUsed(), stats["conf_file"]) + assert.Equal(t, pid, stats["pid"]) + assert.Equal(t, goVersion, stats["go_version"]) + assert.Equal(t, startTimeProvider.UnixNano(), stats["agent_start_nano"]) + assert.Equal(t, "n/a", stats["python_version"]) + assert.Equal(t, arch, stats["build_arch"]) + assert.Equal(t, nowFunc().UnixNano(), stats["time_nano"]) + assert.NotEqual(t, "", stats["title"]) +} + +var expectedTextOutput = fmt.Sprintf(` Status date: 2018-01-05 11:25:15 UTC (1515151515000) + Agent start: 2018-01-05 11:25:15 UTC (1515151515000) + Pid: %d + Go Version: %s + Python Version: n/a + Build arch: %s + Agent flavor: %s + Log Level: info +`, pid, goVersion, arch, agentFlavor) + +func TestCommonHeaderProviderText(t *testing.T) { + nowFunc = func() time.Time { return time.Unix(1515151515, 0) } + startTimeProvider = time.Unix(1515151515, 0) + + defer func() { + nowFunc = time.Now + startTimeProvider = pkgConfig.StartTime + }() + + config := fxutil.Test[config.Component](t, config.MockModule()) + + provider := newCommonHeaderProvider(config) + + buffer := new(bytes.Buffer) + provider.Text(buffer) + + // We replace windows line break by linux so the tests pass on every OS + expectedResult := strings.Replace(expectedTextOutput, "\r\n", "\n", -1) + output := strings.Replace(buffer.String(), "\r\n", "\n", -1) + + assert.Equal(t, expectedResult, output) +} + +func TestCommonHeaderProviderHTML(t *testing.T) { + nowFunc = func() time.Time { return time.Unix(1515151515, 0) } + startTimeProvider = time.Unix(1515151515, 0) + originalTZ := os.Getenv("TZ") + os.Setenv("TZ", "UTC") + + defer func() { + nowFunc = time.Now + startTimeProvider = pkgConfig.StartTime + os.Setenv("TZ", originalTZ) + }() + + config := fxutil.Test[config.Component](t, config.MockModule()) + + provider := newCommonHeaderProvider(config) + + buffer := new(bytes.Buffer) + provider.HTML(buffer) + + // We have to do this strings replacement because html/temaplte escapes the `+` sign + // https://github.com/golang/go/issues/42506 + result := buffer.String() + unescapedResult := strings.Replace(result, "+", "+", -1) + + expectedHTMLOutput := fmt.Sprintf(`
+ Agent Info + + Version: %s +
Flavor: %s +
PID: %d +
Agent start: 2018-01-05 11:25:15 UTC (1515151515000) +
Log Level: info +
Config File: There is no config file +
Conf.d Path: %s +
Checks.d Path: %s + +
+ +
+ System Info + + System time: 2018-01-05 11:25:15 UTC (1515151515000) +
Go Version: %s +
Python Version: n/a +
Build arch: %s + +
+`, version.AgentVersion, agentFlavor, pid, config.GetString("confd_path"), config.GetString("additional_checksd"), goVersion, arch) + + // We replace windows line break by linux so the tests pass on every OS + expectedResult := strings.Replace(expectedHTMLOutput, "\r\n", "\n", -1) + output := strings.Replace(unescapedResult, "\r\n", "\n", -1) + + assert.Equal(t, expectedResult, output) +} diff --git a/comp/core/status/statusimpl/status.go b/comp/core/status/statusimpl/status.go new file mode 100644 index 0000000000000..56072ba7e732d --- /dev/null +++ b/comp/core/status/statusimpl/status.go @@ -0,0 +1,338 @@ +// Unless explicitly stated otherwise all files in this repository are licensed +// under the Apache License Version 2.0. +// This product includes software developed at Datadog (https://www.datadoghq.com/). +// Copyright 2023-present Datadog, Inc. + +// Package statusimpl implements the status component interface +package statusimpl + +import ( + "bytes" + "embed" + "encoding/json" + "io" + "path" + "sort" + "text/template" + "unicode" + + "go.uber.org/fx" + "golang.org/x/text/cases" + "golang.org/x/text/language" + + "github.com/DataDog/datadog-agent/comp/core/config" + "github.com/DataDog/datadog-agent/comp/core/status" + "github.com/DataDog/datadog-agent/pkg/util/fxutil" +) + +//go:embed templates +var templatesFS embed.FS + +type dependencies struct { + fx.In + Config config.Component + + Providers []status.Provider `group:"status"` + HeaderProviders []status.HeaderProvider `group:"header_status"` +} + +type statusImplementation struct { + sortedHeaderProviders []status.HeaderProvider + sortedSectionNames []string + sortedProvidersBySection map[string][]status.Provider +} + +// Module defines the fx options for this component. +func Module() fxutil.Module { + return fxutil.Component( + fx.Provide(newStatus), + ) +} + +func sortByName(providers []status.Provider) []status.Provider { + sort.SliceStable(providers, func(i, j int) bool { + return providers[i].Name() < providers[j].Name() + }) + + return providers +} + +func newStatus(deps dependencies) (status.Component, error) { + // Sections are sorted by name + // The exception is the collector section. We want that to be the first section to be displayed + // We manually insert the collector section in the first place after sorting them alphabetically + sortedSectionNames := []string{} + for _, provider := range deps.Providers { + if !present(provider.Section(), sortedSectionNames) && provider.Section() != status.CollectorSection { + sortedSectionNames = append(sortedSectionNames, provider.Section()) + } + } + sort.Strings(sortedSectionNames) + sortedSectionNames = append([]string{status.CollectorSection}, sortedSectionNames...) + + // Providers of each section are sort alphabetically by name + sortedProvidersBySection := map[string][]status.Provider{} + for _, provider := range deps.Providers { + providers := sortedProvidersBySection[provider.Section()] + sortedProvidersBySection[provider.Section()] = append(providers, provider) + } + for section, providers := range sortedProvidersBySection { + sortedProvidersBySection[section] = sortByName(providers) + } + + // Header providers are sorted by index + // We manually insert the common header provider in the first place after sorting is done + sortedHeaderProviders := deps.HeaderProviders + sort.SliceStable(sortedHeaderProviders, func(i, j int) bool { + return sortedHeaderProviders[i].Index() < sortedHeaderProviders[j].Index() + }) + + sortedHeaderProviders = append([]status.HeaderProvider{newCommonHeaderProvider(deps.Config)}, sortedHeaderProviders...) + + return &statusImplementation{ + sortedSectionNames: sortedSectionNames, + sortedProvidersBySection: sortedProvidersBySection, + sortedHeaderProviders: sortedHeaderProviders, + }, nil +} + +func (s *statusImplementation) GetStatus(format string, _ bool) ([]byte, error) { + var errors []error + + switch format { + case "json": + stats := make(map[string]interface{}) + for _, sc := range s.sortedHeaderProviders { + if err := sc.JSON(stats); err != nil { + errors = append(errors, err) + } + } + + for _, providers := range s.sortedProvidersBySection { + for _, provider := range providers { + if err := provider.JSON(stats); err != nil { + errors = append(errors, err) + } + } + } + + if len(errors) > 0 { + errorsInfo := []string{} + for _, error := range errors { + errorsInfo = append(errorsInfo, error.Error()) + } + stats["errors"] = errorsInfo + } + + return json.Marshal(stats) + case "text": + var b = new(bytes.Buffer) + + for _, sc := range s.sortedHeaderProviders { + printHeader(b, sc.Name()) + newLine(b) + + if err := sc.Text(b); err != nil { + errors = append(errors, err) + } + + newLine(b) + } + + for _, section := range s.sortedSectionNames { + + printHeader(b, section) + newLine(b) + + for _, provider := range s.sortedProvidersBySection[section] { + if err := provider.Text(b); err != nil { + errors = append(errors, err) + } + } + + newLine(b) + } + if len(errors) > 0 { + if err := renderErrors(b, errors); err != nil { + return []byte{}, err + } + + return b.Bytes(), nil + } + + return b.Bytes(), nil + case "html": + var b = new(bytes.Buffer) + + for _, sc := range s.sortedHeaderProviders { + err := sc.HTML(b) + if err != nil { + return b.Bytes(), err + } + } + + for _, section := range s.sortedSectionNames { + for _, provider := range s.sortedProvidersBySection[section] { + err := provider.HTML(b) + if err != nil { + return b.Bytes(), err + } + } + } + return b.Bytes(), nil + default: + return []byte{}, nil + } +} + +func (s *statusImplementation) GetStatusBySection(section string, format string, _ bool) ([]byte, error) { + var errors []error + + switch section { + case "header": + providers := s.sortedHeaderProviders + switch format { + case "json": + stats := make(map[string]interface{}) + + for _, sc := range providers { + if err := sc.JSON(stats); err != nil { + errors = append(errors, err) + } + } + + if len(errors) > 0 { + errorsInfo := []string{} + for _, error := range errors { + errorsInfo = append(errorsInfo, error.Error()) + } + stats["errors"] = errorsInfo + } + + return json.Marshal(stats) + case "text": + var b = new(bytes.Buffer) + + for i, sc := range providers { + if i == 0 { + printHeader(b, sc.Name()) + newLine(b) + } + + err := sc.Text(b) + if err != nil { + return b.Bytes(), err + } + } + + return b.Bytes(), nil + case "html": + var b = new(bytes.Buffer) + + for _, sc := range providers { + err := sc.HTML(b) + if err != nil { + return b.Bytes(), err + } + } + return b.Bytes(), nil + default: + return []byte{}, nil + } + default: + providers := s.sortedProvidersBySection[section] + switch format { + case "json": + stats := make(map[string]interface{}) + + for _, sc := range providers { + if err := sc.JSON(stats); err != nil { + errors = append(errors, err) + } + } + + if len(errors) > 0 { + errorsInfo := []string{} + for _, error := range errors { + errorsInfo = append(errorsInfo, error.Error()) + } + stats["errors"] = errorsInfo + } + + return json.Marshal(stats) + case "text": + var b = new(bytes.Buffer) + + for i, sc := range providers { + if i == 0 { + printHeader(b, section) + newLine(b) + } + + if err := sc.Text(b); err != nil { + errors = append(errors, err) + } + } + + if len(errors) > 0 { + if err := renderErrors(b, errors); err != nil { + return []byte{}, err + } + + return b.Bytes(), nil + } + + return b.Bytes(), nil + case "html": + var b = new(bytes.Buffer) + + for _, sc := range providers { + err := sc.HTML(b) + if err != nil { + return b.Bytes(), err + } + } + return b.Bytes(), nil + default: + return []byte{}, nil + } + } +} + +func present(value string, container []string) bool { + for _, v := range container { + if v == value { + return true + } + } + + return false +} + +func printHeader(buffer *bytes.Buffer, section string) { + dashes := []byte(status.PrintDashes(section, "=")) + buffer.Write(dashes) + newLine(buffer) + + runes := []rune(section) + if unicode.IsUpper(runes[0]) { + buffer.Write([]byte(section)) + } else { + buffer.Write([]byte(cases.Title(language.Und).String(section))) + } + newLine(buffer) + buffer.Write(dashes) +} + +func newLine(buffer *bytes.Buffer) { + buffer.Write([]byte("\n")) +} + +func renderErrors(w io.Writer, errs []error) error { + tmpl, tmplErr := templatesFS.ReadFile(path.Join("templates", "errors.tmpl")) + if tmplErr != nil { + return tmplErr + } + t := template.Must(template.New("errors").Parse(string(tmpl))) + return t.Execute(w, errs) +} diff --git a/comp/core/status/statusimpl/status_mock.go b/comp/core/status/statusimpl/status_mock.go new file mode 100644 index 0000000000000..bc404c7b0bbfe --- /dev/null +++ b/comp/core/status/statusimpl/status_mock.go @@ -0,0 +1,38 @@ +// Unless explicitly stated otherwise all files in this repository are licensed +// under the Apache License Version 2.0. +// This product includes software developed at Datadog (https://www.datadoghq.com/). +// Copyright 2023-present Datadog, Inc. + +//go:build test + +package statusimpl + +import ( + "go.uber.org/fx" + + "github.com/DataDog/datadog-agent/comp/core/status" + "github.com/DataDog/datadog-agent/pkg/util/fxutil" +) + +type statusMock struct { +} + +func (s *statusMock) GetStatus(string, bool) ([]byte, error) { + return []byte{}, nil +} + +func (s *statusMock) GetStatusBySection(string, string, bool) ([]byte, error) { + return []byte{}, nil +} + +// newMock returns a status Mock +func newMock() status.Mock { + return &statusMock{} +} + +// MockModule defines the fx options for the mock component. +func MockModule() fxutil.Module { + return fxutil.Component( + fx.Provide(newMock), + ) +} diff --git a/comp/core/status/statusimpl/status_test.go b/comp/core/status/statusimpl/status_test.go new file mode 100644 index 0000000000000..47aaa3e2ee833 --- /dev/null +++ b/comp/core/status/statusimpl/status_test.go @@ -0,0 +1,671 @@ +// Unless explicitly stated otherwise all files in this repository are licensed +// under the Apache License Version 2.0. +// This product includes software developed at Datadog (https://www.datadoghq.com/). +// Copyright 2016-present Datadog, Inc. + +package statusimpl + +import ( + "encoding/json" + "fmt" + "io" + "os" + "runtime" + "strings" + "testing" + "time" + + "github.com/DataDog/datadog-agent/comp/core/config" + "github.com/DataDog/datadog-agent/comp/core/status" + pkgConfig "github.com/DataDog/datadog-agent/pkg/config" + "github.com/DataDog/datadog-agent/pkg/util/flavor" + "github.com/DataDog/datadog-agent/pkg/util/fxutil" + "github.com/DataDog/datadog-agent/pkg/version" + "github.com/stretchr/testify/assert" + "go.uber.org/fx" +) + +type mockProvider struct { + data map[string]interface{} + text string + html string + name string + section string +} + +func (m mockProvider) Name() string { + return m.name +} + +func (m mockProvider) Section() string { + return m.section +} + +func (m mockProvider) JSON(stats map[string]interface{}) error { + for key, value := range m.data { + stats[key] = value + } + + return nil +} + +func (m mockProvider) Text(buffer io.Writer) error { + _, err := buffer.Write([]byte(m.text)) + return err +} + +func (m mockProvider) HTML(buffer io.Writer) error { + _, err := buffer.Write([]byte(m.html)) + return err +} + +type mockHeaderProvider struct { + data map[string]interface{} + text string + html string + index int + name string +} + +func (m mockHeaderProvider) Index() int { + return m.index +} + +func (m mockHeaderProvider) Name() string { + return m.name +} + +func (m mockHeaderProvider) JSON(stats map[string]interface{}) error { + for key, value := range m.data { + stats[key] = value + } + + return nil +} + +func (m mockHeaderProvider) Text(buffer io.Writer) error { + _, err := buffer.Write([]byte(m.text)) + return err +} + +func (m mockHeaderProvider) HTML(buffer io.Writer) error { + _, err := buffer.Write([]byte(m.html)) + return err +} + +type errorMockProvider struct{} + +func (m errorMockProvider) Name() string { + return "error mock" +} + +func (m errorMockProvider) Section() string { + return "error section" +} + +func (m errorMockProvider) JSON(map[string]interface{}) error { + return fmt.Errorf("testing JSON errors") +} + +func (m errorMockProvider) Text(io.Writer) error { + return fmt.Errorf("testing Text errors") +} + +func (m errorMockProvider) HTML(io.Writer) error { + return fmt.Errorf("testing HTML errors") +} + +var ( + humanReadbaleFlavor = flavor.GetHumanReadableFlavor() + agentVersion = version.AgentVersion + pid = os.Getpid() + goVersion = runtime.Version() + arch = runtime.GOARCH + agentFlavor = flavor.GetFlavor() + testTitle = fmt.Sprintf("%s (v%s)", humanReadbaleFlavor, agentVersion) +) + +var testTextHeader = fmt.Sprintf(`%s +%s +%s`, status.PrintDashes(testTitle, "="), testTitle, status.PrintDashes(testTitle, "=")) + +var expectedStatusTextOutput = fmt.Sprintf(`%s + Status date: 2018-01-05 11:25:15 UTC (1515151515000) + Agent start: 2018-01-05 11:25:15 UTC (1515151515000) + Pid: %d + Go Version: %s + Python Version: n/a + Build arch: %s + Agent flavor: %s + Log Level: info + +========== +Header Foo +========== + header foo: header bar + header foo2: header bar 2 + +========= +Collector +========= + text from a + text from b + +========= +A Section +========= + text from a + +========= +X Section +========= + text from a + text from x + +`, testTextHeader, pid, goVersion, arch, agentFlavor) + +func TestGetStatus(t *testing.T) { + nowFunc = func() time.Time { return time.Unix(1515151515, 0) } + startTimeProvider = time.Unix(1515151515, 0) + originalTZ := os.Getenv("TZ") + os.Setenv("TZ", "UTC") + + defer func() { + nowFunc = time.Now + startTimeProvider = pkgConfig.StartTime + os.Setenv("TZ", originalTZ) + }() + + deps := fxutil.Test[dependencies](t, fx.Options( + config.MockModule(), + fx.Supply( + status.NewInformationProvider(mockProvider{ + data: map[string]interface{}{ + "foo": "bar", + }, + name: "a", + text: " text from a\n", + html: `
+ Foo + +
Bar: bar + +
+`, + section: status.CollectorSection, + }), + status.NewInformationProvider(mockProvider{ + data: map[string]interface{}{ + "foo2": "bar2", + }, + name: "b", + text: " text from b\n", + section: status.CollectorSection, + }), + status.NewInformationProvider(mockProvider{ + data: map[string]interface{}{ + "foo3": "bar3", + }, + name: "x", + text: " text from x\n", + section: "x section", + }), + status.NewInformationProvider(mockProvider{ + data: map[string]interface{}{ + "foo3": "bar3", + }, + name: "a", + text: " text from a\n", + section: "a section", + }), + status.NewInformationProvider(mockProvider{ + data: map[string]interface{}{ + "foo3": "bar3", + }, + name: "a", + text: " text from a\n", + section: "x section", + }), + status.NewHeaderInformationProvider(mockHeaderProvider{ + name: "header foo", + data: map[string]interface{}{ + "header_foo": "header_bar", + }, + text: ` header foo: header bar + header foo2: header bar 2 +`, + html: `
+ Header Foo + +
Header Bar: bar + +
+`, + index: 2, + }), + ), + )) + + statusComponent, err := newStatus(deps) + + assert.NoError(t, err) + + testCases := []struct { + name string + format string + assertFunc func(*testing.T, []byte) + }{ + { + name: "JSON", + format: "json", + assertFunc: func(t *testing.T, bytes []byte) { + result := map[string]interface{}{} + err = json.Unmarshal(bytes, &result) + + assert.NoError(t, err) + + assert.Equal(t, "bar", result["foo"]) + assert.Equal(t, "header_bar", result["header_foo"]) + }, + }, + { + name: "Text", + format: "text", + assertFunc: func(t *testing.T, bytes []byte) { + // We replace windows line break by linux so the tests pass on every OS + expectedResult := strings.Replace(expectedStatusTextOutput, "\r\n", "\n", -1) + output := strings.Replace(string(bytes), "\r\n", "\n", -1) + + assert.Equal(t, expectedResult, output) + }, + }, + { + name: "HTML", + format: "html", + assertFunc: func(t *testing.T, bytes []byte) { + // We have to do this strings replacement because html/temaplte escapes the `+` sign + // https://github.com/golang/go/issues/42506 + result := string(bytes) + unescapedResult := strings.Replace(result, "+", "+", -1) + + expectedStatusHTMLOutput := fmt.Sprintf(`
+ Agent Info + + Version: %s +
Flavor: %s +
PID: %d +
Agent start: 2018-01-05 11:25:15 UTC (1515151515000) +
Log Level: info +
Config File: There is no config file +
Conf.d Path: %s +
Checks.d Path: %s + +
+ +
+ System Info + + System time: 2018-01-05 11:25:15 UTC (1515151515000) +
Go Version: %s +
Python Version: n/a +
Build arch: %s + +
+
+ Header Foo + +
Header Bar: bar + +
+
+ Foo + +
Bar: bar + +
+`, agentVersion, agentFlavor, pid, deps.Config.GetString("confd_path"), deps.Config.GetString("additional_checksd"), goVersion, arch) + + // We replace windows line break by linux so the tests pass on every OS + expectedResult := strings.Replace(expectedStatusHTMLOutput, "\r\n", "\n", -1) + output := strings.Replace(unescapedResult, "\r\n", "\n", -1) + + assert.Equal(t, expectedResult, output) + }, + }, + } + + for _, testCase := range testCases { + t.Run(testCase.name, func(t *testing.T) { + bytesResult, err := statusComponent.GetStatus(testCase.format, false) + + assert.NoError(t, err) + + testCase.assertFunc(t, bytesResult) + }) + } +} + +var expectedStatusTextErrorOutput = fmt.Sprintf(`%s + Status date: 2018-01-05 11:25:15 UTC (1515151515000) + Agent start: 2018-01-05 11:25:15 UTC (1515151515000) + Pid: %d + Go Version: %s + Python Version: n/a + Build arch: %s + Agent flavor: agent + Log Level: info + +========= +Collector +========= + text from b + +============= +Error Section +============= + +==================== +Status render errors +==================== + - testing Text errors + +`, testTextHeader, pid, goVersion, arch) + +func TestGetStatusWithErrors(t *testing.T) { + nowFunc = func() time.Time { return time.Unix(1515151515, 0) } + startTimeProvider = time.Unix(1515151515, 0) + originalTZ := os.Getenv("TZ") + os.Setenv("TZ", "UTC") + + defer func() { + nowFunc = time.Now + startTimeProvider = pkgConfig.StartTime + os.Setenv("TZ", originalTZ) + }() + + deps := fxutil.Test[dependencies](t, fx.Options( + config.MockModule(), + fx.Supply( + status.NewInformationProvider(errorMockProvider{}), + status.NewInformationProvider(mockProvider{ + data: map[string]interface{}{ + "foo2": "bar2", + }, + name: "b", + text: " text from b\n", + section: status.CollectorSection, + }), + ), + )) + + statusComponent, err := newStatus(deps) + + assert.NoError(t, err) + + testCases := []struct { + name string + format string + assertFunc func(*testing.T, []byte) + }{ + { + name: "JSON", + format: "json", + assertFunc: func(t *testing.T, bytes []byte) { + result := map[string]interface{}{} + err = json.Unmarshal(bytes, &result) + + assert.NoError(t, err) + + assert.Equal(t, "testing JSON errors", result["errors"].([]interface{})[0].(string)) + }, + }, + { + name: "Text", + format: "text", + assertFunc: func(t *testing.T, bytes []byte) { + // We replace windows line break by linux so the tests pass on every OS + expectedResult := strings.Replace(expectedStatusTextErrorOutput, "\r\n", "\n", -1) + output := strings.Replace(string(bytes), "\r\n", "\n", -1) + + assert.Equal(t, expectedResult, output) + }, + }, + } + + for _, testCase := range testCases { + t.Run(testCase.name, func(t *testing.T) { + bytesResult, err := statusComponent.GetStatus(testCase.format, false) + + assert.NoError(t, err) + + testCase.assertFunc(t, bytesResult) + }) + } +} + +func TestGetStatusBySection(t *testing.T) { + deps := fxutil.Test[dependencies](t, fx.Options( + config.MockModule(), + fx.Supply( + status.NewInformationProvider(mockProvider{ + data: map[string]interface{}{ + "foo": "bar", + }, + name: "a", + text: " text from a\n", + html: `
+ Foo + +
Bar: bar + +
+`, + section: status.CollectorSection, + }), + status.NewInformationProvider(mockProvider{ + data: map[string]interface{}{ + "foo2": "bar2", + }, + name: "b", + text: " text from b\n", + section: status.CollectorSection, + }), + status.NewInformationProvider(mockProvider{ + data: map[string]interface{}{ + "foo3": "bar3", + }, + name: "x", + text: " text from x\n", + section: "x section", + }), + status.NewInformationProvider(mockProvider{ + data: map[string]interface{}{ + "foo3": "bar3", + }, + name: "a", + text: " text from a\n", + section: "a section", + }), + status.NewInformationProvider(mockProvider{ + data: map[string]interface{}{ + "foo3": "bar3", + }, + name: "a", + text: " text from a\n", + section: "x section", + }), + status.NewHeaderInformationProvider(mockHeaderProvider{ + data: map[string]interface{}{ + "header_foo": "header_bar", + }, + text: ` header foo: header bar + header foo2: header bar 2 +`, + html: `
+ Header Foo + +
Header Bar: bar + +
+`, + index: 2, + }), + ), + )) + + statusComponent, err := newStatus(deps) + + assert.NoError(t, err) + + testCases := []struct { + name string + section string + format string + assertFunc func(*testing.T, []byte) + }{ + { + name: "JSON", + section: "header", + format: "json", + assertFunc: func(t *testing.T, bytes []byte) { + result := map[string]interface{}{} + err = json.Unmarshal(bytes, &result) + + assert.NoError(t, err) + + assert.Nil(t, result["foo"]) + assert.Equal(t, "header_bar", result["header_foo"]) + }, + }, + { + name: "Text", + format: "text", + section: "x section", + assertFunc: func(t *testing.T, bytes []byte) { + result := `========= +X Section +========= + text from a + text from x +` + + // We replace windows line break by linux so the tests pass on every OS + expectedResult := strings.Replace(result, "\r\n", "\n", -1) + output := strings.Replace(string(bytes), "\r\n", "\n", -1) + + assert.Equal(t, expectedResult, output) + }, + }, + { + name: "HTML", + section: "collector", + format: "html", + assertFunc: func(t *testing.T, bytes []byte) { + result := `
+ Foo + +
Bar: bar + +
+` + // We replace windows line break by linux so the tests pass on every OS + expectedResult := strings.Replace(result, "\r\n", "\n", -1) + output := strings.Replace(string(bytes), "\r\n", "\n", -1) + + assert.Equal(t, expectedResult, output) + }, + }, + } + + for _, testCase := range testCases { + t.Run(testCase.name, func(t *testing.T) { + bytesResult, err := statusComponent.GetStatusBySection(testCase.section, testCase.format, false) + + assert.NoError(t, err) + + testCase.assertFunc(t, bytesResult) + }) + } +} + +func TestGetStatusBySectionsWithErrors(t *testing.T) { + nowFunc = func() time.Time { return time.Unix(1515151515, 0) } + startTimeProvider = time.Unix(1515151515, 0) + originalTZ := os.Getenv("TZ") + os.Setenv("TZ", "UTC") + + defer func() { + nowFunc = time.Now + startTimeProvider = pkgConfig.StartTime + os.Setenv("TZ", originalTZ) + }() + + deps := fxutil.Test[dependencies](t, fx.Options( + config.MockModule(), + fx.Supply( + status.NewInformationProvider(errorMockProvider{}), + status.NewInformationProvider(mockProvider{ + data: map[string]interface{}{ + "foo2": "bar2", + }, + name: "b", + text: " text from b\n", + section: status.CollectorSection, + }), + ), + )) + + statusComponent, err := newStatus(deps) + + assert.NoError(t, err) + + testCases := []struct { + name string + format string + assertFunc func(*testing.T, []byte) + }{ + { + name: "JSON", + format: "json", + assertFunc: func(t *testing.T, bytes []byte) { + result := map[string]interface{}{} + err = json.Unmarshal(bytes, &result) + + assert.NoError(t, err) + + assert.Equal(t, "testing JSON errors", result["errors"].([]interface{})[0].(string)) + }, + }, + { + name: "Text", + format: "text", + assertFunc: func(t *testing.T, bytes []byte) { + expected := `============= +Error Section +============= +==================== +Status render errors +==================== + - testing Text errors + +` + + // We replace windows line break by linux so the tests pass on every OS + expectedResult := strings.Replace(expected, "\r\n", "\n", -1) + output := strings.Replace(string(bytes), "\r\n", "\n", -1) + + assert.Equal(t, expectedResult, output) + }, + }, + } + + for _, testCase := range testCases { + t.Run(testCase.name, func(t *testing.T) { + bytesResult, err := statusComponent.GetStatusBySection("error section", testCase.format, false) + + assert.NoError(t, err) + + testCase.assertFunc(t, bytesResult) + }) + } +} diff --git a/comp/core/status/statusimpl/templates/errors.tmpl b/comp/core/status/statusimpl/templates/errors.tmpl new file mode 100644 index 0000000000000..2a89c05b9e218 --- /dev/null +++ b/comp/core/status/statusimpl/templates/errors.tmpl @@ -0,0 +1,6 @@ +==================== +Status render errors +==================== +{{- range $err := . }} + - {{ $err }} +{{ end }} diff --git a/comp/core/status/statusimpl/templates/html.tmpl b/comp/core/status/statusimpl/templates/html.tmpl new file mode 100644 index 0000000000000..e42180ed6bed2 --- /dev/null +++ b/comp/core/status/statusimpl/templates/html.tmpl @@ -0,0 +1,32 @@ +
+ Agent Info + + Version: {{.version}} +
Flavor: {{.flavor}} +
PID: {{.pid}} +
Agent start: {{ formatUnixTime .agent_start_nano }} + {{- if .config.log_file}} +
Log File: {{.config.log_file}} + {{end}} +
Log Level: {{.config.log_level}} +
Config File: {{if .conf_file}}{{.conf_file}}{{else}}There is no config file{{end}} +
Conf.d Path: {{.config.confd_path}} +
Checks.d Path: {{.config.additional_checksd}} + +
+ +
+ System Info + + System time: {{ formatUnixTime .time_nano }} + {{- if .ntpOffset}} +
NTP Offset: {{ humanizeDuration .ntpOffset "s"}} + {{- if ntpWarning .ntpOffset}} +
NTP Offset is high. Datadog may ignore metrics sent by this Agent. + {{- end}} + {{end}} +
Go Version: {{.go_version}} +
Python Version: {{.python_version}} +
Build arch: {{.build_arch}} + +
diff --git a/comp/core/status/statusimpl/templates/text.tmpl b/comp/core/status/statusimpl/templates/text.tmpl new file mode 100644 index 0000000000000..acb0168ab750b --- /dev/null +++ b/comp/core/status/statusimpl/templates/text.tmpl @@ -0,0 +1,13 @@ + Status date: {{ formatUnixTime .time_nano }} + Agent start: {{ formatUnixTime .agent_start_nano }} + Pid: {{.pid}} + Go Version: {{.go_version}} + {{- if .python_version }} + Python Version: {{.python_version}} + {{- end }} + Build arch: {{.build_arch}} + Agent flavor: {{.flavor}} + {{- if .config.log_file}} + Log File: {{.config.log_file}} + {{- end }} + Log Level: {{.config.log_level}} diff --git a/docs/components/README.md b/docs/components/README.md index 280d86a4b8871..a21c1df34a302 100644 --- a/docs/components/README.md +++ b/docs/components/README.md @@ -4,7 +4,7 @@ * [Overview of Components](./components.md) * [Guidelines](./guidelines.md) * [Defining Components](./defining-components.md) - * [Using Components](./using.md) + * [Using Components](./usage.md) * [Defining Bundles](./defining-bundles.md) * [Defining Apps and Binaries](./defining-apps.md) * [Registrations](./registrations.md) diff --git a/docs/components/defining-components.md b/docs/components/defining-components.md index 155523e279279..ec3a4a833880f 100644 --- a/docs/components/defining-components.md +++ b/docs/components/defining-components.md @@ -1,6 +1,6 @@ # Defining Components -You can use the invoke task `inv new-component comp//` to generate a scaffold for your new component. +You can use the invoke task `inv components.new-component comp//` to generate a scaffold for your new component. Below is a description of the different folders and files of your component. diff --git a/pkg/util/flavor/flavor.go b/pkg/util/flavor/flavor.go index a0348ce44f181..b81dcb3d9e493 100644 --- a/pkg/util/flavor/flavor.go +++ b/pkg/util/flavor/flavor.go @@ -29,6 +29,20 @@ const ( TraceAgent = "trace_agent" ) +var agentFlavors = map[string]string{ + DefaultAgent: "Agent", + IotAgent: "IoT Agent", + ClusterAgent: "Cluster Agent", + Dogstatsd: "DogStatsD", + SecurityAgent: "Security Agent", + ServerlessAgent: "Serverless Agent", + HerokuAgent: "Heroku Agent", + ProcessAgent: "Process Agent", + TraceAgent: "Trace Agent", +} + +const unknownAgent = "Unknown Agent" + var agentFlavor = DefaultAgent // SetFlavor sets the Agent flavor @@ -46,3 +60,12 @@ func SetFlavor(flavor string) { func GetFlavor() string { return agentFlavor } + +// GetHumanReadableFlavor gets the running Agent flavor in a human readable form +func GetHumanReadableFlavor() string { + if val, ok := agentFlavors[agentFlavor]; ok { + return val + } + + return unknownAgent +} diff --git a/pkg/util/flavor/flavor_test.go b/pkg/util/flavor/flavor_test.go new file mode 100644 index 0000000000000..b18d14f84170c --- /dev/null +++ b/pkg/util/flavor/flavor_test.go @@ -0,0 +1,29 @@ +// Unless explicitly stated otherwise all files in this repository are licensed +// under the Apache License Version 2.0. +// This product includes software developed at Datadog (https://www.datadoghq.com/). +// Copyright 2016-present Datadog, Inc. + +package flavor + +import ( + "fmt" + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestGetHumanReadableFlavor(t *testing.T) { + for k, v := range agentFlavors { + t.Run(fmt.Sprintf("%s: %s", k, v), func(t *testing.T) { + SetFlavor(k) + + assert.Equal(t, v, GetHumanReadableFlavor()) + }) + } + + t.Run("Unknown Agent", func(t *testing.T) { + SetFlavor("foo") + + assert.Equal(t, "Unknown Agent", GetHumanReadableFlavor()) + }) +} From 2b6a9c17d5c18b258ffdb5b9d479c7ce8dfba63f Mon Sep 17 00:00:00 2001 From: Kevin Fairise <132568982+KevinFairise2@users.noreply.github.com> Date: Mon, 18 Dec 2023 10:45:09 +0100 Subject: [PATCH 13/17] Change Pulumi logger to write to t.Log (#21580) Change Pulumi logger to write to t.Log --- test/new-e2e/pkg/utils/e2e/e2e.go | 14 +++++-- test/new-e2e/pkg/utils/infra/stack_manager.go | 42 +++++++++++++------ .../system-probe/system-probe-test-env.go | 4 +- .../install-script/install_script_test.go | 4 +- test/new-e2e/tests/containers/eks_test.go | 4 +- test/new-e2e/tests/containers/kindvm_test.go | 4 +- test/new-e2e/tests/ndm/snmp/snmpTestEnv.go | 2 +- 7 files changed, 50 insertions(+), 24 deletions(-) diff --git a/test/new-e2e/pkg/utils/e2e/e2e.go b/test/new-e2e/pkg/utils/e2e/e2e.go index 9fd41e981c1cc..bbd491d3972ab 100644 --- a/test/new-e2e/pkg/utils/e2e/e2e.go +++ b/test/new-e2e/pkg/utils/e2e/e2e.go @@ -370,6 +370,15 @@ const ( deleteTimeout = 30 * time.Minute ) +type testWriter struct { + t *testing.T +} + +func (tw testWriter) Write(p []byte) (n int, err error) { + tw.t.Log(string(p)) + return len(p), nil +} + // Suite manages the environment creation and runs E2E tests. type Suite[Env any] struct { suite.Suite @@ -510,7 +519,7 @@ func (suite *Suite[Env]) TearDownSuite() { // TODO: Implement retry on delete ctx, cancel := context.WithTimeout(context.Background(), deleteTimeout) defer cancel() - err := infra.GetStackManager().DeleteStack(ctx, suite.params.StackName) + err := infra.GetStackManager().DeleteStack(ctx, suite.params.StackName, testWriter{t: suite.T()}) if err != nil { suite.T().Errorf("unable to delete stack: %s, err :%v", suite.params.StackName, err) suite.T().Fail() @@ -520,7 +529,6 @@ func (suite *Suite[Env]) TearDownSuite() { func createEnv[Env any](suite *Suite[Env], stackDef *StackDefinition[Env]) (*Env, auto.UpResult, error) { var env *Env ctx := context.Background() - _, stackOutput, err := infra.GetStackManager().GetStackNoDeleteOnFailure( ctx, suite.params.StackName, @@ -529,7 +537,7 @@ func createEnv[Env any](suite *Suite[Env], stackDef *StackDefinition[Env]) (*Env var err error env, err = stackDef.envFactory(ctx) return err - }, false) + }, false, testWriter{t: suite.T()}) return env, stackOutput, err } diff --git a/test/new-e2e/pkg/utils/infra/stack_manager.go b/test/new-e2e/pkg/utils/infra/stack_manager.go index 8f12645b53e53..e0baed5700d2d 100644 --- a/test/new-e2e/pkg/utils/infra/stack_manager.go +++ b/test/new-e2e/pkg/utils/infra/stack_manager.go @@ -10,6 +10,7 @@ import ( "context" "errors" "fmt" + "io" "os" "runtime" "strings" @@ -102,10 +103,10 @@ func newStackManager() (*StackManager, error) { // GetStack creates or return a stack based on stack name and config, if error occurs during stack creation it destroy all the resources created func (sm *StackManager) GetStack(ctx context.Context, name string, config runner.ConfigMap, deployFunc pulumi.RunFunc, failOnMissing bool) (*auto.Stack, auto.UpResult, error) { - stack, upResult, err := sm.getStack(ctx, name, config, deployFunc, failOnMissing) + stack, upResult, err := sm.getStack(ctx, name, config, deployFunc, failOnMissing, nil) if err != nil { - errDestroy := sm.deleteStack(ctx, name, stack) + errDestroy := sm.deleteStack(ctx, name, stack, nil) if errDestroy != nil { return stack, upResult, errors.Join(err, errDestroy) } @@ -115,13 +116,13 @@ func (sm *StackManager) GetStack(ctx context.Context, name string, config runner } // GetStackNoDeleteOnFailure creates or return a stack based on stack name and config, if error occurs during stack creation, it will not destroy the created resources. Using this can lead to resource leaks. -func (sm *StackManager) GetStackNoDeleteOnFailure(ctx context.Context, name string, config runner.ConfigMap, deployFunc pulumi.RunFunc, failOnMissing bool) (*auto.Stack, auto.UpResult, error) { +func (sm *StackManager) GetStackNoDeleteOnFailure(ctx context.Context, name string, config runner.ConfigMap, deployFunc pulumi.RunFunc, failOnMissing bool, logWriter io.Writer) (*auto.Stack, auto.UpResult, error) { - return sm.getStack(ctx, name, config, deployFunc, failOnMissing) + return sm.getStack(ctx, name, config, deployFunc, failOnMissing, logWriter) } // DeleteStack safely deletes a stack -func (sm *StackManager) DeleteStack(ctx context.Context, name string) error { +func (sm *StackManager) DeleteStack(ctx context.Context, name string, logWriter io.Writer) error { stack, ok := sm.stacks.Get(name) if !ok { @@ -141,7 +142,7 @@ func (sm *StackManager) DeleteStack(ctx context.Context, name string) error { stack = &newStack } - return sm.deleteStack(ctx, name, stack) + return sm.deleteStack(ctx, name, stack, logWriter) } // ForceRemoveStackConfiguration removes the configuration files pulumi creates for managing a stack. @@ -164,7 +165,7 @@ func (sm *StackManager) Cleanup(ctx context.Context) []error { var errors []error sm.stacks.Range(func(stackID string, stack *auto.Stack) { - err := sm.deleteStack(ctx, stackID, stack) + err := sm.deleteStack(ctx, stackID, stack, nil) if err != nil { errors = append(errors, err) } @@ -173,13 +174,23 @@ func (sm *StackManager) Cleanup(ctx context.Context) []error { return errors } -func (sm *StackManager) deleteStack(ctx context.Context, stackID string, stack *auto.Stack) error { +func (sm *StackManager) deleteStack(ctx context.Context, stackID string, stack *auto.Stack, logWriter io.Writer) error { if stack == nil { return fmt.Errorf("unable to find stack, skipping deletion of: %s", stackID) } destroyContext, cancel := context.WithTimeout(ctx, stackDestroyTimeout) - _, err := stack.Destroy(destroyContext, optdestroy.ProgressStreams(os.Stdout)) + + var logger io.Writer + + if logWriter == nil { + logger = os.Stdout + } else { + logger = logWriter + } + _, err := stack.Destroy(destroyContext, optdestroy.ProgressStreams(logger), optdestroy.DebugLogging(debug.LoggingOptions{ + FlowToPlugins: true, + })) cancel() if err != nil { return err @@ -191,7 +202,7 @@ func (sm *StackManager) deleteStack(ctx context.Context, stackID string, stack * return err } -func (sm *StackManager) getStack(ctx context.Context, name string, config runner.ConfigMap, deployFunc pulumi.RunFunc, failOnMissing bool) (*auto.Stack, auto.UpResult, error) { +func (sm *StackManager) getStack(ctx context.Context, name string, config runner.ConfigMap, deployFunc pulumi.RunFunc, failOnMissing bool, logWriter io.Writer) (*auto.Stack, auto.UpResult, error) { // Build configuration from profile profile := runner.GetProfile() stackName := buildStackName(profile.NamePrefix(), name) @@ -231,8 +242,15 @@ func (sm *StackManager) getStack(ctx context.Context, name string, config runner upCtx, cancel := context.WithTimeout(ctx, stackUpTimeout) var loglevel uint = 1 defer cancel() - upResult, err := stack.Up(upCtx, optup.ProgressStreams(os.Stderr), optup.DebugLogging(debug.LoggingOptions{ - LogToStdErr: true, + var logger io.Writer + + if logWriter == nil { + logger = os.Stderr + } else { + logger = logWriter + } + + upResult, err := stack.Up(upCtx, optup.ProgressStreams(logger), optup.DebugLogging(debug.LoggingOptions{ FlowToPlugins: true, LogLevel: &loglevel, })) diff --git a/test/new-e2e/system-probe/system-probe-test-env.go b/test/new-e2e/system-probe/system-probe-test-env.go index e19cbb749626e..191c14ea413b3 100644 --- a/test/new-e2e/system-probe/system-probe-test-env.go +++ b/test/new-e2e/system-probe/system-probe-test-env.go @@ -227,7 +227,7 @@ func NewTestEnv(name, x86InstanceType, armInstanceType string, opts *EnvOpts) (* return fmt.Errorf("setup micro-vms in remote instance: %w", err) } return nil - }, opts.FailOnMissing) + }, opts.FailOnMissing, nil) if err != nil { return handleScenarioFailure(err, func(possibleError handledError) { // handle the following errors by trying in a different availability zone @@ -255,7 +255,7 @@ func NewTestEnv(name, x86InstanceType, armInstanceType string, opts *EnvOpts) (* // Destroy deletes the stack with the provided name func Destroy(name string) error { - return infra.GetStackManager().DeleteStack(context.Background(), name) + return infra.GetStackManager().DeleteStack(context.Background(), name, nil) } // RemoveStack removes the stack configuration with the provided name diff --git a/test/new-e2e/tests/agent-platform/install-script/install_script_test.go b/test/new-e2e/tests/agent-platform/install-script/install_script_test.go index 0867bdefc2b45..e4dd94c61a6f0 100644 --- a/test/new-e2e/tests/agent-platform/install-script/install_script_test.go +++ b/test/new-e2e/tests/agent-platform/install-script/install_script_test.go @@ -69,7 +69,7 @@ func TestInstallScript(t *testing.T) { osVersions := strings.Split(*osVersion, ",") cwsSupportedOsVersionList := strings.Split(*cwsSupportedOsVersion, ",") - fmt.Println("Parsed platform json file: ", platformJSON) + t.Log("Parsed platform json file: ", platformJSON) for _, osVers := range osVersions { vmOpts := []ec2params.Option{} @@ -99,7 +99,7 @@ func TestInstallScript(t *testing.T) { } t.Run(fmt.Sprintf("test install script on %s %s %s agent %s", osVers, *architecture, *flavor, *majorVersion), func(tt *testing.T) { tt.Parallel() - fmt.Printf("Testing %s", osVers) + tt.Logf("Testing %s", osVers) e2e.Run(tt, &installScriptSuite{cwsSupported: cwsSupported}, e2e.EC2VMStackDef(vmOpts...), params.WithStackName(fmt.Sprintf("install-script-test-%v-%v-%s-%s-%v", os.Getenv("CI_PIPELINE_ID"), osVers, *architecture, *flavor, *majorVersion))) }) } diff --git a/test/new-e2e/tests/containers/eks_test.go b/test/new-e2e/tests/containers/eks_test.go index b0e5328e7bd67..20a9dd3d5d68a 100644 --- a/test/new-e2e/tests/containers/eks_test.go +++ b/test/new-e2e/tests/containers/eks_test.go @@ -42,13 +42,13 @@ func (suite *eksSuite) SetupSuite() { "dddogstatsd:deploy": auto.ConfigValue{Value: "true"}, } - _, stackOutput, err := infra.GetStackManager().GetStackNoDeleteOnFailure(ctx, "eks-cluster", stackConfig, eks.Run, false) + _, stackOutput, err := infra.GetStackManager().GetStackNoDeleteOnFailure(ctx, "eks-cluster", stackConfig, eks.Run, false, nil) if !suite.Assert().NoError(err) { stackName, err := infra.GetStackManager().GetPulumiStackName("eks-cluster") suite.Require().NoError(err) suite.T().Log(dumpEKSClusterState(ctx, stackName)) if !runner.GetProfile().AllowDevMode() || !*keepStacks { - infra.GetStackManager().DeleteStack(ctx, "eks-cluster") + infra.GetStackManager().DeleteStack(ctx, "eks-cluster", nil) } suite.T().FailNow() } diff --git a/test/new-e2e/tests/containers/kindvm_test.go b/test/new-e2e/tests/containers/kindvm_test.go index 624b459ed2239..322eabfd0ca4d 100644 --- a/test/new-e2e/tests/containers/kindvm_test.go +++ b/test/new-e2e/tests/containers/kindvm_test.go @@ -41,13 +41,13 @@ func (suite *kindSuite) SetupSuite() { "dddogstatsd:deploy": auto.ConfigValue{Value: "true"}, } - _, stackOutput, err := infra.GetStackManager().GetStackNoDeleteOnFailure(ctx, "kind-cluster", stackConfig, kindvm.Run, false) + _, stackOutput, err := infra.GetStackManager().GetStackNoDeleteOnFailure(ctx, "kind-cluster", stackConfig, kindvm.Run, false, nil) if !suite.Assert().NoError(err) { stackName, err := infra.GetStackManager().GetPulumiStackName("kind-cluster") suite.Require().NoError(err) suite.T().Log(dumpKindClusterState(ctx, stackName)) if !runner.GetProfile().AllowDevMode() || !*keepStacks { - infra.GetStackManager().DeleteStack(ctx, "kind-cluster") + infra.GetStackManager().DeleteStack(ctx, "kind-cluster", nil) } suite.T().FailNow() } diff --git a/test/new-e2e/tests/ndm/snmp/snmpTestEnv.go b/test/new-e2e/tests/ndm/snmp/snmpTestEnv.go index 4b1c2ce87b302..c18dc830ddbae 100644 --- a/test/new-e2e/tests/ndm/snmp/snmpTestEnv.go +++ b/test/new-e2e/tests/ndm/snmp/snmpTestEnv.go @@ -131,7 +131,7 @@ func NewTestEnv() (*TestEnv, error) { // Destroy delete the NDM stack. Deprecated, should port to NDM func (testEnv *TestEnv) Destroy() error { - return infra.GetStackManager().DeleteStack(testEnv.context, testEnv.name) + return infra.GetStackManager().DeleteStack(testEnv.context, testEnv.name, nil) } //go:embed compose/data From bff8530633834a17ca68dab0c6a1792dc2444d79 Mon Sep 17 00:00:00 2001 From: Florent Clarret Date: Mon, 18 Dec 2023 12:58:14 +0100 Subject: [PATCH 14/17] Cryptography build (#21463) * test * test * test * update branch * revert to main * Revert "test" This reverts commit 591b10b0e50899bdb078b79080e6b220c18b8534. --- omnibus/config/software/datadog-agent-integrations-py3.rb | 2 ++ 1 file changed, 2 insertions(+) diff --git a/omnibus/config/software/datadog-agent-integrations-py3.rb b/omnibus/config/software/datadog-agent-integrations-py3.rb index 82d6063e6e214..08ed137187222 100644 --- a/omnibus/config/software/datadog-agent-integrations-py3.rb +++ b/omnibus/config/software/datadog-agent-integrations-py3.rb @@ -174,6 +174,8 @@ { "RUSTFLAGS" => "-C link-arg=-Wl,-rpath,#{install_dir}/embedded/lib", "OPENSSL_DIR" => "#{install_dir}/embedded/", + "PIP_NO_CACHE_DIR" => "off", + "PIP_FORCE_REINSTALL" => "1", } ) end From a87ec42695734e9f6129a7c05c0faf05ec3d12ce Mon Sep 17 00:00:00 2001 From: Marethyu <45374460+Pythyu@users.noreply.github.com> Date: Mon, 18 Dec 2023 13:06:14 +0100 Subject: [PATCH 15/17] Updates current tests with fakeintake loadbalancer's new default state (#21535) Updates current tests with fakeintake loadbalancer's new default state --- .gitlab-ci.yml | 2 +- test/new-e2e/examples/agentenv_metrics_test.go | 2 -- .../agent-shared-components/forwarder/nss_failover_test.go | 4 ++-- test/new-e2e/tests/agent-subcommands/diagnose_test.go | 3 +-- test/new-e2e/tests/agent-subcommands/flare/flare_nix_test.go | 3 +-- test/new-e2e/tests/agent-subcommands/flare/flare_win_test.go | 3 +-- test/new-e2e/tests/agent-subcommands/subcommands_test.go | 4 +--- test/new-e2e/tests/ndm/snmp/snmpTestEnv.go | 3 +-- test/new-e2e/tests/process/linux_test.go | 2 -- 9 files changed, 8 insertions(+), 18 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 6b42308704575..9313b5d4a6400 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -176,7 +176,7 @@ variables: # To use images from test-infra-definitions dev branches, set the SUFFIX variable to -dev # and check the job creating the image to make sure you have the right SHA prefix TEST_INFRA_DEFINITIONS_BUILDIMAGES_SUFFIX: "" - TEST_INFRA_DEFINITIONS_BUILDIMAGES: 281b2a324002 + TEST_INFRA_DEFINITIONS_BUILDIMAGES: 67d2009bcd81 DATADOG_AGENT_BUILDERS: v22276738-b36b132 DATADOG_AGENT_EMBEDDED_PATH: /opt/datadog-agent/embedded diff --git a/test/new-e2e/examples/agentenv_metrics_test.go b/test/new-e2e/examples/agentenv_metrics_test.go index 5370c184d06e6..d181e021d80c0 100644 --- a/test/new-e2e/examples/agentenv_metrics_test.go +++ b/test/new-e2e/examples/agentenv_metrics_test.go @@ -11,7 +11,6 @@ import ( "github.com/DataDog/datadog-agent/test/fakeintake/client" "github.com/DataDog/datadog-agent/test/new-e2e/pkg/utils/e2e" - "github.com/DataDog/test-infra-definitions/scenarios/aws/fakeintake/fakeintakeparams" "github.com/DataDog/test-infra-definitions/scenarios/aws/vm/ec2os" "github.com/DataDog/test-infra-definitions/scenarios/aws/vm/ec2params" "github.com/stretchr/testify/assert" @@ -24,7 +23,6 @@ type fakeintakeSuiteMetrics struct { func TestVMSuiteEx5(t *testing.T) { e2e.Run(t, &fakeintakeSuiteMetrics{}, e2e.FakeIntakeStackDef( - e2e.WithFakeIntakeParams(fakeintakeparams.WithoutLoadBalancer()), e2e.WithVMParams(ec2params.WithOS(ec2os.CentOS)), )) } diff --git a/test/new-e2e/tests/agent-shared-components/forwarder/nss_failover_test.go b/test/new-e2e/tests/agent-shared-components/forwarder/nss_failover_test.go index 225439084adbc..df97fec97f461 100644 --- a/test/new-e2e/tests/agent-shared-components/forwarder/nss_failover_test.go +++ b/test/new-e2e/tests/agent-shared-components/forwarder/nss_failover_test.go @@ -80,12 +80,12 @@ func multiFakeintakeStackDef(agentOptions ...agentparams.Option) *e2e.StackDefin return nil, err } - fiExporter1, err := aws.NewEcsFakeintake(awsEnv, fakeintakeparams.WithName(fakeintake1Name), fakeintakeparams.WithoutLoadBalancer()) + fiExporter1, err := aws.NewEcsFakeintake(awsEnv, fakeintakeparams.WithName(fakeintake1Name)) if err != nil { return nil, err } - fiExporter2, err := aws.NewEcsFakeintake(awsEnv, fakeintakeparams.WithName(fakeintake2Name), fakeintakeparams.WithoutLoadBalancer()) + fiExporter2, err := aws.NewEcsFakeintake(awsEnv, fakeintakeparams.WithName(fakeintake2Name)) if err != nil { return nil, err } diff --git a/test/new-e2e/tests/agent-subcommands/diagnose_test.go b/test/new-e2e/tests/agent-subcommands/diagnose_test.go index ee4e8647aa302..a7896f64b4f90 100644 --- a/test/new-e2e/tests/agent-subcommands/diagnose_test.go +++ b/test/new-e2e/tests/agent-subcommands/diagnose_test.go @@ -17,7 +17,6 @@ import ( "github.com/DataDog/datadog-agent/test/new-e2e/pkg/utils/e2e/client" svcmanager "github.com/DataDog/datadog-agent/test/new-e2e/tests/agent-platform/common/svc-manager" "github.com/DataDog/test-infra-definitions/components/datadog/agentparams" - "github.com/DataDog/test-infra-definitions/scenarios/aws/fakeintake/fakeintakeparams" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) @@ -34,7 +33,7 @@ var allSuites = []string{ } func TestAgentDiagnoseEC2Suite(t *testing.T) { - e2e.Run(t, &agentDiagnoseSuite{}, e2e.FakeIntakeStackDef(e2e.WithFakeIntakeParams(fakeintakeparams.WithoutLoadBalancer()))) + e2e.Run(t, &agentDiagnoseSuite{}, e2e.FakeIntakeStackDef()) } // type summary represents the number of success, fail, warnings and errors of a diagnose command diff --git a/test/new-e2e/tests/agent-subcommands/flare/flare_nix_test.go b/test/new-e2e/tests/agent-subcommands/flare/flare_nix_test.go index b639913ab2fd0..9ec2a45b830a2 100644 --- a/test/new-e2e/tests/agent-subcommands/flare/flare_nix_test.go +++ b/test/new-e2e/tests/agent-subcommands/flare/flare_nix_test.go @@ -14,7 +14,6 @@ import ( "github.com/DataDog/datadog-agent/test/new-e2e/pkg/utils/e2e" "github.com/DataDog/datadog-agent/test/new-e2e/pkg/utils/e2e/client" "github.com/DataDog/test-infra-definitions/components/datadog/agentparams" - "github.com/DataDog/test-infra-definitions/scenarios/aws/fakeintake/fakeintakeparams" "github.com/DataDog/test-infra-definitions/scenarios/aws/vm/ec2os" "github.com/DataDog/test-infra-definitions/scenarios/aws/vm/ec2params" ) @@ -34,7 +33,7 @@ type linuxFlareSuite struct { func TestLinuxFlareSuite(t *testing.T) { t.Parallel() - e2e.Run(t, &linuxFlareSuite{}, e2e.FakeIntakeStackDef(e2e.WithVMParams(ec2params.WithOS(ec2os.UbuntuOS)), e2e.WithFakeIntakeParams(fakeintakeparams.WithoutLoadBalancer()))) + e2e.Run(t, &linuxFlareSuite{}, e2e.FakeIntakeStackDef(e2e.WithVMParams(ec2params.WithOS(ec2os.UbuntuOS)))) } func (v *linuxFlareSuite) TestFlareWithAllConfiguration() { diff --git a/test/new-e2e/tests/agent-subcommands/flare/flare_win_test.go b/test/new-e2e/tests/agent-subcommands/flare/flare_win_test.go index 3379d8c899ab4..add1882fc2cf5 100644 --- a/test/new-e2e/tests/agent-subcommands/flare/flare_win_test.go +++ b/test/new-e2e/tests/agent-subcommands/flare/flare_win_test.go @@ -11,7 +11,6 @@ import ( "github.com/DataDog/datadog-agent/test/new-e2e/pkg/utils/e2e" "github.com/DataDog/datadog-agent/test/new-e2e/pkg/utils/e2e/client" - "github.com/DataDog/test-infra-definitions/scenarios/aws/fakeintake/fakeintakeparams" "github.com/DataDog/test-infra-definitions/scenarios/aws/vm/ec2os" "github.com/DataDog/test-infra-definitions/scenarios/aws/vm/ec2params" "github.com/stretchr/testify/assert" @@ -23,7 +22,7 @@ type windowsFlareSuite struct { func TestWindowsFlareSuite(t *testing.T) { t.Parallel() - e2e.Run(t, &windowsFlareSuite{}, e2e.FakeIntakeStackDef(e2e.WithVMParams(ec2params.WithOS(ec2os.WindowsOS)), e2e.WithFakeIntakeParams(fakeintakeparams.WithoutLoadBalancer()))) + e2e.Run(t, &windowsFlareSuite{}, e2e.FakeIntakeStackDef(e2e.WithVMParams(ec2params.WithOS(ec2os.WindowsOS)))) } func (v *windowsFlareSuite) TestFlareWindows() { diff --git a/test/new-e2e/tests/agent-subcommands/subcommands_test.go b/test/new-e2e/tests/agent-subcommands/subcommands_test.go index 6f7457ddc788e..1891fa3ec5983 100644 --- a/test/new-e2e/tests/agent-subcommands/subcommands_test.go +++ b/test/new-e2e/tests/agent-subcommands/subcommands_test.go @@ -14,8 +14,6 @@ import ( "github.com/DataDog/datadog-agent/test/new-e2e/pkg/utils/e2e" "github.com/DataDog/datadog-agent/test/new-e2e/pkg/utils/e2e/client" "github.com/DataDog/test-infra-definitions/components/datadog/agentparams" - "github.com/DataDog/test-infra-definitions/scenarios/aws/fakeintake/fakeintakeparams" - "github.com/cenkalti/backoff" "github.com/stretchr/testify/assert" ) @@ -30,7 +28,7 @@ type subcommandWithFakeIntakeSuite struct { func TestSubcommandSuite(t *testing.T) { e2e.Run(t, &subcommandSuite{}, e2e.AgentStackDef()) - e2e.Run(t, &subcommandWithFakeIntakeSuite{}, e2e.FakeIntakeStackDef(e2e.WithFakeIntakeParams(fakeintakeparams.WithoutLoadBalancer()))) + e2e.Run(t, &subcommandWithFakeIntakeSuite{}, e2e.FakeIntakeStackDef()) } // section contains the content status of a specific section (e.g. Forwarder) diff --git a/test/new-e2e/tests/ndm/snmp/snmpTestEnv.go b/test/new-e2e/tests/ndm/snmp/snmpTestEnv.go index c18dc830ddbae..449b3decb9dcc 100644 --- a/test/new-e2e/tests/ndm/snmp/snmpTestEnv.go +++ b/test/new-e2e/tests/ndm/snmp/snmpTestEnv.go @@ -16,7 +16,6 @@ import ( "github.com/DataDog/test-infra-definitions/components/datadog/agent" "github.com/DataDog/test-infra-definitions/components/datadog/dockeragentparams" "github.com/DataDog/test-infra-definitions/scenarios/aws" - "github.com/DataDog/test-infra-definitions/scenarios/aws/fakeintake/fakeintakeparams" "github.com/DataDog/test-infra-definitions/scenarios/aws/vm/ec2vm" "github.com/pulumi/pulumi/sdk/v3/go/auto" @@ -58,7 +57,7 @@ func NewTestEnv() (*TestEnv, error) { return err } - fakeintakeExporter, err := aws.NewEcsFakeintake(vm.GetAwsEnvironment(), fakeintakeparams.WithoutLoadBalancer()) + fakeintakeExporter, err := aws.NewEcsFakeintake(vm.GetAwsEnvironment()) if err != nil { return err } diff --git a/test/new-e2e/tests/process/linux_test.go b/test/new-e2e/tests/process/linux_test.go index 369abf2d2827e..1eb1b9e6b8757 100644 --- a/test/new-e2e/tests/process/linux_test.go +++ b/test/new-e2e/tests/process/linux_test.go @@ -10,7 +10,6 @@ import ( "time" "github.com/DataDog/test-infra-definitions/components/datadog/agentparams" - "github.com/DataDog/test-infra-definitions/scenarios/aws/fakeintake/fakeintakeparams" "github.com/stretchr/testify/assert" "github.com/DataDog/datadog-agent/test/fakeintake/aggregator" @@ -24,7 +23,6 @@ type linuxTestSuite struct { func TestLinuxTestSuite(t *testing.T) { e2e.Run(t, &linuxTestSuite{}, e2e.FakeIntakeStackDef( - e2e.WithFakeIntakeParams(fakeintakeparams.WithoutLoadBalancer()), e2e.WithAgentParams(agentparams.WithAgentConfig(processCheckConfigStr)), )) } From 39cf73177a2d2ff57d7df65714aabe137cf64ccd Mon Sep 17 00:00:00 2001 From: Nicolas Schweitzer Date: Mon, 18 Dec 2023 13:22:40 +0100 Subject: [PATCH 16/17] Add definition of new labels (#21566) Co-authored-by: Brett Blue <84536271+brett0000FF@users.noreply.github.com> Co-authored-by: Bryce Kahle --- .circleci/config.yml | 20 +++++- .ddqa/config.toml | 2 + .github/PULL_REQUEST_TEMPLATE.md | 2 +- .github/dependabot.yaml | 7 ++ .github/workflows/cws-btfhub-sync.yml | 2 +- docs/dev/contributing.md | 6 +- tasks/__init__.py | 6 +- tasks/{test.py => go_test.py} | 38 ++++++++++- tasks/new_e2e_tests.py | 2 +- tasks/release.py | 1 + tasks/security_agent.py | 2 +- tasks/system_probe.py | 2 +- tasks/unit-tests/linters_tests.py | 97 +++++++++++++++++++++++++++ 13 files changed, 174 insertions(+), 13 deletions(-) rename tasks/{test.py => go_test.py} (96%) create mode 100644 tasks/unit-tests/linters_tests.py diff --git a/.circleci/config.yml b/.circleci/config.yml index e44f826a9b224..8cab47ee58b39 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -135,6 +135,15 @@ jobs: command: inv -e lint-teamassignment name: run PR check for team assignment labels + skip_qa: + <<: *job_template + steps: + - restore_cache: *restore_source + - restore_cache: *restore_deps + - run: + command: inv -e lint-skip-qa + name: run PR check for skip-qa labels + milestone: <<: *job_template steps: @@ -208,8 +217,8 @@ jobs: - run: name: setting env vars for click command: | - echo 'export LC_ALL="C.UTF-8"' >> $BASH_ENV - echo 'export LANG="C.UTF-8"' >> $BASH_ENV + echo 'export LC_ALL="C.UTF-8"' >> $BASH_ENV + echo 'export LANG="C.UTF-8"' >> $BASH_ENV - run: name: lint python files command: inv -e lint-python @@ -307,6 +316,13 @@ workflows: - main requires: - dependencies + - skip_qa: + filters: + branches: + ignore: + - main + requires: + - dependencies - milestone: filters: branches: diff --git a/.ddqa/config.toml b/.ddqa/config.toml index 63bca53c98c08..d408e718a1cd4 100644 --- a/.ddqa/config.toml +++ b/.ddqa/config.toml @@ -6,6 +6,8 @@ qa_statuses = [ ] ignored_labels = [ "qa/skip-qa", + "qa/done", + "qa/no-code-change", ] [teams."Agent Metrics Logs"] diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md index 1d0e2ce7226fe..4faeaff8349ba 100644 --- a/.github/PULL_REQUEST_TEMPLATE.md +++ b/.github/PULL_REQUEST_TEMPLATE.md @@ -57,7 +57,7 @@ Note: Adding GitHub labels is only possible for contributors with write access. - [ ] Use the `major_change` label if your change either has a major impact on the code base, is impacting multiple teams or is changing important well-established internals of the Agent. This label will be use during QA to make sure each team pay extra attention to the changed behavior. For any customer facing change use a releasenote. - [ ] A [release note](https://github.com/DataDog/datadog-agent/blob/main/docs/dev/contributing.md#reno) has been added or the `changelog/no-changelog` label has been applied. - [ ] Changed code has automated tests for its functionality. -- [ ] Adequate QA/testing plan information is provided if the `qa/skip-qa` label is not applied. +- [ ] Adequate QA/testing plan information is provided. Except if the `qa/skip-qa` label, with required either `qa/done` or `qa/no-code-change` labels, are applied. - [ ] At least one `team/..` label has been applied, indicating the team(s) that should QA this change. - [ ] If applicable, docs team has been notified or [an issue has been opened on the documentation repo](https://github.com/DataDog/documentation/issues/new). - [ ] If applicable, the `need-change/operator` and `need-change/helm` labels have been applied. diff --git a/.github/dependabot.yaml b/.github/dependabot.yaml index b8c0d9f11b92e..ce32c4ef15ae1 100644 --- a/.github/dependabot.yaml +++ b/.github/dependabot.yaml @@ -113,6 +113,7 @@ updates: - team/agent-platform - changelog/no-changelog - qa/skip-qa + - qa/no-code-change milestone: 22 schedule: interval: monthly @@ -139,6 +140,7 @@ updates: - team/agent-e2e-test - changelog/no-changelog - qa/skip-qa + - qa/no-code-change - dev/testing milestone: 22 ignore: @@ -157,6 +159,7 @@ updates: - team/agent-e2e-test - changelog/no-changelog - qa/skip-qa + - qa/no-code-change - dev/testing milestone: 22 schedule: @@ -170,6 +173,7 @@ updates: - team/agent-platform - changelog/no-changelog - qa/skip-qa + - qa/no-code-change - dev/tooling milestone: 22 schedule: @@ -183,6 +187,7 @@ updates: - team/agent-security - changelog/no-changelog - qa/skip-qa + - qa/no-code-change - dev/tooling milestone: 22 schedule: @@ -195,6 +200,7 @@ updates: - team/agent-e2e-test - changelog/no-changelog - qa/skip-qa + - qa/no-code-change - dev/testing milestone: 22 schedule: @@ -208,6 +214,7 @@ updates: - team/agent-platform - changelog/no-changelog - qa/skip-qa + - qa/no-code-change - dev/tooling milestone: 22 schedule: diff --git a/.github/workflows/cws-btfhub-sync.yml b/.github/workflows/cws-btfhub-sync.yml index 0aee1b35c2265..4e9a6ed1e86e8 100644 --- a/.github/workflows/cws-btfhub-sync.yml +++ b/.github/workflows/cws-btfhub-sync.yml @@ -85,5 +85,5 @@ jobs: owner, repo, issue_number: result.data.number, - labels: ['changelog/no-changelog', 'qa/skip-qa', 'team/agent-security'] + labels: ['changelog/no-changelog', 'qa/skip-qa', 'qa/no-code-change', 'team/agent-security'] }); diff --git a/docs/dev/contributing.md b/docs/dev/contributing.md index 5225511793b50..8a23d7ad4eb33 100644 --- a/docs/dev/contributing.md +++ b/docs/dev/contributing.md @@ -258,8 +258,10 @@ labels that can be use: - `community`: for community PRs. - `changelog/no-changelog`: for PRs that don't require a reno releasenote (useful for PRs only changing documentation or tests). -- `qa/skip-qa`: this will skip creating a QA card for the PR during the release - process (example: for a documentation only PRs). +- `qa/skip-qa`, `qa/done`, `qa/no-code-change`: if the `qa/skip-qa` label is set with + an additional required `qa/done` or `qa/no-code-change`, it will skip the creation + of a QA card related to this PR during next release process (example: + documentation-only PRs). - `major_change`: to flag the PR as a major change impacting many/all teams working on the agent and will require deeper QA (example: when we change the Python version shipped in the agent). diff --git a/tasks/__init__.py b/tasks/__init__.py index f07f7698ce384..a7d50df7a79ca 100644 --- a/tasks/__init__.py +++ b/tasks/__init__.py @@ -50,8 +50,7 @@ reset, tidy_all, ) -from .show_linters_issues import show_linters_issues -from .test import ( +from .go_test import ( codecov, download_tools, e2e_tests, @@ -68,10 +67,12 @@ lint_milestone, lint_python, lint_releasenote, + lint_skip_qa, lint_teamassignment, send_unit_tests_stats, test, ) +from .show_linters_issues import show_linters_issues from .update_go import go_version, update_go from .utils import generate_config from .windows_resources import build_messagetable @@ -95,6 +96,7 @@ ns.add_task(lint_copyrights), ns.add_task(lint_teamassignment) ns.add_task(lint_releasenote) +ns.add_task(lint_skip_qa) ns.add_task(lint_milestone) ns.add_task(lint_filenames) ns.add_task(lint_python) diff --git a/tasks/test.py b/tasks/go_test.py similarity index 96% rename from tasks/test.py rename to tasks/go_test.py index 66fcbaab7f119..1c3e50b9e3e70 100644 --- a/tasks/test.py +++ b/tasks/go_test.py @@ -854,8 +854,9 @@ def lint_teamassignment(_): issue = res.json() labels = {l['name'] for l in issue.get('labels', [])} - if "qa/skip-qa" in labels: - print("qa/skip-qa label set -- no need for team assignment") + skip_qa_labels = ["qa/skip-qa", "qa/done", "qa/no-code-change"] + if any(skip_label in labels for skip_label in skip_qa_labels): + print("A label to skip QA is set -- no need for team assignment") return for label in labels: @@ -874,6 +875,39 @@ def lint_teamassignment(_): print("PR not found, skipping check for team assignment.") +@task +def lint_skip_qa(_): + """ + Ensure that when qa/skip-qa is used, we have one of [qa/done , qa/no-code-change]. Error if not valid. + """ + branch = os.environ.get("CIRCLE_BRANCH") + pr_url = os.environ.get("CIRCLE_PULL_REQUEST") + + if branch == DEFAULT_BRANCH: + print(f"Running on {DEFAULT_BRANCH}, skipping check for skip-qa label.") + elif pr_url: + import requests + + pr_id = pr_url.rsplit('/')[-1] + + res = requests.get(f"https://api.github.com/repos/DataDog/datadog-agent/issues/{pr_id}") + issue = res.json() + + labels = {l['name'] for l in issue.get('labels', [])} + skip_qa = "qa/skip-qa" + new_qa_labels = ["qa/done", "qa/no-code-change"] + if skip_qa in labels and not any(skip_label in labels for skip_label in new_qa_labels): + print( + f"PR {pr_url} request to skip QA without justification. Requires an additional `qa/done` or `qa/no-code-change`." + ) + raise Exit(code=1) + return + # No PR is associated with this build: given that we have the "run only on PRs" setting activated, + # this can only happen when we're building on a tag. We don't need to check for skip-qa. + else: + print("PR not found, skipping check for skip-qa.") + + @task def lint_milestone(_): """ diff --git a/tasks/new_e2e_tests.py b/tasks/new_e2e_tests.py index e2ce8bcbebb1e..69503338bf77e 100644 --- a/tasks/new_e2e_tests.py +++ b/tasks/new_e2e_tests.py @@ -15,9 +15,9 @@ from invoke.tasks import task from .flavor import AgentFlavor +from .go_test import test_flavor from .libs.junit_upload import produce_junit_tar from .modules import DEFAULT_MODULES -from .test import test_flavor from .utils import REPO_PATH, get_git_commit diff --git a/tasks/release.py b/tasks/release.py index 075c2b9d6002d..3ff5ec0acc235 100644 --- a/tasks/release.py +++ b/tasks/release.py @@ -1135,6 +1135,7 @@ def create_rc(ctx, major_versions="6,7", patch_version=False, upstream="origin") labels=[ "changelog/no-changelog", "qa/skip-qa", + "qa/no-code-change", "team/agent-platform", "team/agent-release-management", "category/release_operations", diff --git a/tasks/security_agent.py b/tasks/security_agent.py index a573ac657b19b..c18673a684d42 100644 --- a/tasks/security_agent.py +++ b/tasks/security_agent.py @@ -13,6 +13,7 @@ from .build_tags import get_default_build_tags from .go import run_golangci_lint +from .go_test import environ from .libs.ninja_syntax import NinjaWriter from .process_agent import TempDir from .system_probe import ( @@ -22,7 +23,6 @@ ninja_define_ebpf_compiler, ninja_define_exe_compiler, ) -from .test import environ from .utils import ( REPO_PATH, bin_name, diff --git a/tasks/system_probe.py b/tasks/system_probe.py index 32a600de46799..40f8fd255ef5a 100644 --- a/tasks/system_probe.py +++ b/tasks/system_probe.py @@ -17,9 +17,9 @@ from invoke.exceptions import Exit from .build_tags import UNIT_TEST_TAGS, get_default_build_tags +from .go_test import environ from .libs.common.color import color_message from .libs.ninja_syntax import NinjaWriter -from .test import environ from .utils import REPO_PATH, bin_name, get_build_flags, get_gobin, get_version_numeric_only from .windows_resources import MESSAGESTRINGS_MC_PATH, arch_to_windres_target diff --git a/tasks/unit-tests/linters_tests.py b/tasks/unit-tests/linters_tests.py new file mode 100644 index 0000000000000..97a36c7e4a534 --- /dev/null +++ b/tasks/unit-tests/linters_tests.py @@ -0,0 +1,97 @@ +import os +import unittest +from unittest.mock import MagicMock, patch + +from invoke import MockContext +from invoke.exceptions import Exit + +from .. import go_test + + +class TestLintSkipQA(unittest.TestCase): + @patch('builtins.print') + def test_on_default(self, mock_print): + os.environ["CIRCLE_BRANCH"] = "main" + os.environ["CIRCLE_PULL_REQUEST"] = "42" + go_test.lint_skip_qa(MockContext()) + mock_print.assert_called_with(f"Running on {go_test.DEFAULT_BRANCH}, skipping check for skip-qa label.") + + @patch('builtins.print') + def test_no_pr(self, mock_print): + os.environ["CIRCLE_BRANCH"] = "pied" + go_test.lint_skip_qa(MockContext()) + mock_print.assert_called_with("PR not found, skipping check for skip-qa.") + + @patch('builtins.print') + @patch('requests.get') + def test_no_skip_qa(self, mock_requests_get, mock_print): + os.environ["CIRCLE_BRANCH"] = "oak" + os.environ["CIRCLE_PULL_REQUEST"] = "51" + issue = {'labels': [{'name': 'de_cadix_a_des_yeux_de_velours'}]} + mock_response = MagicMock() + mock_response.json.return_value = issue + mock_requests_get.return_value = mock_response + go_test.lint_skip_qa(MockContext()) + mock_print.assert_not_called() + + @patch('requests.get') + def test_skip_qa_alone(self, mock_requests_get): + os.environ["CIRCLE_BRANCH"] = "mapple" + os.environ["CIRCLE_PULL_REQUEST"] = "69" + issue = {'labels': [{'name': 'qa/skip-qa'}]} + mock_response = MagicMock() + mock_response.json.return_value = issue + mock_requests_get.return_value = mock_response + with self.assertRaises(Exit): + go_test.lint_skip_qa(MockContext()) + + @patch('requests.get') + def test_skip_qa_bad_label(self, mock_requests_get): + os.environ["CIRCLE_BRANCH"] = "ash" + os.environ["CIRCLE_PULL_REQUEST"] = "666" + issue = {'labels': [{'name': 'qa/skip-qa'}, {"name": "qa/lity-streets"}]} + mock_response = MagicMock() + mock_response.json.return_value = issue + mock_requests_get.return_value = mock_response + with self.assertRaises(Exit): + go_test.lint_skip_qa(MockContext()) + + @patch('builtins.print') + @patch('requests.get') + def test_skip_qa_done(self, mock_requests_get, mock_print): + os.environ["CIRCLE_BRANCH"] = "gingko" + os.environ["CIRCLE_PULL_REQUEST"] = "1337" + issue = {'labels': [{'name': 'qa/skip-qa'}, {'name': 'qa/done'}]} + mock_response = MagicMock() + mock_response.json.return_value = issue + mock_requests_get.return_value = mock_response + go_test.lint_skip_qa(MockContext()) + mock_print.assert_not_called() + + @patch('builtins.print') + @patch('requests.get') + def test_skip_qa_done_alone(self, mock_requests_get, mock_print): + os.environ["CIRCLE_BRANCH"] = "beech" + os.environ["CIRCLE_PULL_REQUEST"] = "1515" + issue = {'labels': [{'name': 'qa/done'}]} + mock_response = MagicMock() + mock_response.json.return_value = issue + mock_requests_get.return_value = mock_response + go_test.lint_skip_qa(MockContext()) + mock_print.assert_not_called() + + @patch('builtins.print') + @patch('requests.get') + def test_skip_qa_no_code(self, mock_requests_get, mock_print): + os.environ["CIRCLE_BRANCH"] = "sequoia" + os.environ["CIRCLE_PULL_REQUEST"] = "1664" + issue = {'labels': [{'name': 'qa/skip-qa'}, {'name': 'qa/no-code-change'}]} + mock_response = MagicMock() + mock_response.json.return_value = issue + mock_requests_get.return_value = mock_response + go_test.lint_skip_qa(MockContext()) + mock_print.assert_not_called() + + +if __name__ == "__main__": + unittest.main() From 6bb4dd62a3a399417cccfe28fd7559557f3680e4 Mon Sep 17 00:00:00 2001 From: Nicolas Guerguadj <35628945+Kaderinho@users.noreply.github.com> Date: Mon, 18 Dec 2023 14:21:44 +0100 Subject: [PATCH 17/17] fix: add dsd replay, metadata host and inventoryhost back to the run() signature (#21605) Add dsd replay, metadata host and inventoryhost back to the run() signature --- cmd/agent/subcommands/run/command.go | 6 ++++++ cmd/agent/subcommands/run/command_windows.go | 6 ++++++ 2 files changed, 12 insertions(+) diff --git a/cmd/agent/subcommands/run/command.go b/cmd/agent/subcommands/run/command.go index 07b4eadb3a7c7..78a0082cf21e7 100644 --- a/cmd/agent/subcommands/run/command.go +++ b/cmd/agent/subcommands/run/command.go @@ -53,6 +53,7 @@ import ( "github.com/DataDog/datadog-agent/comp/core/workloadmeta" "github.com/DataDog/datadog-agent/comp/core/workloadmeta/collectors" "github.com/DataDog/datadog-agent/comp/dogstatsd" + "github.com/DataDog/datadog-agent/comp/dogstatsd/replay" dogstatsdServer "github.com/DataDog/datadog-agent/comp/dogstatsd/server" dogstatsddebug "github.com/DataDog/datadog-agent/comp/dogstatsd/serverDebug" "github.com/DataDog/datadog-agent/comp/forwarder" @@ -66,8 +67,10 @@ import ( "github.com/DataDog/datadog-agent/comp/logs" logsAgent "github.com/DataDog/datadog-agent/comp/logs/agent" "github.com/DataDog/datadog-agent/comp/metadata" + "github.com/DataDog/datadog-agent/comp/metadata/host" "github.com/DataDog/datadog-agent/comp/metadata/inventoryagent" "github.com/DataDog/datadog-agent/comp/metadata/inventorychecks" + "github.com/DataDog/datadog-agent/comp/metadata/inventoryhost" "github.com/DataDog/datadog-agent/comp/metadata/runner" "github.com/DataDog/datadog-agent/comp/ndmtmp" "github.com/DataDog/datadog-agent/comp/netflow" @@ -198,6 +201,7 @@ func run(log log.Component, telemetry telemetry.Component, sysprobeconfig sysprobeconfig.Component, server dogstatsdServer.Component, + _ replay.Component, serverDebug dogstatsddebug.Component, forwarder defaultforwarder.Component, wmeta workloadmeta.Component, @@ -209,7 +213,9 @@ func run(log log.Component, cliParams *cliParams, logsAgent optional.Option[logsAgent.Component], otelcollector otelcollector.Component, + _ host.Component, invAgent inventoryagent.Component, + _ inventoryhost.Component, _ secrets.Component, invChecks inventorychecks.Component, _ netflowServer.Component, diff --git a/cmd/agent/subcommands/run/command_windows.go b/cmd/agent/subcommands/run/command_windows.go index 6b928d17e0c8b..26a36d9ac7e89 100644 --- a/cmd/agent/subcommands/run/command_windows.go +++ b/cmd/agent/subcommands/run/command_windows.go @@ -44,12 +44,15 @@ import ( "github.com/DataDog/datadog-agent/comp/core/sysprobeconfig/sysprobeconfigimpl" "github.com/DataDog/datadog-agent/comp/core/telemetry" "github.com/DataDog/datadog-agent/comp/core/workloadmeta" + "github.com/DataDog/datadog-agent/comp/dogstatsd/replay" dogstatsdServer "github.com/DataDog/datadog-agent/comp/dogstatsd/server" dogstatsddebug "github.com/DataDog/datadog-agent/comp/dogstatsd/serverDebug" "github.com/DataDog/datadog-agent/comp/forwarder/defaultforwarder" logsAgent "github.com/DataDog/datadog-agent/comp/logs/agent" + "github.com/DataDog/datadog-agent/comp/metadata/host" "github.com/DataDog/datadog-agent/comp/metadata/inventoryagent" "github.com/DataDog/datadog-agent/comp/metadata/inventorychecks" + "github.com/DataDog/datadog-agent/comp/metadata/inventoryhost" "github.com/DataDog/datadog-agent/comp/metadata/runner" netflowServer "github.com/DataDog/datadog-agent/comp/netflow/server" otelcollector "github.com/DataDog/datadog-agent/comp/otelcol/collector" @@ -80,6 +83,7 @@ func StartAgentWithDefaults(ctxChan <-chan context.Context) (<-chan error, error telemetry telemetry.Component, sysprobeconfig sysprobeconfig.Component, server dogstatsdServer.Component, + _ replay.Component, serverDebug dogstatsddebug.Component, wmeta workloadmeta.Component, rcclient rcclient.Component, @@ -89,7 +93,9 @@ func StartAgentWithDefaults(ctxChan <-chan context.Context) (<-chan error, error sharedSerializer serializer.MetricSerializer, otelcollector otelcollector.Component, demultiplexer demultiplexer.Component, + _ host.Component, invAgent inventoryagent.Component, + _ inventoryhost.Component, _ secrets.Component, invChecks inventorychecks.Component, _ netflowServer.Component,