From e92df0ab7af2969d5945d75ab92b82fce34bd1e3 Mon Sep 17 00:00:00 2001 From: Raphael Gavache Date: Wed, 18 Dec 2024 19:33:19 +0100 Subject: [PATCH 01/14] [fleet] use custom tracer (#32301) --- LICENSE-3rdparty.csv | 3 - cmd/installer-downloader/main.go | 6 +- .../subcommands/installer/command.go | 12 +- .../subcommands/installer/umask_nix.go | 2 +- .../subcommands/installer/umask_windows.go | 2 +- .../agentcrashdetectimpl/agentcrashdetect.go | 13 +- .../telemetry/telemetryimpl/telemetry.go | 12 +- pkg/fleet/daemon/daemon.go | 2 +- pkg/fleet/installer/setup/common/setup.go | 4 +- pkg/fleet/internal/exec/installer_exec.go | 2 +- pkg/fleet/telemetry/http_wrapper.go | 84 +++++ pkg/fleet/telemetry/span.go | 138 +++++++- pkg/fleet/telemetry/telemetry.go | 312 ++++++------------ pkg/fleet/telemetry/telemetry_test.go | 222 +++++++++++++ pkg/fleet/telemetry/tracer.go | 92 ++++++ pkg/internaltelemetry/client.go | 24 +- pkg/internaltelemetry/traces.go | 41 +++ 17 files changed, 715 insertions(+), 256 deletions(-) create mode 100644 pkg/fleet/telemetry/http_wrapper.go create mode 100644 pkg/fleet/telemetry/telemetry_test.go create mode 100644 pkg/fleet/telemetry/tracer.go create mode 100644 pkg/internaltelemetry/traces.go diff --git a/LICENSE-3rdparty.csv b/LICENSE-3rdparty.csv index 92e5c072e3c8e..13955c7e8a940 100644 --- a/LICENSE-3rdparty.csv +++ b/LICENSE-3rdparty.csv @@ -2906,9 +2906,6 @@ core,google.golang.org/protobuf/types/known/timestamppb,BSD-3-Clause,Copyright ( core,google.golang.org/protobuf/types/known/wrapperspb,BSD-3-Clause,Copyright (c) 2018 The Go Authors. All rights reserved core,google.golang.org/protobuf/types/pluginpb,BSD-3-Clause,Copyright (c) 2018 The Go Authors. All rights reserved core,gopkg.in/DataDog/dd-trace-go.v1/appsec/events,Apache-2.0,"Copyright 2016-Present Datadog, Inc." -core,gopkg.in/DataDog/dd-trace-go.v1/contrib/internal/httptrace,Apache-2.0,"Copyright 2016-Present Datadog, Inc." -core,gopkg.in/DataDog/dd-trace-go.v1/contrib/internal/options,Apache-2.0,"Copyright 2016-Present Datadog, Inc." -core,gopkg.in/DataDog/dd-trace-go.v1/contrib/net/http,Apache-2.0,"Copyright 2016-Present Datadog, Inc." core,gopkg.in/DataDog/dd-trace-go.v1/datastreams/options,Apache-2.0,"Copyright 2016-Present Datadog, Inc." core,gopkg.in/DataDog/dd-trace-go.v1/ddtrace,Apache-2.0,"Copyright 2016-Present Datadog, Inc." core,gopkg.in/DataDog/dd-trace-go.v1/ddtrace/ext,Apache-2.0,"Copyright 2016-Present Datadog, Inc." diff --git a/cmd/installer-downloader/main.go b/cmd/installer-downloader/main.go index f7b67910ae94e..60e2141af944e 100644 --- a/cmd/installer-downloader/main.go +++ b/cmd/installer-downloader/main.go @@ -44,12 +44,12 @@ func main() { ctx := context.Background() t := telemetry.NewTelemetry(env.HTTPClient(), env.APIKey, env.Site, fmt.Sprintf("datadog-installer-downloader-%s", Flavor)) - _ = t.Start(ctx) - defer func() { _ = t.Stop(ctx) }() var err error span, ctx := telemetry.StartSpanFromEnv(ctx, fmt.Sprintf("downloader-%s", Flavor)) - defer func() { span.Finish(err) }() err = runDownloader(ctx, env, Version, Flavor) + + span.Finish(err) + t.Stop() if err != nil { fmt.Fprintf(os.Stderr, "Installation failed: %v\n", err) os.Exit(1) diff --git a/cmd/installer/subcommands/installer/command.go b/cmd/installer/subcommands/installer/command.go index 66ec6b1c74c57..f394d9955d407 100644 --- a/cmd/installer/subcommands/installer/command.go +++ b/cmd/installer/subcommands/installer/command.go @@ -87,7 +87,7 @@ func UnprivilegedCommands(_ *command.GlobalParams) []*cobra.Command { type cmd struct { t *telemetry.Telemetry ctx context.Context - span telemetry.Span + span *telemetry.Span env *env.Env } @@ -107,10 +107,7 @@ func newCmd(operation string) *cmd { func (c *cmd) Stop(err error) { c.span.Finish(err) if c.t != nil { - err := c.t.Stop(context.Background()) - if err != nil { - fmt.Fprintf(os.Stderr, "failed to stop telemetry: %v\n", err) - } + c.t.Stop() } } @@ -225,11 +222,6 @@ func newTelemetry(env *env.Env) *telemetry.Telemetry { site = config.Site } t := telemetry.NewTelemetry(env.HTTPClient(), apiKey, site, "datadog-installer") // No sampling rules for commands - err := t.Start(context.Background()) - if err != nil { - fmt.Printf("failed to start telemetry: %v\n", err) - return nil - } return t } diff --git a/cmd/installer/subcommands/installer/umask_nix.go b/cmd/installer/subcommands/installer/umask_nix.go index 1fd44c01ac405..dec64eccd6860 100644 --- a/cmd/installer/subcommands/installer/umask_nix.go +++ b/cmd/installer/subcommands/installer/umask_nix.go @@ -14,7 +14,7 @@ import ( ) // setInstallerUmask sets umask 0 to override any inherited umask -func setInstallerUmask(span telemetry.Span) { +func setInstallerUmask(span *telemetry.Span) { oldmask := syscall.Umask(0) span.SetTag("inherited_umask", oldmask) } diff --git a/cmd/installer/subcommands/installer/umask_windows.go b/cmd/installer/subcommands/installer/umask_windows.go index 1b076f92bc389..d8661700cd56e 100644 --- a/cmd/installer/subcommands/installer/umask_windows.go +++ b/cmd/installer/subcommands/installer/umask_windows.go @@ -10,4 +10,4 @@ package installer import "github.com/DataDog/datadog-agent/pkg/fleet/telemetry" // setInstallerUmask no-op on Windows -func setInstallerUmask(_ telemetry.Span) {} +func setInstallerUmask(_ *telemetry.Span) {} diff --git a/comp/checks/agentcrashdetect/agentcrashdetectimpl/agentcrashdetect.go b/comp/checks/agentcrashdetect/agentcrashdetectimpl/agentcrashdetect.go index d8631536c3a44..6098dcf98f629 100644 --- a/comp/checks/agentcrashdetect/agentcrashdetectimpl/agentcrashdetect.go +++ b/comp/checks/agentcrashdetect/agentcrashdetectimpl/agentcrashdetect.go @@ -168,11 +168,22 @@ func (wcd *AgentCrashDetect) Run() error { } log.Infof("Sending crash: %v", formatText(crash)) - lts := internaltelemetry.NewClient(wcd.tconfig.NewHTTPClient(), wcd.tconfig.TelemetryConfig.Endpoints, "ddnpm", true) + lts := internaltelemetry.NewClient(wcd.tconfig.NewHTTPClient(), toTelemEndpoints(wcd.tconfig.TelemetryConfig.Endpoints), "ddnpm", true) lts.SendLog("WARN", formatText(crash)) return nil } +func toTelemEndpoints(endpoints []*traceconfig.Endpoint) []*internaltelemetry.Endpoint { + telemEndpoints := make([]*internaltelemetry.Endpoint, 0, len(endpoints)) + for _, e := range endpoints { + telemEndpoints = append(telemEndpoints, &internaltelemetry.Endpoint{ + Host: e.Host, + APIKey: e.APIKey, + }) + } + return telemEndpoints +} + func newAgentCrashComponent(deps dependencies) agentcrashdetect.Component { instance := &agentCrashComponent{} instance.tconfig = deps.TConfig.Object() diff --git a/comp/updater/telemetry/telemetryimpl/telemetry.go b/comp/updater/telemetry/telemetryimpl/telemetry.go index d8f961e8398e6..ee1bab4efdd56 100644 --- a/comp/updater/telemetry/telemetryimpl/telemetry.go +++ b/comp/updater/telemetry/telemetryimpl/telemetry.go @@ -7,10 +7,10 @@ package telemetryimpl import ( + "context" "net/http" "go.uber.org/fx" - "gopkg.in/DataDog/dd-trace-go.v1/ddtrace/tracer" "github.com/DataDog/datadog-agent/comp/core/config" "github.com/DataDog/datadog-agent/comp/updater/telemetry" @@ -38,13 +38,7 @@ func newTelemetry(deps dependencies) (telemetry.Component, error) { client := &http.Client{ Transport: httputils.CreateHTTPTransport(deps.Config), } - telemetry := fleettelemetry.NewTelemetry(client, utils.SanitizeAPIKey(deps.Config.GetString("api_key")), deps.Config.GetString("site"), "datadog-installer-daemon", - fleettelemetry.WithSamplingRules( - tracer.NameServiceRule("cdn.*", "datadog-installer-daemon", 0.1), - tracer.NameServiceRule("*garbage_collect*", "datadog-installer-daemon", 0.05), - tracer.NameServiceRule("HTTPClient.*", "datadog-installer-daemon", 0.05), - ), - ) - deps.Lc.Append(fx.Hook{OnStart: telemetry.Start, OnStop: telemetry.Stop}) + telemetry := fleettelemetry.NewTelemetry(client, utils.SanitizeAPIKey(deps.Config.GetString("api_key")), deps.Config.GetString("site"), "datadog-installer-daemon") + deps.Lc.Append(fx.Hook{OnStop: func(context.Context) error { telemetry.Stop(); return nil }}) return telemetry, nil } diff --git a/pkg/fleet/daemon/daemon.go b/pkg/fleet/daemon/daemon.go index ba79d5456dd3e..5023393d99e5b 100644 --- a/pkg/fleet/daemon/daemon.go +++ b/pkg/fleet/daemon/daemon.go @@ -570,7 +570,7 @@ type requestState struct { ErrorCode installerErrors.InstallerErrorCode } -func newRequestContext(request remoteAPIRequest) (telemetry.Span, context.Context) { +func newRequestContext(request remoteAPIRequest) (*telemetry.Span, context.Context) { ctx := context.WithValue(context.Background(), requestStateKey, &requestState{ Package: request.Package, ID: request.ID, diff --git a/pkg/fleet/installer/setup/common/setup.go b/pkg/fleet/installer/setup/common/setup.go index 0a7b93447c22f..b6e512d7dc5d6 100644 --- a/pkg/fleet/installer/setup/common/setup.go +++ b/pkg/fleet/installer/setup/common/setup.go @@ -41,7 +41,7 @@ type Setup struct { Out *Output Env *env.Env Ctx context.Context - Span telemetry.Span + Span *telemetry.Span Packages Packages Config Config } @@ -130,7 +130,7 @@ func (s *Setup) installPackage(name string, url string) (err error) { span, ctx := telemetry.StartSpanFromContext(s.Ctx, "install") defer func() { span.Finish(err) }() span.SetTag("url", url) - span.SetTag("_top_level", 1) + span.SetTopLevel() s.Out.WriteString(fmt.Sprintf("Installing %s...\n", name)) err = s.installer.Install(ctx, url, nil) diff --git a/pkg/fleet/internal/exec/installer_exec.go b/pkg/fleet/internal/exec/installer_exec.go index 834f54b3d8a53..92f5880723fa9 100644 --- a/pkg/fleet/internal/exec/installer_exec.go +++ b/pkg/fleet/internal/exec/installer_exec.go @@ -40,7 +40,7 @@ func NewInstallerExec(env *env.Env, installerBinPath string) *InstallerExec { type installerCmd struct { *exec.Cmd - span telemetry.Span + span *telemetry.Span ctx context.Context } diff --git a/pkg/fleet/telemetry/http_wrapper.go b/pkg/fleet/telemetry/http_wrapper.go new file mode 100644 index 0000000000000..2f94bda79fe1f --- /dev/null +++ b/pkg/fleet/telemetry/http_wrapper.go @@ -0,0 +1,84 @@ +// Unless explicitly stated otherwise all files in this repository are licensed +// under the Apache License Version 2.0. +// This product includes software developed at Datadog (https://www.datadoghq.com/). +// Copyright 2016-present Datadog, Inc. + +// Package telemetry provides the telemetry for fleet components. +package telemetry + +import ( + "net/http" + "strconv" + "strings" +) + +// WrapRoundTripper wraps the round tripper with the telemetry round tripper. +func WrapRoundTripper(rt http.RoundTripper) http.RoundTripper { + if rt == nil { + rt = http.DefaultTransport + } + if wrapped, ok := rt.(*roundTripper); ok { + rt = wrapped.base + } + return &roundTripper{ + base: rt, + } +} + +type roundTripper struct { + base http.RoundTripper +} + +func (rt *roundTripper) RoundTrip(req *http.Request) (res *http.Response, err error) { + span, _ := StartSpanFromContext(req.Context(), "http.request") + defer func() { span.Finish(err) }() + + url := *req.URL + url.User = nil + + span.span.Type = "http" + span.SetResourceName(req.Method + " " + urlFromRequest(req)) + span.span.Meta["http.method"] = req.Method + span.span.Meta["http.url"] = req.URL.String() + span.span.Meta["span.kind"] = "client" + span.span.Meta["network.destination.name"] = url.Hostname() + res, err = rt.base.RoundTrip(req) + if err != nil { + span.SetTag("http.errors", err.Error()) + return res, err + } + span.SetTag("http.status_code", strconv.Itoa(res.StatusCode)) + if res.StatusCode >= 400 { + span.SetTag("http.errors", res.Status) + } + return res, err +} + +// urlFromRequest returns the URL from the HTTP request. The URL query string is included in the return object iff queryString is true +// See https://docs.datadoghq.com/tracing/configure_data_security#redacting-the-query-in-the-url for more information. +func urlFromRequest(r *http.Request) string { + // Quoting net/http comments about net.Request.URL on server requests: + // "For most requests, fields other than Path and RawQuery will be + // empty. (See RFC 7230, Section 5.3)" + // This is why we don't rely on url.URL.String(), url.URL.Host, url.URL.Scheme, etc... + var url string + path := r.URL.EscapedPath() + scheme := r.URL.Scheme + if r.TLS != nil { + scheme = "https" + } + if r.Host != "" { + url = strings.Join([]string{scheme, "://", r.Host, path}, "") + } else { + url = path + } + // Collect the query string if we are allowed to report it and obfuscate it if possible/allowed + if r.URL.RawQuery != "" { + query := r.URL.RawQuery + url = strings.Join([]string{url, query}, "?") + } + if frag := r.URL.EscapedFragment(); frag != "" { + url = strings.Join([]string{url, frag}, "#") + } + return url +} diff --git a/pkg/fleet/telemetry/span.go b/pkg/fleet/telemetry/span.go index ac8f79516c736..ac5f00b9ade0a 100644 --- a/pkg/fleet/telemetry/span.go +++ b/pkg/fleet/telemetry/span.go @@ -7,20 +7,144 @@ package telemetry import ( - "gopkg.in/DataDog/dd-trace-go.v1/ddtrace" - "gopkg.in/DataDog/dd-trace-go.v1/ddtrace/ext" - "gopkg.in/DataDog/dd-trace-go.v1/ddtrace/tracer" + "context" + "fmt" + "math/rand/v2" + "runtime/debug" + "strconv" + "sync" + "sync/atomic" + "time" + + "github.com/DataDog/datadog-agent/pkg/internaltelemetry" ) -// Span is an alias for ddtrace.Span until we phase ddtrace out. -type Span struct{ ddtrace.Span } +const spanKey = spanContextKey("span_context") + +type spanContextKey string + +// Span represents a span. +type Span struct { + mu sync.Mutex + span internaltelemetry.Span + finished atomic.Bool +} + +func newSpan(name string, parentID, traceID uint64) *Span { + if traceID == 0 { + traceID = rand.Uint64() + if !headSamplingKeep(name, traceID) { + traceID = dropTraceID + } + } + s := &Span{ + span: internaltelemetry.Span{ + TraceID: traceID, + ParentID: parentID, + SpanID: rand.Uint64(), + Name: name, + Resource: name, + Start: time.Now().UnixNano(), + Meta: make(map[string]string), + Metrics: make(map[string]float64), + }, + } + if parentID == 0 { + s.SetTopLevel() + } + + globalTracer.registerSpan(s) + return s +} // Finish finishes the span with an error. func (s *Span) Finish(err error) { - s.Span.Finish(tracer.WithError(err)) + s.finished.Store(true) + s.mu.Lock() + defer s.mu.Unlock() + s.span.Duration = time.Now().UnixNano() - s.span.Start + if err != nil { + s.span.Error = 1 + s.span.Meta = map[string]string{ + "error.message": err.Error(), + "error.stack": string(debug.Stack()), + } + } + globalTracer.finishSpan(s) } // SetResourceName sets the resource name of the span. func (s *Span) SetResourceName(name string) { - s.Span.SetTag(ext.ResourceName, name) + if s.finished.Load() { + return + } + s.mu.Lock() + defer s.mu.Unlock() + s.span.Resource = name +} + +// SetTopLevel sets the span as a top level span. +func (s *Span) SetTopLevel() { + s.SetTag("_top_level", 1) +} + +// SetTag sets a tag on the span. +func (s *Span) SetTag(key string, value interface{}) { + if s.finished.Load() { + return + } + s.mu.Lock() + defer s.mu.Unlock() + if value == nil { + s.span.Meta[key] = "nil" + } + switch v := value.(type) { + case string: + s.span.Meta[key] = v + case bool: + s.span.Meta[key] = strconv.FormatBool(v) + case int: + s.span.Metrics[key] = float64(v) + case int8: + s.span.Metrics[key] = float64(v) + case int16: + s.span.Metrics[key] = float64(v) + case int32: + s.span.Metrics[key] = float64(v) + case int64: + s.span.Metrics[key] = float64(v) + case uint: + s.span.Metrics[key] = float64(v) + case uint8: + s.span.Metrics[key] = float64(v) + case uint16: + s.span.Metrics[key] = float64(v) + case uint32: + s.span.Metrics[key] = float64(v) + case uint64: + s.span.Metrics[key] = float64(v) + case float32: + s.span.Metrics[key] = float64(v) + case float64: + s.span.Metrics[key] = v + default: + s.span.Meta[key] = fmt.Sprintf("not_supported_type %T", v) + } +} + +type spanIDs struct { + traceID uint64 + spanID uint64 +} + +func getSpanIDsFromContext(ctx context.Context) (spanIDs, bool) { + sIDs, ok := ctx.Value(spanKey).(spanIDs) + if !ok { + return spanIDs{}, false + } + return sIDs, true +} + +func setSpanIDsInContext(ctx context.Context, span *Span) context.Context { + return context.WithValue(ctx, spanKey, spanIDs{traceID: span.span.TraceID, spanID: span.span.SpanID}) } diff --git a/pkg/fleet/telemetry/telemetry.go b/pkg/fleet/telemetry/telemetry.go index 5bbfd8ca773c4..66174ad18fc32 100644 --- a/pkg/fleet/telemetry/telemetry.go +++ b/pkg/fleet/telemetry/telemetry.go @@ -8,280 +8,178 @@ package telemetry import ( "context" - "errors" "fmt" - "io" - "math/rand/v2" - "net" "net/http" "os" "strconv" "strings" - "sync" - - httptrace "gopkg.in/DataDog/dd-trace-go.v1/contrib/net/http" - "gopkg.in/DataDog/dd-trace-go.v1/ddtrace" - "gopkg.in/DataDog/dd-trace-go.v1/ddtrace/tracer" - - "github.com/gorilla/mux" + "time" "github.com/DataDog/datadog-agent/pkg/internaltelemetry" - pb "github.com/DataDog/datadog-agent/pkg/proto/pbgo/trace" - traceconfig "github.com/DataDog/datadog-agent/pkg/trace/config" - "github.com/DataDog/datadog-agent/pkg/util/log" "github.com/DataDog/datadog-agent/pkg/version" ) const ( - // EnvTraceID is the environment variable key for the trace ID - EnvTraceID = "DATADOG_TRACE_ID" - // EnvParentID is the environment variable key for the parent ID - EnvParentID = "DATADOG_PARENT_ID" -) - -const ( + envTraceID = "DATADOG_TRACE_ID" + envParentID = "DATADOG_PARENT_ID" telemetrySubdomain = "instrumentation-telemetry-intake" - telemetryEndpoint = "/v0.4/traces" ) // Telemetry handles the telemetry for fleet components. type Telemetry struct { telemetryClient internaltelemetry.Client + done chan struct{} + flushed chan struct{} - site string + env string service string - - listener *telemetryListener - server *http.Server - client *http.Client - - samplingRules []tracer.SamplingRule } -// Option is a functional option for telemetry. -type Option func(*Telemetry) - // NewTelemetry creates a new telemetry instance -func NewTelemetry(client *http.Client, apiKey string, site string, service string, opts ...Option) *Telemetry { - endpoint := &traceconfig.Endpoint{ +func NewTelemetry(client *http.Client, apiKey string, site string, service string) *Telemetry { + t := newTelemetry(client, apiKey, site, service) + t.Start() + return t +} + +func newTelemetry(client *http.Client, apiKey string, site string, service string) *Telemetry { + endpoint := &internaltelemetry.Endpoint{ Host: fmt.Sprintf("https://%s.%s", telemetrySubdomain, strings.TrimSpace(site)), APIKey: apiKey, } - listener := newTelemetryListener() - t := &Telemetry{ - telemetryClient: internaltelemetry.NewClient(client, []*traceconfig.Endpoint{endpoint}, service, site == "datad0g.com"), - site: site, - service: service, - listener: listener, - server: &http.Server{}, - client: &http.Client{ - Transport: &http.Transport{ - Dial: listener.Dial, - }, - }, + env := "prod" + if site == "datad0g.com" { + env = "staging" } - for _, opt := range opts { - opt(t) + + return &Telemetry{ + telemetryClient: internaltelemetry.NewClient(client, []*internaltelemetry.Endpoint{endpoint}, service, site == "datad0g.com"), + done: make(chan struct{}), + flushed: make(chan struct{}), + env: env, + service: service, } - t.server.Handler = t.handler() - return t } // Start starts the telemetry -func (t *Telemetry) Start(_ context.Context) error { +func (t *Telemetry) Start() { + ticker := time.Tick(1 * time.Minute) go func() { - err := t.server.Serve(t.listener) - if err != nil { - log.Infof("telemetry server stopped: %v", err) + for { + select { + case <-ticker: + t.sendCompletedSpans() + case <-t.done: + t.sendCompletedSpans() + close(t.flushed) + return + } } }() - env := "prod" - if t.site == "datad0g.com" { - env = "staging" - } - - tracer.Start( - tracer.WithService(t.service), - tracer.WithServiceVersion(version.AgentVersion), - tracer.WithEnv(env), - tracer.WithGlobalTag("site", t.site), - tracer.WithHTTPClient(t.client), - tracer.WithLogStartup(false), - - // We don't need the value, we just need to enforce that it's not - // the default. If it is, then the tracer will try to use the socket - // if it exists -- and it always exists for newer agents. - // If the agent address is the socket, the tracer overrides WithHTTPClient to use it. - tracer.WithAgentAddr("192.0.2.42:12345"), // 192.0.2.0/24 is reserved - tracer.WithSamplingRules(t.samplingRules), - ) - return nil } // Stop stops the telemetry -func (t *Telemetry) Stop(ctx context.Context) error { - tracer.Flush() - tracer.Stop() - t.listener.Close() - err := t.server.Shutdown(ctx) - if err != nil { - log.Errorf("error shutting down telemetry server: %v", err) - } - return nil -} - -func (t *Telemetry) handler() http.Handler { - r := mux.NewRouter().Headers("Content-Type", "application/msgpack").Subrouter() - r.HandleFunc(telemetryEndpoint, func(w http.ResponseWriter, r *http.Request) { - defer r.Body.Close() - body, err := io.ReadAll(r.Body) - if err != nil { - log.Errorf("error reading request body: %v", err) - w.WriteHeader(http.StatusInternalServerError) - return - } - var traces pb.Traces - _, err = traces.UnmarshalMsg(body) - if err != nil { - log.Errorf("error unmarshalling traces: %v", err) - w.WriteHeader(http.StatusBadRequest) - return - } - t.telemetryClient.SendTraces(traces) - w.WriteHeader(http.StatusOK) - }) - return r +func (t *Telemetry) Stop() { + close(t.done) + <-t.flushed } -type telemetryListener struct { - conns chan net.Conn - - close chan struct{} - closeOnce sync.Once -} - -func newTelemetryListener() *telemetryListener { - return &telemetryListener{ - conns: make(chan net.Conn), - close: make(chan struct{}), +func (t *Telemetry) extractCompletedSpans() internaltelemetry.Traces { + spans := globalTracer.flushCompletedSpans() + if len(spans) == 0 { + return internaltelemetry.Traces{} } -} - -func (l *telemetryListener) Close() error { - l.closeOnce.Do(func() { - close(l.close) - }) - return nil -} - -func (l *telemetryListener) Accept() (net.Conn, error) { - select { - case <-l.close: - return nil, errors.New("listener closed") - case conn := <-l.conns: - return conn, nil + traces := make(map[uint64][]*internaltelemetry.Span) + for _, span := range spans { + span.span.Service = t.service + span.span.Meta["env"] = t.env + span.span.Meta["version"] = version.AgentVersion + span.span.Metrics["_sampling_priority_v1"] = 2 + traces[span.span.TraceID] = append(traces[span.span.TraceID], &span.span) + } + tracesArray := make([]internaltelemetry.Trace, 0, len(traces)) + for _, trace := range traces { + tracesArray = append(tracesArray, internaltelemetry.Trace(trace)) } + return internaltelemetry.Traces(tracesArray) } -func (l *telemetryListener) Addr() net.Addr { - return addr(0) +func (t *Telemetry) sendCompletedSpans() { + tracesArray := t.extractCompletedSpans() + if len(tracesArray) == 0 { + return + } + t.telemetryClient.SendTraces(tracesArray) } -func (l *telemetryListener) Dial(_, _ string) (net.Conn, error) { - select { - case <-l.close: - return nil, errors.New("listener closed") - default: +// SpanFromContext returns the span from the context if available. +func SpanFromContext(ctx context.Context) (*Span, bool) { + spanIDs, ok := getSpanIDsFromContext(ctx) + if !ok { + return nil, false } - server, client := net.Pipe() - l.conns <- server - return client, nil + return globalTracer.getSpan(spanIDs.spanID) } -type addr int - -func (addr) Network() string { - return "memory" +// StartSpanFromEnv starts a span using the environment variables to find the parent span. +func StartSpanFromEnv(ctx context.Context, operationName string) (*Span, context.Context) { + traceID, parentID := extractIDsFromEnv() + return StartSpanFromIDs(ctx, operationName, traceID, parentID) } -func (addr) String() string { - return "local" +func extractIDsFromEnv() (string, string) { + parentID, ok := os.LookupEnv(envParentID) + if !ok { + return "0", "0" + } + traceID, ok := os.LookupEnv(envTraceID) + if !ok { + return "0", "0" + } + return traceID, parentID } -// StartSpanFromIDs starts a span using the trace and parent -// IDs provided. -func StartSpanFromIDs(ctx context.Context, operationName, traceID, parentID string, spanOptions ...ddtrace.StartSpanOption) (Span, context.Context) { - ctxCarrier := tracer.TextMapCarrier{ - tracer.DefaultTraceIDHeader: traceID, - tracer.DefaultParentIDHeader: parentID, - tracer.DefaultPriorityHeader: "2", +func converIDsToUint64(traceID, parentID string) (uint64, uint64) { + traceIDInt, err := strconv.ParseUint(traceID, 10, 64) + if err != nil { + return 0, 0 } - spanCtx, err := tracer.Extract(ctxCarrier) + parentIDInt, err := strconv.ParseUint(parentID, 10, 64) if err != nil { - log.Debugf("failed to extract span context from install script params: %v", err) - return StartSpanFromContext(ctx, operationName, spanOptions...) + return 0, 0 } - spanOptions = append([]ddtrace.StartSpanOption{tracer.ChildOf(spanCtx)}, spanOptions...) - return StartSpanFromContext(ctx, operationName, spanOptions...) + return traceIDInt, parentIDInt } -// SpanFromContext returns the span from the context if available. -func SpanFromContext(ctx context.Context) (Span, bool) { - span, ok := tracer.SpanFromContext(ctx) - if !ok { - return Span{}, false - } - return Span{span}, true +// StartSpanFromIDs starts a span using the trace and parent +// IDs provided. +func StartSpanFromIDs(ctx context.Context, operationName, traceID, parentID string) (*Span, context.Context) { + traceIDInt, parentIDInt := converIDsToUint64(traceID, parentID) + span, ctx := startSpanFromIDs(ctx, operationName, traceIDInt, parentIDInt) + span.SetTopLevel() + return span, ctx } -// StartSpanFromContext starts a span using the context to find the parent span. -func StartSpanFromContext(ctx context.Context, operationName string, spanOptions ...ddtrace.StartSpanOption) (Span, context.Context) { - span, ctx := tracer.StartSpanFromContext(ctx, operationName, spanOptions...) - return Span{span}, ctx +func startSpanFromIDs(ctx context.Context, operationName string, traceID, parentID uint64) (*Span, context.Context) { + s := newSpan(operationName, parentID, traceID) + ctx = setSpanIDsInContext(ctx, s) + return s, ctx } -// StartSpanFromEnv starts a span using the environment variables to find the parent span. -func StartSpanFromEnv(ctx context.Context, operationName string, spanOptions ...ddtrace.StartSpanOption) (Span, context.Context) { - traceID, ok := os.LookupEnv(EnvTraceID) - if !ok { - traceID = strconv.FormatUint(rand.Uint64(), 10) - } - parentID, ok := os.LookupEnv(EnvParentID) - if !ok { - parentID = "0" - } - return StartSpanFromIDs(ctx, operationName, traceID, parentID, spanOptions...) +// StartSpanFromContext starts a span using the context to find the parent span. +func StartSpanFromContext(ctx context.Context, operationName string) (*Span, context.Context) { + spanIDs, _ := getSpanIDsFromContext(ctx) + return startSpanFromIDs(ctx, operationName, spanIDs.traceID, spanIDs.spanID) } // EnvFromContext returns the environment variables for the context. func EnvFromContext(ctx context.Context) []string { - spanCtx, ok := SpanContextFromContext(ctx) + sIDs, ok := getSpanIDsFromContext(ctx) if !ok { return []string{} } return []string{ - fmt.Sprintf("%s=%d", EnvTraceID, spanCtx.TraceID()), - fmt.Sprintf("%s=%d", EnvParentID, spanCtx.SpanID()), + fmt.Sprintf("%s=%s", envTraceID, strconv.FormatUint(sIDs.traceID, 10)), + fmt.Sprintf("%s=%s", envParentID, strconv.FormatUint(sIDs.spanID, 10)), } } - -// SpanContextFromContext extracts the span context from the context if available. -func SpanContextFromContext(ctx context.Context) (ddtrace.SpanContext, bool) { - span, ok := tracer.SpanFromContext(ctx) - if !ok { - return nil, false - } - return span.Context(), true -} - -// WithSamplingRules sets the sampling rules for the telemetry. -func WithSamplingRules(rules ...tracer.SamplingRule) Option { - return func(t *Telemetry) { - t.samplingRules = rules - } -} - -// WrapRoundTripper wraps the round tripper with the telemetry round tripper. -func WrapRoundTripper(rt http.RoundTripper) http.RoundTripper { - return httptrace.WrapRoundTripper(rt) -} diff --git a/pkg/fleet/telemetry/telemetry_test.go b/pkg/fleet/telemetry/telemetry_test.go new file mode 100644 index 0000000000000..d0866ee4d4a13 --- /dev/null +++ b/pkg/fleet/telemetry/telemetry_test.go @@ -0,0 +1,222 @@ +// Unless explicitly stated otherwise all files in this repository are licensed +// under the Apache License Version 2.0. +// This product includes software developed at Datadog (https://www.datadoghq.com/). +// Copyright 2016-present Datadog, Inc. + +// Package telemetry provides the telemetry for fleet components. +package telemetry + +import ( + "context" + "errors" + "net/http" + "os" + "testing" + + "github.com/DataDog/datadog-agent/pkg/internaltelemetry" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestFreshSpan(t *testing.T) { + ctx := context.Background() + _, ok := SpanFromContext(ctx) + require.False(t, ok) + + s, ctx := StartSpanFromContext(ctx, "test") + require.NotNil(t, s) + s.SetResourceName("new") + + span, ok := SpanFromContext(ctx) + require.True(t, ok) + require.Equal(t, s, span) + + assert.Equal(t, "test", s.span.Name) + assert.Equal(t, "new", s.span.Resource) + assert.Equal(t, "new", s.span.Resource) + assert.Equal(t, "new", span.span.Resource) +} + +func TestInheritence(t *testing.T) { + ctx := context.Background() + s, ctx := StartSpanFromContext(ctx, "test") + require.NotNil(t, s) + + child, _ := StartSpanFromContext(ctx, "child") + require.NotNil(t, child) + + assert.Equal(t, s.span.SpanID, child.span.ParentID) + assert.Equal(t, s.span.TraceID, child.span.TraceID) +} + +func TestStartSpanFromIDs(t *testing.T) { + ctx := context.Background() + traceID := "100" + parentID := "200" + + span, ctx := StartSpanFromIDs(ctx, "ids-operation", traceID, parentID) + require.NotNil(t, span, "Expected a span") + require.Equal(t, uint64(100), span.span.TraceID) + require.Equal(t, uint64(200), span.span.ParentID) + + val, ok := span.span.Metrics["_top_level"] + require.True(t, ok) + require.Equal(t, 1.0, val) + + spanFromCtx, ok := SpanFromContext(ctx) + require.True(t, ok) + require.Equal(t, span, spanFromCtx) +} + +func strPtr(s string) *string { + return &s +} + +func TestSpanFromEnv(t *testing.T) { + randTraceID := uint64(9) + tt := []struct { + name string + envTraceID *string + envParentID *string + expectedTraceID uint64 + expectedParentID uint64 + }{ + { + name: "no parent env", + envTraceID: strPtr("100"), + envParentID: nil, + expectedTraceID: randTraceID, + expectedParentID: 0, + }, + { + name: "no trace env", + envTraceID: nil, + envParentID: strPtr("100"), + expectedTraceID: randTraceID, + expectedParentID: 0, + }, + { + name: "traceID malformed", + envTraceID: strPtr("not-a-number"), + envParentID: strPtr("200"), + expectedTraceID: randTraceID, + expectedParentID: 0, + }, + { + name: "parentID malformed", + envTraceID: strPtr("100"), + envParentID: strPtr("not-a-number"), + expectedTraceID: randTraceID, + expectedParentID: 0, + }, + { + name: "inheritance", + envTraceID: strPtr("100"), + envParentID: strPtr("200"), + expectedTraceID: 100, + expectedParentID: 200, + }, + } + for _, tc := range tt { + t.Run(tc.name, func(t *testing.T) { + if tc.envTraceID != nil { + os.Setenv(envTraceID, *tc.envTraceID) + defer os.Unsetenv(envTraceID) + } + if tc.envParentID != nil { + os.Setenv(envParentID, *tc.envParentID) + defer os.Unsetenv(envParentID) + } + + span, ctx := StartSpanFromEnv(context.Background(), "env-operation") + require.NotNil(t, span, "Expected a span") + s, ok := SpanFromContext(ctx) + assert.True(t, ok) + assert.Equal(t, span, s) + + assert.Equal(t, tc.expectedParentID, span.span.ParentID) + if tc.expectedTraceID != randTraceID { + assert.Equal(t, tc.expectedTraceID, span.span.TraceID) + } else { + assert.NotEqual(t, 0, span.span.TraceID) + } + + }) + } +} + +func TestLimit(t *testing.T) { + totalSpans := maxSpansInFlight + 2 + ctx := context.Background() + for i := 0; i < totalSpans; i++ { + _, ctx = StartSpanFromContext(ctx, "test") + } + assert.Len(t, globalTracer.spans, maxSpansInFlight) +} + +func TestEnvFromContext(t *testing.T) { + s, ctx := StartSpanFromContext(context.Background(), "test") + s.span.TraceID = 456 + s.span.SpanID = 123 + ctx = setSpanIDsInContext(ctx, s) + env := EnvFromContext(ctx) + assert.ElementsMatch(t, []string{"DATADOG_TRACE_ID=456", "DATADOG_PARENT_ID=123"}, env) + + env = EnvFromContext(context.Background()) + assert.ElementsMatch(t, []string{}, env) +} + +func TestSpanFinished(t *testing.T) { + s, _ := StartSpanFromContext(context.Background(), "test") + s.Finish(nil) + s.SetResourceName("new") + s.SetTag("key", "value") + + assert.Equal(t, "test", s.span.Resource) + _, ok := s.span.Meta["key"] + assert.False(t, ok) +} + +func TestRemapOnFlush(t *testing.T) { + const testService = "test-service" + const numTraces = 10 + telem := newTelemetry(&http.Client{}, "api", "datad0g.com", testService) + globalTracer = &tracer{spans: make(map[uint64]*Span)} + + // traces with 2 spans + for i := 0; i < numTraces; i++ { + parentSpan, ctx := StartSpanFromContext(context.Background(), "parent") + childSpan, _ := StartSpanFromContext(ctx, "child") + childSpan.Finish(errors.New("test_error")) + parentSpan.Finish(nil) + } + resTraces := telem.extractCompletedSpans() + require.Len(t, resTraces, numTraces) + + for _, trace := range resTraces { + assert.Len(t, trace, 2) + for _, span := range trace { + assert.Equal(t, testService, span.Service) + assert.Equal(t, "staging", span.Meta["env"]) + assert.Equal(t, 2.0, span.Metrics["_sampling_priority_v1"]) + } + var parent, child *internaltelemetry.Span + if trace[0].Name == "parent" { + parent = trace[0] + child = trace[1] + } else { + parent = trace[1] + child = trace[0] + } + assert.Equal(t, parent.SpanID, child.ParentID) + val, ok := parent.Metrics["_top_level"] + require.True(t, ok) + require.Equal(t, 1.0, val) + _, ok = child.Metrics["_top_level"] + require.False(t, ok) + + require.Equal(t, int32(1), child.Error) + require.Equal(t, "test_error", child.Meta["error.message"]) + require.Contains(t, child.Meta["error.stack"], "telemetry_test.go") + } +} diff --git a/pkg/fleet/telemetry/tracer.go b/pkg/fleet/telemetry/tracer.go new file mode 100644 index 0000000000000..0923b3563e457 --- /dev/null +++ b/pkg/fleet/telemetry/tracer.go @@ -0,0 +1,92 @@ +// Unless explicitly stated otherwise all files in this repository are licensed +// under the Apache License Version 2.0. +// This product includes software developed at Datadog (https://www.datadoghq.com/). +// Copyright 2016-present Datadog, Inc. + +// Package telemetry provides the telemetry for fleet components. +package telemetry + +import ( + "math" + "strings" + "sync" +) + +const ( + dropTraceID = 1 + maxSpansInFlight = 1000 +) + +var ( + globalTracer *tracer + samplingRates = map[string]float64{ + "cdn": 0.1, + "garbage_collect": 0.05, + "HTTPClient": 0.05, + } +) + +func init() { + globalTracer = &tracer{ + spans: make(map[uint64]*Span), + } +} + +type tracer struct { + mu sync.Mutex + spans map[uint64]*Span + completedSpans []*Span +} + +func (t *tracer) registerSpan(span *Span) { + if span.span.TraceID == dropTraceID { + return + } + t.mu.Lock() + defer t.mu.Unlock() + // naive maxSpansInFlight check as this is just telemetry + // next iteration if needed would be to flush long running spans to troubleshoot + if len(t.spans) >= maxSpansInFlight { + return + } + t.spans[span.span.SpanID] = span +} + +func (t *tracer) getSpan(spanID uint64) (*Span, bool) { + t.mu.Lock() + defer t.mu.Unlock() + span, ok := t.spans[spanID] + return span, ok +} + +func (t *tracer) finishSpan(span *Span) { + t.mu.Lock() + defer t.mu.Unlock() + delete(t.spans, span.span.SpanID) + t.completedSpans = append(t.completedSpans, span) +} + +func (t *tracer) flushCompletedSpans() []*Span { + t.mu.Lock() + defer t.mu.Unlock() + newSpanArray := make([]*Span, 0) + completedSpans := t.completedSpans + t.completedSpans = newSpanArray + return completedSpans +} + +func headSamplingKeep(spanName string, traceID uint64) bool { + for k, r := range samplingRates { + if strings.Contains(spanName, k) { + return sampledByRate(traceID, r) + } + } + return true +} + +func sampledByRate(n uint64, rate float64) bool { + if rate < 1 { + return n*uint64(1111111111111111111) < uint64(rate*math.MaxUint64) + } + return true +} diff --git a/pkg/internaltelemetry/client.go b/pkg/internaltelemetry/client.go index 52b1b2010edf0..ee7754fad190b 100644 --- a/pkg/internaltelemetry/client.go +++ b/pkg/internaltelemetry/client.go @@ -21,8 +21,6 @@ import ( "go.uber.org/atomic" - pb "github.com/DataDog/datadog-agent/pkg/proto/pbgo/trace" - "github.com/DataDog/datadog-agent/pkg/trace/config" "github.com/DataDog/datadog-agent/pkg/util/log" "github.com/DataDog/datadog-agent/pkg/version" "github.com/shirou/gopsutil/v4/host" @@ -36,13 +34,19 @@ const ( // Client defines the interface for a telemetry client type Client interface { SendLog(level string, message string) - SendTraces(traces pb.Traces) + SendTraces(traces Traces) +} + +// Endpoint defines the endpoint object +type Endpoint struct { + APIKey string `json:"-"` + Host string } type client struct { m sync.Mutex client httpClient - endpoints []*config.Endpoint + endpoints []*Endpoint sendPayloadTimeout time.Duration // we can pre-calculate the host payload structure at init time @@ -94,7 +98,7 @@ type Application struct { // TracePayload defines the trace payload object type TracePayload struct { - Traces []pb.Trace `json:"traces"` + Traces []Trace `json:"traces"` } // LogPayload defines the log payload object @@ -115,7 +119,7 @@ type httpClient interface { } // NewClient creates a new telemetry client -func NewClient(httpClient httpClient, endpoints []*config.Endpoint, service string, debug bool) Client { +func NewClient(httpClient httpClient, endpoints []*Endpoint, service string, debug bool) Client { info, err := host.Info() if err != nil { log.Errorf("failed to retrieve host info: %v", err) @@ -158,7 +162,7 @@ func (c *client) SendLog(level, message string) { c.sendPayload(RequestTypeLogs, payload) } -func (c *client) SendTraces(traces pb.Traces) { +func (c *client) SendTraces(traces Traces) { c.m.Lock() defer c.m.Unlock() payload := TracePayload{ @@ -170,8 +174,8 @@ func (c *client) SendTraces(traces pb.Traces) { // sampleTraces is a simple uniform sampling function that samples traces based // on the sampling rate, given that there is no trace agent to sample the traces // We try to keep the tracer behaviour: the first rule that matches apply its rate to the whole trace -func (c *client) sampleTraces(traces pb.Traces) pb.Traces { - tracesWithSampling := pb.Traces{} +func (c *client) sampleTraces(traces Traces) Traces { + tracesWithSampling := Traces{} for _, trace := range traces { samplingRate := 1.0 for _, span := range trace { @@ -206,7 +210,7 @@ func (c *client) sendPayload(requestType RequestType, payload interface{}) { group := sync.WaitGroup{} for _, endpoint := range c.endpoints { group.Add(1) - go func(endpoint *config.Endpoint) { + go func(endpoint *Endpoint) { defer group.Done() url := fmt.Sprintf("%s%s", endpoint.Host, telemetryEndpoint) req, err := http.NewRequest("POST", url, bytes.NewReader(serializedPayload)) diff --git a/pkg/internaltelemetry/traces.go b/pkg/internaltelemetry/traces.go new file mode 100644 index 0000000000000..54c8545a604b7 --- /dev/null +++ b/pkg/internaltelemetry/traces.go @@ -0,0 +1,41 @@ +// Unless explicitly stated otherwise all files in this repository are licensed +// under the Apache License Version 2.0. +// This product includes software developed at Datadog (https://www.datadoghq.com/). +// Copyright 2023-present Datadog, Inc. + +// Package internaltelemetry full description in README.md +package internaltelemetry + +// Traces is a collection of traces +type Traces []Trace + +// Trace is a collection of spans with the same trace ID +type Trace []*Span + +// Span used for installation telemetry +type Span struct { + // Service is the name of the service that handled this span. + Service string `json:"service"` + // Name is the name of the operation this span represents. + Name string `json:"name"` + // Resource is the name of the resource this span represents. + Resource string `json:"resource"` + // TraceID is the ID of the trace to which this span belongs. + TraceID uint64 `json:"trace_id"` + // SpanID is the ID of this span. + SpanID uint64 `json:"span_id"` + // ParentID is the ID of the parent span. + ParentID uint64 `json:"parent_id"` + // Start is the start time of this span in nanoseconds since the Unix epoch. + Start int64 `json:"start"` + // Duration is the duration of this span in nanoseconds. + Duration int64 `json:"duration"` + // Error is the error status of this span. + Error int32 `json:"error"` + // Meta is a mapping from tag name to tag value for string-valued tags. + Meta map[string]string `json:"meta,omitempty"` + // Metrics is a mapping from metric name to metric value for numeric metrics. + Metrics map[string]float64 `json:"metrics,omitempty"` + // Type is the type of the span. + Type string `json:"type"` +} From f236edf97bffd669b155ef3bef109563a5690afe Mon Sep 17 00:00:00 2001 From: Baptiste Foy Date: Wed, 18 Dec 2024 19:33:36 +0100 Subject: [PATCH 02/14] upgrade(installer): Retry more network errors (#32346) --- pkg/fleet/installer/oci/download.go | 127 +++++++++++++++++----------- 1 file changed, 79 insertions(+), 48 deletions(-) diff --git a/pkg/fleet/installer/oci/download.go b/pkg/fleet/installer/oci/download.go index 7fcce0234512c..e557cf8b53d7f 100644 --- a/pkg/fleet/installer/oci/download.go +++ b/pkg/fleet/installer/oci/download.go @@ -62,8 +62,8 @@ const ( ) const ( - layerMaxSize = 3 << 30 // 3GiB - extractLayerRetries = 3 + layerMaxSize = 3 << 30 // 3GiB + networkRetries = 3 ) var ( @@ -318,30 +318,32 @@ func (d *DownloadedPackage) ExtractLayers(mediaType types.MediaType, dir string) return fmt.Errorf("could not get layer media type: %w", err) } if layerMediaType == mediaType { - // Retry stream reset errors - for i := 0; i < extractLayerRetries; i++ { - if i > 0 { - time.Sleep(time.Second) - } - uncompressedLayer, err := layer.Uncompressed() - if err != nil { - return fmt.Errorf("could not uncompress layer: %w", err) - } - err = tar.Extract(uncompressedLayer, dir, layerMaxSize) - uncompressedLayer.Close() - if err != nil { - if !isStreamResetError(err) && !isConnectionResetByPeerError(err) { - return fmt.Errorf("could not extract layer: %w", err) + err = withNetworkRetries( + func() error { + var err error + defer func() { + if err != nil { + deferErr := tar.Clean(dir) + if deferErr != nil { + err = deferErr + } + } + }() + uncompressedLayer, err := layer.Uncompressed() + if err != nil { + return err } - log.Warnf("network error while extracting layer, retrying") - // Clean up the directory before retrying to avoid partial extraction - err = tar.Clean(dir) + err = tar.Extract(uncompressedLayer, dir, layerMaxSize) + uncompressedLayer.Close() if err != nil { - return fmt.Errorf("could not clean directory: %w", err) + return err } - } else { - break - } + + return nil + }, + ) + if err != nil { + return fmt.Errorf("could not extract layer: %w", err) } } } @@ -349,16 +351,22 @@ func (d *DownloadedPackage) ExtractLayers(mediaType types.MediaType, dir string) } // WriteOCILayout writes the image as an OCI layout to the given directory. -func (d *DownloadedPackage) WriteOCILayout(dir string) error { - layoutPath, err := layout.Write(dir, empty.Index) - if err != nil { - return fmt.Errorf("could not write layout: %w", err) - } - err = layoutPath.AppendImage(d.Image) - if err != nil { - return fmt.Errorf("could not append image to layout: %w", err) - } - return nil +func (d *DownloadedPackage) WriteOCILayout(dir string) (err error) { + var layoutPath layout.Path + return withNetworkRetries( + func() error { + layoutPath, err = layout.Write(dir, empty.Index) + if err != nil { + return fmt.Errorf("could not write layout: %w", err) + } + + err = layoutPath.AppendImage(d.Image) + if err != nil { + return fmt.Errorf("could not append image to layout: %w", err) + } + return nil + }, + ) } // PackageURL returns the package URL for the given site, package and version. @@ -371,15 +379,50 @@ func PackageURL(env *env.Env, pkg string, version string) string { } } +func withNetworkRetries(f func() error) error { + var err error + for i := 0; i < networkRetries; i++ { + err = f() + if err == nil { + return nil + } + if !isRetryableNetworkError(err) { + return err + } + log.Warnf("retrying after network error: %s", err) + time.Sleep(time.Second) + } + return err +} + +// isRetryableNetworkError returns true if the error is a network error we should retry on +func isRetryableNetworkError(err error) bool { + if err == nil { + return false + } + + if netErr, ok := err.(*net.OpError); ok { + if netErr.Temporary() { + // Temporary errors, such as "connection timed out" + return true + } + if syscallErr, ok := netErr.Err.(*os.SyscallError); ok { + if errno, ok := syscallErr.Err.(syscall.Errno); ok { + // Connection reset errors, such as "connection reset by peer" + return errno == syscall.ECONNRESET + } + } + } + + return isStreamResetError(err) +} + // isStreamResetError returns true if the given error is a stream reset error. // Sometimes, in GCR, the tar extract fails with "stream error: stream ID x; INTERNAL_ERROR; received from peer". // This happens because the uncompressed layer reader is a http/2 response body under the hood. That body is // streamed and receives a "reset stream frame", with the code 0x2 (INTERNAL_ERROR). This is an error from the server // that we need to retry. func isStreamResetError(err error) bool { - if err == nil { - return false - } serr := http2.StreamError{} if errors.As(err, &serr) { return serr.Code == http2.ErrCodeInternal @@ -391,18 +434,6 @@ func isStreamResetError(err error) bool { return false } -// isConnectionResetByPeer returns true if the error is a connection reset by peer error -func isConnectionResetByPeerError(err error) bool { - if netErr, ok := err.(*net.OpError); ok { - if syscallErr, ok := netErr.Err.(*os.SyscallError); ok { - if errno, ok := syscallErr.Err.(syscall.Errno); ok { - return errno == syscall.ECONNRESET - } - } - } - return false -} - type usernamePasswordKeychain struct { username string password string From 86c077224161ff41e04f438e7ddb0cc9152af0a1 Mon Sep 17 00:00:00 2001 From: Zhengda Lu Date: Wed, 18 Dec 2024 13:33:49 -0500 Subject: [PATCH 03/14] bump github.com/DataDog/go-sqllexer to v0.0.18 (#32315) --- comp/otelcol/ddflareextension/impl/go.mod | 2 +- comp/otelcol/ddflareextension/impl/go.sum | 4 ++-- comp/otelcol/otlp/components/exporter/datadogexporter/go.mod | 2 +- comp/otelcol/otlp/components/exporter/datadogexporter/go.sum | 4 ++-- comp/otelcol/otlp/components/statsprocessor/go.mod | 2 +- comp/otelcol/otlp/components/statsprocessor/go.sum | 4 ++-- go.mod | 2 +- go.sum | 4 ++-- pkg/config/remote/go.mod | 2 +- pkg/config/remote/go.sum | 4 ++-- pkg/obfuscate/go.mod | 2 +- pkg/obfuscate/go.sum | 4 ++-- pkg/trace/go.mod | 2 +- pkg/trace/go.sum | 4 ++-- pkg/trace/stats/oteltest/go.mod | 2 +- pkg/trace/stats/oteltest/go.sum | 4 ++-- test/otel/go.mod | 2 +- test/otel/go.sum | 4 ++-- 18 files changed, 27 insertions(+), 27 deletions(-) diff --git a/comp/otelcol/ddflareextension/impl/go.mod b/comp/otelcol/ddflareextension/impl/go.mod index 21d3f17b9066a..ce08a7999f072 100644 --- a/comp/otelcol/ddflareextension/impl/go.mod +++ b/comp/otelcol/ddflareextension/impl/go.mod @@ -267,7 +267,7 @@ require ( github.com/DataDog/datadog-api-client-go/v2 v2.33.0 // indirect github.com/DataDog/datadog-go/v5 v5.6.0 // indirect github.com/DataDog/dd-sensitive-data-scanner/sds-go/go v0.0.0-20240816154533-f7f9beb53a42 // indirect - github.com/DataDog/go-sqllexer v0.0.17 // indirect + github.com/DataDog/go-sqllexer v0.0.18 // indirect github.com/DataDog/go-tuf v1.1.0-0.5.2 // indirect github.com/DataDog/mmh3 v0.0.0-20210722141835-012dc69a9e49 // indirect github.com/DataDog/opentelemetry-mapping-go/pkg/otlp/attributes v0.22.0 // indirect diff --git a/comp/otelcol/ddflareextension/impl/go.sum b/comp/otelcol/ddflareextension/impl/go.sum index d4f8e67e79da5..7aed291a64674 100644 --- a/comp/otelcol/ddflareextension/impl/go.sum +++ b/comp/otelcol/ddflareextension/impl/go.sum @@ -70,8 +70,8 @@ github.com/DataDog/datadog-go/v5 v5.6.0 h1:2oCLxjF/4htd55piM75baflj/KoE6VYS7alEU github.com/DataDog/datadog-go/v5 v5.6.0/go.mod h1:K9kcYBlxkcPP8tvvjZZKs/m1edNAUFzBbdpTUKfCsuw= github.com/DataDog/dd-sensitive-data-scanner/sds-go/go v0.0.0-20240816154533-f7f9beb53a42 h1:RoH7VLzTnxHEugRPIgnGlxwDFszFGI7b3WZZUtWuPRM= github.com/DataDog/dd-sensitive-data-scanner/sds-go/go v0.0.0-20240816154533-f7f9beb53a42/go.mod h1:TX7CTOQ3LbQjfAi4SwqUoR5gY1zfUk7VRBDTuArjaDc= -github.com/DataDog/go-sqllexer v0.0.17 h1:u47fJAVg/+5DA74ZW3w0Qu+3qXHd3GtnA8ZBYixdPrM= -github.com/DataDog/go-sqllexer v0.0.17/go.mod h1:KwkYhpFEVIq+BfobkTC1vfqm4gTi65skV/DpDBXtexc= +github.com/DataDog/go-sqllexer v0.0.18 h1:ErBvoO7/srJLdA2ebwd+HPqD4g1kN++BP64A8qvmh9U= +github.com/DataDog/go-sqllexer v0.0.18/go.mod h1:KwkYhpFEVIq+BfobkTC1vfqm4gTi65skV/DpDBXtexc= github.com/DataDog/go-tuf v1.1.0-0.5.2 h1:4CagiIekonLSfL8GMHRHcHudo1fQnxELS9g4tiAupQ4= github.com/DataDog/go-tuf v1.1.0-0.5.2/go.mod h1:zBcq6f654iVqmkk8n2Cx81E1JnNTMOAx1UEO/wZR+P0= github.com/DataDog/gohai v0.0.0-20230524154621-4316413895ee h1:tXibLZk3G6HncIFJKaNItsdzcrk4YqILNDZlXPTNt4k= diff --git a/comp/otelcol/otlp/components/exporter/datadogexporter/go.mod b/comp/otelcol/otlp/components/exporter/datadogexporter/go.mod index 301ef0f3b1f48..dfe091b9d4b79 100644 --- a/comp/otelcol/otlp/components/exporter/datadogexporter/go.mod +++ b/comp/otelcol/otlp/components/exporter/datadogexporter/go.mod @@ -201,7 +201,7 @@ require ( github.com/DataDog/datadog-agent/pkg/version v0.59.1 // indirect github.com/DataDog/datadog-api-client-go/v2 v2.33.0 // indirect github.com/DataDog/dd-sensitive-data-scanner/sds-go/go v0.0.0-20240816154533-f7f9beb53a42 // indirect - github.com/DataDog/go-sqllexer v0.0.17 // indirect + github.com/DataDog/go-sqllexer v0.0.18 // indirect github.com/DataDog/go-tuf v1.1.0-0.5.2 // indirect github.com/DataDog/mmh3 v0.0.0-20210722141835-012dc69a9e49 // indirect github.com/DataDog/opentelemetry-mapping-go/pkg/otlp/logs v0.22.0 // indirect diff --git a/comp/otelcol/otlp/components/exporter/datadogexporter/go.sum b/comp/otelcol/otlp/components/exporter/datadogexporter/go.sum index e9c507fbafa8a..c32ab7009cbeb 100644 --- a/comp/otelcol/otlp/components/exporter/datadogexporter/go.sum +++ b/comp/otelcol/otlp/components/exporter/datadogexporter/go.sum @@ -8,8 +8,8 @@ github.com/DataDog/datadog-go/v5 v5.6.0 h1:2oCLxjF/4htd55piM75baflj/KoE6VYS7alEU github.com/DataDog/datadog-go/v5 v5.6.0/go.mod h1:K9kcYBlxkcPP8tvvjZZKs/m1edNAUFzBbdpTUKfCsuw= github.com/DataDog/dd-sensitive-data-scanner/sds-go/go v0.0.0-20240816154533-f7f9beb53a42 h1:RoH7VLzTnxHEugRPIgnGlxwDFszFGI7b3WZZUtWuPRM= github.com/DataDog/dd-sensitive-data-scanner/sds-go/go v0.0.0-20240816154533-f7f9beb53a42/go.mod h1:TX7CTOQ3LbQjfAi4SwqUoR5gY1zfUk7VRBDTuArjaDc= -github.com/DataDog/go-sqllexer v0.0.17 h1:u47fJAVg/+5DA74ZW3w0Qu+3qXHd3GtnA8ZBYixdPrM= -github.com/DataDog/go-sqllexer v0.0.17/go.mod h1:KwkYhpFEVIq+BfobkTC1vfqm4gTi65skV/DpDBXtexc= +github.com/DataDog/go-sqllexer v0.0.18 h1:ErBvoO7/srJLdA2ebwd+HPqD4g1kN++BP64A8qvmh9U= +github.com/DataDog/go-sqllexer v0.0.18/go.mod h1:KwkYhpFEVIq+BfobkTC1vfqm4gTi65skV/DpDBXtexc= github.com/DataDog/go-tuf v1.1.0-0.5.2 h1:4CagiIekonLSfL8GMHRHcHudo1fQnxELS9g4tiAupQ4= github.com/DataDog/go-tuf v1.1.0-0.5.2/go.mod h1:zBcq6f654iVqmkk8n2Cx81E1JnNTMOAx1UEO/wZR+P0= github.com/DataDog/mmh3 v0.0.0-20210722141835-012dc69a9e49 h1:EbzDX8HPk5uE2FsJYxD74QmMw0/3CqSKhEr6teh0ncQ= diff --git a/comp/otelcol/otlp/components/statsprocessor/go.mod b/comp/otelcol/otlp/components/statsprocessor/go.mod index 18606500a04d2..c03fc39e3ca59 100644 --- a/comp/otelcol/otlp/components/statsprocessor/go.mod +++ b/comp/otelcol/otlp/components/statsprocessor/go.mod @@ -43,7 +43,7 @@ require ( github.com/DataDog/datadog-agent/pkg/util/pointer v0.59.0 // indirect github.com/DataDog/datadog-agent/pkg/util/scrubber v0.59.0 // indirect github.com/DataDog/datadog-agent/pkg/version v0.59.1 // indirect - github.com/DataDog/go-sqllexer v0.0.17 // indirect + github.com/DataDog/go-sqllexer v0.0.18 // indirect github.com/DataDog/go-tuf v1.1.0-0.5.2 // indirect github.com/DataDog/sketches-go v1.4.6 // indirect github.com/Microsoft/go-winio v0.6.2 // indirect diff --git a/comp/otelcol/otlp/components/statsprocessor/go.sum b/comp/otelcol/otlp/components/statsprocessor/go.sum index a19fb370fa6c4..22ef689018918 100644 --- a/comp/otelcol/otlp/components/statsprocessor/go.sum +++ b/comp/otelcol/otlp/components/statsprocessor/go.sum @@ -1,7 +1,7 @@ github.com/DataDog/datadog-go/v5 v5.6.0 h1:2oCLxjF/4htd55piM75baflj/KoE6VYS7alEUqFvRDw= github.com/DataDog/datadog-go/v5 v5.6.0/go.mod h1:K9kcYBlxkcPP8tvvjZZKs/m1edNAUFzBbdpTUKfCsuw= -github.com/DataDog/go-sqllexer v0.0.17 h1:u47fJAVg/+5DA74ZW3w0Qu+3qXHd3GtnA8ZBYixdPrM= -github.com/DataDog/go-sqllexer v0.0.17/go.mod h1:KwkYhpFEVIq+BfobkTC1vfqm4gTi65skV/DpDBXtexc= +github.com/DataDog/go-sqllexer v0.0.18 h1:ErBvoO7/srJLdA2ebwd+HPqD4g1kN++BP64A8qvmh9U= +github.com/DataDog/go-sqllexer v0.0.18/go.mod h1:KwkYhpFEVIq+BfobkTC1vfqm4gTi65skV/DpDBXtexc= github.com/DataDog/go-tuf v1.1.0-0.5.2 h1:4CagiIekonLSfL8GMHRHcHudo1fQnxELS9g4tiAupQ4= github.com/DataDog/go-tuf v1.1.0-0.5.2/go.mod h1:zBcq6f654iVqmkk8n2Cx81E1JnNTMOAx1UEO/wZR+P0= github.com/DataDog/opentelemetry-mapping-go/pkg/otlp/attributes v0.22.0 h1:yfk2cF8Bx98fSFpGrehEHh1FRqewfxcCTAbUDt5r3F8= diff --git a/go.mod b/go.mod index c8de7db9cc5f0..959651ab0bcce 100644 --- a/go.mod +++ b/go.mod @@ -735,7 +735,7 @@ require ( github.com/DataDog/datadog-agent/pkg/util/winutil v0.59.1 github.com/DataDog/datadog-agent/pkg/version v0.59.1 github.com/DataDog/go-libddwaf/v3 v3.5.1 - github.com/DataDog/go-sqllexer v0.0.17 + github.com/DataDog/go-sqllexer v0.0.18 github.com/Datadog/dublin-traceroute v0.0.2 github.com/aquasecurity/trivy v0.49.2-0.20240227072422-e1ea02c7b80d github.com/aws/aws-sdk-go-v2/service/kms v1.37.6 diff --git a/go.sum b/go.sum index 26375333ef0cf..ffa368593f411 100644 --- a/go.sum +++ b/go.sum @@ -142,8 +142,8 @@ github.com/DataDog/go-grpc-bidirectional-streaming-example v0.0.0-20221024060302 github.com/DataDog/go-grpc-bidirectional-streaming-example v0.0.0-20221024060302-b9cf785c02fe/go.mod h1:90sqV0j7E8wYCyqIp5d9HmYWLTFQttqPFFtNYDyAybQ= github.com/DataDog/go-libddwaf/v3 v3.5.1 h1:GWA4ln4DlLxiXm+X7HA/oj0ZLcdCwOS81KQitegRTyY= github.com/DataDog/go-libddwaf/v3 v3.5.1/go.mod h1:n98d9nZ1gzenRSk53wz8l6d34ikxS+hs62A31Fqmyi4= -github.com/DataDog/go-sqllexer v0.0.17 h1:u47fJAVg/+5DA74ZW3w0Qu+3qXHd3GtnA8ZBYixdPrM= -github.com/DataDog/go-sqllexer v0.0.17/go.mod h1:KwkYhpFEVIq+BfobkTC1vfqm4gTi65skV/DpDBXtexc= +github.com/DataDog/go-sqllexer v0.0.18 h1:ErBvoO7/srJLdA2ebwd+HPqD4g1kN++BP64A8qvmh9U= +github.com/DataDog/go-sqllexer v0.0.18/go.mod h1:KwkYhpFEVIq+BfobkTC1vfqm4gTi65skV/DpDBXtexc= github.com/DataDog/go-tuf v1.1.0-0.5.2 h1:4CagiIekonLSfL8GMHRHcHudo1fQnxELS9g4tiAupQ4= github.com/DataDog/go-tuf v1.1.0-0.5.2/go.mod h1:zBcq6f654iVqmkk8n2Cx81E1JnNTMOAx1UEO/wZR+P0= github.com/DataDog/gohai v0.0.0-20230524154621-4316413895ee h1:tXibLZk3G6HncIFJKaNItsdzcrk4YqILNDZlXPTNt4k= diff --git a/pkg/config/remote/go.mod b/pkg/config/remote/go.mod index a1c4a4ebbc561..72ed5685445f8 100644 --- a/pkg/config/remote/go.mod +++ b/pkg/config/remote/go.mod @@ -87,7 +87,7 @@ require ( github.com/DataDog/datadog-agent/pkg/version v0.59.1 // indirect github.com/DataDog/datadog-go/v5 v5.6.0 // indirect github.com/DataDog/go-libddwaf/v3 v3.5.1 // indirect - github.com/DataDog/go-sqllexer v0.0.17 // indirect + github.com/DataDog/go-sqllexer v0.0.18 // indirect github.com/DataDog/sketches-go v1.4.6 // indirect github.com/Microsoft/go-winio v0.6.2 // indirect github.com/cespare/xxhash/v2 v2.3.0 // indirect diff --git a/pkg/config/remote/go.sum b/pkg/config/remote/go.sum index ca537b0d1660e..512cb6bacbab2 100644 --- a/pkg/config/remote/go.sum +++ b/pkg/config/remote/go.sum @@ -11,8 +11,8 @@ github.com/DataDog/datadog-go/v5 v5.6.0 h1:2oCLxjF/4htd55piM75baflj/KoE6VYS7alEU github.com/DataDog/datadog-go/v5 v5.6.0/go.mod h1:K9kcYBlxkcPP8tvvjZZKs/m1edNAUFzBbdpTUKfCsuw= github.com/DataDog/go-libddwaf/v3 v3.5.1 h1:GWA4ln4DlLxiXm+X7HA/oj0ZLcdCwOS81KQitegRTyY= github.com/DataDog/go-libddwaf/v3 v3.5.1/go.mod h1:n98d9nZ1gzenRSk53wz8l6d34ikxS+hs62A31Fqmyi4= -github.com/DataDog/go-sqllexer v0.0.17 h1:u47fJAVg/+5DA74ZW3w0Qu+3qXHd3GtnA8ZBYixdPrM= -github.com/DataDog/go-sqllexer v0.0.17/go.mod h1:KwkYhpFEVIq+BfobkTC1vfqm4gTi65skV/DpDBXtexc= +github.com/DataDog/go-sqllexer v0.0.18 h1:ErBvoO7/srJLdA2ebwd+HPqD4g1kN++BP64A8qvmh9U= +github.com/DataDog/go-sqllexer v0.0.18/go.mod h1:KwkYhpFEVIq+BfobkTC1vfqm4gTi65skV/DpDBXtexc= github.com/DataDog/go-tuf v1.1.0-0.5.2 h1:4CagiIekonLSfL8GMHRHcHudo1fQnxELS9g4tiAupQ4= github.com/DataDog/go-tuf v1.1.0-0.5.2/go.mod h1:zBcq6f654iVqmkk8n2Cx81E1JnNTMOAx1UEO/wZR+P0= github.com/DataDog/gostackparse v0.7.0 h1:i7dLkXHvYzHV308hnkvVGDL3BR4FWl7IsXNPz/IGQh4= diff --git a/pkg/obfuscate/go.mod b/pkg/obfuscate/go.mod index b34df591521b5..709579d414b7f 100644 --- a/pkg/obfuscate/go.mod +++ b/pkg/obfuscate/go.mod @@ -4,7 +4,7 @@ go 1.22.0 require ( github.com/DataDog/datadog-go/v5 v5.6.0 - github.com/DataDog/go-sqllexer v0.0.17 + github.com/DataDog/go-sqllexer v0.0.18 github.com/outcaste-io/ristretto v0.2.3 github.com/stretchr/testify v1.10.0 go.uber.org/atomic v1.11.0 diff --git a/pkg/obfuscate/go.sum b/pkg/obfuscate/go.sum index 063bd88a07005..940c4e5438e4a 100644 --- a/pkg/obfuscate/go.sum +++ b/pkg/obfuscate/go.sum @@ -1,7 +1,7 @@ github.com/DataDog/datadog-go/v5 v5.6.0 h1:2oCLxjF/4htd55piM75baflj/KoE6VYS7alEUqFvRDw= github.com/DataDog/datadog-go/v5 v5.6.0/go.mod h1:K9kcYBlxkcPP8tvvjZZKs/m1edNAUFzBbdpTUKfCsuw= -github.com/DataDog/go-sqllexer v0.0.17 h1:u47fJAVg/+5DA74ZW3w0Qu+3qXHd3GtnA8ZBYixdPrM= -github.com/DataDog/go-sqllexer v0.0.17/go.mod h1:KwkYhpFEVIq+BfobkTC1vfqm4gTi65skV/DpDBXtexc= +github.com/DataDog/go-sqllexer v0.0.18 h1:ErBvoO7/srJLdA2ebwd+HPqD4g1kN++BP64A8qvmh9U= +github.com/DataDog/go-sqllexer v0.0.18/go.mod h1:KwkYhpFEVIq+BfobkTC1vfqm4gTi65skV/DpDBXtexc= github.com/Microsoft/go-winio v0.5.0/go.mod h1:JPGBdM1cNvN/6ISo+n8V5iA4v8pBzdOpzfwIujj1a84= github.com/Microsoft/go-winio v0.6.2 h1:F2VQgta7ecxGYO8k3ZZz3RS8fVIXVxONVUPlNERoyfY= github.com/Microsoft/go-winio v0.6.2/go.mod h1:yd8OoFMLzJbo9gZq8j5qaps8bJ9aShtEA8Ipt1oGCvU= diff --git a/pkg/trace/go.mod b/pkg/trace/go.mod index 29bf241afd7b8..c3f102a944494 100644 --- a/pkg/trace/go.mod +++ b/pkg/trace/go.mod @@ -61,7 +61,7 @@ require go.opentelemetry.io/collector/processor v0.115.0 // indirect require ( github.com/DataDog/datadog-agent/pkg/version v0.59.1 // indirect - github.com/DataDog/go-sqllexer v0.0.17 // indirect + github.com/DataDog/go-sqllexer v0.0.18 // indirect github.com/DataDog/go-tuf v1.1.0-0.5.2 // indirect github.com/DataDog/zstd v1.5.6 // indirect github.com/cespare/xxhash/v2 v2.3.0 // indirect diff --git a/pkg/trace/go.sum b/pkg/trace/go.sum index e1b7335487809..efd28e50f6d16 100644 --- a/pkg/trace/go.sum +++ b/pkg/trace/go.sum @@ -1,7 +1,7 @@ github.com/DataDog/datadog-go/v5 v5.6.0 h1:2oCLxjF/4htd55piM75baflj/KoE6VYS7alEUqFvRDw= github.com/DataDog/datadog-go/v5 v5.6.0/go.mod h1:K9kcYBlxkcPP8tvvjZZKs/m1edNAUFzBbdpTUKfCsuw= -github.com/DataDog/go-sqllexer v0.0.17 h1:u47fJAVg/+5DA74ZW3w0Qu+3qXHd3GtnA8ZBYixdPrM= -github.com/DataDog/go-sqllexer v0.0.17/go.mod h1:KwkYhpFEVIq+BfobkTC1vfqm4gTi65skV/DpDBXtexc= +github.com/DataDog/go-sqllexer v0.0.18 h1:ErBvoO7/srJLdA2ebwd+HPqD4g1kN++BP64A8qvmh9U= +github.com/DataDog/go-sqllexer v0.0.18/go.mod h1:KwkYhpFEVIq+BfobkTC1vfqm4gTi65skV/DpDBXtexc= github.com/DataDog/go-tuf v1.1.0-0.5.2 h1:4CagiIekonLSfL8GMHRHcHudo1fQnxELS9g4tiAupQ4= github.com/DataDog/go-tuf v1.1.0-0.5.2/go.mod h1:zBcq6f654iVqmkk8n2Cx81E1JnNTMOAx1UEO/wZR+P0= github.com/DataDog/opentelemetry-mapping-go/pkg/otlp/attributes v0.22.0 h1:yfk2cF8Bx98fSFpGrehEHh1FRqewfxcCTAbUDt5r3F8= diff --git a/pkg/trace/stats/oteltest/go.mod b/pkg/trace/stats/oteltest/go.mod index 071e7c2c2eba0..1259dbf55e8bf 100644 --- a/pkg/trace/stats/oteltest/go.mod +++ b/pkg/trace/stats/oteltest/go.mod @@ -30,7 +30,7 @@ require ( github.com/DataDog/datadog-agent/pkg/util/pointer v0.59.0 // indirect github.com/DataDog/datadog-agent/pkg/util/scrubber v0.59.0 // indirect github.com/DataDog/datadog-agent/pkg/version v0.59.1 // indirect - github.com/DataDog/go-sqllexer v0.0.17 // indirect + github.com/DataDog/go-sqllexer v0.0.18 // indirect github.com/DataDog/go-tuf v1.1.0-0.5.2 // indirect github.com/DataDog/sketches-go v1.4.6 // indirect github.com/Microsoft/go-winio v0.6.2 // indirect diff --git a/pkg/trace/stats/oteltest/go.sum b/pkg/trace/stats/oteltest/go.sum index a19fb370fa6c4..22ef689018918 100644 --- a/pkg/trace/stats/oteltest/go.sum +++ b/pkg/trace/stats/oteltest/go.sum @@ -1,7 +1,7 @@ github.com/DataDog/datadog-go/v5 v5.6.0 h1:2oCLxjF/4htd55piM75baflj/KoE6VYS7alEUqFvRDw= github.com/DataDog/datadog-go/v5 v5.6.0/go.mod h1:K9kcYBlxkcPP8tvvjZZKs/m1edNAUFzBbdpTUKfCsuw= -github.com/DataDog/go-sqllexer v0.0.17 h1:u47fJAVg/+5DA74ZW3w0Qu+3qXHd3GtnA8ZBYixdPrM= -github.com/DataDog/go-sqllexer v0.0.17/go.mod h1:KwkYhpFEVIq+BfobkTC1vfqm4gTi65skV/DpDBXtexc= +github.com/DataDog/go-sqllexer v0.0.18 h1:ErBvoO7/srJLdA2ebwd+HPqD4g1kN++BP64A8qvmh9U= +github.com/DataDog/go-sqllexer v0.0.18/go.mod h1:KwkYhpFEVIq+BfobkTC1vfqm4gTi65skV/DpDBXtexc= github.com/DataDog/go-tuf v1.1.0-0.5.2 h1:4CagiIekonLSfL8GMHRHcHudo1fQnxELS9g4tiAupQ4= github.com/DataDog/go-tuf v1.1.0-0.5.2/go.mod h1:zBcq6f654iVqmkk8n2Cx81E1JnNTMOAx1UEO/wZR+P0= github.com/DataDog/opentelemetry-mapping-go/pkg/otlp/attributes v0.22.0 h1:yfk2cF8Bx98fSFpGrehEHh1FRqewfxcCTAbUDt5r3F8= diff --git a/test/otel/go.mod b/test/otel/go.mod index 88d5f3a127be5..beb5f560e099e 100644 --- a/test/otel/go.mod +++ b/test/otel/go.mod @@ -179,7 +179,7 @@ require ( github.com/DataDog/datadog-api-client-go/v2 v2.33.0 // indirect github.com/DataDog/datadog-go/v5 v5.6.0 // indirect github.com/DataDog/dd-sensitive-data-scanner/sds-go/go v0.0.0-20240816154533-f7f9beb53a42 // indirect - github.com/DataDog/go-sqllexer v0.0.17 // indirect + github.com/DataDog/go-sqllexer v0.0.18 // indirect github.com/DataDog/go-tuf v1.1.0-0.5.2 // indirect github.com/DataDog/opentelemetry-mapping-go/pkg/otlp/attributes v0.22.0 // indirect github.com/DataDog/opentelemetry-mapping-go/pkg/otlp/logs v0.22.0 // indirect diff --git a/test/otel/go.sum b/test/otel/go.sum index bce88ff15bfa8..4ccec6f105aa4 100644 --- a/test/otel/go.sum +++ b/test/otel/go.sum @@ -8,8 +8,8 @@ github.com/DataDog/datadog-go/v5 v5.6.0 h1:2oCLxjF/4htd55piM75baflj/KoE6VYS7alEU github.com/DataDog/datadog-go/v5 v5.6.0/go.mod h1:K9kcYBlxkcPP8tvvjZZKs/m1edNAUFzBbdpTUKfCsuw= github.com/DataDog/dd-sensitive-data-scanner/sds-go/go v0.0.0-20240816154533-f7f9beb53a42 h1:RoH7VLzTnxHEugRPIgnGlxwDFszFGI7b3WZZUtWuPRM= github.com/DataDog/dd-sensitive-data-scanner/sds-go/go v0.0.0-20240816154533-f7f9beb53a42/go.mod h1:TX7CTOQ3LbQjfAi4SwqUoR5gY1zfUk7VRBDTuArjaDc= -github.com/DataDog/go-sqllexer v0.0.17 h1:u47fJAVg/+5DA74ZW3w0Qu+3qXHd3GtnA8ZBYixdPrM= -github.com/DataDog/go-sqllexer v0.0.17/go.mod h1:KwkYhpFEVIq+BfobkTC1vfqm4gTi65skV/DpDBXtexc= +github.com/DataDog/go-sqllexer v0.0.18 h1:ErBvoO7/srJLdA2ebwd+HPqD4g1kN++BP64A8qvmh9U= +github.com/DataDog/go-sqllexer v0.0.18/go.mod h1:KwkYhpFEVIq+BfobkTC1vfqm4gTi65skV/DpDBXtexc= github.com/DataDog/go-tuf v1.1.0-0.5.2 h1:4CagiIekonLSfL8GMHRHcHudo1fQnxELS9g4tiAupQ4= github.com/DataDog/go-tuf v1.1.0-0.5.2/go.mod h1:zBcq6f654iVqmkk8n2Cx81E1JnNTMOAx1UEO/wZR+P0= github.com/DataDog/opentelemetry-mapping-go/pkg/inframetadata v0.22.0 h1:r1Dx2cRHCBWkVluSZA41i4eoI/nOGbcrrZdkqWjoFCc= From 73f4ec93904fed8ed102a15fba2132199a012c04 Mon Sep 17 00:00:00 2001 From: "Brian L. Troutwine" Date: Wed, 18 Dec 2024 10:47:21 -0800 Subject: [PATCH 04/14] Update lading to 0.25.2 (#32271) Signed-off-by: Brian L. Troutwine --- test/regression/config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/regression/config.yaml b/test/regression/config.yaml index 362301357f864..4dcef653249aa 100644 --- a/test/regression/config.yaml +++ b/test/regression/config.yaml @@ -1,5 +1,5 @@ lading: - version: 0.24.0 + version: 0.25.2 target: cpu_allotment: 8 From 32aabeeeca86051037569dfe8a0ab411124ab09d Mon Sep 17 00:00:00 2001 From: eugene kirillov <3404064+krlv@users.noreply.github.com> Date: Wed, 18 Dec 2024 11:05:57 -0800 Subject: [PATCH 05/14] Bump otel-agent BYOC flow to use 7.59.0-v1.1.0 agent version (#31986) --- Dockerfiles/agent-ot/Dockerfile.agent-otel | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Dockerfiles/agent-ot/Dockerfile.agent-otel b/Dockerfiles/agent-ot/Dockerfile.agent-otel index 78534598a581c..49ea309af7019 100644 --- a/Dockerfiles/agent-ot/Dockerfile.agent-otel +++ b/Dockerfiles/agent-ot/Dockerfile.agent-otel @@ -1,5 +1,5 @@ -ARG AGENT_VERSION=7.57.0-v1.0-ot-beta-jmx -ARG AGENT_BRANCH=7.57.x-otel-beta-v1 +ARG AGENT_VERSION=7.59.0-v1.1.0-ot-beta-jmx +ARG AGENT_BRANCH=7.59.x # Use the Ubuntu Slim AMD64 base image FROM ubuntu:24.04 AS builder From 47cad73468d7a08c8319e24440326659dfe6992b Mon Sep 17 00:00:00 2001 From: Stephen Wakely Date: Wed, 18 Dec 2024 19:06:08 +0000 Subject: [PATCH 06/14] [APR-190] Change the default metric compression kind to be `zstd`. (#32087) Co-authored-by: blt Co-authored-by: scottopell --- pkg/config/setup/config.go | 2 +- .../internal/metrics/service_checks_test.go | 2 +- ...s-compression-default-zstd-c786c2d28eb51b1f.yaml | 13 +++++++++++++ 3 files changed, 15 insertions(+), 2 deletions(-) create mode 100644 releasenotes/notes/metrics-compression-default-zstd-c786c2d28eb51b1f.yaml diff --git a/pkg/config/setup/config.go b/pkg/config/setup/config.go index 24f8d81503459..fdb1d427e9d15 100644 --- a/pkg/config/setup/config.go +++ b/pkg/config/setup/config.go @@ -79,7 +79,7 @@ const ( DefaultRuntimePoliciesDir = "/etc/datadog-agent/runtime-security.d" // DefaultCompressorKind is the default compressor. Options available are 'zlib' and 'zstd' - DefaultCompressorKind = "zlib" + DefaultCompressorKind = "zstd" // DefaultZstdCompressionLevel is the default compression level for `zstd`. // Compression level 1 provides the lowest compression ratio, but uses much less RSS especially diff --git a/pkg/serializer/internal/metrics/service_checks_test.go b/pkg/serializer/internal/metrics/service_checks_test.go index 2a96f3d10072a..2559d7c9ed2cc 100644 --- a/pkg/serializer/internal/metrics/service_checks_test.go +++ b/pkg/serializer/internal/metrics/service_checks_test.go @@ -126,7 +126,7 @@ func TestPayloadsEmptyServiceCheck(t *testing.T) { func TestPayloadsServiceChecks(t *testing.T) { config := mock.New(t) - config.Set("serializer_max_payload_size", 200, pkgconfigmodel.SourceAgentRuntime) + config.Set("serializer_max_payload_size", 250, pkgconfigmodel.SourceAgentRuntime) serviceCheckCollection := []ServiceChecks{ {createServiceCheck("1"), createServiceCheck("2"), createServiceCheck("3")}, diff --git a/releasenotes/notes/metrics-compression-default-zstd-c786c2d28eb51b1f.yaml b/releasenotes/notes/metrics-compression-default-zstd-c786c2d28eb51b1f.yaml new file mode 100644 index 0000000000000..7d9a8653d8f0d --- /dev/null +++ b/releasenotes/notes/metrics-compression-default-zstd-c786c2d28eb51b1f.yaml @@ -0,0 +1,13 @@ +# Each section from every release note are combined when the +# CHANGELOG.rst is rendered. So the text needs to be worded so that +# it does not depend on any information only available in another +# section. This may mean repeating some details, but each section +# must be readable independently of the other. +# +# Each section note must be formatted as reStructuredText. +--- +enhancements: + - | + Metric payloads are compressed using `zstd` compression by default. + This can be reverted to the previous compression kind by adding + ``serializer_compressor_kind: zlib`` to the configuration. From ba0629fd7798a2e063213a5af520a621d9e0d163 Mon Sep 17 00:00:00 2001 From: Raphael Gavache Date: Wed, 18 Dec 2024 20:06:15 +0100 Subject: [PATCH 07/14] [fleet] fix codeowner (#32357) --- .github/CODEOWNERS | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index ec5ac92f6a2e5..50f8ff18a0c96 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -176,6 +176,7 @@ /.gitlab/package_build/ @DataDog/agent-delivery /.gitlab/package_build/windows.yml @DataDog/agent-delivery @DataDog/windows-agent +/.gitlab/package_build/installer.yml @DataDog/agent-delivery @DataDog/fleet /.gitlab/packaging/ @DataDog/agent-delivery /.gitlab/benchmarks/benchmarks.yml @DataDog/agent-apm From 7c1dae666e77bddee4ded3e2cbb5f822bb39764b Mon Sep 17 00:00:00 2001 From: Gabriel Dos Santos <91925154+gabedos@users.noreply.github.com> Date: Wed, 18 Dec 2024 14:21:36 -0500 Subject: [PATCH 08/14] [CONTP-517] Support None card in tagger (#31897) --- .../tagger/collectors/pod_tag_extractor.go | 2 + comp/core/tagger/impl/tagger.go | 5 +-- .../subscriber/subscription_manager_test.go | 37 +++++++++++++++++++ comp/core/tagger/tagstore/entity_tags.go | 12 ++++-- comp/core/tagger/tagstore/entity_tags_test.go | 6 +++ comp/core/tagger/tagstore/tagstore_test.go | 4 ++ comp/core/tagger/telemetry/telemetry.go | 4 ++ comp/core/tagger/types/types.go | 16 ++++++++ ...none-cardinality-tag-f2ceec7ec571387e.yaml | 11 ++++++ 9 files changed, 91 insertions(+), 6 deletions(-) create mode 100644 releasenotes/notes/none-cardinality-tag-f2ceec7ec571387e.yaml diff --git a/comp/core/tagger/collectors/pod_tag_extractor.go b/comp/core/tagger/collectors/pod_tag_extractor.go index 669a71584f609..aded502edaf4c 100644 --- a/comp/core/tagger/collectors/pod_tag_extractor.go +++ b/comp/core/tagger/collectors/pod_tag_extractor.go @@ -34,6 +34,8 @@ func (p *PodTagExtractor) Extract(podEntity *workloadmeta.KubernetesPod, cardina return append(tagInfos.LowCardTags, tagInfos.OrchestratorCardTags...) case types.LowCardinality: return tagInfos.LowCardTags + case types.NoneCardinality: + return []string{} default: log.Errorf("unsupported tag cardinality %v", cardinality) return []string{} diff --git a/comp/core/tagger/impl/tagger.go b/comp/core/tagger/impl/tagger.go index 7023106b1c118..de05b9f6eaa09 100644 --- a/comp/core/tagger/impl/tagger.go +++ b/comp/core/tagger/impl/tagger.go @@ -422,7 +422,7 @@ func (t *TaggerWrapper) EnrichTags(tb tagset.TagsAccumulator, originInfo taggert // | none | empty || empty | // | empty | not empty || container prefix + originFromMsg | // | none | not empty || container prefix + originFromMsg | - if t.datadogConfig.dogstatsdOptOutEnabled && originInfo.Cardinality == "none" { + if t.datadogConfig.dogstatsdOptOutEnabled && originInfo.Cardinality == types.NoneCardinalityString { originInfo.ContainerIDFromSocket = packets.NoOrigin originInfo.PodUID = "" originInfo.ContainerID = "" @@ -460,8 +460,7 @@ func (t *TaggerWrapper) EnrichTags(tb tagset.TagsAccumulator, originInfo taggert } default: // Disable origin detection if cardinality is none - // TODO: The `none` cardinality should be directly supported by the Tagger. - if originInfo.Cardinality == "none" { + if originInfo.Cardinality == types.NoneCardinalityString { originInfo.ContainerIDFromSocket = packets.NoOrigin originInfo.PodUID = "" originInfo.ContainerID = "" diff --git a/comp/core/tagger/subscriber/subscription_manager_test.go b/comp/core/tagger/subscriber/subscription_manager_test.go index b9684af30e2d4..dbebd8ed23d98 100644 --- a/comp/core/tagger/subscriber/subscription_manager_test.go +++ b/comp/core/tagger/subscriber/subscription_manager_test.go @@ -109,6 +109,21 @@ func TestSubscriptionManager(t *testing.T) { highCardSubscription.Unsubscribe() + // None Cardinality Subscriber + noneCardSubID := "none-card-sub" + noneCardSubscription, err := sm.Subscribe(noneCardSubID, types.NewFilterBuilder().Include(types.EntityIDPrefix("foo")).Build(types.NoneCardinality), nil) + require.NoError(t, err) + + sm.Notify([]types.EntityEvent{ + events["added"], + events["modified"], + events["deleted"], + events["added-with-no-id"], + events["added-with-unmatched-prefix"], + }) + + noneCardSubscription.Unsubscribe() + // Verify low cardinality subscriber received events assertReceivedEvents(t, lowCardSubscription.EventsChan(), []types.EntityEvent{ { @@ -192,6 +207,28 @@ func TestSubscriptionManager(t *testing.T) { }, }, }) + + // Verify none cardinality subscriber received events + assertReceivedEvents(t, noneCardSubscription.EventsChan(), []types.EntityEvent{ + { + EventType: types.EventTypeAdded, + Entity: types.Entity{ + ID: entityID, + }, + }, + { + EventType: types.EventTypeModified, + Entity: types.Entity{ + ID: entityID, + }, + }, + { + EventType: types.EventTypeDeleted, + Entity: types.Entity{ + ID: entityID, + }, + }, + }) } func assertReceivedEvents(t *testing.T, ch chan []types.EntityEvent, expectedEvents []types.EntityEvent) { diff --git a/comp/core/tagger/tagstore/entity_tags.go b/comp/core/tagger/tagstore/entity_tags.go index 3488a8e345c16..ed62d45849109 100644 --- a/comp/core/tagger/tagstore/entity_tags.go +++ b/comp/core/tagger/tagstore/entity_tags.go @@ -109,12 +109,16 @@ func (e *EntityTagsWithMultipleSources) getStandard() []string { func (e *EntityTagsWithMultipleSources) getHashedTags(cardinality types.TagCardinality) tagset.HashedTags { e.computeCache() - if cardinality == types.HighCardinality { + switch cardinality { + case types.HighCardinality: return e.cachedAll - } else if cardinality == types.OrchestratorCardinality { + case types.OrchestratorCardinality: return e.cachedOrchestrator + case types.NoneCardinality: + return tagset.HashedTags{} + default: + return e.cachedLow } - return e.cachedLow } func (e *EntityTagsWithMultipleSources) computeCache() { @@ -302,6 +306,8 @@ func (e *EntityTagsWithSingleSource) getHashedTags(cardinality types.TagCardinal return e.cachedAll case types.OrchestratorCardinality: return e.cachedOrchestrator + case types.NoneCardinality: + return tagset.HashedTags{} default: return e.cachedLow } diff --git a/comp/core/tagger/tagstore/entity_tags_test.go b/comp/core/tagger/tagstore/entity_tags_test.go index c93d764377aaf..16e2d4b077f86 100644 --- a/comp/core/tagger/tagstore/entity_tags_test.go +++ b/comp/core/tagger/tagstore/entity_tags_test.go @@ -100,6 +100,12 @@ func TestGetHashedTags(t *testing.T) { []string{"l1:v1", "l2:v2", "service:s1", "o1:v1", "o2:v2", "h1:v1", "h2:v2"}, entityTags.getHashedTags(types.HighCardinality).Get(), ) + + assert.Equal( + t, + []string(nil), + entityTags.getHashedTags(types.NoneCardinality).Get(), + ) } func TestTagsForSource(t *testing.T) { diff --git a/comp/core/tagger/tagstore/tagstore_test.go b/comp/core/tagger/tagstore/tagstore_test.go index 4699aa6ad56cb..0d992b682ca68 100644 --- a/comp/core/tagger/tagstore/tagstore_test.go +++ b/comp/core/tagger/tagstore/tagstore_test.go @@ -88,10 +88,12 @@ func (s *StoreTestSuite) TestLookup() { tagsHigh := s.tagstore.Lookup(entityID, types.HighCardinality) tagsOrch := s.tagstore.Lookup(entityID, types.OrchestratorCardinality) tagsLow := s.tagstore.Lookup(entityID, types.LowCardinality) + tagsNone := s.tagstore.Lookup(entityID, types.NoneCardinality) assert.Len(s.T(), tagsHigh, 4) assert.Len(s.T(), tagsLow, 2) assert.Len(s.T(), tagsOrch, 3) + assert.Nil(s.T(), tagsNone) } func (s *StoreTestSuite) TestLookupHashedWithEntityStr() { @@ -118,10 +120,12 @@ func (s *StoreTestSuite) TestLookupHashedWithEntityStr() { tagsLow := s.tagstore.LookupHashedWithEntityStr(entityID, types.LowCardinality) tagsOrch := s.tagstore.LookupHashedWithEntityStr(entityID, types.OrchestratorCardinality) tagsHigh := s.tagstore.LookupHashedWithEntityStr(entityID, types.HighCardinality) + tagsNone := s.tagstore.LookupHashedWithEntityStr(entityID, types.NoneCardinality) assert.ElementsMatch(s.T(), tagsLow.Get(), []string{"low1", "low2"}) assert.ElementsMatch(s.T(), tagsOrch.Get(), []string{"low1", "low2", "orchestrator1"}) assert.ElementsMatch(s.T(), tagsHigh.Get(), []string{"low1", "low2", "orchestrator1", "high1"}) + assert.ElementsMatch(s.T(), tagsNone.Get(), []string{}) } func (s *StoreTestSuite) TestLookupStandard() { diff --git a/comp/core/tagger/telemetry/telemetry.go b/comp/core/tagger/telemetry/telemetry.go index 97c7153d9c052..0682475eb6d8e 100644 --- a/comp/core/tagger/telemetry/telemetry.go +++ b/comp/core/tagger/telemetry/telemetry.go @@ -68,6 +68,7 @@ type Store struct { LowCardinalityQueries CardinalityTelemetry OrchestratorCardinalityQueries CardinalityTelemetry HighCardinalityQueries CardinalityTelemetry + NoneCardinalityQueries CardinalityTelemetry UnknownCardinalityQueries CardinalityTelemetry } @@ -144,6 +145,7 @@ func NewStore(telemetryComp telemetry.Component) *Store { LowCardinalityQueries: newCardinalityTelemetry(queries, types.LowCardinalityString), OrchestratorCardinalityQueries: newCardinalityTelemetry(queries, types.OrchestratorCardinalityString), HighCardinalityQueries: newCardinalityTelemetry(queries, types.HighCardinalityString), + NoneCardinalityQueries: newCardinalityTelemetry(queries, types.NoneCardinalityString), UnknownCardinalityQueries: newCardinalityTelemetry(queries, types.UnknownCardinalityString), } }) @@ -160,6 +162,8 @@ func (s *Store) QueriesByCardinality(card types.TagCardinality) *CardinalityTele return &s.OrchestratorCardinalityQueries case types.HighCardinality: return &s.HighCardinalityQueries + case types.NoneCardinality: + return &s.NoneCardinalityQueries default: return &s.UnknownCardinalityQueries } diff --git a/comp/core/tagger/types/types.go b/comp/core/tagger/types/types.go index e45f9e6daa084..dc53af7baf27b 100644 --- a/comp/core/tagger/types/types.go +++ b/comp/core/tagger/types/types.go @@ -78,6 +78,7 @@ const ( LowCardinality TagCardinality = iota OrchestratorCardinality HighCardinality + NoneCardinality ) // Entity is an entity ID + tags. @@ -92,6 +93,10 @@ type Entity struct { // GetTags flattens all tags from all cardinalities into a single slice of tag // strings. func (e Entity) GetTags(cardinality TagCardinality) []string { + if cardinality == NoneCardinality { + return []string{} + } + tagArrays := make([][]string, 0, 3) tagArrays = append(tagArrays, e.LowCardinalityTags) @@ -117,6 +122,11 @@ func (e Entity) Copy(cardinality TagCardinality) Entity { case LowCardinality: newEntity.HighCardinalityTags = nil newEntity.OrchestratorCardinalityTags = nil + case NoneCardinality: + newEntity.HighCardinalityTags = nil + newEntity.OrchestratorCardinalityTags = nil + newEntity.LowCardinalityTags = nil + newEntity.StandardTags = nil } return newEntity @@ -131,6 +141,8 @@ const ( ShortOrchestratorCardinalityString = "orch" // HighCardinalityString is the string representation of the high cardinality HighCardinalityString = "high" + // NoneCardinalityString is the string representation of the none cardinality + NoneCardinalityString = "none" // UnknownCardinalityString represents an unknown level of cardinality UnknownCardinalityString = "unknown" ) @@ -145,6 +157,8 @@ func StringToTagCardinality(c string) (TagCardinality, error) { return OrchestratorCardinality, nil case LowCardinalityString: return LowCardinality, nil + case NoneCardinalityString: + return NoneCardinality, nil default: return LowCardinality, fmt.Errorf("unsupported value %s received for tag cardinality", c) } @@ -160,6 +174,8 @@ func TagCardinalityToString(c TagCardinality) string { return OrchestratorCardinalityString case LowCardinality: return LowCardinalityString + case NoneCardinality: + return NoneCardinalityString default: return UnknownCardinalityString } diff --git a/releasenotes/notes/none-cardinality-tag-f2ceec7ec571387e.yaml b/releasenotes/notes/none-cardinality-tag-f2ceec7ec571387e.yaml new file mode 100644 index 0000000000000..2957ae7f9441d --- /dev/null +++ b/releasenotes/notes/none-cardinality-tag-f2ceec7ec571387e.yaml @@ -0,0 +1,11 @@ +# Each section from every release note are combined when the +# CHANGELOG.rst is rendered. So the text needs to be worded so that +# it does not depend on any information only available in another +# section. This may mean repeating some details, but each section +# must be readable independently of the other. +# +# Each section note must be formatted as reStructuredText. +--- +enhancements: + - | + Adds support for the `none` cardinality type in the tagger. From a2eda15d0208e92ed2288b0efcd05808bb23236c Mon Sep 17 00:00:00 2001 From: Paul Cacheux Date: Wed, 18 Dec 2024 20:35:49 +0100 Subject: [PATCH 09/14] [CWS] cut allocation in `GetProcContainerContext` (#32183) --- pkg/security/utils/cgroup.go | 53 ++++++++++++++++++++++++++++++++---- 1 file changed, 48 insertions(+), 5 deletions(-) diff --git a/pkg/security/utils/cgroup.go b/pkg/security/utils/cgroup.go index 5dbbb97c58af1..df59e317006a5 100644 --- a/pkg/security/utils/cgroup.go +++ b/pkg/security/utils/cgroup.go @@ -12,6 +12,7 @@ import ( "bufio" "bytes" "crypto/sha256" + "fmt" "os" "strconv" "strings" @@ -49,6 +50,49 @@ func (cg ControlGroup) GetContainerID() containerutils.ContainerID { return containerutils.ContainerID(id) } +// GetLastProcControlGroups returns the first cgroup membership of the specified task. +func GetLastProcControlGroups(tgid, pid uint32) (ControlGroup, error) { + data, err := os.ReadFile(CgroupTaskPath(tgid, pid)) + if err != nil { + return ControlGroup{}, err + } + + data = bytes.TrimSpace(data) + + index := bytes.LastIndexByte(data, '\n') + if index < 0 { + index = 0 + } else { + index++ // to skip the \n + } + if index >= len(data) { + return ControlGroup{}, fmt.Errorf("invalid cgroup data: %s", data) + } + + lastLine := string(data[index:]) + + idstr, rest, ok := strings.Cut(lastLine, ":") + if !ok { + return ControlGroup{}, fmt.Errorf("invalid cgroup line: %s", lastLine) + } + + id, err := strconv.Atoi(idstr) + if err != nil { + return ControlGroup{}, err + } + + controllers, path, ok := strings.Cut(rest, ":") + if !ok { + return ControlGroup{}, fmt.Errorf("invalid cgroup line: %s", lastLine) + } + + return ControlGroup{ + ID: id, + Controllers: strings.Split(controllers, ","), + Path: path, + }, nil +} + // GetProcControlGroups returns the cgroup membership of the specified task. func GetProcControlGroups(tgid, pid uint32) ([]ControlGroup, error) { data, err := os.ReadFile(CgroupTaskPath(tgid, pid)) @@ -85,15 +129,14 @@ func GetProcContainerID(tgid, pid uint32) (containerutils.ContainerID, error) { // GetProcContainerContext returns the container ID which the process belongs to along with its manager. Returns "" if the process does not belong // to a container. func GetProcContainerContext(tgid, pid uint32) (containerutils.ContainerID, model.CGroupContext, error) { - cgroups, err := GetProcControlGroups(tgid, pid) - if err != nil || len(cgroups) == 0 { + cgroup, err := GetLastProcControlGroups(tgid, pid) + if err != nil { return "", model.CGroupContext{}, err } - lastCgroup := len(cgroups) - 1 - containerID, runtime := cgroups[lastCgroup].GetContainerContext() + containerID, runtime := cgroup.GetContainerContext() cgroupContext := model.CGroupContext{ - CGroupID: containerutils.CGroupID(cgroups[lastCgroup].Path), + CGroupID: containerutils.CGroupID(cgroup.Path), CGroupFlags: runtime, } From 23a03806e9998192fbe5fcd1d38e193acf8d7713 Mon Sep 17 00:00:00 2001 From: shreyamalpani Date: Wed, 18 Dec 2024 14:49:23 -0500 Subject: [PATCH 10/14] [SLES-2001] decode lambda error.msg and error.type (#32231) --- pkg/serverless/daemon/routes.go | 10 ++++++ pkg/serverless/daemon/routes_test.go | 48 +++++++++++++++++++++++++++- 2 files changed, 57 insertions(+), 1 deletion(-) diff --git a/pkg/serverless/daemon/routes.go b/pkg/serverless/daemon/routes.go index 733b4a49b050c..fe78651ac4d2f 100644 --- a/pkg/serverless/daemon/routes.go +++ b/pkg/serverless/daemon/routes.go @@ -104,7 +104,17 @@ func (e *EndInvocation) ServeHTTP(w http.ResponseWriter, r *http.Request) { } errorMsg := r.Header.Get(invocationlifecycle.InvocationErrorMsgHeader) + if decodedMsg, err := base64.StdEncoding.DecodeString(errorMsg); err != nil { + log.Debug("Error message header may not be encoded, setting as is") + } else { + errorMsg = string(decodedMsg) + } errorType := r.Header.Get(invocationlifecycle.InvocationErrorTypeHeader) + if decodedType, err := base64.StdEncoding.DecodeString(errorType); err != nil { + log.Debug("Error type header may not be encoded, setting as is") + } else { + errorType = string(decodedType) + } errorStack := r.Header.Get(invocationlifecycle.InvocationErrorStackHeader) if decodedStack, err := base64.StdEncoding.DecodeString(errorStack); err != nil { log.Debug("Could not decode error stack header") diff --git a/pkg/serverless/daemon/routes_test.go b/pkg/serverless/daemon/routes_test.go index 25231204af6c8..e630d2829bd61 100644 --- a/pkg/serverless/daemon/routes_test.go +++ b/pkg/serverless/daemon/routes_test.go @@ -7,6 +7,7 @@ package daemon import ( "bytes" + "encoding/base64" "fmt" "io" "net/http" @@ -104,7 +105,7 @@ func TestEndInvocation(t *testing.T) { assert.Equal(m.lastEndDetails.Runtime, d.ExecutionContext.GetCurrentState().Runtime) } -func TestEndInvocationWithError(t *testing.T) { +func TestEndInvocationWithErrorEncodedHeaders(t *testing.T) { assert := assert.New(t) port := testutil.FreeTCPPort(t) d := StartDaemon(fmt.Sprintf("127.0.0.1:%d", port)) @@ -114,10 +115,52 @@ func TestEndInvocationWithError(t *testing.T) { m := &mockLifecycleProcessor{} d.InvocationProcessor = m + errorMessage := "Error message" + errorType := "System.Exception" + errorStack := "System.Exception: Error message \n at TestFunction.Handle(ILambdaContext context)" + + client := &http.Client{} + body := bytes.NewBuffer([]byte(`{}`)) + request, err := http.NewRequest(http.MethodPost, fmt.Sprintf("http://127.0.0.1:%d/lambda/end-invocation", port), body) + request.Header.Set("x-datadog-invocation-error", "true") + request.Header.Set(invocationlifecycle.InvocationErrorMsgHeader, base64.StdEncoding.EncodeToString([]byte(errorMessage))) + request.Header.Set(invocationlifecycle.InvocationErrorTypeHeader, base64.StdEncoding.EncodeToString([]byte(errorType))) + request.Header.Set(invocationlifecycle.InvocationErrorStackHeader, base64.StdEncoding.EncodeToString([]byte(errorStack))) + assert.Nil(err) + res, err := client.Do(request) + assert.Nil(err) + if res != nil { + res.Body.Close() + assert.Equal(res.StatusCode, 200) + } + assert.True(m.OnInvokeEndCalled) + assert.True(m.isError) + assert.Equal(m.lastEndDetails.ErrorMsg, errorMessage) + assert.Equal(m.lastEndDetails.ErrorType, errorType) + assert.Equal(m.lastEndDetails.ErrorStack, errorStack) +} + +func TestEndInvocationWithErrorNonEncodedHeaders(t *testing.T) { + assert := assert.New(t) + port := testutil.FreeTCPPort(t) + d := StartDaemon(fmt.Sprintf("127.0.0.1:%d", port)) + time.Sleep(100 * time.Millisecond) + defer d.Stop() + + m := &mockLifecycleProcessor{} + d.InvocationProcessor = m + + errorMessage := "Error message" + errorType := "System.Exception" + errorStack := "System.Exception: Error message at TestFunction.Handle(ILambdaContext context)" + client := &http.Client{} body := bytes.NewBuffer([]byte(`{}`)) request, err := http.NewRequest(http.MethodPost, fmt.Sprintf("http://127.0.0.1:%d/lambda/end-invocation", port), body) request.Header.Set("x-datadog-invocation-error", "true") + request.Header.Set(invocationlifecycle.InvocationErrorMsgHeader, errorMessage) + request.Header.Set(invocationlifecycle.InvocationErrorTypeHeader, errorType) + request.Header.Set(invocationlifecycle.InvocationErrorStackHeader, errorStack) assert.Nil(err) res, err := client.Do(request) assert.Nil(err) @@ -127,6 +170,9 @@ func TestEndInvocationWithError(t *testing.T) { } assert.True(m.OnInvokeEndCalled) assert.True(m.isError) + assert.Equal(m.lastEndDetails.ErrorMsg, errorMessage) + assert.Equal(m.lastEndDetails.ErrorType, errorType) + assert.Equal(m.lastEndDetails.ErrorStack, errorStack) } func TestTraceContext(t *testing.T) { From f803a52458da24266aa502d72d27dfdafdef6fcf Mon Sep 17 00:00:00 2001 From: Kylian Serrania Date: Wed, 18 Dec 2024 21:38:42 +0100 Subject: [PATCH 11/14] Revert "bump github.com/DataDog/go-sqllexer to v0.0.18" (#32365) This PR broke a KMT test (pkg/network/protocols/postgres.TestExtractTableFunction/single_table_name_with_mixed_caps). #incident-33427 --- comp/otelcol/ddflareextension/impl/go.mod | 2 +- comp/otelcol/ddflareextension/impl/go.sum | 4 ++-- comp/otelcol/otlp/components/exporter/datadogexporter/go.mod | 2 +- comp/otelcol/otlp/components/exporter/datadogexporter/go.sum | 4 ++-- comp/otelcol/otlp/components/statsprocessor/go.mod | 2 +- comp/otelcol/otlp/components/statsprocessor/go.sum | 4 ++-- go.mod | 2 +- go.sum | 4 ++-- pkg/config/remote/go.mod | 2 +- pkg/config/remote/go.sum | 4 ++-- pkg/obfuscate/go.mod | 2 +- pkg/obfuscate/go.sum | 4 ++-- pkg/trace/go.mod | 2 +- pkg/trace/go.sum | 4 ++-- pkg/trace/stats/oteltest/go.mod | 2 +- pkg/trace/stats/oteltest/go.sum | 4 ++-- test/otel/go.mod | 2 +- test/otel/go.sum | 4 ++-- 18 files changed, 27 insertions(+), 27 deletions(-) diff --git a/comp/otelcol/ddflareextension/impl/go.mod b/comp/otelcol/ddflareextension/impl/go.mod index ce08a7999f072..21d3f17b9066a 100644 --- a/comp/otelcol/ddflareextension/impl/go.mod +++ b/comp/otelcol/ddflareextension/impl/go.mod @@ -267,7 +267,7 @@ require ( github.com/DataDog/datadog-api-client-go/v2 v2.33.0 // indirect github.com/DataDog/datadog-go/v5 v5.6.0 // indirect github.com/DataDog/dd-sensitive-data-scanner/sds-go/go v0.0.0-20240816154533-f7f9beb53a42 // indirect - github.com/DataDog/go-sqllexer v0.0.18 // indirect + github.com/DataDog/go-sqllexer v0.0.17 // indirect github.com/DataDog/go-tuf v1.1.0-0.5.2 // indirect github.com/DataDog/mmh3 v0.0.0-20210722141835-012dc69a9e49 // indirect github.com/DataDog/opentelemetry-mapping-go/pkg/otlp/attributes v0.22.0 // indirect diff --git a/comp/otelcol/ddflareextension/impl/go.sum b/comp/otelcol/ddflareextension/impl/go.sum index 7aed291a64674..d4f8e67e79da5 100644 --- a/comp/otelcol/ddflareextension/impl/go.sum +++ b/comp/otelcol/ddflareextension/impl/go.sum @@ -70,8 +70,8 @@ github.com/DataDog/datadog-go/v5 v5.6.0 h1:2oCLxjF/4htd55piM75baflj/KoE6VYS7alEU github.com/DataDog/datadog-go/v5 v5.6.0/go.mod h1:K9kcYBlxkcPP8tvvjZZKs/m1edNAUFzBbdpTUKfCsuw= github.com/DataDog/dd-sensitive-data-scanner/sds-go/go v0.0.0-20240816154533-f7f9beb53a42 h1:RoH7VLzTnxHEugRPIgnGlxwDFszFGI7b3WZZUtWuPRM= github.com/DataDog/dd-sensitive-data-scanner/sds-go/go v0.0.0-20240816154533-f7f9beb53a42/go.mod h1:TX7CTOQ3LbQjfAi4SwqUoR5gY1zfUk7VRBDTuArjaDc= -github.com/DataDog/go-sqllexer v0.0.18 h1:ErBvoO7/srJLdA2ebwd+HPqD4g1kN++BP64A8qvmh9U= -github.com/DataDog/go-sqllexer v0.0.18/go.mod h1:KwkYhpFEVIq+BfobkTC1vfqm4gTi65skV/DpDBXtexc= +github.com/DataDog/go-sqllexer v0.0.17 h1:u47fJAVg/+5DA74ZW3w0Qu+3qXHd3GtnA8ZBYixdPrM= +github.com/DataDog/go-sqllexer v0.0.17/go.mod h1:KwkYhpFEVIq+BfobkTC1vfqm4gTi65skV/DpDBXtexc= github.com/DataDog/go-tuf v1.1.0-0.5.2 h1:4CagiIekonLSfL8GMHRHcHudo1fQnxELS9g4tiAupQ4= github.com/DataDog/go-tuf v1.1.0-0.5.2/go.mod h1:zBcq6f654iVqmkk8n2Cx81E1JnNTMOAx1UEO/wZR+P0= github.com/DataDog/gohai v0.0.0-20230524154621-4316413895ee h1:tXibLZk3G6HncIFJKaNItsdzcrk4YqILNDZlXPTNt4k= diff --git a/comp/otelcol/otlp/components/exporter/datadogexporter/go.mod b/comp/otelcol/otlp/components/exporter/datadogexporter/go.mod index dfe091b9d4b79..301ef0f3b1f48 100644 --- a/comp/otelcol/otlp/components/exporter/datadogexporter/go.mod +++ b/comp/otelcol/otlp/components/exporter/datadogexporter/go.mod @@ -201,7 +201,7 @@ require ( github.com/DataDog/datadog-agent/pkg/version v0.59.1 // indirect github.com/DataDog/datadog-api-client-go/v2 v2.33.0 // indirect github.com/DataDog/dd-sensitive-data-scanner/sds-go/go v0.0.0-20240816154533-f7f9beb53a42 // indirect - github.com/DataDog/go-sqllexer v0.0.18 // indirect + github.com/DataDog/go-sqllexer v0.0.17 // indirect github.com/DataDog/go-tuf v1.1.0-0.5.2 // indirect github.com/DataDog/mmh3 v0.0.0-20210722141835-012dc69a9e49 // indirect github.com/DataDog/opentelemetry-mapping-go/pkg/otlp/logs v0.22.0 // indirect diff --git a/comp/otelcol/otlp/components/exporter/datadogexporter/go.sum b/comp/otelcol/otlp/components/exporter/datadogexporter/go.sum index c32ab7009cbeb..e9c507fbafa8a 100644 --- a/comp/otelcol/otlp/components/exporter/datadogexporter/go.sum +++ b/comp/otelcol/otlp/components/exporter/datadogexporter/go.sum @@ -8,8 +8,8 @@ github.com/DataDog/datadog-go/v5 v5.6.0 h1:2oCLxjF/4htd55piM75baflj/KoE6VYS7alEU github.com/DataDog/datadog-go/v5 v5.6.0/go.mod h1:K9kcYBlxkcPP8tvvjZZKs/m1edNAUFzBbdpTUKfCsuw= github.com/DataDog/dd-sensitive-data-scanner/sds-go/go v0.0.0-20240816154533-f7f9beb53a42 h1:RoH7VLzTnxHEugRPIgnGlxwDFszFGI7b3WZZUtWuPRM= github.com/DataDog/dd-sensitive-data-scanner/sds-go/go v0.0.0-20240816154533-f7f9beb53a42/go.mod h1:TX7CTOQ3LbQjfAi4SwqUoR5gY1zfUk7VRBDTuArjaDc= -github.com/DataDog/go-sqllexer v0.0.18 h1:ErBvoO7/srJLdA2ebwd+HPqD4g1kN++BP64A8qvmh9U= -github.com/DataDog/go-sqllexer v0.0.18/go.mod h1:KwkYhpFEVIq+BfobkTC1vfqm4gTi65skV/DpDBXtexc= +github.com/DataDog/go-sqllexer v0.0.17 h1:u47fJAVg/+5DA74ZW3w0Qu+3qXHd3GtnA8ZBYixdPrM= +github.com/DataDog/go-sqllexer v0.0.17/go.mod h1:KwkYhpFEVIq+BfobkTC1vfqm4gTi65skV/DpDBXtexc= github.com/DataDog/go-tuf v1.1.0-0.5.2 h1:4CagiIekonLSfL8GMHRHcHudo1fQnxELS9g4tiAupQ4= github.com/DataDog/go-tuf v1.1.0-0.5.2/go.mod h1:zBcq6f654iVqmkk8n2Cx81E1JnNTMOAx1UEO/wZR+P0= github.com/DataDog/mmh3 v0.0.0-20210722141835-012dc69a9e49 h1:EbzDX8HPk5uE2FsJYxD74QmMw0/3CqSKhEr6teh0ncQ= diff --git a/comp/otelcol/otlp/components/statsprocessor/go.mod b/comp/otelcol/otlp/components/statsprocessor/go.mod index c03fc39e3ca59..18606500a04d2 100644 --- a/comp/otelcol/otlp/components/statsprocessor/go.mod +++ b/comp/otelcol/otlp/components/statsprocessor/go.mod @@ -43,7 +43,7 @@ require ( github.com/DataDog/datadog-agent/pkg/util/pointer v0.59.0 // indirect github.com/DataDog/datadog-agent/pkg/util/scrubber v0.59.0 // indirect github.com/DataDog/datadog-agent/pkg/version v0.59.1 // indirect - github.com/DataDog/go-sqllexer v0.0.18 // indirect + github.com/DataDog/go-sqllexer v0.0.17 // indirect github.com/DataDog/go-tuf v1.1.0-0.5.2 // indirect github.com/DataDog/sketches-go v1.4.6 // indirect github.com/Microsoft/go-winio v0.6.2 // indirect diff --git a/comp/otelcol/otlp/components/statsprocessor/go.sum b/comp/otelcol/otlp/components/statsprocessor/go.sum index 22ef689018918..a19fb370fa6c4 100644 --- a/comp/otelcol/otlp/components/statsprocessor/go.sum +++ b/comp/otelcol/otlp/components/statsprocessor/go.sum @@ -1,7 +1,7 @@ github.com/DataDog/datadog-go/v5 v5.6.0 h1:2oCLxjF/4htd55piM75baflj/KoE6VYS7alEUqFvRDw= github.com/DataDog/datadog-go/v5 v5.6.0/go.mod h1:K9kcYBlxkcPP8tvvjZZKs/m1edNAUFzBbdpTUKfCsuw= -github.com/DataDog/go-sqllexer v0.0.18 h1:ErBvoO7/srJLdA2ebwd+HPqD4g1kN++BP64A8qvmh9U= -github.com/DataDog/go-sqllexer v0.0.18/go.mod h1:KwkYhpFEVIq+BfobkTC1vfqm4gTi65skV/DpDBXtexc= +github.com/DataDog/go-sqllexer v0.0.17 h1:u47fJAVg/+5DA74ZW3w0Qu+3qXHd3GtnA8ZBYixdPrM= +github.com/DataDog/go-sqllexer v0.0.17/go.mod h1:KwkYhpFEVIq+BfobkTC1vfqm4gTi65skV/DpDBXtexc= github.com/DataDog/go-tuf v1.1.0-0.5.2 h1:4CagiIekonLSfL8GMHRHcHudo1fQnxELS9g4tiAupQ4= github.com/DataDog/go-tuf v1.1.0-0.5.2/go.mod h1:zBcq6f654iVqmkk8n2Cx81E1JnNTMOAx1UEO/wZR+P0= github.com/DataDog/opentelemetry-mapping-go/pkg/otlp/attributes v0.22.0 h1:yfk2cF8Bx98fSFpGrehEHh1FRqewfxcCTAbUDt5r3F8= diff --git a/go.mod b/go.mod index 959651ab0bcce..c8de7db9cc5f0 100644 --- a/go.mod +++ b/go.mod @@ -735,7 +735,7 @@ require ( github.com/DataDog/datadog-agent/pkg/util/winutil v0.59.1 github.com/DataDog/datadog-agent/pkg/version v0.59.1 github.com/DataDog/go-libddwaf/v3 v3.5.1 - github.com/DataDog/go-sqllexer v0.0.18 + github.com/DataDog/go-sqllexer v0.0.17 github.com/Datadog/dublin-traceroute v0.0.2 github.com/aquasecurity/trivy v0.49.2-0.20240227072422-e1ea02c7b80d github.com/aws/aws-sdk-go-v2/service/kms v1.37.6 diff --git a/go.sum b/go.sum index ffa368593f411..26375333ef0cf 100644 --- a/go.sum +++ b/go.sum @@ -142,8 +142,8 @@ github.com/DataDog/go-grpc-bidirectional-streaming-example v0.0.0-20221024060302 github.com/DataDog/go-grpc-bidirectional-streaming-example v0.0.0-20221024060302-b9cf785c02fe/go.mod h1:90sqV0j7E8wYCyqIp5d9HmYWLTFQttqPFFtNYDyAybQ= github.com/DataDog/go-libddwaf/v3 v3.5.1 h1:GWA4ln4DlLxiXm+X7HA/oj0ZLcdCwOS81KQitegRTyY= github.com/DataDog/go-libddwaf/v3 v3.5.1/go.mod h1:n98d9nZ1gzenRSk53wz8l6d34ikxS+hs62A31Fqmyi4= -github.com/DataDog/go-sqllexer v0.0.18 h1:ErBvoO7/srJLdA2ebwd+HPqD4g1kN++BP64A8qvmh9U= -github.com/DataDog/go-sqllexer v0.0.18/go.mod h1:KwkYhpFEVIq+BfobkTC1vfqm4gTi65skV/DpDBXtexc= +github.com/DataDog/go-sqllexer v0.0.17 h1:u47fJAVg/+5DA74ZW3w0Qu+3qXHd3GtnA8ZBYixdPrM= +github.com/DataDog/go-sqllexer v0.0.17/go.mod h1:KwkYhpFEVIq+BfobkTC1vfqm4gTi65skV/DpDBXtexc= github.com/DataDog/go-tuf v1.1.0-0.5.2 h1:4CagiIekonLSfL8GMHRHcHudo1fQnxELS9g4tiAupQ4= github.com/DataDog/go-tuf v1.1.0-0.5.2/go.mod h1:zBcq6f654iVqmkk8n2Cx81E1JnNTMOAx1UEO/wZR+P0= github.com/DataDog/gohai v0.0.0-20230524154621-4316413895ee h1:tXibLZk3G6HncIFJKaNItsdzcrk4YqILNDZlXPTNt4k= diff --git a/pkg/config/remote/go.mod b/pkg/config/remote/go.mod index 72ed5685445f8..a1c4a4ebbc561 100644 --- a/pkg/config/remote/go.mod +++ b/pkg/config/remote/go.mod @@ -87,7 +87,7 @@ require ( github.com/DataDog/datadog-agent/pkg/version v0.59.1 // indirect github.com/DataDog/datadog-go/v5 v5.6.0 // indirect github.com/DataDog/go-libddwaf/v3 v3.5.1 // indirect - github.com/DataDog/go-sqllexer v0.0.18 // indirect + github.com/DataDog/go-sqllexer v0.0.17 // indirect github.com/DataDog/sketches-go v1.4.6 // indirect github.com/Microsoft/go-winio v0.6.2 // indirect github.com/cespare/xxhash/v2 v2.3.0 // indirect diff --git a/pkg/config/remote/go.sum b/pkg/config/remote/go.sum index 512cb6bacbab2..ca537b0d1660e 100644 --- a/pkg/config/remote/go.sum +++ b/pkg/config/remote/go.sum @@ -11,8 +11,8 @@ github.com/DataDog/datadog-go/v5 v5.6.0 h1:2oCLxjF/4htd55piM75baflj/KoE6VYS7alEU github.com/DataDog/datadog-go/v5 v5.6.0/go.mod h1:K9kcYBlxkcPP8tvvjZZKs/m1edNAUFzBbdpTUKfCsuw= github.com/DataDog/go-libddwaf/v3 v3.5.1 h1:GWA4ln4DlLxiXm+X7HA/oj0ZLcdCwOS81KQitegRTyY= github.com/DataDog/go-libddwaf/v3 v3.5.1/go.mod h1:n98d9nZ1gzenRSk53wz8l6d34ikxS+hs62A31Fqmyi4= -github.com/DataDog/go-sqllexer v0.0.18 h1:ErBvoO7/srJLdA2ebwd+HPqD4g1kN++BP64A8qvmh9U= -github.com/DataDog/go-sqllexer v0.0.18/go.mod h1:KwkYhpFEVIq+BfobkTC1vfqm4gTi65skV/DpDBXtexc= +github.com/DataDog/go-sqllexer v0.0.17 h1:u47fJAVg/+5DA74ZW3w0Qu+3qXHd3GtnA8ZBYixdPrM= +github.com/DataDog/go-sqllexer v0.0.17/go.mod h1:KwkYhpFEVIq+BfobkTC1vfqm4gTi65skV/DpDBXtexc= github.com/DataDog/go-tuf v1.1.0-0.5.2 h1:4CagiIekonLSfL8GMHRHcHudo1fQnxELS9g4tiAupQ4= github.com/DataDog/go-tuf v1.1.0-0.5.2/go.mod h1:zBcq6f654iVqmkk8n2Cx81E1JnNTMOAx1UEO/wZR+P0= github.com/DataDog/gostackparse v0.7.0 h1:i7dLkXHvYzHV308hnkvVGDL3BR4FWl7IsXNPz/IGQh4= diff --git a/pkg/obfuscate/go.mod b/pkg/obfuscate/go.mod index 709579d414b7f..b34df591521b5 100644 --- a/pkg/obfuscate/go.mod +++ b/pkg/obfuscate/go.mod @@ -4,7 +4,7 @@ go 1.22.0 require ( github.com/DataDog/datadog-go/v5 v5.6.0 - github.com/DataDog/go-sqllexer v0.0.18 + github.com/DataDog/go-sqllexer v0.0.17 github.com/outcaste-io/ristretto v0.2.3 github.com/stretchr/testify v1.10.0 go.uber.org/atomic v1.11.0 diff --git a/pkg/obfuscate/go.sum b/pkg/obfuscate/go.sum index 940c4e5438e4a..063bd88a07005 100644 --- a/pkg/obfuscate/go.sum +++ b/pkg/obfuscate/go.sum @@ -1,7 +1,7 @@ github.com/DataDog/datadog-go/v5 v5.6.0 h1:2oCLxjF/4htd55piM75baflj/KoE6VYS7alEUqFvRDw= github.com/DataDog/datadog-go/v5 v5.6.0/go.mod h1:K9kcYBlxkcPP8tvvjZZKs/m1edNAUFzBbdpTUKfCsuw= -github.com/DataDog/go-sqllexer v0.0.18 h1:ErBvoO7/srJLdA2ebwd+HPqD4g1kN++BP64A8qvmh9U= -github.com/DataDog/go-sqllexer v0.0.18/go.mod h1:KwkYhpFEVIq+BfobkTC1vfqm4gTi65skV/DpDBXtexc= +github.com/DataDog/go-sqllexer v0.0.17 h1:u47fJAVg/+5DA74ZW3w0Qu+3qXHd3GtnA8ZBYixdPrM= +github.com/DataDog/go-sqllexer v0.0.17/go.mod h1:KwkYhpFEVIq+BfobkTC1vfqm4gTi65skV/DpDBXtexc= github.com/Microsoft/go-winio v0.5.0/go.mod h1:JPGBdM1cNvN/6ISo+n8V5iA4v8pBzdOpzfwIujj1a84= github.com/Microsoft/go-winio v0.6.2 h1:F2VQgta7ecxGYO8k3ZZz3RS8fVIXVxONVUPlNERoyfY= github.com/Microsoft/go-winio v0.6.2/go.mod h1:yd8OoFMLzJbo9gZq8j5qaps8bJ9aShtEA8Ipt1oGCvU= diff --git a/pkg/trace/go.mod b/pkg/trace/go.mod index c3f102a944494..29bf241afd7b8 100644 --- a/pkg/trace/go.mod +++ b/pkg/trace/go.mod @@ -61,7 +61,7 @@ require go.opentelemetry.io/collector/processor v0.115.0 // indirect require ( github.com/DataDog/datadog-agent/pkg/version v0.59.1 // indirect - github.com/DataDog/go-sqllexer v0.0.18 // indirect + github.com/DataDog/go-sqllexer v0.0.17 // indirect github.com/DataDog/go-tuf v1.1.0-0.5.2 // indirect github.com/DataDog/zstd v1.5.6 // indirect github.com/cespare/xxhash/v2 v2.3.0 // indirect diff --git a/pkg/trace/go.sum b/pkg/trace/go.sum index efd28e50f6d16..e1b7335487809 100644 --- a/pkg/trace/go.sum +++ b/pkg/trace/go.sum @@ -1,7 +1,7 @@ github.com/DataDog/datadog-go/v5 v5.6.0 h1:2oCLxjF/4htd55piM75baflj/KoE6VYS7alEUqFvRDw= github.com/DataDog/datadog-go/v5 v5.6.0/go.mod h1:K9kcYBlxkcPP8tvvjZZKs/m1edNAUFzBbdpTUKfCsuw= -github.com/DataDog/go-sqllexer v0.0.18 h1:ErBvoO7/srJLdA2ebwd+HPqD4g1kN++BP64A8qvmh9U= -github.com/DataDog/go-sqllexer v0.0.18/go.mod h1:KwkYhpFEVIq+BfobkTC1vfqm4gTi65skV/DpDBXtexc= +github.com/DataDog/go-sqllexer v0.0.17 h1:u47fJAVg/+5DA74ZW3w0Qu+3qXHd3GtnA8ZBYixdPrM= +github.com/DataDog/go-sqllexer v0.0.17/go.mod h1:KwkYhpFEVIq+BfobkTC1vfqm4gTi65skV/DpDBXtexc= github.com/DataDog/go-tuf v1.1.0-0.5.2 h1:4CagiIekonLSfL8GMHRHcHudo1fQnxELS9g4tiAupQ4= github.com/DataDog/go-tuf v1.1.0-0.5.2/go.mod h1:zBcq6f654iVqmkk8n2Cx81E1JnNTMOAx1UEO/wZR+P0= github.com/DataDog/opentelemetry-mapping-go/pkg/otlp/attributes v0.22.0 h1:yfk2cF8Bx98fSFpGrehEHh1FRqewfxcCTAbUDt5r3F8= diff --git a/pkg/trace/stats/oteltest/go.mod b/pkg/trace/stats/oteltest/go.mod index 1259dbf55e8bf..071e7c2c2eba0 100644 --- a/pkg/trace/stats/oteltest/go.mod +++ b/pkg/trace/stats/oteltest/go.mod @@ -30,7 +30,7 @@ require ( github.com/DataDog/datadog-agent/pkg/util/pointer v0.59.0 // indirect github.com/DataDog/datadog-agent/pkg/util/scrubber v0.59.0 // indirect github.com/DataDog/datadog-agent/pkg/version v0.59.1 // indirect - github.com/DataDog/go-sqllexer v0.0.18 // indirect + github.com/DataDog/go-sqllexer v0.0.17 // indirect github.com/DataDog/go-tuf v1.1.0-0.5.2 // indirect github.com/DataDog/sketches-go v1.4.6 // indirect github.com/Microsoft/go-winio v0.6.2 // indirect diff --git a/pkg/trace/stats/oteltest/go.sum b/pkg/trace/stats/oteltest/go.sum index 22ef689018918..a19fb370fa6c4 100644 --- a/pkg/trace/stats/oteltest/go.sum +++ b/pkg/trace/stats/oteltest/go.sum @@ -1,7 +1,7 @@ github.com/DataDog/datadog-go/v5 v5.6.0 h1:2oCLxjF/4htd55piM75baflj/KoE6VYS7alEUqFvRDw= github.com/DataDog/datadog-go/v5 v5.6.0/go.mod h1:K9kcYBlxkcPP8tvvjZZKs/m1edNAUFzBbdpTUKfCsuw= -github.com/DataDog/go-sqllexer v0.0.18 h1:ErBvoO7/srJLdA2ebwd+HPqD4g1kN++BP64A8qvmh9U= -github.com/DataDog/go-sqllexer v0.0.18/go.mod h1:KwkYhpFEVIq+BfobkTC1vfqm4gTi65skV/DpDBXtexc= +github.com/DataDog/go-sqllexer v0.0.17 h1:u47fJAVg/+5DA74ZW3w0Qu+3qXHd3GtnA8ZBYixdPrM= +github.com/DataDog/go-sqllexer v0.0.17/go.mod h1:KwkYhpFEVIq+BfobkTC1vfqm4gTi65skV/DpDBXtexc= github.com/DataDog/go-tuf v1.1.0-0.5.2 h1:4CagiIekonLSfL8GMHRHcHudo1fQnxELS9g4tiAupQ4= github.com/DataDog/go-tuf v1.1.0-0.5.2/go.mod h1:zBcq6f654iVqmkk8n2Cx81E1JnNTMOAx1UEO/wZR+P0= github.com/DataDog/opentelemetry-mapping-go/pkg/otlp/attributes v0.22.0 h1:yfk2cF8Bx98fSFpGrehEHh1FRqewfxcCTAbUDt5r3F8= diff --git a/test/otel/go.mod b/test/otel/go.mod index beb5f560e099e..88d5f3a127be5 100644 --- a/test/otel/go.mod +++ b/test/otel/go.mod @@ -179,7 +179,7 @@ require ( github.com/DataDog/datadog-api-client-go/v2 v2.33.0 // indirect github.com/DataDog/datadog-go/v5 v5.6.0 // indirect github.com/DataDog/dd-sensitive-data-scanner/sds-go/go v0.0.0-20240816154533-f7f9beb53a42 // indirect - github.com/DataDog/go-sqllexer v0.0.18 // indirect + github.com/DataDog/go-sqllexer v0.0.17 // indirect github.com/DataDog/go-tuf v1.1.0-0.5.2 // indirect github.com/DataDog/opentelemetry-mapping-go/pkg/otlp/attributes v0.22.0 // indirect github.com/DataDog/opentelemetry-mapping-go/pkg/otlp/logs v0.22.0 // indirect diff --git a/test/otel/go.sum b/test/otel/go.sum index 4ccec6f105aa4..bce88ff15bfa8 100644 --- a/test/otel/go.sum +++ b/test/otel/go.sum @@ -8,8 +8,8 @@ github.com/DataDog/datadog-go/v5 v5.6.0 h1:2oCLxjF/4htd55piM75baflj/KoE6VYS7alEU github.com/DataDog/datadog-go/v5 v5.6.0/go.mod h1:K9kcYBlxkcPP8tvvjZZKs/m1edNAUFzBbdpTUKfCsuw= github.com/DataDog/dd-sensitive-data-scanner/sds-go/go v0.0.0-20240816154533-f7f9beb53a42 h1:RoH7VLzTnxHEugRPIgnGlxwDFszFGI7b3WZZUtWuPRM= github.com/DataDog/dd-sensitive-data-scanner/sds-go/go v0.0.0-20240816154533-f7f9beb53a42/go.mod h1:TX7CTOQ3LbQjfAi4SwqUoR5gY1zfUk7VRBDTuArjaDc= -github.com/DataDog/go-sqllexer v0.0.18 h1:ErBvoO7/srJLdA2ebwd+HPqD4g1kN++BP64A8qvmh9U= -github.com/DataDog/go-sqllexer v0.0.18/go.mod h1:KwkYhpFEVIq+BfobkTC1vfqm4gTi65skV/DpDBXtexc= +github.com/DataDog/go-sqllexer v0.0.17 h1:u47fJAVg/+5DA74ZW3w0Qu+3qXHd3GtnA8ZBYixdPrM= +github.com/DataDog/go-sqllexer v0.0.17/go.mod h1:KwkYhpFEVIq+BfobkTC1vfqm4gTi65skV/DpDBXtexc= github.com/DataDog/go-tuf v1.1.0-0.5.2 h1:4CagiIekonLSfL8GMHRHcHudo1fQnxELS9g4tiAupQ4= github.com/DataDog/go-tuf v1.1.0-0.5.2/go.mod h1:zBcq6f654iVqmkk8n2Cx81E1JnNTMOAx1UEO/wZR+P0= github.com/DataDog/opentelemetry-mapping-go/pkg/inframetadata v0.22.0 h1:r1Dx2cRHCBWkVluSZA41i4eoI/nOGbcrrZdkqWjoFCc= From 919ff87361c87fa18c5e7498565f26e4fe2271f9 Mon Sep 17 00:00:00 2001 From: Yang Song Date: Wed, 18 Dec 2024 16:20:11 -0500 Subject: [PATCH 12/14] [OTEL-2125] Add configsync to otel-agent without DD exporter config (#32363) --- cmd/otel-agent/subcommands/run/command.go | 15 ++++++++++++++- comp/otelcol/ddflareextension/impl/server.go | 2 +- 2 files changed, 15 insertions(+), 2 deletions(-) diff --git a/cmd/otel-agent/subcommands/run/command.go b/cmd/otel-agent/subcommands/run/command.go index a414674620e44..08c1472998157 100644 --- a/cmd/otel-agent/subcommands/run/command.go +++ b/cmd/otel-agent/subcommands/run/command.go @@ -25,6 +25,7 @@ import ( "github.com/DataDog/datadog-agent/comp/core/hostname/hostnameinterface" "github.com/DataDog/datadog-agent/comp/core/hostname/remotehostnameimpl" log "github.com/DataDog/datadog-agent/comp/core/log/def" + logfx "github.com/DataDog/datadog-agent/comp/core/log/fx" logtracefx "github.com/DataDog/datadog-agent/comp/core/log/fx-trace" "github.com/DataDog/datadog-agent/comp/core/secrets" tagger "github.com/DataDog/datadog-agent/comp/core/tagger/def" @@ -106,8 +107,20 @@ func runOTelAgentCommand(ctx context.Context, params *subcommands.GlobalParams, fx.Provide(func() coreconfig.Component { return acfg }), + fx.Provide(func(_ coreconfig.Component) log.Params { + return log.ForDaemon(params.LoggerName, "log_file", pkgconfigsetup.DefaultOTelAgentLogFile) + }), + logfx.Module(), + fetchonlyimpl.Module(), + // TODO: don't rely on this pattern; remove this `OptionalModuleWithParams` thing + // and instead adapt OptionalModule to allow parameter passing naturally. + // See: https://github.com/DataDog/datadog-agent/pull/28386 + configsyncimpl.OptionalModuleWithParams(), + fx.Provide(func() configsyncimpl.Params { + return configsyncimpl.NewParams(params.SyncTimeout, params.SyncDelay, true) + }), converterfx.Module(), - fx.Provide(func(cp converter.Component) confmap.Converter { + fx.Provide(func(cp converter.Component, _ optional.Option[configsync.Component]) confmap.Converter { return cp }), collectorcontribFx.Module(), diff --git a/comp/otelcol/ddflareextension/impl/server.go b/comp/otelcol/ddflareextension/impl/server.go index ce1d2ceadef52..233dcddee1977 100644 --- a/comp/otelcol/ddflareextension/impl/server.go +++ b/comp/otelcol/ddflareextension/impl/server.go @@ -116,7 +116,7 @@ func newServer(endpoint string, handler http.Handler, auth bool) (*server, error // no easy way currently to pass required bearer auth token to OSS collector; // skip the validation if running inside a separate collector // TODO: determine way to allow OSS collector to authenticate with agent, OTEL-2226 - if auth { + if auth && util.GetAuthToken() != "" { r.Use(validateToken) } From f4ae8f0fd12ff1bc2c42f54f07dc00223f492a22 Mon Sep 17 00:00:00 2001 From: Paul Cacheux Date: Wed, 18 Dec 2024 23:25:20 +0100 Subject: [PATCH 13/14] [CWS] turn fentry on by default (#31630) --- .../kernel_matrix_testing/security_agent.yml | 42 ------------------- pkg/config/setup/system_probe.go | 6 +-- .../test-runner/files/cws_fentry.json | 10 ----- 3 files changed, 3 insertions(+), 55 deletions(-) delete mode 100644 test/new-e2e/system-probe/test-runner/files/cws_fentry.json diff --git a/.gitlab/kernel_matrix_testing/security_agent.yml b/.gitlab/kernel_matrix_testing/security_agent.yml index b7f4b80cc6ed1..769ebe10138b8 100644 --- a/.gitlab/kernel_matrix_testing/security_agent.yml +++ b/.gitlab/kernel_matrix_testing/security_agent.yml @@ -216,26 +216,6 @@ kmt_run_secagent_tests_x64_ebpfless: - !reference [.collect_outcomes_kmt] - !reference [.upload_junit_kmt] -kmt_run_secagent_tests_x64_fentry: - extends: - - .kmt_run_secagent_tests - image: registry.ddbuild.io/ci/datadog-agent-buildimages/system-probe_x64$DATADOG_AGENT_SYSPROBE_BUILDIMAGES_SUFFIX:$DATADOG_AGENT_SYSPROBE_BUILDIMAGES - tags: ["arch:amd64"] - needs: - - kmt_setup_env_secagent_x64 - - upload_dependencies_secagent_x64 - - upload_secagent_tests_x64 - variables: - ARCH: "x86_64" - parallel: - matrix: - - TAG: - - "amazon_2023" - TEST_SET: [cws_fentry] - after_script: - - !reference [.collect_outcomes_kmt] - - !reference [.upload_junit_kmt] - kmt_run_secagent_tests_x64_docker: extends: - .kmt_run_secagent_tests @@ -350,26 +330,6 @@ kmt_run_secagent_tests_arm64_ebpfless: - !reference [.collect_outcomes_kmt] - !reference [.upload_junit_kmt] -kmt_run_secagent_tests_arm64_fentry: - extends: - - .kmt_run_secagent_tests - image: registry.ddbuild.io/ci/datadog-agent-buildimages/system-probe_arm64$DATADOG_AGENT_SYSPROBE_BUILDIMAGES_SUFFIX:$DATADOG_AGENT_SYSPROBE_BUILDIMAGES - tags: ["arch:arm64"] - needs: - - kmt_setup_env_secagent_arm64 - - upload_dependencies_secagent_arm64 - - upload_secagent_tests_arm64 - variables: - ARCH: "arm64" - parallel: - matrix: - - TAG: - - "ubuntu_24.04" - TEST_SET: [cws_fentry] - after_script: - - !reference [.collect_outcomes_kmt] - - !reference [.upload_junit_kmt] - kmt_run_secagent_tests_arm64_docker: extends: - .kmt_run_secagent_tests @@ -426,7 +386,6 @@ kmt_secagent_tests_join_arm64: - kmt_run_secagent_tests_arm64 - kmt_run_secagent_tests_arm64_ad - kmt_run_secagent_tests_arm64_ebpfless - - kmt_run_secagent_tests_arm64_fentry - kmt_run_secagent_tests_arm64_docker kmt_secagent_cleanup_arm64: @@ -449,7 +408,6 @@ kmt_secagent_tests_join_x64: - kmt_run_secagent_tests_x64_required - kmt_run_secagent_tests_x64_ad - kmt_run_secagent_tests_x64_ebpfless - - kmt_run_secagent_tests_x64_fentry - kmt_run_secagent_tests_x64_docker kmt_secagent_cleanup_x64: diff --git a/pkg/config/setup/system_probe.go b/pkg/config/setup/system_probe.go index bdc97ee3d902a..1ca54550cec5a 100644 --- a/pkg/config/setup/system_probe.go +++ b/pkg/config/setup/system_probe.go @@ -371,9 +371,9 @@ func InitSystemProbeConfig(cfg pkgconfigmodel.Config) { eventMonitorBindEnvAndSetDefault(cfg, join(evNS, "network.classifier_handle"), 0) eventMonitorBindEnvAndSetDefault(cfg, join(evNS, "network.raw_classifier_handle"), 0) eventMonitorBindEnvAndSetDefault(cfg, join(evNS, "event_stream.use_ring_buffer"), true) - eventMonitorBindEnvAndSetDefault(cfg, join(evNS, "event_stream.use_fentry"), false) - eventMonitorBindEnvAndSetDefault(cfg, join(evNS, "event_stream.use_fentry_amd64"), false) - eventMonitorBindEnvAndSetDefault(cfg, join(evNS, "event_stream.use_fentry_arm64"), false) + eventMonitorBindEnvAndSetDefault(cfg, join(evNS, "event_stream.use_fentry"), true) + eventMonitorBindEnv(cfg, join(evNS, "event_stream.use_fentry_amd64")) + eventMonitorBindEnv(cfg, join(evNS, "event_stream.use_fentry_arm64")) eventMonitorBindEnv(cfg, join(evNS, "event_stream.buffer_size")) eventMonitorBindEnvAndSetDefault(cfg, join(evNS, "envs_with_value"), []string{"LD_PRELOAD", "LD_LIBRARY_PATH", "PATH", "HISTSIZE", "HISTFILESIZE", "GLIBC_TUNABLES"}) eventMonitorBindEnvAndSetDefault(cfg, join(evNS, "runtime_compilation.enabled"), false) diff --git a/test/new-e2e/system-probe/test-runner/files/cws_fentry.json b/test/new-e2e/system-probe/test-runner/files/cws_fentry.json deleted file mode 100644 index 8330f6d6048af..0000000000000 --- a/test/new-e2e/system-probe/test-runner/files/cws_fentry.json +++ /dev/null @@ -1,10 +0,0 @@ -{ - "filters": { - "*": { - "exclude": false - } - }, - "additional_env_vars": [ - "DD_EVENT_MONITORING_CONFIG_EVENT_STREAM_USE_FENTRY=true" - ] -} From f51b26c3ad1c5d705ea360562c6376f05a741578 Mon Sep 17 00:00:00 2001 From: Stuart Geipel Date: Wed, 18 Dec 2024 17:44:10 -0500 Subject: [PATCH 14/14] [NPM-3665] Include semodule -l in agent flare (#32189) Co-authored-by: DeForest Richards <56796055+drichards-87@users.noreply.github.com> --- cmd/system-probe/api/debug/handlers_linux.go | 29 ++++++++++++++----- .../api/debug/handlers_nolinux.go | 6 ++++ cmd/system-probe/api/server.go | 1 + pkg/flare/archive_linux.go | 7 +++++ .../flare-semodule-list-883aecc886cd62ac.yaml | 11 +++++++ 5 files changed, 47 insertions(+), 7 deletions(-) create mode 100644 releasenotes/notes/flare-semodule-list-883aecc886cd62ac.yaml diff --git a/cmd/system-probe/api/debug/handlers_linux.go b/cmd/system-probe/api/debug/handlers_linux.go index d2bd7dfbd5f48..07ba06c49354f 100644 --- a/cmd/system-probe/api/debug/handlers_linux.go +++ b/cmd/system-probe/api/debug/handlers_linux.go @@ -17,19 +17,18 @@ import ( "time" ) -// HandleSelinuxSestatus reports the output of sestatus as an http result -func HandleSelinuxSestatus(w http.ResponseWriter, r *http.Request) { - ctx, cancel := context.WithTimeout(r.Context(), 5*time.Second) - defer cancel() - - cmd := exec.CommandContext(ctx, "sestatus") +// handleCommand runs commandName with the provided arguments and writes it to the HTTP response. +// If the command exits with a failure or doesn't exist in the PATH, it will still 200 but report the failure. +// Any other kind of error will 500. +func handleCommand(ctx context.Context, w http.ResponseWriter, commandName string, args ...string) { + cmd := exec.CommandContext(ctx, commandName, args...) output, err := cmd.CombinedOutput() var execError *exec.Error var exitErr *exec.ExitError if err != nil { - // don't 500 for ExitErrors etc, to report "normal" failures to the selinux_sestatus.log file + // don't 500 for ExitErrors etc, to report "normal" failures to the flare log file if !errors.As(err, &execError) && !errors.As(err, &exitErr) { w.WriteHeader(500) } @@ -39,3 +38,19 @@ func HandleSelinuxSestatus(w http.ResponseWriter, r *http.Request) { w.Write(output) } + +// HandleSelinuxSestatus reports the output of sestatus as an http result +func HandleSelinuxSestatus(w http.ResponseWriter, r *http.Request) { + ctx, cancel := context.WithTimeout(r.Context(), 5*time.Second) + defer cancel() + + handleCommand(ctx, w, "sestatus") +} + +// HandleSelinuxSemoduleList reports the output of semodule -l as an http result +func HandleSelinuxSemoduleList(w http.ResponseWriter, r *http.Request) { + ctx, cancel := context.WithTimeout(r.Context(), 5*time.Second) + defer cancel() + + handleCommand(ctx, w, "semodule", "-l") +} diff --git a/cmd/system-probe/api/debug/handlers_nolinux.go b/cmd/system-probe/api/debug/handlers_nolinux.go index 1475d821c1e6e..246f4a3a7c78a 100644 --- a/cmd/system-probe/api/debug/handlers_nolinux.go +++ b/cmd/system-probe/api/debug/handlers_nolinux.go @@ -18,3 +18,9 @@ func HandleSelinuxSestatus(w http.ResponseWriter, _ *http.Request) { w.WriteHeader(500) io.WriteString(w, "HandleSelinuxSestatus is not supported on this platform") } + +// HandleSelinuxSemoduleList is not supported +func HandleSelinuxSemoduleList(w http.ResponseWriter, _ *http.Request) { + w.WriteHeader(500) + io.WriteString(w, "HandleSelinuxSemoduleList is not supported on this platform") +} diff --git a/cmd/system-probe/api/server.go b/cmd/system-probe/api/server.go index d81007a0c8f0d..f4d9e85522d91 100644 --- a/cmd/system-probe/api/server.go +++ b/cmd/system-probe/api/server.go @@ -60,6 +60,7 @@ func StartServer(cfg *sysconfigtypes.Config, telemetry telemetry.Component, wmet if runtime.GOOS == "linux" { mux.HandleFunc("/debug/ebpf_btf_loader_info", ebpf.HandleBTFLoaderInfo) mux.HandleFunc("/debug/selinux_sestatus", debug.HandleSelinuxSestatus) + mux.HandleFunc("/debug/selinux_semodule_list", debug.HandleSelinuxSemoduleList) } go func() { diff --git a/pkg/flare/archive_linux.go b/pkg/flare/archive_linux.go index dafe8bd41d1bc..9a3aea87a0ac0 100644 --- a/pkg/flare/archive_linux.go +++ b/pkg/flare/archive_linux.go @@ -39,6 +39,7 @@ func addSystemProbePlatformSpecificEntries(fb flaretypes.FlareBuilder) { _ = fb.AddFileFromFunc(filepath.Join("system-probe", "conntrack_host.log"), getSystemProbeConntrackHost) _ = fb.AddFileFromFunc(filepath.Join("system-probe", "ebpf_btf_loader.log"), getSystemProbeBTFLoaderInfo) _ = fb.AddFileFromFunc(filepath.Join("system-probe", "selinux_sestatus.log"), getSystemProbeSelinuxSestatus) + _ = fb.AddFileFromFunc(filepath.Join("system-probe", "selinux_semodule_list.log"), getSystemProbeSelinuxSemoduleList) } } @@ -155,3 +156,9 @@ func getSystemProbeSelinuxSestatus() ([]byte, error) { url := sysprobeclient.DebugURL("/selinux_sestatus") return getHTTPData(sysProbeClient, url) } + +func getSystemProbeSelinuxSemoduleList() ([]byte, error) { + sysProbeClient := sysprobeclient.Get(getSystemProbeSocketPath()) + url := sysprobeclient.DebugURL("/selinux_semodule_list") + return getHTTPData(sysProbeClient, url) +} diff --git a/releasenotes/notes/flare-semodule-list-883aecc886cd62ac.yaml b/releasenotes/notes/flare-semodule-list-883aecc886cd62ac.yaml new file mode 100644 index 0000000000000..2baa2dea73281 --- /dev/null +++ b/releasenotes/notes/flare-semodule-list-883aecc886cd62ac.yaml @@ -0,0 +1,11 @@ +# Each section from every release note are combined when the +# CHANGELOG.rst is rendered. So the text needs to be worded so that +# it does not depend on any information only available in another +# section. This may mean repeating some details, but each section +# must be readable independently of the other. +# +# Each section note must be formatted as reStructuredText. +--- +enhancements: + - | + Added the output of ``semodule -l`` to the Agent flare; this information appears in ``system-probe/selinux_semodule_list.log``.