From 0f53c9a45030daca82ab159c340c36ec51890a88 Mon Sep 17 00:00:00 2001 From: Arthur Bellal Date: Fri, 13 Dec 2024 20:38:34 +0100 Subject: [PATCH 1/4] (fleet) basic logging in setup scripts (#32133) --- pkg/fleet/installer/setup/common/output.go | 18 +++++++++ pkg/fleet/installer/setup/common/setup.go | 43 ++++++++++++++++++---- pkg/fleet/installer/setup/setup.go | 26 +++++++------ 3 files changed, 69 insertions(+), 18 deletions(-) create mode 100644 pkg/fleet/installer/setup/common/output.go diff --git a/pkg/fleet/installer/setup/common/output.go b/pkg/fleet/installer/setup/common/output.go new file mode 100644 index 0000000000000..cc42f5e4c8d02 --- /dev/null +++ b/pkg/fleet/installer/setup/common/output.go @@ -0,0 +1,18 @@ +// Unless explicitly stated otherwise all files in this repository are licensed +// under the Apache License Version 2.0. +// This product includes software developed at Datadog (https://www.datadoghq.com/). +// Copyright 2016-present Datadog, Inc. + +package common + +import "io" + +// Output is a writer for the output. It will support some ANSI escape sequences to format the output. +type Output struct { + tty io.Writer +} + +// WriteString writes a string to the output. +func (o *Output) WriteString(s string) { + _, _ = o.tty.Write([]byte(s)) +} diff --git a/pkg/fleet/installer/setup/common/setup.go b/pkg/fleet/installer/setup/common/setup.go index 159b1b69b0d36..7abab785021f6 100644 --- a/pkg/fleet/installer/setup/common/setup.go +++ b/pkg/fleet/installer/setup/common/setup.go @@ -10,11 +10,14 @@ import ( "context" "errors" "fmt" + "io" "os" + "time" "github.com/DataDog/datadog-agent/pkg/fleet/installer" "github.com/DataDog/datadog-agent/pkg/fleet/installer/env" "github.com/DataDog/datadog-agent/pkg/fleet/installer/oci" + "github.com/DataDog/datadog-agent/pkg/version" "gopkg.in/DataDog/dd-trace-go.v1/ddtrace" "gopkg.in/DataDog/dd-trace-go.v1/ddtrace/tracer" ) @@ -32,7 +35,10 @@ var ( type Setup struct { configDir string installer installer.Installer + start time.Time + flavor string + Out *Output Env *env.Env Ctx context.Context Span ddtrace.Span @@ -41,7 +47,13 @@ type Setup struct { } // NewSetup creates a new Setup structure with some default values. -func NewSetup(ctx context.Context, env *env.Env, name string) (*Setup, error) { +func NewSetup(ctx context.Context, env *env.Env, flavor string, flavorPath string, logOutput io.Writer) (*Setup, error) { + header := `Datadog Installer %s - https://www.datadoghq.com +Running the %s installation script (https://github.com/DataDog/datadog-agent/tree/%s/pkg/fleet/installer/setup/%s) - %s +` + start := time.Now() + output := &Output{tty: logOutput} + output.WriteString(fmt.Sprintf(header, version.AgentVersion, flavor, version.Commit, flavorPath, start.Format(time.RFC3339))) if env.APIKey == "" { return nil, ErrNoAPIKey } @@ -49,10 +61,13 @@ func NewSetup(ctx context.Context, env *env.Env, name string) (*Setup, error) { if err != nil { return nil, fmt.Errorf("failed to create installer: %w", err) } - span, ctx := tracer.StartSpanFromContext(ctx, fmt.Sprintf("setup.%s", name)) + span, ctx := tracer.StartSpanFromContext(ctx, fmt.Sprintf("setup.%s", flavor)) s := &Setup{ configDir: configDir, installer: installer, + start: start, + flavor: flavor, + Out: output, Env: env, Ctx: ctx, Span: span, @@ -75,30 +90,44 @@ func NewSetup(ctx context.Context, env *env.Env, name string) (*Setup, error) { // Run installs the packages and writes the configurations func (s *Setup) Run() (err error) { defer func() { s.Span.Finish(tracer.WithError(err)) }() + s.Out.WriteString("Applying configurations...\n") err = writeConfigs(s.Config, s.configDir) if err != nil { return fmt.Errorf("failed to write configuration: %w", err) } - err = s.installPackage(installerOCILayoutURL) + packages := resolvePackages(s.Packages) + s.Out.WriteString("The following packages will be installed:\n") + s.Out.WriteString(fmt.Sprintf(" - %s / %s\n", "datadog-installer", version.AgentVersion)) + for _, p := range packages { + s.Out.WriteString(fmt.Sprintf(" - %s / %s\n", p.name, p.version)) + } + err = s.installPackage("datadog-installer", installerOCILayoutURL) if err != nil { return fmt.Errorf("failed to install installer: %w", err) } - packages := resolvePackages(s.Packages) for _, p := range packages { url := oci.PackageURL(s.Env, p.name, p.version) - err = s.installPackage(url) + err = s.installPackage(p.name, url) if err != nil { return fmt.Errorf("failed to install package %s: %w", url, err) } } + s.Out.WriteString(fmt.Sprintf("Successfully ran the %s install script in %s!\n", s.flavor, time.Since(s.start).Round(time.Second))) return nil } // installPackage mimicks the telemetry of calling the install package command -func (s *Setup) installPackage(url string) (err error) { +func (s *Setup) installPackage(name string, url string) (err error) { span, ctx := tracer.StartSpanFromContext(s.Ctx, "install") defer func() { span.Finish(tracer.WithError(err)) }() span.SetTag("url", url) span.SetTag("_top_level", 1) - return s.installer.Install(ctx, url, nil) + + s.Out.WriteString(fmt.Sprintf("Installing %s...\n", name)) + err = s.installer.Install(ctx, url, nil) + if err != nil { + return err + } + s.Out.WriteString(fmt.Sprintf("Successfully installed %s\n", name)) + return nil } diff --git a/pkg/fleet/installer/setup/setup.go b/pkg/fleet/installer/setup/setup.go index 3b01d320c9975..93a57fc575a5b 100644 --- a/pkg/fleet/installer/setup/setup.go +++ b/pkg/fleet/installer/setup/setup.go @@ -9,6 +9,7 @@ package setup import ( "context" "fmt" + "os" "github.com/DataDog/datadog-agent/pkg/fleet/installer/env" "github.com/DataDog/datadog-agent/pkg/fleet/installer/setup/common" @@ -17,23 +18,26 @@ import ( "github.com/DataDog/datadog-agent/pkg/fleet/internal/paths" ) -const ( - // FlavorDatabricks is the flavor for the Data Jobs Monitoring databricks setup. - FlavorDatabricks = "databricks" -) +type flavor struct { + path string // path is used to print the path to the setup script for users. + run func(*common.Setup) error +} + +var flavors = map[string]flavor{ + "databricks": {path: "djm/databricks.go", run: djm.SetupDatabricks}, +} // Setup installs Datadog. func Setup(ctx context.Context, env *env.Env, flavor string) error { - s, err := common.NewSetup(ctx, env, flavor) + f, ok := flavors[flavor] + if !ok { + return fmt.Errorf("unknown flavor %s", flavor) + } + s, err := common.NewSetup(ctx, env, flavor, f.path, os.Stdout) if err != nil { return err } - switch flavor { - case FlavorDatabricks: - err = djm.SetupDatabricks(s) - default: - return fmt.Errorf("unknown setup flavor %s", flavor) - } + err = f.run(s) if err != nil { return err } From e87eaecb8358a40207a043f80e85cc173500a992 Mon Sep 17 00:00:00 2001 From: Marco Costa Date: Fri, 13 Dec 2024 12:39:23 -0800 Subject: [PATCH 2/4] [AIDM-459] Remap OTel GraphQL resource to use query data (#31731) --- pkg/trace/api/otlp.go | 7 +++++++ pkg/trace/api/otlp_test.go | 12 +++++++++++ pkg/trace/traceutil/otel_util.go | 17 +++++++++++++++ pkg/trace/traceutil/otel_util_test.go | 21 +++++++++++++++++++ ...tel-graphql-resource-e31ae2262b52a873.yaml | 11 ++++++++++ 5 files changed, 68 insertions(+) create mode 100644 releasenotes/notes/otel-graphql-resource-e31ae2262b52a873.yaml diff --git a/pkg/trace/api/otlp.go b/pkg/trace/api/otlp.go index de53f14d369ec..88491ea9052ec 100644 --- a/pkg/trace/api/otlp.go +++ b/pkg/trace/api/otlp.go @@ -684,6 +684,13 @@ func resourceFromTags(meta map[string]string) string { return m + " " + svc } return m + } else if typ := meta[semconv117.AttributeGraphqlOperationType]; typ != "" { + // Enrich GraphQL query resource names. + // See https://github.com/open-telemetry/semantic-conventions/blob/v1.29.0/docs/graphql/graphql-spans.md + if name := meta[semconv117.AttributeGraphqlOperationName]; name != "" { + return typ + " " + name + } + return typ } return "" } diff --git a/pkg/trace/api/otlp_test.go b/pkg/trace/api/otlp_test.go index bd6377ff4612d..0966ee75ff1e9 100644 --- a/pkg/trace/api/otlp_test.go +++ b/pkg/trace/api/otlp_test.go @@ -1573,6 +1573,18 @@ func TestOTLPHelpers(t *testing.T) { meta: map[string]string{semconv.AttributeRPCMethod: "M"}, out: "M", }, + { + meta: map[string]string{"graphql.operation.name": "myQuery"}, + out: "", + }, + { + meta: map[string]string{"graphql.operation.type": "query"}, + out: "query", + }, + { + meta: map[string]string{"graphql.operation.type": "query", "graphql.operation.name": "myQuery"}, + out: "query myQuery", + }, } { assert.Equal(t, tt.out, resourceFromTags(tt.meta)) } diff --git a/pkg/trace/traceutil/otel_util.go b/pkg/trace/traceutil/otel_util.go index bb3e03783c4dc..5e645f7b6673b 100644 --- a/pkg/trace/traceutil/otel_util.go +++ b/pkg/trace/traceutil/otel_util.go @@ -195,6 +195,13 @@ func GetOTelResourceV1(span ptrace.Span, res pcommon.Resource) (resName string) // ...and service if available resName = resName + " " + svc } + } else if m := GetOTelAttrValInResAndSpanAttrs(span, res, false, semconv117.AttributeGraphqlOperationType); m != "" { + // Enrich GraphQL query resource names. + // See https://github.com/open-telemetry/semantic-conventions/blob/v1.29.0/docs/graphql/graphql-spans.md + resName = m + if name := GetOTelAttrValInResAndSpanAttrs(span, res, false, semconv117.AttributeGraphqlOperationName); name != "" { + resName = resName + " " + name + } } else { resName = span.Name() } @@ -249,6 +256,16 @@ func GetOTelResourceV2(span ptrace.Span, res pcommon.Resource) (resName string) } return } + + if m := GetOTelAttrValInResAndSpanAttrs(span, res, false, semconv117.AttributeGraphqlOperationType); m != "" { + // Enrich GraphQL query resource names. + // See https://github.com/open-telemetry/semantic-conventions/blob/v1.29.0/docs/graphql/graphql-spans.md + resName = m + if name := GetOTelAttrValInResAndSpanAttrs(span, res, false, semconv117.AttributeGraphqlOperationName); name != "" { + resName = resName + " " + name + } + return + } resName = span.Name() return diff --git a/pkg/trace/traceutil/otel_util_test.go b/pkg/trace/traceutil/otel_util_test.go index 9b6934caf3a7a..201c64a0745b8 100644 --- a/pkg/trace/traceutil/otel_util_test.go +++ b/pkg/trace/traceutil/otel_util_test.go @@ -293,6 +293,27 @@ func TestGetOTelResource(t *testing.T) { expectedV1: strings.Repeat("a", MaxResourceLen), expectedV2: strings.Repeat("a", MaxResourceLen), }, + { + name: "GraphQL with no type", + sattrs: map[string]string{"graphql.operation.name": "myQuery"}, + normalize: false, + expectedV1: "span_name", + expectedV2: "span_name", + }, + { + name: "GraphQL with only type", + sattrs: map[string]string{"graphql.operation.type": "query"}, + normalize: false, + expectedV1: "query", + expectedV2: "query", + }, + { + name: "GraphQL with only type", + sattrs: map[string]string{"graphql.operation.type": "query", "graphql.operation.name": "myQuery"}, + normalize: false, + expectedV1: "query myQuery", + expectedV2: "query myQuery", + }, } { t.Run(tt.name, func(t *testing.T) { span := ptrace.NewSpan() diff --git a/releasenotes/notes/otel-graphql-resource-e31ae2262b52a873.yaml b/releasenotes/notes/otel-graphql-resource-e31ae2262b52a873.yaml new file mode 100644 index 0000000000000..8962c355b585a --- /dev/null +++ b/releasenotes/notes/otel-graphql-resource-e31ae2262b52a873.yaml @@ -0,0 +1,11 @@ +# Each section from every release note are combined when the +# CHANGELOG.rst is rendered. So the text needs to be worded so that +# it does not depend on any information only available in another +# section. This may mean repeating some details, but each section +# must be readable independently of the other. +# +# Each section note must be formatted as reStructuredText. +--- +enhancements: + - | + For OpenTelemetry GraphQL request spans, the span resource name is now the GraphQL operation type and name. From 37fbbc826dfb608e5b4f6fb7d9058bae8d27ce22 Mon Sep 17 00:00:00 2001 From: Stuart Geipel Date: Fri, 13 Dec 2024 16:59:46 -0500 Subject: [PATCH 3/4] [NPM-3662] Add sestatus to agent flare (#32068) --- cmd/system-probe/api/debug/handlers_linux.go | 41 +++++++++++++++++++ .../api/debug/handlers_nolinux.go | 20 +++++++++ cmd/system-probe/api/server.go | 2 + pkg/ebpf/debug_handlers.go | 5 +-- pkg/flare/archive_linux.go | 7 ++++ ...agent-flare-sestatus-5820cfc79ec91d1f.yaml | 11 +++++ 6 files changed, 83 insertions(+), 3 deletions(-) create mode 100644 cmd/system-probe/api/debug/handlers_linux.go create mode 100644 cmd/system-probe/api/debug/handlers_nolinux.go create mode 100644 releasenotes/notes/agent-flare-sestatus-5820cfc79ec91d1f.yaml diff --git a/cmd/system-probe/api/debug/handlers_linux.go b/cmd/system-probe/api/debug/handlers_linux.go new file mode 100644 index 0000000000000..d2bd7dfbd5f48 --- /dev/null +++ b/cmd/system-probe/api/debug/handlers_linux.go @@ -0,0 +1,41 @@ +// Unless explicitly stated otherwise all files in this repository are licensed +// under the Apache License Version 2.0. +// This product includes software developed at Datadog (https://www.datadoghq.com/). +// Copyright 2024-present Datadog, Inc. + +//go:build linux + +// Package debug contains handlers for debug information global to all of system-probe +package debug + +import ( + "context" + "errors" + "fmt" + "net/http" + "os/exec" + "time" +) + +// HandleSelinuxSestatus reports the output of sestatus as an http result +func HandleSelinuxSestatus(w http.ResponseWriter, r *http.Request) { + ctx, cancel := context.WithTimeout(r.Context(), 5*time.Second) + defer cancel() + + cmd := exec.CommandContext(ctx, "sestatus") + output, err := cmd.CombinedOutput() + + var execError *exec.Error + var exitErr *exec.ExitError + + if err != nil { + // don't 500 for ExitErrors etc, to report "normal" failures to the selinux_sestatus.log file + if !errors.As(err, &execError) && !errors.As(err, &exitErr) { + w.WriteHeader(500) + } + fmt.Fprintf(w, "command failed: %s\n%s", err, output) + return + } + + w.Write(output) +} diff --git a/cmd/system-probe/api/debug/handlers_nolinux.go b/cmd/system-probe/api/debug/handlers_nolinux.go new file mode 100644 index 0000000000000..1475d821c1e6e --- /dev/null +++ b/cmd/system-probe/api/debug/handlers_nolinux.go @@ -0,0 +1,20 @@ +// Unless explicitly stated otherwise all files in this repository are licensed +// under the Apache License Version 2.0. +// This product includes software developed at Datadog (https://www.datadoghq.com/). +// Copyright 2024-present Datadog, Inc. + +//go:build !linux + +// Package debug contains handlers for debug information global to all of system-probe +package debug + +import ( + "io" + "net/http" +) + +// HandleSelinuxSestatus is not supported +func HandleSelinuxSestatus(w http.ResponseWriter, _ *http.Request) { + w.WriteHeader(500) + io.WriteString(w, "HandleSelinuxSestatus is not supported on this platform") +} diff --git a/cmd/system-probe/api/server.go b/cmd/system-probe/api/server.go index 3e4a71056f143..d81007a0c8f0d 100644 --- a/cmd/system-probe/api/server.go +++ b/cmd/system-probe/api/server.go @@ -15,6 +15,7 @@ import ( gorilla "github.com/gorilla/mux" + "github.com/DataDog/datadog-agent/cmd/system-probe/api/debug" "github.com/DataDog/datadog-agent/cmd/system-probe/api/module" "github.com/DataDog/datadog-agent/cmd/system-probe/api/server" sysconfigtypes "github.com/DataDog/datadog-agent/cmd/system-probe/config/types" @@ -58,6 +59,7 @@ func StartServer(cfg *sysconfigtypes.Config, telemetry telemetry.Component, wmet if runtime.GOOS == "linux" { mux.HandleFunc("/debug/ebpf_btf_loader_info", ebpf.HandleBTFLoaderInfo) + mux.HandleFunc("/debug/selinux_sestatus", debug.HandleSelinuxSestatus) } go func() { diff --git a/pkg/ebpf/debug_handlers.go b/pkg/ebpf/debug_handlers.go index ea10d22a844c2..04cba9faed556 100644 --- a/pkg/ebpf/debug_handlers.go +++ b/pkg/ebpf/debug_handlers.go @@ -6,10 +6,9 @@ package ebpf import ( + "fmt" "io" "net/http" - - "github.com/DataDog/datadog-agent/pkg/util/log" ) // HandleBTFLoaderInfo responds with where the system-probe found BTF data (and @@ -17,7 +16,7 @@ import ( func HandleBTFLoaderInfo(w http.ResponseWriter, _ *http.Request) { info, err := GetBTFLoaderInfo() if err != nil { - log.Errorf("unable to get ebpf_btf_loader info: %s", err) + fmt.Fprintf(w, "unable to get ebpf_btf_loader info: %s", err) w.WriteHeader(500) return } diff --git a/pkg/flare/archive_linux.go b/pkg/flare/archive_linux.go index 1c9b5d7a4ad48..dafe8bd41d1bc 100644 --- a/pkg/flare/archive_linux.go +++ b/pkg/flare/archive_linux.go @@ -38,6 +38,7 @@ func addSystemProbePlatformSpecificEntries(fb flaretypes.FlareBuilder) { _ = fb.AddFileFromFunc(filepath.Join("system-probe", "conntrack_cached.log"), getSystemProbeConntrackCached) _ = fb.AddFileFromFunc(filepath.Join("system-probe", "conntrack_host.log"), getSystemProbeConntrackHost) _ = fb.AddFileFromFunc(filepath.Join("system-probe", "ebpf_btf_loader.log"), getSystemProbeBTFLoaderInfo) + _ = fb.AddFileFromFunc(filepath.Join("system-probe", "selinux_sestatus.log"), getSystemProbeSelinuxSestatus) } } @@ -148,3 +149,9 @@ func getSystemProbeBTFLoaderInfo() ([]byte, error) { url := sysprobeclient.DebugURL("/ebpf_btf_loader_info") return getHTTPData(sysProbeClient, url) } + +func getSystemProbeSelinuxSestatus() ([]byte, error) { + sysProbeClient := sysprobeclient.Get(getSystemProbeSocketPath()) + url := sysprobeclient.DebugURL("/selinux_sestatus") + return getHTTPData(sysProbeClient, url) +} diff --git a/releasenotes/notes/agent-flare-sestatus-5820cfc79ec91d1f.yaml b/releasenotes/notes/agent-flare-sestatus-5820cfc79ec91d1f.yaml new file mode 100644 index 0000000000000..e7bac3330c728 --- /dev/null +++ b/releasenotes/notes/agent-flare-sestatus-5820cfc79ec91d1f.yaml @@ -0,0 +1,11 @@ +# Each section from every release note are combined when the +# CHANGELOG.rst is rendered. So the text needs to be worded so that +# it does not depend on any information only available in another +# section. This may mean repeating some details, but each section +# must be readable independently of the other. +# +# Each section note must be formatted as reStructuredText. +--- +enhancements: + - | + Added the output of ``sestatus`` into the Agent flare. This information will appear in ``system-probe/selinux_sestatus.log``. From 786bce8532868cc9f21e4f45233c4c517d36bdd9 Mon Sep 17 00:00:00 2001 From: Branden Clark Date: Fri, 13 Dec 2024 17:37:35 -0500 Subject: [PATCH 4/4] Enter repo root (#32169) --- tasks/winbuildscripts/common.ps1 | 23 ++++++++++++++++++++++- 1 file changed, 22 insertions(+), 1 deletion(-) diff --git a/tasks/winbuildscripts/common.ps1 b/tasks/winbuildscripts/common.ps1 index 35ef863d57535..d1e51fc5b2b1d 100644 --- a/tasks/winbuildscripts/common.ps1 +++ b/tasks/winbuildscripts/common.ps1 @@ -42,6 +42,23 @@ function Exit-BuildRoot() { Pop-Location -StackName AgentBuildRoot } +<# +.SYNOPSIS +Sets the current directory to the root of the repository. +#> +function Enter-RepoRoot() { + # Expected PSScriptRoot: datadog-agent\tasks\winbuildscripts\ + Push-Location "$PSScriptRoot\..\.." -ErrorAction Stop -StackName AgentRepoRoot | Out-Null +} + +<# +.SYNOPSIS +Leaves the repository root directory and returns to the original working directory. +#> +function Exit-RepoRoot() { + Pop-Location -StackName AgentRepoRoot +} + <# .SYNOPSIS Expands the Go module cache from an archive file. @@ -209,6 +226,8 @@ function Invoke-BuildScript { if ($BuildOutOfSource) { Enter-BuildRoot + } else { + Enter-RepoRoot } Expand-ModCache -modcache modcache @@ -234,9 +253,11 @@ function Invoke-BuildScript { # This finally block is executed regardless of whether the try block completes successfully, throws an exception, # or uses `exit` to terminate the script. + # Restore the original working directory if ($BuildOutOfSource) { - # Restore the original working directory Exit-BuildRoot + } else { + Exit-RepoRoot } } }