Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[pull] main from DataDog:main #106

Merged
merged 14 commits into from
Dec 19, 2024
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions .github/CODEOWNERS
Original file line number Diff line number Diff line change
Expand Up @@ -176,6 +176,7 @@

/.gitlab/package_build/ @DataDog/agent-delivery
/.gitlab/package_build/windows.yml @DataDog/agent-delivery @DataDog/windows-agent
/.gitlab/package_build/installer.yml @DataDog/agent-delivery @DataDog/fleet
/.gitlab/packaging/ @DataDog/agent-delivery

/.gitlab/benchmarks/benchmarks.yml @DataDog/agent-apm
Expand Down
42 changes: 0 additions & 42 deletions .gitlab/kernel_matrix_testing/security_agent.yml
Original file line number Diff line number Diff line change
Expand Up @@ -216,26 +216,6 @@ kmt_run_secagent_tests_x64_ebpfless:
- !reference [.collect_outcomes_kmt]
- !reference [.upload_junit_kmt]

kmt_run_secagent_tests_x64_fentry:
extends:
- .kmt_run_secagent_tests
image: registry.ddbuild.io/ci/datadog-agent-buildimages/system-probe_x64$DATADOG_AGENT_SYSPROBE_BUILDIMAGES_SUFFIX:$DATADOG_AGENT_SYSPROBE_BUILDIMAGES
tags: ["arch:amd64"]
needs:
- kmt_setup_env_secagent_x64
- upload_dependencies_secagent_x64
- upload_secagent_tests_x64
variables:
ARCH: "x86_64"
parallel:
matrix:
- TAG:
- "amazon_2023"
TEST_SET: [cws_fentry]
after_script:
- !reference [.collect_outcomes_kmt]
- !reference [.upload_junit_kmt]

kmt_run_secagent_tests_x64_docker:
extends:
- .kmt_run_secagent_tests
Expand Down Expand Up @@ -350,26 +330,6 @@ kmt_run_secagent_tests_arm64_ebpfless:
- !reference [.collect_outcomes_kmt]
- !reference [.upload_junit_kmt]

kmt_run_secagent_tests_arm64_fentry:
extends:
- .kmt_run_secagent_tests
image: registry.ddbuild.io/ci/datadog-agent-buildimages/system-probe_arm64$DATADOG_AGENT_SYSPROBE_BUILDIMAGES_SUFFIX:$DATADOG_AGENT_SYSPROBE_BUILDIMAGES
tags: ["arch:arm64"]
needs:
- kmt_setup_env_secagent_arm64
- upload_dependencies_secagent_arm64
- upload_secagent_tests_arm64
variables:
ARCH: "arm64"
parallel:
matrix:
- TAG:
- "ubuntu_24.04"
TEST_SET: [cws_fentry]
after_script:
- !reference [.collect_outcomes_kmt]
- !reference [.upload_junit_kmt]

kmt_run_secagent_tests_arm64_docker:
extends:
- .kmt_run_secagent_tests
Expand Down Expand Up @@ -426,7 +386,6 @@ kmt_secagent_tests_join_arm64:
- kmt_run_secagent_tests_arm64
- kmt_run_secagent_tests_arm64_ad
- kmt_run_secagent_tests_arm64_ebpfless
- kmt_run_secagent_tests_arm64_fentry
- kmt_run_secagent_tests_arm64_docker

kmt_secagent_cleanup_arm64:
Expand All @@ -449,7 +408,6 @@ kmt_secagent_tests_join_x64:
- kmt_run_secagent_tests_x64_required
- kmt_run_secagent_tests_x64_ad
- kmt_run_secagent_tests_x64_ebpfless
- kmt_run_secagent_tests_x64_fentry
- kmt_run_secagent_tests_x64_docker

kmt_secagent_cleanup_x64:
Expand Down
4 changes: 2 additions & 2 deletions Dockerfiles/agent-ot/Dockerfile.agent-otel
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
ARG AGENT_VERSION=7.57.0-v1.0-ot-beta-jmx
ARG AGENT_BRANCH=7.57.x-otel-beta-v1
ARG AGENT_VERSION=7.59.0-v1.1.0-ot-beta-jmx
ARG AGENT_BRANCH=7.59.x
# Use the Ubuntu Slim AMD64 base image
FROM ubuntu:24.04 AS builder

Expand Down
3 changes: 0 additions & 3 deletions LICENSE-3rdparty.csv
Original file line number Diff line number Diff line change
Expand Up @@ -2906,9 +2906,6 @@ core,google.golang.org/protobuf/types/known/timestamppb,BSD-3-Clause,Copyright (
core,google.golang.org/protobuf/types/known/wrapperspb,BSD-3-Clause,Copyright (c) 2018 The Go Authors. All rights reserved
core,google.golang.org/protobuf/types/pluginpb,BSD-3-Clause,Copyright (c) 2018 The Go Authors. All rights reserved
core,gopkg.in/DataDog/dd-trace-go.v1/appsec/events,Apache-2.0,"Copyright 2016-Present Datadog, Inc."
core,gopkg.in/DataDog/dd-trace-go.v1/contrib/internal/httptrace,Apache-2.0,"Copyright 2016-Present Datadog, Inc."
core,gopkg.in/DataDog/dd-trace-go.v1/contrib/internal/options,Apache-2.0,"Copyright 2016-Present Datadog, Inc."
core,gopkg.in/DataDog/dd-trace-go.v1/contrib/net/http,Apache-2.0,"Copyright 2016-Present Datadog, Inc."
core,gopkg.in/DataDog/dd-trace-go.v1/datastreams/options,Apache-2.0,"Copyright 2016-Present Datadog, Inc."
core,gopkg.in/DataDog/dd-trace-go.v1/ddtrace,Apache-2.0,"Copyright 2016-Present Datadog, Inc."
core,gopkg.in/DataDog/dd-trace-go.v1/ddtrace/ext,Apache-2.0,"Copyright 2016-Present Datadog, Inc."
Expand Down
6 changes: 3 additions & 3 deletions cmd/installer-downloader/main.go
Original file line number Diff line number Diff line change
Expand Up @@ -44,12 +44,12 @@ func main() {
ctx := context.Background()

t := telemetry.NewTelemetry(env.HTTPClient(), env.APIKey, env.Site, fmt.Sprintf("datadog-installer-downloader-%s", Flavor))
_ = t.Start(ctx)
defer func() { _ = t.Stop(ctx) }()
var err error
span, ctx := telemetry.StartSpanFromEnv(ctx, fmt.Sprintf("downloader-%s", Flavor))
defer func() { span.Finish(err) }()
err = runDownloader(ctx, env, Version, Flavor)

span.Finish(err)
t.Stop()
if err != nil {
fmt.Fprintf(os.Stderr, "Installation failed: %v\n", err)
os.Exit(1)
Expand Down
12 changes: 2 additions & 10 deletions cmd/installer/subcommands/installer/command.go
Original file line number Diff line number Diff line change
Expand Up @@ -87,7 +87,7 @@ func UnprivilegedCommands(_ *command.GlobalParams) []*cobra.Command {
type cmd struct {
t *telemetry.Telemetry
ctx context.Context
span telemetry.Span
span *telemetry.Span
env *env.Env
}

Expand All @@ -107,10 +107,7 @@ func newCmd(operation string) *cmd {
func (c *cmd) Stop(err error) {
c.span.Finish(err)
if c.t != nil {
err := c.t.Stop(context.Background())
if err != nil {
fmt.Fprintf(os.Stderr, "failed to stop telemetry: %v\n", err)
}
c.t.Stop()
}
}

Expand Down Expand Up @@ -225,11 +222,6 @@ func newTelemetry(env *env.Env) *telemetry.Telemetry {
site = config.Site
}
t := telemetry.NewTelemetry(env.HTTPClient(), apiKey, site, "datadog-installer") // No sampling rules for commands
err := t.Start(context.Background())
if err != nil {
fmt.Printf("failed to start telemetry: %v\n", err)
return nil
}
return t
}

Expand Down
2 changes: 1 addition & 1 deletion cmd/installer/subcommands/installer/umask_nix.go
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,7 @@ import (
)

// setInstallerUmask sets umask 0 to override any inherited umask
func setInstallerUmask(span telemetry.Span) {
func setInstallerUmask(span *telemetry.Span) {
oldmask := syscall.Umask(0)
span.SetTag("inherited_umask", oldmask)
}
2 changes: 1 addition & 1 deletion cmd/installer/subcommands/installer/umask_windows.go
Original file line number Diff line number Diff line change
Expand Up @@ -10,4 +10,4 @@ package installer
import "github.com/DataDog/datadog-agent/pkg/fleet/telemetry"

// setInstallerUmask no-op on Windows
func setInstallerUmask(_ telemetry.Span) {}
func setInstallerUmask(_ *telemetry.Span) {}
15 changes: 14 additions & 1 deletion cmd/otel-agent/subcommands/run/command.go
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,7 @@ import (
"github.com/DataDog/datadog-agent/comp/core/hostname/hostnameinterface"
"github.com/DataDog/datadog-agent/comp/core/hostname/remotehostnameimpl"
log "github.com/DataDog/datadog-agent/comp/core/log/def"
logfx "github.com/DataDog/datadog-agent/comp/core/log/fx"
logtracefx "github.com/DataDog/datadog-agent/comp/core/log/fx-trace"
"github.com/DataDog/datadog-agent/comp/core/secrets"
tagger "github.com/DataDog/datadog-agent/comp/core/tagger/def"
Expand Down Expand Up @@ -106,8 +107,20 @@ func runOTelAgentCommand(ctx context.Context, params *subcommands.GlobalParams,
fx.Provide(func() coreconfig.Component {
return acfg
}),
fx.Provide(func(_ coreconfig.Component) log.Params {
return log.ForDaemon(params.LoggerName, "log_file", pkgconfigsetup.DefaultOTelAgentLogFile)
}),
logfx.Module(),
fetchonlyimpl.Module(),
// TODO: don't rely on this pattern; remove this `OptionalModuleWithParams` thing
// and instead adapt OptionalModule to allow parameter passing naturally.
// See: https://github.com/DataDog/datadog-agent/pull/28386
configsyncimpl.OptionalModuleWithParams(),
fx.Provide(func() configsyncimpl.Params {
return configsyncimpl.NewParams(params.SyncTimeout, params.SyncDelay, true)
}),
converterfx.Module(),
fx.Provide(func(cp converter.Component) confmap.Converter {
fx.Provide(func(cp converter.Component, _ optional.Option[configsync.Component]) confmap.Converter {
return cp
}),
collectorcontribFx.Module(),
Expand Down
29 changes: 22 additions & 7 deletions cmd/system-probe/api/debug/handlers_linux.go
Original file line number Diff line number Diff line change
Expand Up @@ -17,19 +17,18 @@ import (
"time"
)

// HandleSelinuxSestatus reports the output of sestatus as an http result
func HandleSelinuxSestatus(w http.ResponseWriter, r *http.Request) {
ctx, cancel := context.WithTimeout(r.Context(), 5*time.Second)
defer cancel()

cmd := exec.CommandContext(ctx, "sestatus")
// handleCommand runs commandName with the provided arguments and writes it to the HTTP response.
// If the command exits with a failure or doesn't exist in the PATH, it will still 200 but report the failure.
// Any other kind of error will 500.
func handleCommand(ctx context.Context, w http.ResponseWriter, commandName string, args ...string) {
cmd := exec.CommandContext(ctx, commandName, args...)
output, err := cmd.CombinedOutput()

var execError *exec.Error
var exitErr *exec.ExitError

if err != nil {
// don't 500 for ExitErrors etc, to report "normal" failures to the selinux_sestatus.log file
// don't 500 for ExitErrors etc, to report "normal" failures to the flare log file
if !errors.As(err, &execError) && !errors.As(err, &exitErr) {
w.WriteHeader(500)
}
Expand All @@ -39,3 +38,19 @@ func HandleSelinuxSestatus(w http.ResponseWriter, r *http.Request) {

w.Write(output)
}

// HandleSelinuxSestatus reports the output of sestatus as an http result
func HandleSelinuxSestatus(w http.ResponseWriter, r *http.Request) {
ctx, cancel := context.WithTimeout(r.Context(), 5*time.Second)
defer cancel()

handleCommand(ctx, w, "sestatus")
}

// HandleSelinuxSemoduleList reports the output of semodule -l as an http result
func HandleSelinuxSemoduleList(w http.ResponseWriter, r *http.Request) {
ctx, cancel := context.WithTimeout(r.Context(), 5*time.Second)
defer cancel()

handleCommand(ctx, w, "semodule", "-l")
}
6 changes: 6 additions & 0 deletions cmd/system-probe/api/debug/handlers_nolinux.go
Original file line number Diff line number Diff line change
Expand Up @@ -18,3 +18,9 @@ func HandleSelinuxSestatus(w http.ResponseWriter, _ *http.Request) {
w.WriteHeader(500)
io.WriteString(w, "HandleSelinuxSestatus is not supported on this platform")
}

// HandleSelinuxSemoduleList is not supported
func HandleSelinuxSemoduleList(w http.ResponseWriter, _ *http.Request) {
w.WriteHeader(500)
io.WriteString(w, "HandleSelinuxSemoduleList is not supported on this platform")
}
1 change: 1 addition & 0 deletions cmd/system-probe/api/server.go
Original file line number Diff line number Diff line change
Expand Up @@ -60,6 +60,7 @@ func StartServer(cfg *sysconfigtypes.Config, telemetry telemetry.Component, wmet
if runtime.GOOS == "linux" {
mux.HandleFunc("/debug/ebpf_btf_loader_info", ebpf.HandleBTFLoaderInfo)
mux.HandleFunc("/debug/selinux_sestatus", debug.HandleSelinuxSestatus)
mux.HandleFunc("/debug/selinux_semodule_list", debug.HandleSelinuxSemoduleList)
}

go func() {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -168,11 +168,22 @@ func (wcd *AgentCrashDetect) Run() error {
}

log.Infof("Sending crash: %v", formatText(crash))
lts := internaltelemetry.NewClient(wcd.tconfig.NewHTTPClient(), wcd.tconfig.TelemetryConfig.Endpoints, "ddnpm", true)
lts := internaltelemetry.NewClient(wcd.tconfig.NewHTTPClient(), toTelemEndpoints(wcd.tconfig.TelemetryConfig.Endpoints), "ddnpm", true)
lts.SendLog("WARN", formatText(crash))
return nil
}

func toTelemEndpoints(endpoints []*traceconfig.Endpoint) []*internaltelemetry.Endpoint {
telemEndpoints := make([]*internaltelemetry.Endpoint, 0, len(endpoints))
for _, e := range endpoints {
telemEndpoints = append(telemEndpoints, &internaltelemetry.Endpoint{
Host: e.Host,
APIKey: e.APIKey,
})
}
return telemEndpoints
}

func newAgentCrashComponent(deps dependencies) agentcrashdetect.Component {
instance := &agentCrashComponent{}
instance.tconfig = deps.TConfig.Object()
Expand Down
2 changes: 2 additions & 0 deletions comp/core/tagger/collectors/pod_tag_extractor.go
Original file line number Diff line number Diff line change
Expand Up @@ -34,6 +34,8 @@ func (p *PodTagExtractor) Extract(podEntity *workloadmeta.KubernetesPod, cardina
return append(tagInfos.LowCardTags, tagInfos.OrchestratorCardTags...)
case types.LowCardinality:
return tagInfos.LowCardTags
case types.NoneCardinality:
return []string{}
default:
log.Errorf("unsupported tag cardinality %v", cardinality)
return []string{}
Expand Down
5 changes: 2 additions & 3 deletions comp/core/tagger/impl/tagger.go
Original file line number Diff line number Diff line change
Expand Up @@ -422,7 +422,7 @@ func (t *TaggerWrapper) EnrichTags(tb tagset.TagsAccumulator, originInfo taggert
// | none | empty || empty |
// | empty | not empty || container prefix + originFromMsg |
// | none | not empty || container prefix + originFromMsg |
if t.datadogConfig.dogstatsdOptOutEnabled && originInfo.Cardinality == "none" {
if t.datadogConfig.dogstatsdOptOutEnabled && originInfo.Cardinality == types.NoneCardinalityString {
originInfo.ContainerIDFromSocket = packets.NoOrigin
originInfo.PodUID = ""
originInfo.ContainerID = ""
Expand Down Expand Up @@ -460,8 +460,7 @@ func (t *TaggerWrapper) EnrichTags(tb tagset.TagsAccumulator, originInfo taggert
}
default:
// Disable origin detection if cardinality is none
// TODO: The `none` cardinality should be directly supported by the Tagger.
if originInfo.Cardinality == "none" {
if originInfo.Cardinality == types.NoneCardinalityString {
originInfo.ContainerIDFromSocket = packets.NoOrigin
originInfo.PodUID = ""
originInfo.ContainerID = ""
Expand Down
37 changes: 37 additions & 0 deletions comp/core/tagger/subscriber/subscription_manager_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -109,6 +109,21 @@ func TestSubscriptionManager(t *testing.T) {

highCardSubscription.Unsubscribe()

// None Cardinality Subscriber
noneCardSubID := "none-card-sub"
noneCardSubscription, err := sm.Subscribe(noneCardSubID, types.NewFilterBuilder().Include(types.EntityIDPrefix("foo")).Build(types.NoneCardinality), nil)
require.NoError(t, err)

sm.Notify([]types.EntityEvent{
events["added"],
events["modified"],
events["deleted"],
events["added-with-no-id"],
events["added-with-unmatched-prefix"],
})

noneCardSubscription.Unsubscribe()

// Verify low cardinality subscriber received events
assertReceivedEvents(t, lowCardSubscription.EventsChan(), []types.EntityEvent{
{
Expand Down Expand Up @@ -192,6 +207,28 @@ func TestSubscriptionManager(t *testing.T) {
},
},
})

// Verify none cardinality subscriber received events
assertReceivedEvents(t, noneCardSubscription.EventsChan(), []types.EntityEvent{
{
EventType: types.EventTypeAdded,
Entity: types.Entity{
ID: entityID,
},
},
{
EventType: types.EventTypeModified,
Entity: types.Entity{
ID: entityID,
},
},
{
EventType: types.EventTypeDeleted,
Entity: types.Entity{
ID: entityID,
},
},
})
}

func assertReceivedEvents(t *testing.T, ch chan []types.EntityEvent, expectedEvents []types.EntityEvent) {
Expand Down
12 changes: 9 additions & 3 deletions comp/core/tagger/tagstore/entity_tags.go
Original file line number Diff line number Diff line change
Expand Up @@ -109,12 +109,16 @@ func (e *EntityTagsWithMultipleSources) getStandard() []string {
func (e *EntityTagsWithMultipleSources) getHashedTags(cardinality types.TagCardinality) tagset.HashedTags {
e.computeCache()

if cardinality == types.HighCardinality {
switch cardinality {
case types.HighCardinality:
return e.cachedAll
} else if cardinality == types.OrchestratorCardinality {
case types.OrchestratorCardinality:
return e.cachedOrchestrator
case types.NoneCardinality:
return tagset.HashedTags{}
default:
return e.cachedLow
}
return e.cachedLow
}

func (e *EntityTagsWithMultipleSources) computeCache() {
Expand Down Expand Up @@ -302,6 +306,8 @@ func (e *EntityTagsWithSingleSource) getHashedTags(cardinality types.TagCardinal
return e.cachedAll
case types.OrchestratorCardinality:
return e.cachedOrchestrator
case types.NoneCardinality:
return tagset.HashedTags{}
default:
return e.cachedLow
}
Expand Down
6 changes: 6 additions & 0 deletions comp/core/tagger/tagstore/entity_tags_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -100,6 +100,12 @@ func TestGetHashedTags(t *testing.T) {
[]string{"l1:v1", "l2:v2", "service:s1", "o1:v1", "o2:v2", "h1:v1", "h2:v2"},
entityTags.getHashedTags(types.HighCardinality).Get(),
)

assert.Equal(
t,
[]string(nil),
entityTags.getHashedTags(types.NoneCardinality).Get(),
)
}

func TestTagsForSource(t *testing.T) {
Expand Down
Loading
Loading