From e9568fe799f1ae38ee4643482890e49295a1a669 Mon Sep 17 00:00:00 2001 From: Ibraheem Aboulnaga Date: Tue, 5 Nov 2024 16:20:00 -0500 Subject: [PATCH] wip --- .../tests/otel/otel-agent/config/minimal.yml | 10 +-- .../receive_resource_spans_v2_test.go | 18 ++--- .../receive_resource_spans_v2_test.go | 60 ++++++++++++++ .../tests/otel/utils/pipelines_utils.go | 78 +------------------ 4 files changed, 74 insertions(+), 92 deletions(-) create mode 100644 test/new-e2e/tests/otel/otlp-ingest/receive_resource_spans_v2_test.go diff --git a/test/new-e2e/tests/otel/otel-agent/config/minimal.yml b/test/new-e2e/tests/otel/otel-agent/config/minimal.yml index 40f795bd5ff65..4815dd394401f 100644 --- a/test/new-e2e/tests/otel/otel-agent/config/minimal.yml +++ b/test/new-e2e/tests/otel/otel-agent/config/minimal.yml @@ -14,8 +14,6 @@ exporters: hostname: "otelcol-docker" api: key: ${DD_API_KEY} - debug: - verbosity: detailed processors: batch: connectors: @@ -31,16 +29,16 @@ service: traces: receivers: [otlp] processors: [batch] - exporters: [datadog/connector, debug] + exporters: [datadog/connector] traces/send: receivers: [otlp] processors: [batch] - exporters: [datadog, debug] + exporters: [datadog] metrics: receivers: [otlp, datadog/connector] processors: [batch] - exporters: [datadog, debug] + exporters: [datadog] logs: receivers: [otlp] processors: [batch] - exporters: [datadog, debug] + exporters: [datadog] diff --git a/test/new-e2e/tests/otel/otel-agent/receive_resource_spans_v2_test.go b/test/new-e2e/tests/otel/otel-agent/receive_resource_spans_v2_test.go index 957bbf59e948b..60a97d0c87581 100644 --- a/test/new-e2e/tests/otel/otel-agent/receive_resource_spans_v2_test.go +++ b/test/new-e2e/tests/otel/otel-agent/receive_resource_spans_v2_test.go @@ -8,17 +8,17 @@ package otelagent import ( "github.com/DataDog/datadog-agent/test/new-e2e/pkg/e2e" "github.com/DataDog/datadog-agent/test/new-e2e/pkg/environments" - localkubernetes "github.com/DataDog/datadog-agent/test/new-e2e/pkg/environments/local/kubernetes" + awskubernetes "github.com/DataDog/datadog-agent/test/new-e2e/pkg/environments/aws/kubernetes" "github.com/DataDog/datadog-agent/test/new-e2e/tests/otel/utils" "github.com/DataDog/test-infra-definitions/components/datadog/kubernetesagentparams" "testing" ) -type otlpIngestTestSuiteWithSpanReceiverV2 struct { +type otelAgentSpanReceiverV2TestSuite struct { e2e.BaseSuite[environments.Kubernetes] } -func TestOTLPIngestWithSpanReceiverV2(t *testing.T) { +func TestOTelAgentSpanReceiverV2(t *testing.T) { values := ` datadog: logs: @@ -32,18 +32,14 @@ agents: value: 'enable_receive_resource_spans_v2' ` t.Parallel() - e2e.Run(t, &otlpIngestTestSuiteWithSpanReceiverV2{}, e2e.WithProvisioner(localkubernetes.Provisioner(localkubernetes.WithAgentOptions(kubernetesagentparams.WithoutDualShipping(), kubernetesagentparams.WithHelmValues(values), kubernetesagentparams.WithOTelAgent(), kubernetesagentparams.WithOTelConfig(minimalConfig))))) + e2e.Run(t, &otelAgentSpanReceiverV2TestSuite{}, e2e.WithProvisioner(awskubernetes.KindProvisioner(awskubernetes.WithAgentOptions(kubernetesagentparams.WithoutDualShipping(), kubernetesagentparams.WithHelmValues(values), kubernetesagentparams.WithOTelAgent(), kubernetesagentparams.WithOTelConfig(minimalConfig))))) } -var otlpIngestParams = utils.IAParams{ - InfraAttributes: false, -} - -func (s *otlpIngestTestSuiteWithSpanReceiverV2) SetupSuite() { +func (s *otelAgentSpanReceiverV2TestSuite) SetupSuite() { s.BaseSuite.SetupSuite() utils.TestSpanReceiverV2(s) } -func (s *otlpIngestTestSuiteWithSpanReceiverV2) TestTracesWithSpanReceiverV2() { - utils.TestTracesWithSpanReceiverV2(s, otlpIngestParams) +func (s *otelAgentSpanReceiverV2TestSuite) TestTracesWithSpanReceiverV2() { + utils.TestTracesWithSpanReceiverV2(s) } diff --git a/test/new-e2e/tests/otel/otlp-ingest/receive_resource_spans_v2_test.go b/test/new-e2e/tests/otel/otlp-ingest/receive_resource_spans_v2_test.go new file mode 100644 index 0000000000000..092e17a8bfe29 --- /dev/null +++ b/test/new-e2e/tests/otel/otlp-ingest/receive_resource_spans_v2_test.go @@ -0,0 +1,60 @@ +// Unless explicitly stated otherwise all files in this repository are licensed +// under the Apache License Version 2.0. +// This product includes software developed at Datadog (https://www.datadoghq.com/). +// Copyright 2016-present Datadog, Inc. + +package otlpingest + +import ( + "github.com/DataDog/datadog-agent/test/new-e2e/pkg/e2e" + "github.com/DataDog/datadog-agent/test/new-e2e/pkg/environments" + awskubernetes "github.com/DataDog/datadog-agent/test/new-e2e/pkg/environments/aws/kubernetes" + "github.com/DataDog/datadog-agent/test/new-e2e/tests/otel/utils" + "github.com/DataDog/test-infra-definitions/components/datadog/kubernetesagentparams" + "testing" +) + +type otelIngestSpanReceiverV2TestSuite struct { + e2e.BaseSuite[environments.Kubernetes] +} + +func TestOTelIngestSpanReceiverV2(t *testing.T) { + values := ` +datadog: + otlp: + receiver: + protocols: + grpc: + enabled: true + logs: + enabled: true + logs: + containerCollectAll: false + containerCollectUsingFiles: false +agents: + containers: + agent: + env: + - name: DD_APM_FEATURES + value: 'enable_receive_resource_spans_v2' + traceAgent: + env: + - name: DD_APM_FEATURES + value: 'enable_receive_resource_spans_v2' + otelAgent: + env: + - name: DD_APM_FEATURES + value: 'enable_receive_resource_spans_v2' +` + t.Parallel() + e2e.Run(t, &otelIngestSpanReceiverV2TestSuite{}, e2e.WithProvisioner(awskubernetes.KindProvisioner(awskubernetes.WithAgentOptions(kubernetesagentparams.WithoutDualShipping(), kubernetesagentparams.WithHelmValues(values))))) +} + +func (s *otelIngestSpanReceiverV2TestSuite) SetupSuite() { + s.BaseSuite.SetupSuite() + utils.TestSpanReceiverV2(s) +} + +func (s *otelIngestSpanReceiverV2TestSuite) TestTracesWithSpanReceiverV2() { + utils.TestTracesWithSpanReceiverV2(s) +} diff --git a/test/new-e2e/tests/otel/utils/pipelines_utils.go b/test/new-e2e/tests/otel/utils/pipelines_utils.go index e4364790293ad..9d6ad42535214 100644 --- a/test/new-e2e/tests/otel/utils/pipelines_utils.go +++ b/test/new-e2e/tests/otel/utils/pipelines_utils.go @@ -124,7 +124,7 @@ func TestTraces(s OTelTestSuite, iaParams IAParams) { } // TestTracesWithSpanReceiverV2 tests that OTLP traces are received through OTel pipelines as expected with updated OTLP span receiver -func TestTracesWithSpanReceiverV2(s OTelTestSuite, iaParams IAParams) { +func TestTracesWithSpanReceiverV2(s OTelTestSuite) { err := s.Env().FakeIntake.Client().FlushServerAndResetAggregators() require.NoError(s.T(), err) @@ -150,11 +150,6 @@ func TestTracesWithSpanReceiverV2(s OTelTestSuite, iaParams IAParams) { return } assert.Equal(s.T(), telemetrygenService, tp.Chunks[0].Spans[0].Service) - if iaParams.InfraAttributes { - ctags, ok := getContainerTags(s.T(), tp) - assert.True(s.T(), ok) - assert.NotNil(s.T(), ctags["kube_ownerref_kind"]) - } }, 5*time.Minute, 10*time.Second) require.NotEmpty(s.T(), traces) s.T().Log("Got traces", s.T().Name(), traces) @@ -197,12 +192,6 @@ func TestTracesWithSpanReceiverV2(s OTelTestSuite, iaParams IAParams) { assert.Equal(s.T(), sp.Meta["k8s.container.name"], ctags["kube_container_name"]) assert.Equal(s.T(), sp.Meta["k8s.namespace.name"], ctags["kube_namespace"]) assert.Equal(s.T(), sp.Meta["k8s.pod.name"], ctags["pod_name"]) - - // Verify container tags from infraattributes processor - if iaParams.InfraAttributes { - maps.Copy(ctags, sp.Meta) - testInfraTags(s.T(), ctags, iaParams) - } } } @@ -410,68 +399,7 @@ func TestSpanReceiverV2(s OTelTestSuite) { numTraces := 10 s.T().Log("Starting telemetrygen") - createTelemetrygenJobForSpanReceiverV2(ctx, s, "traces", []string{"--traces", fmt.Sprint(numTraces)}) -} - -func createTelemetrygenJobForSpanReceiverV2(ctx context.Context, s OTelTestSuite, telemetry string, options []string) { - var ttlSecondsAfterFinished int32 = 0 //nolint:revive // We want to see this is explicitly set to 0 - var backOffLimit int32 = 4 - - otlpEndpoint := fmt.Sprintf("%v:4317", s.Env().Agent.LinuxNodeAgent.LabelSelectors["app"]) - jobSpec := &batchv1.Job{ - ObjectMeta: metav1.ObjectMeta{ - Name: fmt.Sprintf("telemetrygen-job-%v-%v", telemetry, strings.ReplaceAll(strings.ToLower(s.T().Name()), "/", "-")), - Namespace: "datadog", - }, - Spec: batchv1.JobSpec{ - TTLSecondsAfterFinished: &ttlSecondsAfterFinished, - Template: corev1.PodTemplateSpec{ - Spec: corev1.PodSpec{ - Containers: []corev1.Container{ - { - Env: []corev1.EnvVar{{ - Name: "OTEL_SERVICE_NAME", - Value: telemetrygenService, - }, { - Name: "OTEL_K8S_POD_ID", - ValueFrom: &corev1.EnvVarSource{FieldRef: &corev1.ObjectFieldSelector{FieldPath: "metadata.uid"}}, - }, { - Name: "OTEL_K8S_NAMESPACE", - ValueFrom: &corev1.EnvVarSource{FieldRef: &corev1.ObjectFieldSelector{FieldPath: "metadata.namespace"}}, - }, { - Name: "OTEL_K8S_NODE_NAME", - ValueFrom: &corev1.EnvVarSource{FieldRef: &corev1.ObjectFieldSelector{FieldPath: "spec.nodeName"}}, - }, { - Name: "OTEL_K8S_POD_NAME", - ValueFrom: &corev1.EnvVarSource{FieldRef: &corev1.ObjectFieldSelector{FieldPath: "metadata.name"}}, - }}, - Name: "telemetrygen-job", - Image: "ghcr.io/open-telemetry/opentelemetry-collector-contrib/telemetrygen:v0.107.0", - Command: append([]string{ - "/telemetrygen", telemetry, "--otlp-endpoint", otlpEndpoint, "--otlp-insecure", - "--telemetry-attributes", fmt.Sprintf("%v=%v", customAttribute, customAttributeValue), - "--telemetry-attributes", fmt.Sprintf("%v=\"%v\"", "k8s.pod.uid", "did-it-work"), - "--otlp-attributes", "service.name=\"$(OTEL_SERVICE_NAME)\"", - "--otlp-attributes", "host.name=\"$(OTEL_K8S_NODE_NAME)\"", - "--otlp-attributes", fmt.Sprintf("deployment.environment=\"%v\"", env), - "--otlp-attributes", fmt.Sprintf("service.version=\"%v\"", version), - "--otlp-attributes", "k8s.namespace.name=\"$(OTEL_K8S_NAMESPACE)\"", - "--otlp-attributes", "k8s.node.name=\"$(OTEL_K8S_NODE_NAME)\"", - "--otlp-attributes", "k8s.pod.name=\"$(OTEL_K8S_POD_NAME)\"", - //"--otlp-attributes", "k8s.pod.uid=\"$(OTEL_K8S_POD_ID)\"", - "--otlp-attributes", "k8s.container.name=\"telemetrygen-job\"", - }, options...), - }, - }, - RestartPolicy: corev1.RestartPolicyNever, - }, - }, - BackoffLimit: &backOffLimit, - }, - } - - _, err := s.Env().KubernetesCluster.Client().BatchV1().Jobs("datadog").Create(ctx, jobSpec, metav1.CreateOptions{}) - require.NoError(s.T(), err, "Could not properly start job") + createTelemetrygenJob(ctx, s, "traces", []string{"--traces", fmt.Sprint(numTraces)}) } func createTelemetrygenJob(ctx context.Context, s OTelTestSuite, telemetry string, options []string) { @@ -511,6 +439,7 @@ func createTelemetrygenJob(ctx context.Context, s OTelTestSuite, telemetry strin Command: append([]string{ "/telemetrygen", telemetry, "--otlp-endpoint", otlpEndpoint, "--otlp-insecure", "--telemetry-attributes", fmt.Sprintf("%v=%v", customAttribute, customAttributeValue), + "--telemetry-attributes", "k8s.pod.uid=\"$(OTEL_K8S_POD_ID)\"", "--otlp-attributes", "service.name=\"$(OTEL_SERVICE_NAME)\"", "--otlp-attributes", "host.name=\"$(OTEL_K8S_NODE_NAME)\"", "--otlp-attributes", fmt.Sprintf("deployment.environment=\"%v\"", env), @@ -518,7 +447,6 @@ func createTelemetrygenJob(ctx context.Context, s OTelTestSuite, telemetry strin "--otlp-attributes", "k8s.namespace.name=\"$(OTEL_K8S_NAMESPACE)\"", "--otlp-attributes", "k8s.node.name=\"$(OTEL_K8S_NODE_NAME)\"", "--otlp-attributes", "k8s.pod.name=\"$(OTEL_K8S_POD_NAME)\"", - "--otlp-attributes", "k8s.pod.uid=\"$(OTEL_K8S_POD_ID)\"", "--otlp-attributes", "k8s.container.name=\"telemetrygen-job\"", }, options...), },