Merge branch 'main' into NDMII-3154-ha-agent-comp

.circleci/config.yml

@@ -15,7 +15,7 @@ experimental:
 templates:
   job_template: &job_template
     docker:
-      - image: gcr.io/datadoghq/agent-circleci-runner:v48262719-bfb00f80
+      - image: gcr.io/datadoghq/agent-circleci-runner:v48372186-ff395e52
         environment:
           USE_SYSTEM_LIBS: "1"
     working_directory: /go/src/github.com/DataDog/datadog-agent

.ddqa/config.toml

@@ -215,8 +215,8 @@ github_labels = ["team/container-app"]
 jira_project = "INPLAT"
 jira_issue_type = "Task"
 jira_statuses = ["Selected For Development", "In Progress", "Done"]
-github_team = "apm-onboarding"
-github_labels = ["team/apm-onboarding"]
+github_team = "injection-platform"
+github_labels = ["team/injection-platform"]
 
 [teams."Agent Release Management"]
 jira_project = "AGNTR"

.github/CODEOWNERS

@@ -44,6 +44,8 @@
 /service.datadog.yaml                   @DataDog/agent-delivery
 /static-analysis.datadog.yml            @DataDog/software-integrity-and-trust @DataDog/agent-devx-infra
 
+/modules.yml                            @DataDog/agent-shared-components
+
 /.circleci/                             @DataDog/agent-devx-infra
 
 /.github/CODEOWNERS                                 # do not notify anyone
@@ -99,7 +101,7 @@
 /.gitlab/package_deps_build/package_deps_build.yml   @DataDog/agent-devx-infra @DataDog/ebpf-platform
 /.gitlab/powershell_script_signing/powershell_script_signing.yml    @DataDog/agent-delivery @DataDog/windows-agent
 /.gitlab/source_test/golang_deps_diff.yml            @DataDog/agent-devx-infra @DataDog/agent-devx-loops
-/.gitlab/source_test/include.yml                     @DataDog/agent-devx-infra
+/.gitlab/source_test/*                               @DataDog/agent-devx-infra
 /.gitlab/source_test/linux.yml                       @DataDog/agent-devx-infra @DataDog/agent-devx-loops
 /.gitlab/source_test/macos.yml                       @DataDog/agent-devx-infra @DataDog/agent-devx-loops
 /.gitlab/source_test/notify.yml                      @DataDog/agent-devx-infra @DataDog/agent-devx-loops
@@ -306,7 +308,6 @@
 /comp/checks/agentcrashdetect @DataDog/windows-kernel-integrations
 /comp/checks/windowseventlog @DataDog/windows-agent
 /comp/checks/winregistry @DataDog/windows-agent
-/comp/core/agenttelemetry @DataDog/agent-metrics-logs
 /comp/core/autodiscovery @DataDog/container-platform
 /comp/core/sysprobeconfig @DataDog/ebpf-platform
 /comp/core/tagger @DataDog/container-platform
@@ -322,6 +323,9 @@
 /comp/haagent @DataDog/network-device-monitoring @DataDog/remote-config @DataDog/fleet
 # END COMPONENTS
 
+# Additional notification to  @iglendd about Agent Telemetry changes for optional approval and governance acknowledgement
+/comp/core/agenttelemetry               @DataDog/agent-shared-components @iglendd
+
 # pkg
 /pkg/                                   @DataDog/agent-shared-components
 /pkg/api/                               @DataDog/agent-shared-components
@@ -365,7 +369,7 @@
 /pkg/clusteragent/                      @DataDog/container-platform
 /pkg/clusteragent/autoscaling/          @DataDog/container-integrations
 /pkg/clusteragent/admission/mutate/autoscaling          @DataDog/container-integrations
-/pkg/clusteragent/admission/mutate/autoinstrumentation/ @DataDog/container-platform @DataDog/apm-onboarding
+/pkg/clusteragent/admission/mutate/autoinstrumentation/ @DataDog/container-platform @DataDog/injection-platform
 /pkg/clusteragent/orchestrator/         @DataDog/container-app
 /pkg/clusteragent/telemetry/            @DataDog/apm-trace-storage
 /pkg/collector/                         @DataDog/agent-metrics-logs
@@ -443,6 +447,7 @@
 /pkg/util/ecs/                          @DataDog/container-integrations
 /pkg/util/funcs/                        @DataDog/ebpf-platform
 /pkg/util/kernel/                       @DataDog/ebpf-platform
+/pkg/util/safeelf/                      @DataDog/ebpf-platform
 /pkg/util/ktime                         @DataDog/agent-security
 /pkg/util/kubernetes/                   @DataDog/container-integrations @DataDog/container-platform @DataDog/container-app
 /pkg/util/podman/                       @DataDog/container-integrations
@@ -564,6 +569,7 @@
 /tasks/libs/ciproviders/                @DataDog/agent-devx-infra
 /tasks/libs/common/omnibus.py           @DataDog/agent-delivery
 /tasks/omnibus.py                       @DataDog/agent-delivery
+/tasks/release.py                       @DataDog/agent-delivery
 /tasks/unit_tests/components_tests.py   @DataDog/agent-shared-components
 /tasks/unit_tests/omnibus_tests.py      @DataDog/agent-delivery
 /tasks/unit_tests/testdata/components_src/ @DataDog/agent-shared-components
@@ -610,7 +616,7 @@
 /test/new-e2e/tests/otel                      @DataDog/opentelemetry
 /test/new-e2e/tests/process                   @DataDog/processes
 /test/new-e2e/tests/sysprobe-functional     @DataDog/windows-kernel-integrations
-/test/new-e2e/tests/security-agent-functional     @DataDog/windows-kernel-integrations
+/test/new-e2e/tests/security-agent-functional     @DataDog/windows-kernel-integrations @DataDog/agent-security
 /test/new-e2e/tests/cws                       @DataDog/agent-security
 /test/new-e2e/tests/agent-metrics-logs   @DataDog/agent-metrics-logs
 /test/new-e2e/tests/windows                   @DataDog/windows-agent @DataDog/windows-kernel-integrations

.github/workflows/buildimages-update.yml

@@ -37,7 +37,8 @@ jobs:
       - name: Checkout branch
         uses: actions/checkout@0ad4b8fadaa221de15dcec353f45205ec38ea70b # v4.1.4
         with:
-          persist-credentials: false
+          # credentials are needed to create the PR at the end of the workflow
+          persist-credentials: true
 
       - name: Fetch branch
         env:
@@ -89,6 +90,7 @@ jobs:
         id: update_build_images
         env:
           TEST_VERSION_FLAG: ${{ inputs.test_version && '--test' || '--no-test' }}
+          # INCLUDE_OTEL_MODULES must be used without quotes to be ignored when empty
           INCLUDE_OTEL_MODULES: ${{ inputs.include_otel_modules && '--include-otel-modules' || '' }}
           CURRENT_GO_VERSION: ${{ steps.current_go_version.outputs.GO_VERSION }}
           INPUT_GO_VERSION: ${{ inputs.go_version }}
@@ -98,7 +100,7 @@ jobs:
             inv -e buildimages.update --tag "$IMAGES_ID" "$TEST_VERSION_FLAG"
             echo "MESSAGE=Update buildimages ID to $IMAGES_ID" >> $GITHUB_OUTPUT
           else
-            inv -e update-go --image-tag "$IMAGES_ID" "$TEST_VERSION_FLAG" "$INCLUDE_OTEL_MODULES" -v "$INPUT_GO_VERSION"
+            inv -e update-go --image-tag "$IMAGES_ID" "$TEST_VERSION_FLAG" $INCLUDE_OTEL_MODULES -v "$INPUT_GO_VERSION"
             echo "MESSAGE=Update Go version to $INPUT_GO_VERSION" >> $GITHUB_OUTPUT
           fi
 
@@ -145,6 +147,7 @@ jobs:
           IMAGES_ID: ${{ inputs.images_id }}
           CURRENT_GO_VERSION: ${{ steps.current_go_version.outputs.GO_VERSION }}
           INPUT_GO_VERSION: ${{ inputs.go_version }}
+          # INPUT_TEST_VERSION must be used without quotes to be ignored when empty
           INPUT_TEST_VERSION: ${{ inputs.test_version && '--test' || '' }}
           GITHUB_REF: ${{ github.ref }}
         run: |
@@ -154,7 +157,7 @@ jobs:
           "$IMAGES_ID" \
           "$CURRENT_GO_VERSION" \
           "$INPUT_GO_VERSION" \
-          "$INPUT_TEST_VERSION" > $TMP_PR_BODY_PATH
+          $INPUT_TEST_VERSION > $TMP_PR_BODY_PATH
 
           # Create the PR
           gh pr create \

.github/workflows/create_rc_pr.yml

@@ -57,6 +57,7 @@ jobs:
       strategy:
         matrix:
           value: ${{fromJSON(needs.find_release_branches.outputs.branches)}}
+        fail-fast: false
       steps:
             - name: Checkout release branch
               uses: actions/checkout@0ad4b8fadaa221de15dcec353f45205ec38ea70b # v4.1.4
@@ -87,7 +88,11 @@ jobs:
                 MATRIX: ${{ matrix.value }}
                 WARNING: ${{ needs.find_release_branches.outputs.warning }}
               run: |
-                echo "CHANGES=$(inv -e release.check-for-changes -r "$MATRIX" "$WARNING")" >> $GITHUB_OUTPUT
+                if [ -n "${{ needs.find_release_branches.outputs.warning }}" ]; then
+                  echo "CHANGES=$(inv -e release.check-for-changes -r "$MATRIX" "$WARNING")" >> $GITHUB_OUTPUT
+                else
+                  echo "CHANGES=$(inv -e release.check-for-changes -r "$MATRIX")" >> $GITHUB_OUTPUT
+                fi
 
             - name: Create RC PR
               if: ${{ steps.check_for_changes.outputs.CHANGES == 'true'}}

.github/workflows/cws-btfhub-sync.yml

@@ -83,11 +83,18 @@ jobs:
           echo "ARTIFACT_NAME=constants-${{ matrix.cone }}" | tr '/' '-' >> $GITHUB_OUTPUT
 
       - name: Sync constants
+        if: ${{ !inputs.force_refresh }}
         env:
           ARTIFACT_NAME: ${{ steps.artifact-name.outputs.ARTIFACT_NAME }}
-          FORCE_REFRESH: ${{ inputs.force_refresh && '--force-refresh' || '' }}
         run: |
-          inv -e security-agent.generate-btfhub-constants --archive-path=./dev/dist/archive --output-path=./"$ARTIFACT_NAME".json "$FORCE_REFRESH"
+          inv -e security-agent.generate-btfhub-constants --archive-path=./dev/dist/archive --output-path=./"$ARTIFACT_NAME".json
+
+      - name: Force sync constants
+        if: ${{ inputs.force_refresh }}
+        env:
+          ARTIFACT_NAME: ${{ steps.artifact-name.outputs.ARTIFACT_NAME }}
+        run: |
+          inv -e security-agent.generate-btfhub-constants --archive-path=./dev/dist/archive --output-path=./"$ARTIFACT_NAME".json --force-refresh
 
       - name: Upload artifact
         uses: actions/upload-artifact@834a144ee995460fba8ed112a2fc961b36a5ec5a # v4.3.6
@@ -106,7 +113,6 @@ jobs:
         uses: actions/checkout@0ad4b8fadaa221de15dcec353f45205ec38ea70b # v4.1.4
         with:
           ref: ${{ inputs.base_branch || 'main' }}
-          persist-credentials: false
 
       - name: Install python
         uses: actions/setup-python@0b93645e9fea7318ecaed2b359559ac225c90a2b # v5.3.0
@@ -158,8 +164,8 @@ jobs:
               title: 'CWS: sync BTFHub constants',
               owner,
               repo,
-              head: "$BRANCH_NAME",
-              base: "$BASE_BRANCH",
+              head: process.env.BRANCH_NAME,
+              base: process.env.BASE_BRANCH,
               body: [
                 '### What does this PR do?',
                 'This PR syncs the BTFHub constants used by CWS',

.github/workflows/go-update-commenter.yml

@@ -5,7 +5,9 @@ on:
     # Only run on PR label events (in particular not on every commit)
     types: [ labeled ]
 
-permissions: {}
+permissions:
+  # write permissions are needed to create the comment
+  pull-requests: write
 
 jobs:
   old-versions-match:
@@ -44,7 +46,7 @@ jobs:
         run: |
           set -euo pipefail
           # build the base of the Github URL to the current commit
-          GITHUB_HEAD_URL="GITHUB_SERVER_URL/$GITHUB_REPOSITORY/blob/$GITHUB_SHA"
+          GITHUB_HEAD_URL="$GITHUB_SERVER_URL/$GITHUB_REPOSITORY/blob/$GITHUB_SHA"
           {
             echo "matches<<EOF"
             echo "Here are potential matches of the former version:"

.github/workflows/go_mod_tidy.yml

@@ -20,7 +20,6 @@ jobs:
       - uses: actions/checkout@0ad4b8fadaa221de15dcec353f45205ec38ea70b # v4.1.4
         with:
           ref: ${{ github.head_ref }}
-          persist-credentials: false
       - name: Checkout PR
         # run only if triggered manually, otherwise we are already on the right branch and we won't have `pr_number`
         if: ${{ github.event_name == 'workflow_dispatch' }}

.github/workflows/label-analysis.yml

@@ -121,3 +121,21 @@ jobs:
           echo "No issue with 'qa/done' and 'qa/no-code-change' labels"
         env:
           LABELS: ${{ needs.fetch-labels.outputs.LABELS}}
+  agenttelemetry-list-change-ack-check:
+    if: github.triggering_actor != 'dd-devflow[bot]'
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@0ad4b8fadaa221de15dcec353f45205ec38ea70b # v4.1.4
+        with:
+          ref: ${{ github.head_ref }}
+      - name: Setup python
+        uses: actions/setup-python@f677139bbe7f9c59b41e40162b753c062f5d49a3 # v5.2.0
+        with:
+          python-version: 3.11
+          cache: 'pip'
+          cache-dependency-path: '**/requirements*.txt'
+      - name: Install dependencies
+        run: pip install -r requirements.txt -r tasks/requirements.txt
+      - name: Check agent telemetry metric list
+        run: inv -e github.agenttelemetry-list-change-ack-check --pr-id=${{ github.event.pull_request.number }}

.gitlab-ci.yml

@@ -172,49 +172,49 @@ variables:
   # To use images from datadog-agent-buildimages dev branches, set the corresponding
   # SUFFIX variable to _test_only
   DATADOG_AGENT_BUILDIMAGES_SUFFIX: ""
-  DATADOG_AGENT_BUILDIMAGES: v48262719-bfb00f80
+  DATADOG_AGENT_BUILDIMAGES: v48372186-ff395e52
   DATADOG_AGENT_WINBUILDIMAGES_SUFFIX: ""
-  DATADOG_AGENT_WINBUILDIMAGES: v48262719-bfb00f80
+  DATADOG_AGENT_WINBUILDIMAGES: v48372186-ff395e52
   DATADOG_AGENT_ARMBUILDIMAGES_SUFFIX: ""
-  DATADOG_AGENT_ARMBUILDIMAGES: v48262719-bfb00f80
+  DATADOG_AGENT_ARMBUILDIMAGES: v48372186-ff395e52
   DATADOG_AGENT_SYSPROBE_BUILDIMAGES_SUFFIX: ""
-  DATADOG_AGENT_SYSPROBE_BUILDIMAGES: v48262719-bfb00f80
+  DATADOG_AGENT_SYSPROBE_BUILDIMAGES: v48372186-ff395e52
   DATADOG_AGENT_BTF_GEN_BUILDIMAGES_SUFFIX: ""
-  DATADOG_AGENT_BTF_GEN_BUILDIMAGES: v48262719-bfb00f80
+  DATADOG_AGENT_BTF_GEN_BUILDIMAGES: v48372186-ff395e52
   # New images to enable different version per image - not used yet
-  CI_IMAGE_BTF_GEN: v48262719-bfb00f80
+  CI_IMAGE_BTF_GEN: v48372186-ff395e52
   CI_IMAGE_BTF_GEN_SUFFIX: ""
-  CI_IMAGE_DEB_X64: v48262719-bfb00f80
+  CI_IMAGE_DEB_X64: v48372186-ff395e52
   CI_IMAGE_DEB_X64_SUFFIX: ""
-  CI_IMAGE_DEB_ARM64: v48262719-bfb00f80
+  CI_IMAGE_DEB_ARM64: v48372186-ff395e52
   CI_IMAGE_DEB_ARM64_SUFFIX: ""
-  CI_IMAGE_DEB_ARMHF: v48262719-bfb00f80
+  CI_IMAGE_DEB_ARMHF: v48372186-ff395e52
   CI_IMAGE_DEB_ARMHF_SUFFIX: ""
-  CI_IMAGE_DD_AGENT_TESTING: v48262719-bfb00f80
+  CI_IMAGE_DD_AGENT_TESTING: v48372186-ff395e52
   CI_IMAGE_DD_AGENT_TESTING_SUFFIX: ""
-  CI_IMAGE_DOCKER_X64: v48262719-bfb00f80
+  CI_IMAGE_DOCKER_X64: v48372186-ff395e52
   CI_IMAGE_DOCKER_X64_SUFFIX: ""
-  CI_IMAGE_DOCKER_ARM64: v48262719-bfb00f80
+  CI_IMAGE_DOCKER_ARM64: v48372186-ff395e52
   CI_IMAGE_DOCKER_ARM64_SUFFIX: ""
-  CI_IMAGE_GITLAB_AGENT_DEPLOY: v48262719-bfb00f80
+  CI_IMAGE_GITLAB_AGENT_DEPLOY: v48372186-ff395e52
   CI_IMAGE_GITLAB_AGENT_DEPLOY_SUFFIX: ""
-  CI_IMAGE_LINUX_GLIBC_2_17_X64: v48262719-bfb00f80
+  CI_IMAGE_LINUX_GLIBC_2_17_X64: v48372186-ff395e52
   CI_IMAGE_LINUX_GLIBC_2_17_X64_SUFFIX: ""
-  CI_IMAGE_LINUX_GLIBC_2_23_ARM64: v48262719-bfb00f80
+  CI_IMAGE_LINUX_GLIBC_2_23_ARM64: v48372186-ff395e52
   CI_IMAGE_LINUX_GLIBC_2_23_ARM64_SUFFIX: ""
-  CI_IMAGE_SYSTEM_PROBE_X64: v48262719-bfb00f80
+  CI_IMAGE_SYSTEM_PROBE_X64: v48372186-ff395e52
   CI_IMAGE_SYSTEM_PROBE_X64_SUFFIX: ""
-  CI_IMAGE_SYSTEM_PROBE_ARM64: v48262719-bfb00f80
+  CI_IMAGE_SYSTEM_PROBE_ARM64: v48372186-ff395e52
   CI_IMAGE_SYSTEM_PROBE_ARM64_SUFFIX: ""
-  CI_IMAGE_RPM_X64: v48262719-bfb00f80
+  CI_IMAGE_RPM_X64: v48372186-ff395e52
   CI_IMAGE_RPM_X64_SUFFIX: ""
-  CI_IMAGE_RPM_ARM64: v48262719-bfb00f80
+  CI_IMAGE_RPM_ARM64: v48372186-ff395e52
   CI_IMAGE_RPM_ARM64_SUFFIX: ""
-  CI_IMAGE_RPM_ARMHF: v48262719-bfb00f80
+  CI_IMAGE_RPM_ARMHF: v48372186-ff395e52
   CI_IMAGE_RPM_ARMHF_SUFFIX: ""
-  CI_IMAGE_WIN_1809_X64: v48262719-bfb00f80
+  CI_IMAGE_WIN_1809_X64: v48372186-ff395e52
   CI_IMAGE_WIN_1809_X64_SUFFIX: ""
-  CI_IMAGE_WIN_LTSC2022_X64: v48262719-bfb00f80
+  CI_IMAGE_WIN_LTSC2022_X64: v48372186-ff395e52
   CI_IMAGE_WIN_LTSC2022_X64_SUFFIX: ""
 
   DATADOG_AGENT_EMBEDDED_PATH: /opt/datadog-agent/embedded
@@ -746,6 +746,7 @@ workflow:
   - .gitlab/kernel_matrix_testing/security_agent.yml
   - .gitlab/kernel_matrix_testing/common.yml
   - .gitlab/source_test/ebpf.yml
+  - test/new-e2e/tests/cws/**/*
   - test/new-e2e/system-probe/**/*
   - test/new-e2e/scenarios/system-probe/**/*
   - test/new-e2e/pkg/runner/**/*
@@ -965,9 +966,7 @@ workflow:
 .on_cws_or_e2e_changes:
   - !reference [.on_e2e_main_release_or_rc]
   - changes:
-      paths:
-        # TODO: Add paths that should trigger tests for CWS
-        - test/new-e2e/tests/cws/**/*
+      paths: *security_agent_change_paths
       compare_to: main # TODO: use a variable, when this is supported https://gitlab.com/gitlab-org/gitlab/-/issues/369916
 
 .on_process_or_e2e_changes:

.gitlab/JOBOWNERS

@@ -132,6 +132,7 @@ trigger_auto_staging_release           @DataDog/agent-delivery
 integration_tests_windows*    @DataDog/windows-agent
 integration_tests_otel        @DataDog/opentelemetry
 docker_image_build_otel       @DataDog/opentelemetry
+ddflare_extension_ocb_build   @DataDog/opentelemetry
 agent_integration_tests       @DataDog/container-integrations
 docker_integration_tests      @DataDog/container-integrations
 

.gitlab/benchmarks/benchmarks.yml

@@ -1,6 +1,6 @@
 benchmark:
   stage: benchmarks
-  # This base image is created here: https://gitlab.ddbuild.io/DataDog/apm-reliability/relenv-microbenchmarking-platform
+  # This base image is created here: https://gitlab.ddbuild.io/DataDog/apm-reliability/benchmarking-platform
   image: 486234852809.dkr.ecr.us-east-1.amazonaws.com/ci/benchmarking-platform:trace-agent_microbenchmarks
   timeout: 1h
   rules:

.gitlab/binary_build/system_probe.yml

@@ -12,7 +12,7 @@
     - find "$CI_BUILDS_DIR" ! -path '*DataDog/datadog-agent*' -delete || true  # Allow failure, we can't remove parent folders of datadog-agent
   script:
     - inv check-go-version
-    - inv -e system-probe.build --strip-object-files --no-bundle
+    - inv -e system-probe.build --strip-object-files
     # fail if references to glibc >= 2.18
     - objdump -p $CI_PROJECT_DIR/$SYSTEM_PROBE_BINARIES_DIR/system-probe | egrep 'GLIBC_2\.(1[8-9]|[2-9][0-9])' && exit 1
     - inv -e system-probe.save-build-outputs $CI_PROJECT_DIR/sysprobe-build-outputs.tar.xz

.gitlab/common/container_publish_job_templates.yml

@@ -6,7 +6,7 @@
   SRC_CWS_INSTRUMENTATION: registry.ddbuild.io/ci/datadog-agent/cws-instrumentation
 
 .docker_publish_job_definition:
-  image: registry.ddbuild.io/ci/datadog-agent-buildimages/deb_x64$DATADOG_AGENT_BUILDIMAGES_SUFFIX:$DATADOG_AGENT_BUILDIMAGES
+  image: registry.ddbuild.io/ci/datadog-agent-buildimages/docker_x64$DATADOG_AGENT_BUILDIMAGES_SUFFIX:$DATADOG_AGENT_BUILDIMAGES
   tags: ["arch:amd64"]
   variables:
     <<: *docker_variables

.gitlab/common/macos.yml

@@ -33,6 +33,11 @@
     fi
   - pyenv activate $VENV_NAME
 
+.vault_login:
+  # Point the CLI to our internal vault
+  - export VAULT_ADDR=https://vault.us1.ddbuild.io
+  - vault login -method=aws -no-print
+
 .macos_gitlab:
   before_script:
     # Selecting the current Go version

.gitlab/common/test_infra_version.yml

@@ -4,4 +4,4 @@ variables:
   # and check the job creating the image to make sure you have the right SHA prefix
   TEST_INFRA_DEFINITIONS_BUILDIMAGES_SUFFIX: ""
   # Make sure to update test-infra-definitions version in go.mod as well
-  TEST_INFRA_DEFINITIONS_BUILDIMAGES: 0269b182e897
+  TEST_INFRA_DEFINITIONS_BUILDIMAGES: b436617374bf