Merge branch 'main' into fix-nil-popinter-serverless-init

.circleci/config.yml

@@ -15,7 +15,7 @@ experimental:
 templates:
   job_template: &job_template
     docker:
-      - image: gcr.io/datadoghq/agent-circleci-runner:v48372186-ff395e52
+      - image: gcr.io/datadoghq/agent-circleci-runner:v50263243-1a30c934
         environment:
           USE_SYSTEM_LIBS: "1"
     working_directory: /go/src/github.com/DataDog/datadog-agent

.custom-gcl.yml

@@ -1,4 +1,4 @@
-version: v1.59.1
+version: v1.60.3
 
 name: golangci-lint
 

.gitattributes

@@ -10,7 +10,7 @@ go.sum -diff -merge linguist-generated=true
 pkg/security/probe/constantfetch/btfhub/constants.json -diff -merge linguist-generated=true
 pkg/security/seclwin/** -diff -merge linguist-generated=true
 # Fixtures should have LF line endings because they are checked against OCI packages built on Linux
-pkg/fleet/internal/fixtures/** text=auto eol=lf
+pkg/fleet/installer/fixtures/** text=auto eol=lf
 
 # Fix `git diff` when running on the below file formats.
 # Our windows build image uses MinGit which tries to use the astextplain diff algorithm (https://git-scm.com/docs/gitattributes#_setting_the_internal_diff_algorithm).
@@ -27,26 +27,25 @@ pkg/fleet/internal/fixtures/** text=auto eol=lf
 #	textconv = astextplain
 # ```
 
-
-*.doc	diff
-*.DOC	diff
-*.docx	diff
-*.DOCX	diff
-*.docm	diff
-*.DOCM	diff
-*.dot	diff
-*.DOT	diff
-*.dotx	diff
-*.DOTX	diff
-*.dotm	diff
-*.DOTM	diff
-*.pdf	diff
-*.PDF	diff
-*.rtf	diff
-*.RTF	diff
-*.ods	diff
-*.ODS	diff
-*.odf	diff
-*.ODF	diff
-*.odt	diff
-*.ODT	diff
+*.doc diff
+*.DOC diff
+*.docx diff
+*.DOCX diff
+*.docm diff
+*.DOCM diff
+*.dot diff
+*.DOT diff
+*.dotx diff
+*.DOTX diff
+*.dotm diff
+*.DOTM diff
+*.pdf diff
+*.PDF diff
+*.rtf diff
+*.RTF diff
+*.ods diff
+*.ODS diff
+*.odf diff
+*.ODF diff
+*.odt diff
+*.ODT diff

.github/CODEOWNERS

@@ -25,7 +25,7 @@
 
 /CHANGELOG.rst                          @DataDog/agent-delivery
 /CHANGELOG-DCA.rst                      @DataDog/container-integrations @DataDog/container-platform
-/CHANGELOG-INSTALLSCRIPT.rst            @DataDog/agent-delivery
+/CHANGELOG-INSTALLSCRIPT.rst            @DataDog/agent-delivery @DataDog/container-ecosystems
 
 /*.md                                   @DataDog/agent-devx-infra @DataDog/documentation
 /NOTICE                                 @DataDog/agent-delivery @DataDog/documentation
@@ -36,7 +36,7 @@
 /Makefile.trace                         @DataDog/agent-delivery
 
 /mkdocs.yml                             @DataDog/agent-devx-infra
-/release.json                           @DataDog/agent-delivery @DataDog/agent-metrics-logs @DataDog/windows-kernel-integrations @DataDog/agent-release-management @DataDog/agent-security
+/release.json                           @DataDog/agent-delivery @DataDog/agent-metrics-logs @DataDog/windows-kernel-integrations @DataDog/agent-security
 /requirements.txt                       @DataDog/agent-devx-infra
 /pyproject.toml                         @DataDog/agent-devx-infra @DataDog/agent-devx-loops
 /repository.datadog.yml                 @DataDog/agent-devx-infra
@@ -45,6 +45,8 @@
 /static-analysis.datadog.yml            @DataDog/software-integrity-and-trust @DataDog/agent-devx-infra
 
 /modules.yml                            @DataDog/agent-shared-components
+# if go.work changes then either .go-version or modules.yml changed too, so ASC might as well own it
+/go.work                                @DataDog/agent-shared-components
 
 /.circleci/                             @DataDog/agent-devx-infra
 
@@ -73,7 +75,8 @@
 /.gitlab/deploy*/*                                   @DataDog/agent-delivery
 /.gitlab/deps_fetch/*                                @DataDog/agent-devx-infra
 /.gitlab/e2e/*                                       @DataDog/agent-devx-infra @DataDog/agent-devx-loops
-/.gitlab/e2e_install_packages/*                      @DataDog/agent-delivery
+/.gitlab/e2e_testing_deploy/*                        @DataDog/agent-devx-loops @DataDog/agent-devx-infra
+/.gitlab/e2e_install_packages/*                      @DataDog/agent-delivery @DataDog/container-ecosystems
 /.gitlab/e2e_pre_test/*                              @DataDog/agent-devx-infra @DataDog/agent-devx-loops
 /.gitlab/kernel_matrix_testing/*                     @DataDog/agent-devx-infra @DataDog/ebpf-platform
 /.gitlab/lint/*                                      @DataDog/agent-devx-infra
@@ -88,8 +91,7 @@
 /.gitlab/binary_build/include.yml                    @DataDog/agent-devx-infra
 /.gitlab/binary_build/linux.yml                      @DataDog/agent-devx-infra @DataDog/agent-delivery
 /.gitlab/functional_test/include.yml                 @DataDog/agent-devx-infra
-/.gitlab/functional_test_cleanup/functional_test_cleanup.yml        @DataDog/agent-devx-infra @DataDog/agent-security
-/.gitlab/install_script_testing/install_script_testing.yml          @DataDog/agent-delivery
+/.gitlab/install_script_testing/install_script_testing.yml          @DataDog/agent-delivery @DataDog/container-ecosystems
 /.gitlab/integration_test/dogstatsd.yml              @DataDog/agent-devx-infra @DataDog/agent-metrics-logs
 /.gitlab/integration_test/include.yml                @DataDog/agent-devx-infra
 /.gitlab/integration_test/linux.yml                  @DataDog/agent-devx-infra
@@ -129,20 +131,16 @@
 
 /.gitlab/deps_build/                                 @DataDog/ebpf-platform @DataDog/agent-delivery @DataDog/windows-agent
 
-/.gitlab/kitchen_*                                   @DataDog/container-ecosystems @DataDog/agent-delivery
-/.gitlab/kitchen_testing/windows.yml                 @DataDog/container-ecosystems @DataDog/agent-delivery @DataDog/windows-agent
-/.gitlab/e2e_install_packages/windows.yml            @DataDog/container-ecosystems @DataDog/agent-delivery @DataDog/windows-agent
+/.gitlab/e2e_install_packages/windows.yml            @DataDog/agent-delivery @DataDog/container-ecosystems @DataDog/windows-agent
 
 /.gitlab/common/                                     @DataDog/agent-devx-infra
 /.gitlab/common/test_infra_version.yml               @DataDog/agent-devx-loops @DataDog/agent-devx-infra
 
 /.gitlab/e2e/e2e.yml                                 @DataDog/container-integrations @DataDog/agent-devx-loops
-/.gitlab/e2e_install_packages                        @DataDog/container-ecosystems @DataDog/agent-delivery
 /.gitlab/container_build/fakeintake.yml              @DataDog/agent-e2e-testing @DataDog/agent-devx-loops
 /.gitlab/binary_build/fakeintake.yml                 @DataDog/agent-e2e-testing @DataDog/agent-devx-loops
 
 /.gitlab/functional_test/serverless.yml              @DataDog/serverless @Datadog/serverless-aws @DataDog/agent-devx-infra
-/.gitlab/functional_test_cleanup                     @DataDog/agent-security @DataDog/windows-kernel-integrations @DataDog/agent-devx-infra
 /.gitlab/functional_test/oracle.yml                  @DataDog/agent-devx-infra @DataDog/database-monitoring
 
 /.gitlab/powershell_script_deploy                    @DataDog/agent-delivery @DataDog/windows-agent
@@ -241,6 +239,7 @@
 /cmd/systray/                           @DataDog/windows-agent
 /cmd/security-agent/                    @DataDog/agent-security
 /cmd/installer/                         @DataDog/fleet @DataDog/windows-agent
+/cmd/installer-downloader/              @DataDog/fleet
 
 /dev/                                   @DataDog/agent-devx-loops
 /devenv/                                @DataDog/agent-devx-loops
@@ -328,6 +327,9 @@
 # Additional notification to  @iglendd about Agent Telemetry changes for optional approval and governance acknowledgement
 /comp/core/agenttelemetry               @DataDog/agent-shared-components @iglendd
 
+# trace-agent logging implementation should also notify agent-apm
+/comp/core/log/impl-trace @DataDog/agent-apm
+
 # pkg
 /pkg/                                   @DataDog/agent-shared-components
 /pkg/api/                               @DataDog/agent-shared-components
@@ -428,7 +430,6 @@
 /pkg/flare/*_win.go                     @Datadog/windows-agent
 /pkg/flare/*_windows.go                 @Datadog/windows-agent
 /pkg/flare/*_windows_test.go            @Datadog/windows-agent
-/pkg/flare/sysprobe                     @DataDog/Networks @Datadog/windows-agent @DataDog/agent-security @DataDog/universal-service-monitoring @DataDog/ebpf-platform
 /pkg/fleet/                             @DataDog/fleet @DataDog/windows-agent
 /pkg/pidfile/                           @DataDog/agent-shared-components
 /pkg/persistentcache/                   @DataDog/agent-metrics-logs
@@ -450,6 +451,7 @@
 /pkg/util/docker/                       @DataDog/container-integrations
 /pkg/util/ecs/                          @DataDog/container-integrations
 /pkg/util/funcs/                        @DataDog/ebpf-platform
+/pkg/util/gpu/                          @DataDog/container-platform
 /pkg/util/kernel/                       @DataDog/ebpf-platform
 /pkg/util/safeelf/                      @DataDog/ebpf-platform
 /pkg/util/ktime                         @DataDog/agent-security
@@ -465,6 +467,7 @@
 /pkg/util/pdhutil/                      @DataDog/windows-agent
 /pkg/util/winutil/                      @DataDog/windows-agent
 /pkg/util/testutil/flake                @DataDog/agent-devx-loops
+/pkg/util/testutil/patternscanner.go    @DataDog/universal-service-monitoring @DataDog/ebpf-platform
 /pkg/util/testutil/docker               @DataDog/universal-service-monitoring @DataDog/ebpf-platform
 /pkg/util/trie                          @DataDog/container-integrations
 /pkg/languagedetection                  @DataDog/processes @DataDog/universal-service-monitoring
@@ -566,6 +569,7 @@
 /tasks/sds.py                           @DataDog/agent-processing-and-routing
 /tasks/systray.py                       @DataDog/windows-agent
 /tasks/winbuildscripts/                 @DataDog/windows-agent
+/tasks/winbuild.py                      @DataDog/windows-agent
 /tasks/windows_resources.py             @DataDog/windows-agent
 /tasks/components.py                    @DataDog/agent-shared-components
 /tasks/components_templates             @DataDog/agent-shared-components
@@ -584,18 +588,6 @@
 /test/integration/docker/otel_agent_build_tests.py @DataDog/opentelemetry
 /test/integration/serverless            @DataDog/serverless @Datadog/serverless-aws
 /test/integration/serverless_perf       @DataDog/serverless @Datadog/serverless-aws
-/test/kitchen/                          @DataDog/agent-devx-loops
-/test/kitchen/test-definitions/         @DataDog/container-ecosystems @DataDog/agent-delivery
-/test/kitchen/test/integration/         @DataDog/container-ecosystems @DataDog/agent-delivery
-/test/kitchen/site-cookbooks/dd-system-probe-check/ @DataDog/windows-kernel-integrations
-/test/kitchen/test/integration/win-all-subservices/ @DataDog/windows-agent
-/test/kitchen/test/integration/win-alt-dir/ @DataDog/windows-agent
-/test/kitchen/test/integration/win-install-fail/ @DataDog/windows-agent
-/test/kitchen/test/integration/win-installopts/ @DataDog/windows-agent
-/test/kitchen/test/integration/win-no-subservices/ @DataDog/windows-agent
-/test/kitchen/test/integration/win-sysprobe-test/ @DataDog/windows-kernel-integrations
-/test/kitchen/test/integration/win-repair/ @DataDog/windows-agent
-/test/kitchen/test/integration/win-user/ @DataDog/windows-agent
 /test/fakeintake/                             @DataDog/agent-e2e-testing @DataDog/agent-devx-loops
 /test/fakeintake/aggregator/ndmflowAggregator.go @DataDog/ndm-integrations
 /test/fakeintake/aggregator/ndmflowAggregator_test.go @DataDog/ndm-integrations
@@ -610,6 +602,7 @@
 /test/new-e2e/tests/agent-subcommands         @DataDog/agent-shared-components
 /test/new-e2e/tests/containers                @DataDog/container-integrations @DataDog/container-platform
 /test/new-e2e/tests/discovery                 @DataDog/universal-service-monitoring
+/test/new-e2e/tests/ha-agent                  @DataDog/ndm-core
 /test/new-e2e/tests/language-detection        @DataDog/processes
 /test/new-e2e/tests/ndm                       @DataDog/ndm-core
 /test/new-e2e/tests/ndm/netflow               @DataDog/ndm-integrations

.github/PULL_REQUEST_TEMPLATE.md

@@ -12,7 +12,17 @@
 
 ### Motivation
 
-### Describe how to test/QA your changes
+### Describe how you validated your changes
+<!--
+Validate your changes before merge, ensuring that:
+* Your PR is tested by static / unit / integrations / e2e tests
+* Your PR description details which e2e tests cover your changes, if any
+* The PR description contains details of how you validated your changes. If you validated changes manually and not through automated tests, add context on why automated tests did not fit your changes validation.
+
+If you want additional validation by a second person, you can ask reviewers to do it. Describe how to set up an environment for manual tests in the PR description. Manual validation is expected to happen on every commit before merge.
+
+Any manual validation step should then map to an automated test. Manual validation should not substitute automation, minus exceptions not supported by test tooling yet.
+-->
 
 ### Possible Drawbacks / Trade-offs
 

.github/workflows/assign_issue.yml

@@ -0,0 +1,31 @@
+---
+name: "Assign issue to a team"
+
+on:
+  issues:
+    types: [opened, reopened]
+
+jobs:
+  auto_assign_issue:
+    runs-on: ubuntu-latest
+    container: 
+      image: ghcr.io/datadog/agent-issue-auto-assign:latest
+      credentials:
+        username: ${{ github.actor }}
+        password: ${{ secrets.GITHUB_TOKEN }}
+    permissions:
+        packages: read
+        issues: write
+    steps:
+    - name: Checkout repository
+      uses: actions/checkout@0ad4b8fadaa221de15dcec353f45205ec38ea70b # v4.1.4
+    - name: Install dependencies
+      # Dependencies are installed at runtime. Otherwise it would create a huge image see https://hub.docker.com/r/pytorch/pytorch/tags
+      run: |
+        pip install --upgrade pip && pip install --no-compile --no-cache-dir torch transformers invoke codeowners slack-sdk PyGithub python-gitlab semver
+    - name: Assign issue
+      env:
+        GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        SLACK_API_TOKEN : ${{ secrets.SLACK_DATADOG_AGENT_BOT_TOKEN }}
+      run: |
+        inv -e issue.assign-owner -i ${{ github.event.issue.number }}

.github/workflows/codeql-analysis.yml

@@ -45,7 +45,7 @@ jobs:
           go-version-file: ".go-version"
 
       - name: Initialize CodeQL
-        uses: github/codeql-action/init@4f3212b61783c3c68e8309a0f18a699764811cda # v3.27.1
+        uses: github/codeql-action/init@f09c1c0a94de965c15400f5634aa42fac8fb8f88 # v3.27.5
         with:
           languages: ${{ matrix.language }}
           config: |
@@ -67,4 +67,4 @@ jobs:
           invoke agent.build --build-exclude=systemd
 
       - name: Perform CodeQL Analysis
-        uses: github/codeql-action/analyze@4f3212b61783c3c68e8309a0f18a699764811cda # v3.27.1
+        uses: github/codeql-action/analyze@f09c1c0a94de965c15400f5634aa42fac8fb8f88 # v3.27.5

.github/workflows/create_rc_pr.yml

@@ -62,8 +62,6 @@ jobs:
             - name: Checkout release branch
               uses: actions/checkout@0ad4b8fadaa221de15dcec353f45205ec38ea70b # v4.1.4
               with:
-                ref: ${{ matrix.value }}
-                fetch-depth: 0
                 persist-credentials: true
 
             - name: Install python
@@ -96,8 +94,10 @@ jobs:
 
             - name: Create RC PR
               if: ${{ steps.check_for_changes.outputs.CHANGES == 'true'}}
+              env:
+                MATRIX: ${{ matrix.value }}
               run: |
                 git config user.name "github-actions[bot]"
                 git config user.email "41898282+github-actions[bot]@users.noreply.github.com"
                 git fetch
-                inv -e release.create-rc --slack-webhook=${{ secrets.AGENT_RELEASE_SYNC_SLACK_WEBHOOK }}
+                inv -e release.create-rc "$MATRIX" --slack-webhook=${{ secrets.AGENT_RELEASE_SYNC_SLACK_WEBHOOK }}

.github/workflows/gohai.yml

@@ -7,9 +7,11 @@ on:
       - main
       - "[0-9]+.[0-9]+.x"
     paths:
+      - ".github/workflows/gohai.yml"
       - "pkg/gohai/**"
   pull_request:
     paths:
+      - ".github/workflows/gohai.yml"
       - "pkg/gohai/**"
 
 permissions: {}
@@ -23,7 +25,7 @@ jobs:
           [
             ubuntu-20.04,
             ubuntu-latest,
-            macos-12,
+            macos-13,
             macos-latest,
             windows-2019,
             windows-latest,

.github/workflows/label-analysis.yml

@@ -102,25 +102,24 @@ jobs:
     if: github.triggering_actor != 'dd-devflow[bot]'
     runs-on: ubuntu-latest
     steps:
+      - name: Checkout repository
+        uses: actions/checkout@0ad4b8fadaa221de15dcec353f45205ec38ea70b # v4.1.4
+        with:
+          persist-credentials: false
+      - name: Setup Python3
+        uses: actions/setup-python@0b93645e9fea7318ecaed2b359559ac225c90a2b # v5.3.0
+        with:
+          python-version: "3.12.6"
+          cache: "pip"
+          cache-dependency-path: '**/requirements*.txt'
+      - name: Install python dependencies
+        run: pip3 install -r requirements.txt
       - name: Check qa/[done|no-code-change] labels are not set together
-        run: |
-          is_qa_done=1
-          is_qa_no_code_change=1
-          for label in $LABELS; do
-            if [[ "$label" == "qa/done" ]]; then
-              is_qa_done=0
-            fi
-            if [[ "$label" == "qa/no-code-change" ]]; then
-              is_qa_no_code_change=0
-            fi
-          done
-          if [ $is_qa_done -eq 0 ] && [ $is_qa_no_code_change -eq 0 ]; then
-            echo "Both 'qa/done' and 'qa/no-code-change' labels are set -- only one of them should be set"
-            exit 1
-          fi
-          echo "No issue with 'qa/done' and 'qa/no-code-change' labels"
         env:
           LABELS: ${{ needs.fetch-labels.outputs.LABELS}}
+        run: |
+          inv -e github.check-qa-labels --labels "${LABELS[@]}"
+
   agenttelemetry-list-change-ack-check:
     if: github.triggering_actor != 'dd-devflow[bot]'
     runs-on: ubuntu-latest

.github/workflows/report-merged-pr.yml

@@ -4,6 +4,8 @@ name: Report Merged PR
 
 on:
   pull_request:
+    branches:
+      - main
     types: [closed]
 
 permissions: {}

.github/workflows/serverless-integration.yml

@@ -9,6 +9,7 @@ on:
       - 'pkg/serverless/**'
       - 'test/integration/serverless/**'
       - '.github/workflows/serverless-integration.yml'
+      - 'go.mod'
   schedule:
     - cron: '0 14 * * *' # cron schedule uses UTC timezone. Run tests at the beginning of the day in US-East
 
@@ -67,6 +68,7 @@ jobs:
         env:
           AWS_ACCESS_KEY_ID: ${{ secrets.SERVERLESS_AWS_ACCESS_KEY_ID }}
           AWS_SECRET_ACCESS_KEY: ${{ secrets.SERVERLESS_AWS_SECRET_ACCESS_KEY }}
+          GOWORK: off
         with:
           timeout_minutes: 60
           max_attempts: 2