Merge branch 'main' into nicholas.hulston/s3-span-pointers

DataDog · Dec 11, 2024 · 6d3d800 · 6d3d800
2 parents 20c1a10 + c7f3db3
commit 6d3d800
Show file tree

Hide file tree

Showing 691 changed files with 12,389 additions and 6,046 deletions.
diff --git a/.ddqa/config.toml b/.ddqa/config.toml
@@ -48,7 +48,7 @@ jira_issue_type = "Task"
 jira_statuses = ["To Do", "In Progress", "Done"]
 github_team = "agent-delivery"
 github_labels = ["team/agent-delivery"]
-exclude_members = ["hithwen"]
+exclude_members = ["KSerrania"]
 
 [teams."Universal Service Monitoring"]
 jira_project = "USMON"

diff --git a/.gitattributes b/.gitattributes
@@ -10,7 +10,7 @@ go.sum -diff -merge linguist-generated=true
 pkg/security/probe/constantfetch/btfhub/constants.json -diff -merge linguist-generated=true
 pkg/security/seclwin/** -diff -merge linguist-generated=true
 # Fixtures should have LF line endings because they are checked against OCI packages built on Linux
-pkg/fleet/internal/fixtures/** text=auto eol=lf
+pkg/fleet/installer/fixtures/** text=auto eol=lf
 
 # Fix `git diff` when running on the below file formats.
 # Our windows build image uses MinGit which tries to use the astextplain diff algorithm (https://git-scm.com/docs/gitattributes#_setting_the_internal_diff_algorithm).
@@ -27,26 +27,25 @@ pkg/fleet/internal/fixtures/** text=auto eol=lf
 #	textconv = astextplain
 # ```
 
-
-*.doc	diff
-*.DOC	diff
-*.docx	diff
-*.DOCX	diff
-*.docm	diff
-*.DOCM	diff
-*.dot	diff
-*.DOT	diff
-*.dotx	diff
-*.DOTX	diff
-*.dotm	diff
-*.DOTM	diff
-*.pdf	diff
-*.PDF	diff
-*.rtf	diff
-*.RTF	diff
-*.ods	diff
-*.ODS	diff
-*.odf	diff
-*.ODF	diff
-*.odt	diff
-*.ODT	diff
+*.doc diff
+*.DOC diff
+*.docx diff
+*.DOCX diff
+*.docm diff
+*.DOCM diff
+*.dot diff
+*.DOT diff
+*.dotx diff
+*.DOTX diff
+*.dotm diff
+*.DOTM diff
+*.pdf diff
+*.PDF diff
+*.rtf diff
+*.RTF diff
+*.ods diff
+*.ODS diff
+*.odf diff
+*.ODF diff
+*.odt diff
+*.ODT diff
diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS
@@ -25,7 +25,7 @@
 
 /CHANGELOG.rst                          @DataDog/agent-delivery
 /CHANGELOG-DCA.rst                      @DataDog/container-integrations @DataDog/container-platform
-/CHANGELOG-INSTALLSCRIPT.rst            @DataDog/agent-delivery
+/CHANGELOG-INSTALLSCRIPT.rst            @DataDog/agent-delivery @DataDog/container-ecosystems
 
 /*.md                                   @DataDog/agent-devx-infra @DataDog/documentation
 /NOTICE                                 @DataDog/agent-delivery @DataDog/documentation
@@ -50,19 +50,20 @@
 
 /.circleci/                             @DataDog/agent-devx-infra
 
-/.github/CODEOWNERS                                 # do not notify anyone
-/.github/*_TEMPLATE.md                              @DataDog/agent-devx-loops
-/.github/dependabot.yaml                            @DataDog/agent-devx-infra
-/.github/workflows/serverless-benchmarks.yml        @DataDog/serverless @Datadog/serverless-aws
-/.github/workflows/serverless-binary-size.yml       @DataDog/serverless @Datadog/serverless-aws
-/.github/workflows/serverless-integration.yml       @DataDog/serverless @Datadog/serverless-aws
-/.github/workflows/cws-btfhub-sync.yml              @DataDog/agent-security
-/.github/workflows/gohai.yml                        @DataDog/agent-shared-components
-/.github/workflows/go-update-commenter.yml          @DataDog/agent-shared-components
-/.github/workflows/buildimages-update.yml           @DataDog/agent-delivery @DataDog/agent-shared-components
-
-/.run                                               @DataDog/agent-devx-loops
-/.run/docker/                                       @DataDog/container-integrations @DataDog/container-platform
+/.github/CODEOWNERS                                  # do not notify anyone
+/.github/*_TEMPLATE.md                               @DataDog/agent-devx-loops
+/.github/dependabot.yaml                             @DataDog/agent-devx-infra
+/.github/workflows/serverless-benchmarks.yml         @DataDog/serverless @Datadog/serverless-aws
+/.github/workflows/serverless-binary-size.yml        @DataDog/serverless @Datadog/serverless-aws
+/.github/workflows/serverless-integration.yml        @DataDog/serverless @Datadog/serverless-aws
+/.github/workflows/cws-btfhub-sync.yml               @DataDog/agent-security
+/.github/workflows/gohai.yml                         @DataDog/agent-shared-components
+/.github/workflows/go-update-commenter.yml           @DataDog/agent-shared-components
+/.github/workflows/buildimages-update.yml            @DataDog/agent-delivery @DataDog/agent-shared-components
+/.github/workflows/collector-generate-and-update.yml @DataDog/opentelemetry
+
+/.run                                                @DataDog/agent-devx-loops
+/.run/docker/                                        @DataDog/container-integrations @DataDog/container-platform
 
 # Gitlab files
 # Files containing job contents are owned by teams in charge of the jobs + agent-devx-infra or agent-delivery
@@ -76,7 +77,7 @@
 /.gitlab/deps_fetch/*                                @DataDog/agent-devx-infra
 /.gitlab/e2e/*                                       @DataDog/agent-devx-infra @DataDog/agent-devx-loops
 /.gitlab/e2e_testing_deploy/*                        @DataDog/agent-devx-loops @DataDog/agent-devx-infra
-/.gitlab/e2e_install_packages/*                      @DataDog/agent-delivery
+/.gitlab/e2e_install_packages/*                      @DataDog/agent-delivery @DataDog/container-ecosystems
 /.gitlab/e2e_pre_test/*                              @DataDog/agent-devx-infra @DataDog/agent-devx-loops
 /.gitlab/kernel_matrix_testing/*                     @DataDog/agent-devx-infra @DataDog/ebpf-platform
 /.gitlab/lint/*                                      @DataDog/agent-devx-infra
@@ -91,7 +92,7 @@
 /.gitlab/binary_build/include.yml                    @DataDog/agent-devx-infra
 /.gitlab/binary_build/linux.yml                      @DataDog/agent-devx-infra @DataDog/agent-delivery
 /.gitlab/functional_test/include.yml                 @DataDog/agent-devx-infra
-/.gitlab/install_script_testing/install_script_testing.yml          @DataDog/agent-delivery
+/.gitlab/install_script_testing/install_script_testing.yml          @DataDog/agent-delivery @DataDog/container-ecosystems
 /.gitlab/integration_test/dogstatsd.yml              @DataDog/agent-devx-infra @DataDog/agent-metrics-logs
 /.gitlab/integration_test/include.yml                @DataDog/agent-devx-infra
 /.gitlab/integration_test/linux.yml                  @DataDog/agent-devx-infra
@@ -118,7 +119,7 @@
 /.gitlab/binary_build/system_probe.yml               @DataDog/ebpf-platform @DataDog/agent-delivery
 /.gitlab/binary_build/windows.yml                    @DataDog/agent-delivery @DataDog/windows-agent
 
-/.gitlab/benchmarks/                                 @DataDog/agent-devx-infra @DataDog/apm-reliability-and-performance @DataDog/agent-apm
+/.gitlab/benchmarks/                                 @DataDog/agent-devx-infra @DataDog/apm-ecosystems-performance @DataDog/agent-apm
 
 /.gitlab/deploy_containers/                          @DataDog/container-integrations @DataDog/agent-delivery
 /.gitlab/deploy_dca/                                 @DataDog/container-integrations @DataDog/agent-delivery
@@ -131,13 +132,12 @@
 
 /.gitlab/deps_build/                                 @DataDog/ebpf-platform @DataDog/agent-delivery @DataDog/windows-agent
 
-/.gitlab/e2e_install_packages/windows.yml            @DataDog/container-ecosystems @DataDog/agent-delivery @DataDog/windows-agent
+/.gitlab/e2e_install_packages/windows.yml            @DataDog/agent-delivery @DataDog/container-ecosystems @DataDog/windows-agent
 
 /.gitlab/common/                                     @DataDog/agent-devx-infra
 /.gitlab/common/test_infra_version.yml               @DataDog/agent-devx-loops @DataDog/agent-devx-infra
 
 /.gitlab/e2e/e2e.yml                                 @DataDog/container-integrations @DataDog/agent-devx-loops
-/.gitlab/e2e_install_packages                        @DataDog/container-ecosystems @DataDog/agent-delivery
 /.gitlab/container_build/fakeintake.yml              @DataDog/agent-e2e-testing @DataDog/agent-devx-loops
 /.gitlab/binary_build/fakeintake.yml                 @DataDog/agent-e2e-testing @DataDog/agent-devx-loops
 
@@ -193,6 +193,7 @@
 /cmd/agent/subcommands/remoteconfig     @Datadog/remote-config
 /cmd/agent/subcommands/snmp             @DataDog/ndm-core
 /cmd/agent/subcommands/streamlogs       @DataDog/agent-metrics-logs
+/cmd/agent/subcommands/analyzelogs      @DataDog/agent-metrics-logs
 /cmd/agent/subcommands/streamep         @DataDog/container-integrations
 /cmd/agent/subcommands/taggerlist       @DataDog/container-platform
 /cmd/agent/subcommands/workloadlist     @DataDog/container-platform
@@ -570,7 +571,9 @@
 /tasks/sds.py                           @DataDog/agent-processing-and-routing
 /tasks/systray.py                       @DataDog/windows-agent
 /tasks/winbuildscripts/                 @DataDog/windows-agent
+/tasks/winbuild.py                      @DataDog/windows-agent
 /tasks/windows_resources.py             @DataDog/windows-agent
+/tasks/collector.py                     @DataDog/opentelemetry
 /tasks/components.py                    @DataDog/agent-shared-components
 /tasks/components_templates             @DataDog/agent-shared-components
 /tasks/libs/ciproviders/                @DataDog/agent-devx-infra
@@ -606,6 +609,7 @@
 /test/new-e2e/tests/language-detection        @DataDog/processes
 /test/new-e2e/tests/ndm                       @DataDog/ndm-core
 /test/new-e2e/tests/ndm/netflow               @DataDog/ndm-integrations
+/test/new-e2e/tests/netpath                   @DataDog/Networks @DataDog/network-device-monitoring
 /test/new-e2e/tests/npm                       @DataDog/Networks
 /test/new-e2e/tests/npm/ec2_1host_wkit_test.go @DataDog/Networks @DataDog/windows-kernel-integrations
 /test/new-e2e/tests/orchestrator              @DataDog/container-app

diff --git a/.github/dependabot.yaml b/.github/dependabot.yaml
@@ -27,6 +27,9 @@ updates:
       - dependency-name: github.com/ugorji/go
       # Ignore internal modules
       - dependency-name: github.com/DataDog/datadog-agent/*
+      # OpenTelemetry collector packages need to be updated with inv rather than dependabot
+      - dependency-name: go.opentelemetry.io/collector/*
+      - dependency-name: github.com/open-telemetry/opentelemetry-collector-contrib/*
     schedule:
       interval: weekly
     open-pull-requests-limit: 100
@@ -52,6 +55,9 @@ updates:
       - dependency-name: github.com/DataDog/datadog-agent/*
       # See https://github.com/DataDog/datadog-agent/pull/10112
       - dependency-name: github.com/mailru/easyjson
+      # OpenTelemetry collector packages need to be updated with inv rather than dependabot
+      - dependency-name: go.opentelemetry.io/collector/*
+      - dependency-name: github.com/open-telemetry/opentelemetry-collector-contrib/*
     schedule:
       interval: weekly
     open-pull-requests-limit: 100

diff --git a/.github/workflows/assign_issue.yml b/.github/workflows/assign_issue.yml
@@ -26,5 +26,6 @@ jobs:
     - name: Assign issue
       env:
         GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        SLACK_API_TOKEN : ${{ secrets.SLACK_DATADOG_AGENT_BOT_TOKEN }}
       run: |
-        inv -e issue.assign-owner --issue ${{ github.event.issue.number }}
+        inv -e issue.assign-owner -i ${{ github.event.issue.number }}
diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml
@@ -45,7 +45,7 @@ jobs:
           go-version-file: ".go-version"
 
       - name: Initialize CodeQL
-        uses: github/codeql-action/init@f09c1c0a94de965c15400f5634aa42fac8fb8f88 # v3.27.5
+        uses: github/codeql-action/init@aa578102511db1f4524ed59b8cc2bae4f6e88195 # v3.27.6
         with:
           languages: ${{ matrix.language }}
           config: |
@@ -67,4 +67,4 @@ jobs:
           invoke agent.build --build-exclude=systemd
 
       - name: Perform CodeQL Analysis
-        uses: github/codeql-action/analyze@f09c1c0a94de965c15400f5634aa42fac8fb8f88 # v3.27.5
+        uses: github/codeql-action/analyze@aa578102511db1f4524ed59b8cc2bae4f6e88195 # v3.27.6
diff --git a/.github/workflows/collector-generate-and-update.yml b/.github/workflows/collector-generate-and-update.yml
@@ -11,62 +11,30 @@ jobs:
     permissions:
       pull-requests: write
     steps:
-      - name: Set date
-        id: date
-        run: echo "date=$(date +'%Y-%m-%d')" >> $GITHUB_OUTPUT
-
       - name: Checkout repository
         uses: actions/checkout@0ad4b8fadaa221de15dcec353f45205ec38ea70b # v4.1.4
 
       - name: Set up Python
         uses: actions/setup-python@0b93645e9fea7318ecaed2b359559ac225c90a2b # v5.3.0
         with:
-          python-version: '3.12.6'
+          python-version: 3.12
+          cache: 'pip'
+
+      - name: Set up Go
+        uses: actions/setup-go@41dfa10bad2bb2ae585af6ee5bb4d7d973ad74ed # v5.1.0
+        with:
+          go-version: '1.22.8'
 
-      - name: Install dependencies
+      - name: Install Dependencies
         run: |
-          python -m pip install --upgrade pip
+          python3 -m pip install --upgrade pip
           pip install -r requirements.txt
 
-      - name: Run update task
-        env: 
-          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-        run: inv -e collector.update
-
-      - name: Run generate task
+      - name: Run Collector Update Script
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-        run: inv -e collector.generate
-
-      - name: Check for changes
-        id: check_changes
-        run: |
-          git config --global user.name "github-actions[bot]"
-          git config --global user.email "github-actions[bot]@users.noreply.github.com"
-          git add .
-          if git diff-index --quiet HEAD; then
-            echo "No changes detected"
-            echo "changes_detected=false" >> $GITHUB_OUTPUT
-          else
-            echo "Changes detected"
-            echo "changes_detected=true" >> $GITHUB_OUTPUT
-          fi
-
-      - name: Commit changes
-        if: steps.check_changes.outputs.changes_detected == 'true'
-        run: |
-          git switch -c update-otel-collector-dependencies-${{ steps.date.outputs.date }}
-          git commit -m "Update OTel Collector dependencies and generate OTel Agent"
-          git push -u origin update-otel-collector-dependencies-${{ steps.date.outputs.date }}
-
-      - name: Install GitHub CLI
-        if: steps.check_changes.outputs.changes_detected == 'true'
-        run: |
-          sudo apt-get update
-          sudo apt-get install gh
-
-      - name: Create draft pull request
-        if: steps.check_changes.outputs.changes_detected == 'true'
         run: |
-          gh auth login --with-token <<< ${{ secrets.GITHUB_TOKEN }}
-          gh pr create --title "Update OTel collector dependencies" --body "This PR updates the OTel Collector dependencies to the latest version. Please ensure that all tests pass before marking ready for review." --base main --head update-otel-collector-dependencies-${{ steps.date.outputs.date }} --draft
+          inv -e collector.update
+          inv -e collector.generate
+          inv -e generate-licenses
+          inv -e collector.pull-request
diff --git a/.github/workflows/create_rc_pr.yml b/.github/workflows/create_rc_pr.yml
@@ -62,8 +62,6 @@ jobs:
             - name: Checkout release branch
               uses: actions/checkout@0ad4b8fadaa221de15dcec353f45205ec38ea70b # v4.1.4
               with:
-                ref: ${{ matrix.value }}
-                fetch-depth: 0
                 persist-credentials: true
 
             - name: Install python

diff --git a/.github/workflows/cws-btfhub-sync.yml b/.github/workflows/cws-btfhub-sync.yml
@@ -97,7 +97,7 @@ jobs:
           inv -e security-agent.generate-btfhub-constants --archive-path=./dev/dist/archive --output-path=./"$ARTIFACT_NAME".json --force-refresh
 
       - name: Upload artifact
-        uses: actions/upload-artifact@834a144ee995460fba8ed112a2fc961b36a5ec5a # v4.3.6
+        uses: actions/upload-artifact@50769540e7f4bd5e21e526ee35c689e35e0d6874 # v4.4.0
         with:
           name: ${{ steps.artifact-name.outputs.ARTIFACT_NAME }}
           path: ./${{ steps.artifact-name.outputs.ARTIFACT_NAME }}.json

diff --git a/.github/workflows/docs-dev.yml b/.github/workflows/docs-dev.yml
@@ -50,7 +50,7 @@ jobs:
     - name: Build documentation
       run: invoke docs.build
 
-    - uses: actions/upload-artifact@834a144ee995460fba8ed112a2fc961b36a5ec5a # v4.3.6
+    - uses: actions/upload-artifact@50769540e7f4bd5e21e526ee35c689e35e0d6874 # v4.4.0
       with:
         name: documentation
         path: site

diff --git a/.github/workflows/report-merged-pr.yml b/.github/workflows/report-merged-pr.yml
@@ -4,6 +4,8 @@ name: Report Merged PR
 
 on:
   pull_request:
+    branches:
+      - main
     types: [closed]
 
 permissions: {}

diff --git a/.github/workflows/serverless-benchmarks.yml b/.github/workflows/serverless-benchmarks.yml
@@ -48,7 +48,7 @@ jobs:
               ./pkg/serverless/... | tee "$TEMP_RUNNER"/benchmark.log
 
       - name: Upload result artifact
-        uses: actions/upload-artifact@834a144ee995460fba8ed112a2fc961b36a5ec5a # v4.3.6
+        uses: actions/upload-artifact@50769540e7f4bd5e21e526ee35c689e35e0d6874 # v4.4.0
         with:
           name: baseline.log
           path: ${{runner.temp}}/benchmark.log
@@ -87,7 +87,7 @@ jobs:
               ./pkg/serverless/... | tee "$TEMP_RUNNER"/benchmark.log
 
       - name: Upload result artifact
-        uses: actions/upload-artifact@834a144ee995460fba8ed112a2fc961b36a5ec5a # v4.3.6
+        uses: actions/upload-artifact@50769540e7f4bd5e21e526ee35c689e35e0d6874 # v4.4.0
         with:
           name: current.log
           path: ${{runner.temp}}/benchmark.log

diff --git a/.github/workflows/serverless-binary-size.yml b/.github/workflows/serverless-binary-size.yml
@@ -152,7 +152,7 @@ jobs:
           done
 
       - name: Archive dependency graphs
-        uses: actions/upload-artifact@834a144ee995460fba8ed112a2fc961b36a5ec5a # v4.3.6
+        uses: actions/upload-artifact@50769540e7f4bd5e21e526ee35c689e35e0d6874 # v4.4.0
         if: steps.should.outputs.should_run == 'true'
         with:
           name: dependency-graphs

diff --git a/.github/workflows/serverless-integration.yml b/.github/workflows/serverless-integration.yml
@@ -80,7 +80,7 @@ jobs:
 
       - name: Archive raw logs
         if: always()
-        uses: actions/upload-artifact@834a144ee995460fba8ed112a2fc961b36a5ec5a # v4.3.6
+        uses: actions/upload-artifact@50769540e7f4bd5e21e526ee35c689e35e0d6874 # v4.4.0
         with:
           name: rawlogs-${{ matrix.suite }}-${{ matrix.architecture }}
           path: ${{ steps.rawlogs.outputs.dir }}
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
@@ -134,6 +134,7 @@ variables:
   BUCKET_BRANCH: dev # path inside the staging s3 buckets to release to: 'dev', 'nightly', 'oldnightly', 'beta' or 'stable'
   DEB_TESTING_S3_BUCKET: apttesting.datad0g.com
   RPM_TESTING_S3_BUCKET: yumtesting.datad0g.com
+  INSTALLER_TESTING_S3_BUCKET: installtesting.datad0g.com
   WINDOWS_TESTING_S3_BUCKET_A6: pipelines/A6/$CI_PIPELINE_ID
   WINDOWS_TESTING_S3_BUCKET_A7: pipelines/A7/$CI_PIPELINE_ID
   WINDOWS_BUILDS_S3_BUCKET: $WIN_S3_BUCKET/builds
@@ -980,6 +981,16 @@ workflow:
       compare_to: main # TODO: use a variable, when this is supported https://gitlab.com/gitlab-org/gitlab/-/issues/369916
     when: on_success
 
+.on_netpath_or_e2e_changes:
+  - !reference [.on_e2e_main_release_or_rc]
+  - changes:
+      paths:
+        - pkg/collector/corechecks/networkpath/**/*
+        - test/new-e2e/tests/netpath/**/*
+        - test/new-e2e/go.mod
+      compare_to: main # TODO: use a variable, when this is supported https://gitlab.com/gitlab-org/gitlab/-/issues/369916
+    when: on_success
+
 .on_otel_or_e2e_changes:
   - !reference [.on_e2e_main_release_or_rc]
   - changes: