update jackgopack4 branch with main

.github/CODEOWNERS

@@ -165,6 +165,7 @@
 /cmd/agent/dist/conf.d/jetson.d/        @DataDog/agent-devx-infra
 /cmd/agent/dist/conf.d/oracle.d/    @DataDog/database-monitoring
 /cmd/agent/dist/conf.d/oracle-dbm.d/    @DataDog/database-monitoring
+/cmd/agent/dist/conf.d/network_path.d/  @DataDog/Networks @DataDog/network-device-monitoring
 /cmd/agent/dist/conf.d/sbom.d/          @DataDog/container-integrations
 /cmd/agent/dist/conf.d/snmp.d/          @DataDog/network-device-monitoring
 /cmd/agent/dist/conf.d/win32_event_log.d/ @DataDog/windows-agent
@@ -207,6 +208,7 @@
 /Dockerfiles/agent/entrypoint.d.windows/ @DataDog/container-integrations @DataDog/windows-agent
 /Dockerfiles/agent/entrypoint.ps1        @DataDog/container-integrations @DataDog/windows-agent
 /Dockerfiles/agent/windows/              @DataDog/container-integrations @DataDog/windows-agent
+/Dockerfiles/agent-ot/Dockerfile.byoc   @DataDog/opentelemetry
 
 /docs/                                  @DataDog/documentation @DataDog/agent-devx-loops
 /docs/dev/checks/                       @DataDog/documentation @DataDog/agent-metrics-logs

.github/dependabot.yaml

@@ -7,7 +7,6 @@ updates:
       - dependencies-go
       - team/triage
       - changelog/no-changelog
-    milestone: 22
     ignore:
       # These dependencies are replaced in the main go.mod file.
       # They are ignored since bumping them would have no effect on the build.
@@ -45,7 +44,6 @@ updates:
       - dependencies-go
       - team/agent-apm
       - changelog/no-changelog
-    milestone: 22
     ignore:
       # Ignore internal modules
       - dependency-name: github.com/DataDog/datadog-agent/*
@@ -61,7 +59,6 @@ updates:
       - dependencies-go
       - team/agent-shared-components
       - changelog/no-changelog
-    milestone: 22
     ignore:
       # Ignore internal modules
       - dependency-name: github.com/DataDog/datadog-agent/*
@@ -77,7 +74,6 @@ updates:
       - dependencies-go
       - team/agent-apm
       - changelog/no-changelog
-    milestone: 22
     ignore:
       # Ignore internal modules
       - dependency-name: github.com/DataDog/datadog-agent/*
@@ -93,7 +89,6 @@ updates:
       - dependencies-go
       - team/agent-security
       - changelog/no-changelog
-    milestone: 22
     ignore:
       # Ignore internal modules
       - dependency-name: github.com/DataDog/datadog-agent/*
@@ -113,7 +108,6 @@ updates:
       - team/agent-delivery
       - changelog/no-changelog
       - qa/no-code-change
-    milestone: 22
     schedule:
       interval: monthly
     open-pull-requests-limit: 100
@@ -124,7 +118,6 @@ updates:
       - dependencies-go
       - team/network-device-monitoring
       - changelog/no-changelog
-    milestone: 22
     ignore:
       # Ignore internal modules
       - dependency-name: github.com/DataDog/datadog-agent/*
@@ -140,7 +133,6 @@ updates:
       - changelog/no-changelog
       - qa/no-code-change
       - dev/testing
-    milestone: 22
     ignore:
       # Ignore test-infra-definitions because bumping the GO package inside `go.mod`
       # requires to also bump `TEST_INFRA_DEFINITIONS_BUILDIMAGES` inside `.gitlab/common/test_infra_version.yml`
@@ -162,7 +154,6 @@ updates:
       - changelog/no-changelog
       - qa/no-code-change
       - dev/testing
-    milestone: 22
     schedule:
       interval: weekly
     open-pull-requests-limit: 100
@@ -175,7 +166,6 @@ updates:
       - changelog/no-changelog
       - qa/no-code-change
       - dev/tooling
-    milestone: 22
     schedule:
       interval: monthly
     open-pull-requests-limit: 100
@@ -187,7 +177,6 @@ updates:
       - changelog/no-changelog
       - qa/no-code-change
       - dev/testing
-    milestone: 22
     schedule:
       interval: weekly
     open-pull-requests-limit: 100
@@ -200,7 +189,6 @@ updates:
       - changelog/no-changelog
       - qa/no-code-change
       - dev/tooling
-    milestone: 22
     schedule:
       interval: weekly
     open-pull-requests-limit: 100

.github/workflows/create_rc_pr.yml

@@ -48,6 +48,9 @@ jobs:
 
             - name: Check for changes since last RC
               id: check_for_changes
+              env:
+                ATLASSIAN_USERNAME: ${{ secrets.ATLASSIAN_USERNAME }}
+                ATLASSIAN_PASSWORD: ${{ secrets.ATLASSIAN_PASSWORD }}
               run: |
                 echo "CHANGES=$(inv -e release.check-for-changes -r ${{ env.RELEASE_BRANCH }} ${{ env.WARNING }})" >> $GITHUB_OUTPUT
 

.gitlab-ci.yml

@@ -219,6 +219,9 @@ variables:
   GITLAB_FULL_API_TOKEN_SSM_NAME: ci.datadog-agent.gitlab_full_api_token  # ci-cd
   INSTALL_SCRIPT_API_KEY_SSM_NAME: ci.agent-linux-install-script.datadog_api_key_2  # agent-delivery
   JIRA_READ_API_TOKEN_SSM_NAME: ci.datadog-agent.jira_read_api_token  # agent-devx-infra
+  AGENT_GITHUB_APP_ID_SSM_NAME: ci.datadog-agent.platform-github-app-id  # agent-devx-infra
+  AGENT_GITHUB_INSTALLATION_ID_SSM_NAME: ci.datadog-agent.platform-github-app-installation-id  # agent-devx-infra
+  AGENT_GITHUB_KEY_SSM_NAME: ci.datadog-agent.platform-github-app-key  # agent-devx-infra
   MACOS_GITHUB_APP_ID_SSM_NAME: ci.datadog-agent.macos_github_app_id  # agent-devx-infra
   MACOS_GITHUB_INSTALLATION_ID_SSM_NAME: ci.datadog-agent.macos_github_installation_id  # agent-devx-infra
   MACOS_GITHUB_KEY_SSM_NAME: ci.datadog-agent.macos_github_key_b64  # agent-devx-infra
@@ -359,14 +362,12 @@ variables:
 .if_mergequeue: &if_mergequeue
   if: $CI_COMMIT_BRANCH =~ /^mq-working-branch-/
 
-.fakeintake_changes: &fakeintake_changes
-  changes:
-    paths:
-      - "test/fakeintake/**/*"
-      - .gitlab/binary_build/fakeintake.yml
-      - .gitlab/container_build/fakeintake.yml
-      - .gitlab/dev_container_deploy/fakeintake.yml
-    compare_to: main # TODO: use a variable, when this is supported https://gitlab.com/gitlab-org/gitlab/-/issues/369916
+.fakeintake_paths: &fakeintake_paths
+  paths:
+    - "test/fakeintake/**/*"
+    - .gitlab/binary_build/fakeintake.yml
+    - .gitlab/container_build/fakeintake.yml
+    - .gitlab/dev_container_deploy/fakeintake.yml
 
 #
 # Workflow rules
@@ -790,7 +791,9 @@ workflow:
 
 .on_e2e_or_fakeintake_changes_or_manual:
   - !reference [.on_e2e_main_release_or_rc]
-  - <<: *fakeintake_changes
+  - changes:
+      <<: *fakeintake_paths
+      compare_to: main # TODO: use a variable, when this is supported https://gitlab.com/gitlab-org/gitlab/-/issues/369916
     variables:
       FAKEINTAKE_IMAGE_OVERRIDE: "public.ecr.aws/datadog/fakeintake:v$CI_COMMIT_SHORT_SHA"
     when: on_success
@@ -1051,10 +1054,13 @@ workflow:
       compare_to: main # TODO: use a variable, when this is supported https://gitlab.com/gitlab-org/gitlab/-/issues/369916
 
 .on_fakeintake_changes:
-  - <<: *fakeintake_changes
+  - changes:
+      <<: *fakeintake_paths
+      compare_to: main # TODO: use a variable, when this is supported https://gitlab.com/gitlab-org/gitlab/-/issues/369916
 
 .on_fakeintake_changes_on_main:
-  - <<: *fakeintake_changes
+  - changes:
+      <<: *fakeintake_paths
     <<: *if_main_branch
 
 .fast_on_dev_branch_only:

.gitlab/.pre/test_gitlab_configuration.yml

@@ -7,5 +7,19 @@ test_gitlab_configuration:
   script:
     - source /root/.bashrc
     - export GITLAB_TOKEN=$($CI_PROJECT_DIR/tools/ci/aws_ssm_get_wrapper.sh $GITLAB_FULL_API_TOKEN_SSM_NAME)
-    - inv linter.gitlab-ci
-    - inv linter.gitlab-change-paths
+    - inv -e linter.gitlab-ci
+    - inv -e linter.job-change-path
+    - inv -e linter.gitlab-change-paths
+
+test_gitlab_compare_to:
+  stage: .pre
+  image: 486234852809.dkr.ecr.us-east-1.amazonaws.com/ci/datadog-agent-buildimages/deb_x64$DATADOG_AGENT_BUILDIMAGES_SUFFIX:$DATADOG_AGENT_BUILDIMAGES
+  tags: ["arch:amd64"]
+  rules:
+    - !reference [.on_gitlab_changes]
+  script:
+    - source /root/.bashrc
+    - export GITLAB_TOKEN=$($CI_PROJECT_DIR/tools/ci/aws_ssm_get_wrapper.sh $GITLAB_FULL_API_TOKEN_SSM_NAME)
+    - !reference [.setup_agent_github_app]
+    - pip install -r tasks/requirements.txt
+    - inv pipeline.compare-to-itself

.gitlab/common/shared.yml

@@ -41,6 +41,12 @@
       echo "Using GitHub App instance 2"
     fi
 
+.setup_agent_github_app:
+  - export GITHUB_KEY_B64=$($CI_PROJECT_DIR/tools/ci/aws_ssm_get_wrapper.sh $AGENT_GITHUB_KEY_SSM_NAME)
+  - export GITHUB_APP_ID=$($CI_PROJECT_DIR/tools/ci/aws_ssm_get_wrapper.sh $AGENT_GITHUB_APP_ID_SSM_NAME)
+  - export GITHUB_INSTALLATION_ID=$($CI_PROJECT_DIR/tools/ci/aws_ssm_get_wrapper.sh $AGENT_GITHUB_INSTALLATION_ID_SSM_NAME)
+  - echo "Using agent GitHub App"
+
 # Install `dd-pkg` and lint packages produced by Omnibus, supports only deb and rpm packages
 .lint_linux_packages:
   - curl -sSL "https://dd-package-tools.s3.amazonaws.com/dd-pkg/${DD_PKG_VERSION}/dd-pkg_Linux_${DD_PKG_ARCH}.tar.gz" | tar -xz -C /usr/local/bin dd-pkg

.gitlab/container_build/fakeintake.yml

@@ -4,6 +4,7 @@ docker_build_fakeintake:
   rules:
     - !reference [.except_mergequeue]
     - !reference [.on_fakeintake_changes]
+    - !reference [.on_fakeintake_changes_on_main]
   needs: []
   image: 486234852809.dkr.ecr.us-east-1.amazonaws.com/docker:20.10-py3
   tags: ["arch:amd64"]

.gitlab/deploy_packages/oci.yml

@@ -21,6 +21,9 @@ include:
     - export VERSION=$(inv agent.version --url-safe)-1
     - git config --global credential.helper '!f() { echo username=x-access-token; echo "password=$GH_TOKEN"; };f'
     - ${CI_PROJECT_DIR}/tools/ci/retry.sh git clone -v --depth=1 https://github.com/DataDog/datadog-packages /tmp/datadog-packages
+    - git config --global url."https://gitlab-ci-token:${CI_JOB_TOKEN}@gitlab.ddbuild.io/DataDog/".insteadOf "https://github.com/DataDog/"
+    - export GOPROXY="binaries.ddbuild.io,proxy.golang.org,direct"
+    - export GONOSUMDB="github.com/DataDog,go.ddbuild.io"
     - cd /tmp/datadog-packages/cmd/datadog-package
     - go build .
     - ./datadog-package push registry.ddbuild.io/ci/remote-updates/${OCI_PRODUCT}:${VERSION} ${OMNIBUS_PACKAGE_DIR}/${OCI_PRODUCT}-${VERSION}.oci.tar

.gitlab/dev_container_deploy/fakeintake.yml

@@ -14,17 +14,17 @@ publish_fakeintake:
     IMG_REGISTRIES: public
     IMG_SIGNING: "false"
 
-publish_fakeintake_latest:
-  extends: .docker_publish_job_definition
-  stage: dev_container_deploy
-  rules:
-    - !reference [.except_mergequeue]
-    - !reference [.on_fakeintake_changes_on_main]
-  needs:
-    - job: docker_build_fakeintake
-      optional: false
-  variables:
-    IMG_SOURCES: 486234852809.dkr.ecr.us-east-1.amazonaws.com/ci/datadog-agent/fakeintake:v${CI_PIPELINE_ID}-${CI_COMMIT_SHORT_SHA}
-    IMG_DESTINATIONS: fakeintake:latest
-    IMG_REGISTRIES: public
-    IMG_SIGNING: "false"
+# publish_fakeintake_latest:
+#   extends: .docker_publish_job_definition
+#   stage: dev_container_deploy
+#   rules:
+#     - !reference [.except_mergequeue]
+#     - !reference [.on_fakeintake_changes_on_main]
+#   needs:
+#     - job: docker_build_fakeintake
+#       optional: false
+#   variables:
+#     IMG_SOURCES: 486234852809.dkr.ecr.us-east-1.amazonaws.com/ci/datadog-agent/fakeintake:v${CI_PIPELINE_ID}-${CI_COMMIT_SHORT_SHA}
+#     IMG_DESTINATIONS: fakeintake:latest
+#     IMG_REGISTRIES: public
+#     IMG_SIGNING: "false"

.gitlab/e2e/e2e.yml

@@ -165,7 +165,9 @@ new-e2e-containers:
     TEAM: container-integrations
   parallel:
     matrix:
-      - EXTRA_PARAMS: "--run TestKindSuite -c ddinfra:kubernetesVersion=1.19"
+      # Temporarily disable old version of Kubernetes
+      # On this version, the reported kubernetes CPU usage appears to be significantly off
+      # - EXTRA_PARAMS: "--run TestKindSuite -c ddinfra:kubernetesVersion=1.19"
       - EXTRA_PARAMS: "--run TestKindSuite -c ddinfra:kubernetesVersion=1.22"
       - EXTRA_PARAMS: "--run TestKindSuite -c ddinfra:kubernetesVersion=1.27"
       - EXTRA_PARAMS: "--run TestKindSuite -c ddinfra:kubernetesVersion=1.29"

.gitlab/kitchen_testing/new-e2e_testing/ubuntu.yml

@@ -18,17 +18,17 @@
 .new-e2e_ubuntu_a6_x86_64:
   variables:
     E2E_ARCH: x86_64
-    E2E_OSVERS: "ubuntu-14-04,ubuntu-16-04,ubuntu-18-04,ubuntu-20-04,ubuntu-22-04"
+    E2E_OSVERS: "ubuntu-14-04,ubuntu-16-04,ubuntu-18-04,ubuntu-20-04,ubuntu-22-04,ubuntu-24-04"
     E2E_CWS_SUPPORTED_OSVERS: "ubuntu-18-04,ubuntu-20-04,ubuntu-22-04"
-    E2E_BRANCH_OSVERS: "ubuntu-22-04"
+    E2E_BRANCH_OSVERS: "ubuntu-24-04"
   needs:
     - !reference [.needs_new_e2e_template]
     - deploy_deb_testing-a6_x64
 
 .new-e2e_ubuntu_a6_arm64:
   variables:
     E2E_ARCH: arm64
-    E2E_OSVERS: "ubuntu-18-04,ubuntu-20-04"
+    E2E_OSVERS: "ubuntu-18-04,ubuntu-20-04,ubuntu-24-04"
     E2E_CWS_SUPPORTED_OSVERS: "ubuntu-18-04,ubuntu-20-04"
     E2E_BRANCH_OSVERS: "ubuntu-20-04"
   needs:
@@ -38,17 +38,17 @@
 .new-e2e_ubuntu_a7_x86_64:
   variables:
     E2E_ARCH: x86_64
-    E2E_OSVERS: "ubuntu-14-04,ubuntu-16-04,ubuntu-18-04,ubuntu-20-04,ubuntu-22-04"
+    E2E_OSVERS: "ubuntu-14-04,ubuntu-16-04,ubuntu-18-04,ubuntu-20-04,ubuntu-22-04,ubuntu-24-04"
     E2E_CWS_SUPPORTED_OSVERS: "ubuntu-18-04,ubuntu-20-04,ubuntu-22-04"
-    E2E_BRANCH_OSVERS: "ubuntu-22-04"
+    E2E_BRANCH_OSVERS: "ubuntu-24-04"
   needs:
     - !reference [.needs_new_e2e_template]
     - deploy_deb_testing-a7_x64
 
 .new-e2e_ubuntu_a7_arm64:
   variables:
     E2E_ARCH: arm64
-    E2E_OSVERS: "ubuntu-18-04,ubuntu-20-04"
+    E2E_OSVERS: "ubuntu-18-04,ubuntu-20-04,ubuntu-24-04"
     E2E_CWS_SUPPORTED_OSVERS: "ubuntu-18-04,ubuntu-20-04"
     E2E_BRANCH_OSVERS: "ubuntu-20-04"
   needs:

.gitlab/package_build/installer.yml

@@ -100,6 +100,9 @@ datadog-agent-oci-arm64-a7:
     - !reference [.upload_sbom_artifacts]
   variables:
     AGENT_MAJOR_VERSION: 7
+    KUBERNETES_CPU_REQUEST: 8
+    KUBERNETES_MEMORY_REQUEST: "16Gi"
+    KUBERNETES_MEMORY_LIMIT: "16Gi"
   artifacts:
     expire_in: 2 weeks
     paths:

.gitlab/packaging/oci.yml

@@ -24,8 +24,11 @@
     - git config --global credential.helper '!f() { echo username=x-access-token; echo "password=$GH_TOKEN"; };f'
     - cd /tmp/
     - ${CI_PROJECT_DIR}/tools/ci/retry.sh git clone --depth=1 https://github.com/DataDog/datadog-packages
+    - git config --global url."https://gitlab-ci-token:${CI_JOB_TOKEN}@gitlab.ddbuild.io/DataDog/".insteadOf "https://github.com/DataDog/"
+    - export GOPROXY="binaries.ddbuild.io,proxy.golang.org,direct"
+    - export GONOSUMDB="github.com/DataDog,go.ddbuild.io"
     - cd datadog-packages/cmd/datadog-package
-    - go build .
+    - ${CI_PROJECT_DIR}/tools/ci/retry.sh go build .
     - OUTPUT_DIR="/tmp/oci_output"
     - mkdir -p ${OUTPUT_DIR}
     - ls $OMNIBUS_PACKAGE_DIR

.gitlab/post_rc_build/post_rc_tasks.yml

@@ -15,4 +15,6 @@ update_rc_build_links:
     - export ATLASSIAN_PASSWORD=$($CI_PROJECT_DIR/tools/ci/aws_ssm_get_wrapper.sh $JIRA_READ_API_TOKEN_SSM_NAME)
     - export ATLASSIAN_USERNAME=robot-jira-agentplatform@datadoghq.com
     - python3 -m pip install -r tasks/requirements_release_tasks.txt
-    - inv -e release.update-build-links ${CI_COMMIT_REF_NAME}
+    - PATCH=$(echo "$CI_COMMIT_REF_NAME" | cut -d'.' -f3 | cut -c1)
+    - if [[ "$PATCH" == "0" ]]; then PATCH_OPTION=""; else PATCH_OPTION="-p"; fi
+    - inv -e release.update-build-links ${CI_COMMIT_REF_NAME} ${PATCH_OPTION}

.golangci.yml

@@ -99,7 +99,7 @@ linters-settings:
   errcheck:
     # Disable warnings for `fmt`, `log` and `seelog` packages. Also ignore `Write` functions from `net/http` package.
     # Disable warnings for select Windows functions
-    ignore: fmt:.*,github.com/DataDog/datadog-agent/pkg/util/log:.*,github.com/DataDog/datadog-agent/comp/core/log:.*,github.com/cihub/seelog:.*,net/http:Write,github.com/DataDog/datadog-agent/pkg/trace/metrics:.*,github.com/DataDog/datadog-agent/pkg/collector/corechecks:Warnf?,golang.org/x/sys/windows:(CloseHandle|FreeLibrary|FreeSid|RegCloseKey|SetEvent|LocalFree),syscall:CloseHandle,golang.org/x/sys/windows/svc/mgr:Disconnect,golang.org/x/sys/windows/svc/debug:(Close|Error|Info|Warning),github.com/lxn/walk:Dispose,github.com/DataDog/datadog-agent/comp/core/flare/types:(AddFile.*|CopyDir.*|CopyFile.*),github.com/DataDog/datadog-agent/comp/core/flare/builder:(AddFile.*|CopyDir.*|CopyFile.*),golang.org/x/sys/windows/registry:Close
+    ignore: fmt:.*,github.com/DataDog/datadog-agent/pkg/util/log:.*,github.com/DataDog/datadog-agent/comp/core/log/def:.*,github.com/DataDog/datadog-agent/comp/core/log/mock:.*,github.com/cihub/seelog:.*,net/http:Write,github.com/DataDog/datadog-agent/pkg/trace/metrics:.*,github.com/DataDog/datadog-agent/pkg/collector/corechecks:Warnf?,golang.org/x/sys/windows:(CloseHandle|FreeLibrary|FreeSid|RegCloseKey|SetEvent|LocalFree),syscall:CloseHandle,golang.org/x/sys/windows/svc/mgr:Disconnect,golang.org/x/sys/windows/svc/debug:(Close|Error|Info|Warning),github.com/lxn/walk:Dispose,github.com/DataDog/datadog-agent/comp/core/flare/types:(AddFile.*|CopyDir.*|CopyFile.*),github.com/DataDog/datadog-agent/comp/core/flare/builder:(AddFile.*|CopyDir.*|CopyFile.*),golang.org/x/sys/windows/registry:Close
   staticcheck:
     checks: ["all",
              "-ST1000", "-ST1003", "-ST1016", "-ST1020", "-ST1021", "-ST1022", # These ones are disabled by default on staticcheck

.vscode/launch.json.template

@@ -35,5 +35,20 @@
             "mode": "test",
             "program": "${fileDirname}"
         },
+        {
+            "name": "Remote Debug Tasks",
+            "type": "debugpy",
+            "request": "attach",
+            "connect": {
+                "host": "localhost",
+                "port": 5678
+            },
+            "pathMappings": [
+                {
+                    "localRoot": "${workspaceFolder}",
+                    "remoteRoot": "."
+                }
+            ]
+        },
     ]
-}
+}

.vscode/settings.json.template

@@ -26,5 +26,14 @@
         "SC2059",
         "SC2028"
     ],
-    "shellcheck.run": "onSave"
+    "shellcheck.run": "onSave",
+    "python.testing.unittestArgs": [
+        "-v",
+        "-s",
+        "./tasks",
+        "-p",
+        "*_tests.py"
+    ],
+    "python.testing.pytestEnabled": false,
+    "python.testing.unittestEnabled": true
 }}