[pull] main from DataDog:main

.github/workflows/cws-btfhub-sync.yml

@@ -20,7 +20,7 @@ permissions: {}
 
 jobs:
   generate:
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-22.04
     strategy:
       fail-fast: false
       matrix:
@@ -104,7 +104,7 @@ jobs:
 
   combine:
     needs: generate
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-22.04
     permissions:
       contents: write
       pull-requests: write

.gitlab/.ci-linters.yml

@@ -57,8 +57,6 @@ job-owners:
     - build_otel_agent_binary_arm64
     - build_otel_agent_binary_x64
     - cancel-prev-pipelines
-    - check_pkg_size-amd64-a7
-    - check_pkg_size-arm64-a7
     - close_failing_tests_stale_issues
     - compute_gitlab_ci_config
     - deploy_cluster_agent_cloudfoundry

.gitlab/JOBOWNERS

@@ -67,7 +67,7 @@ iot_agent_suse*                      @DataDog/agent-delivery
 dogstatsd_suse*                      @DataDog/agent-delivery
 agent_oci*                           @DataDog/agent-delivery
 installer_oci*                       @DataDog/agent-delivery
-new_check_pkg_size                   @DataDog/agent-delivery
+check_pkg_size                       @DataDog/agent-delivery
 
 # Testing package deploy
 deploy_deb_testing*                  @DataDog/agent-delivery

.gitlab/dev_container_deploy/docker_linux.yml

@@ -80,6 +80,24 @@ dev_master-a7:
       - IMG_SOURCES: ${SRC_AGENT}:v${CI_PIPELINE_ID}-${CI_COMMIT_SHORT_SHA}-7-jmx-amd64,${SRC_AGENT}:v${CI_PIPELINE_ID}-${CI_COMMIT_SHORT_SHA}-7-jmx-arm64
         IMG_DESTINATIONS: agent-dev:master-py3-jmx
 
+dev_master-fips:
+  extends: .docker_publish_job_definition
+  stage: dev_container_deploy
+  rules: !reference [.on_main]
+  needs:
+    - docker_build_fips_agent7
+    - docker_build_fips_agent7_arm64
+    - docker_build_fips_agent7_jmx
+    - docker_build_fips_agent7_arm64_jmx
+  variables:
+    IMG_REGISTRIES: dev
+  parallel:
+    matrix:
+      - IMG_SOURCES: ${SRC_AGENT}:v${CI_PIPELINE_ID}-${CI_COMMIT_SHORT_SHA}-7-fips-amd64,${SRC_AGENT}:v${CI_PIPELINE_ID}-${CI_COMMIT_SHORT_SHA}-7-fips-arm64
+        IMG_DESTINATIONS: agent-dev:master-fips
+      - IMG_SOURCES: ${SRC_AGENT}:v${CI_PIPELINE_ID}-${CI_COMMIT_SHORT_SHA}-7-fips-jmx-amd64,${SRC_AGENT}:v${CI_PIPELINE_ID}-${CI_COMMIT_SHORT_SHA}-7-fips-jmx-arm64
+        IMG_DESTINATIONS: agent-dev:master-fips-jmx
+
 dev_master-dogstatsd:
   extends: .docker_publish_job_definition
   stage: dev_container_deploy

.gitlab/pkg_metrics/pkg_metrics.yml

@@ -76,118 +76,14 @@ send_pkg_size:
 
     - inv package.send-size --flavor "agent" --package-os "suse" --package-path $OMNIBUS_PACKAGE_DIR_SUSE/datadog-agent-7.*.aarch64.rpm --major-version "7" --git-ref "${CI_COMMIT_REF_SLUG}" --bucket-branch "${BUCKET_BRANCH}" --arch arm64
 
-.check_pkg_size:
-  stage: pkg_metrics
-  image: registry.ddbuild.io/ci/datadog-agent-buildimages/deb_x64$DATADOG_AGENT_BUILDIMAGES_SUFFIX:$DATADOG_AGENT_BUILDIMAGES
-  tags: ["arch:amd64"]
-  script:
-    - ls -l $OMNIBUS_PACKAGE_DIR
-    - if [[ "${ARCH}" == "amd64" ]]; then ls -l $OMNIBUS_PACKAGE_DIR_SUSE; fi
-
-    - export failures=0
-    - export last_stable=$(inv release.get-release-json-value "last_stable::${MAJOR_VERSION}" --no-worktree)
-    # Get stable packages from S3 buckets, send new package sizes & compare stable and new package sizes
-    # The loop assumes that all flavors start with "da", which is currently the case
-    # We want to run all package size comparisons before failing, so we set +e while doing the comparisons
-    # to get the error codes without exiting the shell.
-    - |
-      if [[ "${ARCH}" == "amd64" ]]; then ARCH_RPM_EXT="x86_64"; else ARCH_RPM_EXT="aarch64"; fi
-      for flavor in ${FLAVORS}; do
-
-          if [[ "${ARCH}" == "amd64" && "$flavor" != "datadog-heroku-agent" ]]; then
-            mkdir -p "/tmp/stable/${flavor}/suse"
-            curl -sSL "https://s3.amazonaws.com/yum.datadoghq.com/suse/stable/${MAJOR_VERSION}/${ARCH_RPM_EXT}/${flavor}-${last_stable}-1.${ARCH_RPM_EXT}.rpm" -o "/tmp/stable/${flavor}/suse/${flavor}-${last_stable}-1.${ARCH_RPM_EXT}.rpm"
-            set +e
-            inv package.compare-size --package-type "${flavor} suse rpm" --last-stable "${last_stable}" --threshold "${max_sizes[${flavor}]}" --new-package "$OMNIBUS_PACKAGE_DIR_SUSE/${flavor}-${MAJOR_VERSION}.*.${ARCH_RPM_EXT}.rpm" --stable-package "/tmp/stable/${flavor}/suse/${flavor}-${last_stable}-1.${ARCH_RPM_EXT}.rpm"
-            failures=$((${failures}+$?))
-            set -e
-          fi
-
-          mkdir -p "/tmp/stable/${flavor}"
-
-          curl -sSL "https://s3.amazonaws.com/apt.datadoghq.com/pool/d/da/${flavor}_${last_stable}-1_${ARCH}.deb" -o "/tmp/stable/${flavor}/${flavor}_${last_stable}-1_${ARCH}.deb"
-
-          set +e
-          inv package.compare-size --package-type "${flavor} deb" --last-stable "${last_stable}" --threshold "${max_sizes[${flavor}]}" --new-package "$OMNIBUS_PACKAGE_DIR/${flavor}_${MAJOR_VERSION}*_${ARCH}.deb" --stable-package "/tmp/stable/${flavor}/${flavor}_${last_stable}-1_${ARCH}.deb"
-          failures=$((${failures}+$?))
-          set -e
-
-          if [[ "$flavor" != "datadog-heroku-agent" && ( "${ARCH}" == "amd64" || "$flavor" != "datadog-dogstatsd") ]]; then
-            # We don't build RPM packages for the heroku flavor
-            curl -sSL "https://s3.amazonaws.com/yum.datadoghq.com/stable/${MAJOR_VERSION}/${ARCH_RPM_EXT}/${flavor}-${last_stable}-1.${ARCH_RPM_EXT}.rpm" -o "/tmp/stable/${flavor}/${flavor}-${last_stable}-1.${ARCH_RPM_EXT}.rpm"
-            set +e
-            inv package.compare-size --package-type "${flavor} rpm" --last-stable "${last_stable}" --threshold "${max_sizes[${flavor}]}" --new-package "$OMNIBUS_PACKAGE_DIR/${flavor}-${MAJOR_VERSION}.*.${ARCH_RPM_EXT}.rpm" --stable-package "/tmp/stable/${flavor}/${flavor}-${last_stable}-1.${ARCH_RPM_EXT}.rpm"
-            failures=$((${failures}+$?))
-            set -e
-          fi
-      done
-
-    # Make the job fail if at least one package is above threshold
-    - if [ "${failures}" -ne "0" ]; then false; fi
-
-check_pkg_size-amd64-a7:
-  extends: .check_pkg_size
-  rules:
-    - !reference [.except_mergequeue]
-    - when: on_success
-  needs:
-    - agent_deb-x64-a7
-    - iot_agent_deb-x64
-    - dogstatsd_deb-x64
-    - agent_heroku_deb-x64-a7
-    - agent_rpm-x64-a7
-    - iot_agent_rpm-x64
-    - dogstatsd_rpm-x64
-    - agent_suse-x64-a7
-    - dogstatsd_suse-x64
-    - iot_agent_suse-x64
-  variables:
-    MAJOR_VERSION: 7
-    FLAVORS: "datadog-agent datadog-iot-agent datadog-dogstatsd datadog-heroku-agent"
-    ARCH: "amd64"
-  before_script:
-    # FIXME: ["datadog-agent"]="140000000" and ["datadog-heroku-agent"]="140000000" should
-    # be replaced by "50000000"
-    # "70000000" is needed as of now because of multiple large additions in 7.45
-    - |
-      declare -Ar max_sizes=(
-          ["datadog-agent"]="140000000"
-          ["datadog-iot-agent"]="10000000"
-          ["datadog-dogstatsd"]="10000000"
-          ["datadog-heroku-agent"]="70000000"
-      )
-
-check_pkg_size-arm64-a7:
-  extends: .check_pkg_size
-  rules: !reference [.on_all_builds]
-  needs:
-    - agent_deb-arm64-a7
-    - iot_agent_deb-arm64
-    - dogstatsd_deb-arm64
-    - agent_rpm-arm64-a7
-    - iot_agent_rpm-arm64
-  variables:
-    MAJOR_VERSION: 7
-    FLAVORS: "datadog-agent datadog-iot-agent datadog-dogstatsd"
-    ARCH: "arm64"
-  before_script:
-    # FIXME: ["datadog-agent"]="140000000" should be replaced by "70000000"
-    # "140000000" is needed as of now because of multiple large additions in 7.45
-    - |
-      declare -Ar max_sizes=(
-          ["datadog-agent"]="140000000"
-          ["datadog-iot-agent"]="10000000"
-          ["datadog-dogstatsd"]="10000000"
-      )
 
-new_check_pkg_size:
+check_pkg_size:
   stage: pkg_metrics
   image: registry.ddbuild.io/ci/datadog-agent-buildimages/deb_x64$DATADOG_AGENT_BUILDIMAGES_SUFFIX:$DATADOG_AGENT_BUILDIMAGES
   tags: ["arch:amd64"]
   rules:
     - !reference [.except_mergequeue]
     - when: on_success
-  allow_failure: true
   needs:
     - agent_deb-x64-a7
     - iot_agent_deb-x64

cmd/process-agent/subcommands/config/config.go

@@ -181,6 +181,11 @@ func getConfigValue(deps dependencies, args []string) error {
 }
 
 func getClient(cfg model.Reader) (settings.Client, error) {
+	err := util.SetAuthToken(cfg)
+	if err != nil {
+		return nil, err
+	}
+
 	httpClient := apiutil.GetClient(false)
 	ipcAddress, err := pkgconfigsetup.GetIPCAddress(pkgconfigsetup.Datadog())
 

cmd/process-agent/subcommands/status/status.go

@@ -21,6 +21,7 @@ import (
 	log "github.com/DataDog/datadog-agent/comp/core/log/def"
 	compStatus "github.com/DataDog/datadog-agent/comp/core/status"
 	"github.com/DataDog/datadog-agent/comp/process"
+	"github.com/DataDog/datadog-agent/pkg/api/util"
 	apiutil "github.com/DataDog/datadog-agent/pkg/api/util"
 	"github.com/DataDog/datadog-agent/pkg/collector/python"
 	pkgconfigsetup "github.com/DataDog/datadog-agent/pkg/config/setup"
@@ -147,6 +148,11 @@ func runStatus(deps dependencies) error {
 		return err
 	}
 
+	err = util.SetAuthToken(deps.Config)
+	if err != nil {
+		return err
+	}
+
 	getAndWriteStatus(deps.Log, statusURL, os.Stdout)
 	return nil
 }

cmd/process-agent/subcommands/taggerlist/tagger_list.go

@@ -18,6 +18,7 @@ import (
 	"github.com/DataDog/datadog-agent/comp/core/config"
 	log "github.com/DataDog/datadog-agent/comp/core/log/def"
 	"github.com/DataDog/datadog-agent/comp/core/tagger/api"
+	"github.com/DataDog/datadog-agent/pkg/api/util"
 	pkgconfigsetup "github.com/DataDog/datadog-agent/pkg/config/setup"
 	"github.com/DataDog/datadog-agent/pkg/util/fxutil"
 )
@@ -58,6 +59,10 @@ func taggerList(deps dependencies) error {
 		return err
 	}
 
+	err = util.SetAuthToken(deps.Config)
+	if err != nil {
+		return err
+	}
 	return api.GetTaggerList(color.Output, taggerURL)
 }
 

comp/process/apiserver/apiserver.go

@@ -16,8 +16,10 @@ import (
 
 	"github.com/DataDog/datadog-agent/cmd/process-agent/api"
 	"github.com/DataDog/datadog-agent/comp/api/authtoken"
-	log "github.com/DataDog/datadog-agent/comp/core/log/def"
+	logComp "github.com/DataDog/datadog-agent/comp/core/log/def"
+	"github.com/DataDog/datadog-agent/pkg/api/util"
 	pkgconfigsetup "github.com/DataDog/datadog-agent/pkg/config/setup"
+	"github.com/DataDog/datadog-agent/pkg/util/log"
 )
 
 var _ Component = (*apiserver)(nil)
@@ -31,7 +33,7 @@ type dependencies struct {
 
 	Lc fx.Lifecycle
 
-	Log log.Component
+	Log logComp.Component
 
 	At authtoken.Component
 
@@ -41,6 +43,7 @@ type dependencies struct {
 //nolint:revive // TODO(PROC) Fix revive linter
 func newApiServer(deps dependencies) Component {
 	r := mux.NewRouter()
+	r.Use(validateToken)
 	api.SetupAPIServerHandlers(deps.APIServerDeps, r) // Set up routes
 
 	addr, err := pkgconfigsetup.GetProcessAPIAddressPort(pkgconfigsetup.Datadog())
@@ -84,3 +87,13 @@ func newApiServer(deps dependencies) Component {
 
 	return apiserver
 }
+
+func validateToken(next http.Handler) http.Handler {
+	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		if err := util.Validate(w, r); err != nil {
+			log.Warnf("invalid auth token for %s request to %s: %s", r.Method, r.RequestURI, err)
+			return
+		}
+		next.ServeHTTP(w, r)
+	})
+}

comp/process/apiserver/apiserver_test.go

@@ -6,29 +6,41 @@
 package apiserver
 
 import (
+	"fmt"
+	"net"
 	"net/http"
 	"testing"
 	"time"
 
 	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
 	"go.uber.org/fx"
 
-	"github.com/DataDog/datadog-agent/comp/api/authtoken/fetchonlyimpl"
+	"github.com/DataDog/datadog-agent/comp/api/authtoken/createandfetchimpl"
 	"github.com/DataDog/datadog-agent/comp/core"
+	"github.com/DataDog/datadog-agent/comp/core/config"
 	"github.com/DataDog/datadog-agent/comp/core/settings/settingsimpl"
 	"github.com/DataDog/datadog-agent/comp/core/status"
 	"github.com/DataDog/datadog-agent/comp/core/status/statusimpl"
 	tagger "github.com/DataDog/datadog-agent/comp/core/tagger/def"
 	taggerfx "github.com/DataDog/datadog-agent/comp/core/tagger/fx"
 	workloadmeta "github.com/DataDog/datadog-agent/comp/core/workloadmeta/def"
 	workloadmetafx "github.com/DataDog/datadog-agent/comp/core/workloadmeta/fx"
+	"github.com/DataDog/datadog-agent/pkg/api/util"
 	"github.com/DataDog/datadog-agent/pkg/util/fxutil"
 )
 
 func TestLifecycle(t *testing.T) {
+	listener, err := net.Listen("tcp", ":0")
+	require.NoError(t, err)
+	port := listener.Addr().(*net.TCPAddr).Port
+
 	_ = fxutil.Test[Component](t, fx.Options(
 		Module(),
 		core.MockBundle(),
+		fx.Replace(config.MockParams{Overrides: map[string]interface{}{
+			"process_config.cmd_port": port,
+		}}),
 		workloadmetafx.Module(workloadmeta.NewParams()),
 		fx.Supply(
 			status.Params{
@@ -40,16 +52,61 @@ func TestLifecycle(t *testing.T) {
 		}),
 		statusimpl.Module(),
 		settingsimpl.MockModule(),
-		fetchonlyimpl.MockModule(),
+		createandfetchimpl.Module(),
 	))
 
-	assert.Eventually(t, func() bool {
-		res, err := http.Get("http://localhost:6162/config")
-		if err != nil {
-			return false
-		}
+	assert.EventuallyWithT(t, func(c *assert.CollectT) {
+		url := fmt.Sprintf("http://localhost:%d/agent/status", port)
+		req, err := http.NewRequest("GET", url, nil)
+		require.NoError(c, err)
+		req.Header.Set("Authorization", "Bearer "+util.GetAuthToken())
+		res, err := util.GetClient(false).Do(req)
+		require.NoError(c, err)
 		defer res.Body.Close()
+		assert.Equal(c, http.StatusOK, res.StatusCode)
+	}, 5*time.Second, time.Second)
+}
 
-		return res.StatusCode == http.StatusOK
+func TestPostAuthentication(t *testing.T) {
+	listener, err := net.Listen("tcp", ":0")
+	require.NoError(t, err)
+	port := listener.Addr().(*net.TCPAddr).Port
+
+	_ = fxutil.Test[Component](t, fx.Options(
+		Module(),
+		core.MockBundle(),
+		fx.Replace(config.MockParams{Overrides: map[string]interface{}{
+			"process_config.cmd_port": port,
+		}}),
+		workloadmetafx.Module(workloadmeta.NewParams()),
+		fx.Supply(
+			status.Params{
+				PythonVersionGetFunc: func() string { return "n/a" },
+			},
+		),
+		taggerfx.Module(tagger.Params{
+			UseFakeTagger: true,
+		}),
+		statusimpl.Module(),
+		settingsimpl.MockModule(),
+		createandfetchimpl.Module(),
+	))
+
+	assert.EventuallyWithT(t, func(c *assert.CollectT) {
+		// No authentication
+		url := fmt.Sprintf("http://localhost:%d/config/log_level?value=debug", port)
+		req, err := http.NewRequest("POST", url, nil)
+		require.NoError(c, err)
+		res, err := util.GetClient(false).Do(req)
+		require.NoError(c, err)
+		defer res.Body.Close()
+		assert.Equal(c, http.StatusUnauthorized, res.StatusCode)
+
+		// With authentication
+		req.Header.Set("Authorization", "Bearer "+util.GetAuthToken())
+		res, err = util.GetClient(false).Do(req)
+		require.NoError(c, err)
+		defer res.Body.Close()
+		assert.Equal(c, http.StatusOK, res.StatusCode)
 	}, 5*time.Second, time.Second)
 }