[CWS][SEC-11696] add Fargate e2e tests (#21667)

[CWS][SEC-11696] add Fargate e2e tests

.gitlab/dev_container_deploy/docker_linux.yml

@@ -279,6 +279,17 @@ dev_nightly-dogstatsd:
     IMG_SOURCES: ${SRC_DSD}:v${CI_PIPELINE_ID}-${CI_COMMIT_SHORT_SHA}-amd64,${SRC_DSD}:v${CI_PIPELINE_ID}-${CI_COMMIT_SHORT_SHA}-arm64
     IMG_DESTINATIONS: dogstatsd:${CI_PIPELINE_ID}-${CI_COMMIT_SHORT_SHA}
 
+.qa_cws_instrumentation:
+  extends: .docker_publish_job_definition
+  stage: dev_container_deploy
+  needs:
+    - docker_build_cws_instrumentation_amd64
+    - docker_build_cws_instrumentation_arm64
+  variables:
+    IMG_REGISTRIES: agent-qa
+    IMG_SOURCES: ${SRC_CWS_INSTRUMENTATION}:v${CI_PIPELINE_ID}-${CI_COMMIT_SHORT_SHA}-amd64,${SRC_CWS_INSTRUMENTATION}:v${CI_PIPELINE_ID}-${CI_COMMIT_SHORT_SHA}-arm64
+    IMG_DESTINATIONS: cws-instrumentation:${CI_PIPELINE_ID}-${CI_COMMIT_SHORT_SHA}
+
 qa_main_agent:
   extends: .qa_agent
   rules: !reference [.on_main_and_no_skip_e2e]
@@ -308,3 +319,14 @@ qa_branch_dogstatsd:
   rules:
     - !reference [.if_run_e2e_tests]
     - !reference [.on_dev_branch_manual]
+
+qa_main_cws_instrumentation:
+  extends: .qa_cws_instrumentation
+  rules:
+    !reference [.on_main_and_no_skip_e2e]
+
+qa_branch_cws_instrumentation:
+  extends: .qa_cws_instrumentation
+  rules:
+    - !reference [.if_run_e2e_tests]
+    - !reference [.on_dev_branch_manual]

.gitlab/e2e.yml

@@ -263,13 +263,23 @@ new-e2e-cws-dev:
   variables:
     TARGETS: ./tests/cws
     TEAM: csm-threats-agent
+    CWS_INSTRUMENTATION_FULLIMAGEPATH: 669783387624.dkr.ecr.us-east-1.amazonaws.com/cws-instrumentation:${CI_PIPELINE_ID}-${CI_COMMIT_SHORT_SHA}
+  parallel:
+    matrix:
+      - EXTRA_PARAMS: --run TestAgentSuite
+      - EXTRA_PARAMS: --run TestECSFargate
 
 new-e2e-cws-main:
   extends: .new_e2e_template
   rules: !reference [.on_main_and_no_skip_e2e]
   variables:
     TARGETS: ./tests/cws
     TEAM: csm-threats-agent
+    CWS_INSTRUMENTATION_FULLIMAGEPATH: 669783387624.dkr.ecr.us-east-1.amazonaws.com/cws-instrumentation:${CI_PIPELINE_ID}-${CI_COMMIT_SHORT_SHA}
+  parallel:
+    matrix:
+      - EXTRA_PARAMS: --run TestAgentSuite
+      - EXTRA_PARAMS: --run TestECSFargate
 
 new-e2e-process-dev:
   extends: .new_e2e_template

test/new-e2e/go.mod

@@ -176,8 +176,8 @@ require (
 	github.com/pmezard/go-difflib v1.0.0 // indirect
 	github.com/pulumi/appdash v0.0.0-20231130102222-75f619a67231 // indirect
 	github.com/pulumi/esc v0.6.2 // indirect
-	github.com/pulumi/pulumi-aws/sdk/v5 v5.42.0 // indirect
-	github.com/pulumi/pulumi-awsx/sdk v1.0.6 // indirect
+	github.com/pulumi/pulumi-aws/sdk/v5 v5.42.0
+	github.com/pulumi/pulumi-awsx/sdk v1.0.6
 	github.com/pulumi/pulumi-command/sdk v0.9.2 // indirect
 	github.com/pulumi/pulumi-docker/sdk/v3 v3.6.1 // indirect
 	github.com/pulumi/pulumi-eks/sdk v1.0.4 // indirect

test/new-e2e/tests/cws/lib/app.go → test/new-e2e/tests/cws/api/api.go

@@ -3,26 +3,27 @@
 // This product includes software developed at Datadog (https://www.datadoghq.com/).
 // Copyright 2016-present Datadog, Inc.
 
-// Package cws holds cws test related functions
-package cws
+// Package api provides test helpers to interact with the Datadog API
+package api
 
 import (
 	"context"
 	"time"
 
+	"github.com/DataDog/datadog-api-client-go/api/v2/datadog"
+
 	"github.com/DataDog/datadog-agent/test/new-e2e/pkg/runner"
 	"github.com/DataDog/datadog-agent/test/new-e2e/pkg/runner/parameters"
-	"github.com/DataDog/datadog-api-client-go/api/v2/datadog"
 )
 
-// APIClient represents the datadog API context
-type APIClient struct {
+// Client represents the datadog API context
+type Client struct {
 	api *datadog.APIClient
 	ctx context.Context
 }
 
-// NewAPIClient initialise a client with the API and APP keys
-func NewAPIClient() *APIClient {
+// NewClient initialise a client with the API and APP keys
+func NewClient() *Client {
 	apiKey, _ := runner.GetProfile().SecretStore().Get(parameters.APIKey)
 	appKey, _ := runner.GetProfile().SecretStore().Get(parameters.APPKey)
 	ctx := context.WithValue(
@@ -40,14 +41,14 @@ func NewAPIClient() *APIClient {
 
 	cfg := datadog.NewConfiguration()
 
-	return &APIClient{
+	return &Client{
 		api: datadog.NewAPIClient(cfg),
 		ctx: ctx,
 	}
 }
 
 // GetAppLog returns the logs corresponding to the query
-func (c *APIClient) GetAppLog(query string) (*datadog.LogsListResponse, error) {
+func (c *Client) GetAppLog(query string) (*datadog.LogsListResponse, error) {
 	sort := datadog.LOGSSORT_TIMESTAMP_ASCENDING
 
 	body := datadog.LogsListRequest{
@@ -74,7 +75,7 @@ func (c *APIClient) GetAppLog(query string) (*datadog.LogsListResponse, error) {
 }
 
 // GetAppSignal returns the signal corresponding to the query
-func (c *APIClient) GetAppSignal(query string) (*datadog.SecurityMonitoringSignalsListResponse, error) {
+func (c *Client) GetAppSignal(query string) (*datadog.SecurityMonitoringSignalsListResponse, error) {
 	now := time.Now().UTC()
 	queryFrom := now.Add(-15 * time.Minute)
 	sort := datadog.SECURITYMONITORINGSIGNALSSORT_TIMESTAMP_ASCENDING
@@ -104,7 +105,7 @@ func (c *APIClient) GetAppSignal(query string) (*datadog.SecurityMonitoringSigna
 }
 
 // CreateCwsSignalRule creates a cws signal rule
-func (c *APIClient) CreateCwsSignalRule(name string, msg string, agentRuleID string, tags []string) (*datadog.SecurityMonitoringRuleResponse, error) {
+func (c *Client) CreateCwsSignalRule(name string, msg string, agentRuleID string, tags []string) (*datadog.SecurityMonitoringRuleResponse, error) {
 	if tags == nil {
 		tags = []string{}
 	}
@@ -156,7 +157,7 @@ func (c *APIClient) CreateCwsSignalRule(name string, msg string, agentRuleID str
 }
 
 // CreateCWSAgentRule creates a cws agent rule
-func (c *APIClient) CreateCWSAgentRule(name string, msg string, secl string) (*datadog.CloudWorkloadSecurityAgentRuleResponse, error) {
+func (c *Client) CreateCWSAgentRule(name string, msg string, secl string) (*datadog.CloudWorkloadSecurityAgentRuleResponse, error) {
 	body := datadog.CloudWorkloadSecurityAgentRuleCreateRequest{
 		Data: datadog.CloudWorkloadSecurityAgentRuleCreateData{
 			Attributes: datadog.CloudWorkloadSecurityAgentRuleCreateAttributes{
@@ -178,13 +179,13 @@ func (c *APIClient) CreateCWSAgentRule(name string, msg string, secl string) (*d
 }
 
 // DeleteSignalRule deletes a signal rule
-func (c *APIClient) DeleteSignalRule(ruleID string) error {
+func (c *Client) DeleteSignalRule(ruleID string) error {
 	_, err := c.api.SecurityMonitoringApi.DeleteSecurityMonitoringRule(c.ctx, ruleID)
 	return err
 }
 
 // DeleteAgentRule deletes an agent rule
-func (c *APIClient) DeleteAgentRule(ruleID string) error {
+func (c *Client) DeleteAgentRule(ruleID string) error {
 	_, err := c.api.CloudWorkloadSecurityApi.DeleteCloudWorkloadSecurityAgentRule(c.ctx, ruleID)
 	return err
 }

test/new-e2e/tests/cws/lib/logs.go → test/new-e2e/tests/cws/api/helpers.go

@@ -3,20 +3,18 @@
 // This product includes software developed at Datadog (https://www.datadoghq.com/).
 // Copyright 2016-present Datadog, Inc.
 
-package cws
+package api
 
 import (
 	"errors"
-	"fmt"
 	"time"
 
 	"github.com/DataDog/datadog-api-client-go/api/v2/datadog"
 	"github.com/cenkalti/backoff"
 )
 
 // WaitAppLogs waits for the app log corresponding to the query
-func WaitAppLogs(apiClient *APIClient, query string) (*datadog.LogAttributes, error) {
-	query = fmt.Sprintf("host:cws-new-e2e-test-host %s", query)
+func WaitAppLogs(apiClient *Client, query string) (*datadog.LogAttributes, error) {
 	var resp *datadog.LogAttributes
 	err := backoff.Retry(func() error {
 		tmpResp, err := apiClient.GetAppLog(query)
@@ -33,7 +31,7 @@ func WaitAppLogs(apiClient *APIClient, query string) (*datadog.LogAttributes, er
 }
 
 // WaitAppSignal waits for the signal corresponding to the query
-func WaitAppSignal(apiClient *APIClient, query string) (*datadog.SecurityMonitoringSignalAttributes, error) {
+func WaitAppSignal(apiClient *Client, query string) (*datadog.SecurityMonitoringSignalAttributes, error) {
 	var resp *datadog.SecurityMonitoringSignalAttributes
 	err := backoff.Retry(func() error {
 		tmpResp, err := apiClient.GetAppSignal(query)

test/new-e2e/tests/cws/e2e_cws_test.go → test/new-e2e/tests/cws/ec2_test.go

@@ -3,6 +3,7 @@
 // This product includes software developed at Datadog (https://www.datadoghq.com/).
 // Copyright 2016-present Datadog, Inc.
 
+// Package cws holds cws e2e tests
 package cws
 
 import (
@@ -23,14 +24,28 @@ import (
 	awshost "github.com/DataDog/datadog-agent/test/new-e2e/pkg/environments/aws/host"
 	"github.com/DataDog/datadog-agent/test/new-e2e/pkg/runner"
 	"github.com/DataDog/datadog-agent/test/new-e2e/pkg/runner/parameters"
-	cws "github.com/DataDog/datadog-agent/test/new-e2e/tests/cws/lib"
+	"github.com/DataDog/datadog-agent/test/new-e2e/tests/cws/api"
 
 	"github.com/DataDog/test-infra-definitions/components/datadog/agentparams"
 )
 
+const (
+	// securityStartLog is the log corresponding to a successful start of the security-agent
+	securityStartLog = "Successfully connected to the runtime security module"
+
+	// systemProbeStartLog is the log corresponding to a successful start of the system-probe
+	systemProbeStartLog = "runtime security started"
+
+	// securityAgentPath is the path of the security-agent binary
+	securityAgentPath = "/opt/datadog-agent/embedded/bin/security-agent"
+
+	// policiesPath is the path of the default runtime security policies
+	policiesPath = "/etc/datadog-agent/runtime-security.d/test.policy"
+)
+
 type agentSuite struct {
 	e2e.BaseSuite[environments.Host]
-	apiClient     *cws.APIClient
+	apiClient     *api.Client
 	signalRuleID  string
 	agentRuleID   string
 	dirname       string
@@ -53,7 +68,7 @@ var securityAgentConfig string
 func TestAgentSuite(t *testing.T) {
 	e2e.Run(t, &agentSuite{}, e2e.WithProvisioner(
 		awshost.Provisioner(
-			awshost.WithName("cws-e2e-tests"),
+			awshost.WithName("cws-agent-e2e-tests"),
 			awshost.WithAgentOptions(
 				agentparams.WithAgentConfig(agentConfig),
 				agentparams.WithSecurityAgentConfig(securityAgentConfig),
@@ -65,7 +80,7 @@ func TestAgentSuite(t *testing.T) {
 
 func (a *agentSuite) SetupSuite() {
 	a.BaseSuite.SetupSuite()
-	a.apiClient = cws.NewAPIClient()
+	a.apiClient = api.NewClient()
 }
 
 func (a *agentSuite) TearDownSuite() {
@@ -104,11 +119,11 @@ func (a *agentSuite) TestOpenSignal() {
 	assert.Equal(a.T(), isReady, true, "Agent should be ready")
 
 	// Check if system-probe has started
-	err = a.waitAgentLogs("system-probe", cws.SystemProbeStartLog)
+	err = a.waitAgentLogs("system-probe", systemProbeStartLog)
 	require.NoError(a.T(), err, "system-probe could not start")
 
 	// Check if security-agent has started
-	err = a.waitAgentLogs("security-agent", cws.SecurityStartLog)
+	err = a.waitAgentLogs("security-agent", securityStartLog)
 	require.NoError(a.T(), err, "security-agent could not start")
 
 	// Download policies
@@ -119,7 +134,7 @@ func (a *agentSuite) TestOpenSignal() {
 	require.NoError(a.T(), err, "Could not get APP KEY")
 
 	a.EventuallyWithT(func(c *assert.CollectT) {
-		policies := a.Env().RemoteHost.MustExecute(fmt.Sprintf("DD_APP_KEY=%s DD_API_KEY=%s %s runtime policy download >| temp.txt && cat temp.txt", appKey, apiKey, cws.SecurityAgentPath))
+		policies := a.Env().RemoteHost.MustExecute(fmt.Sprintf("DD_APP_KEY=%s DD_API_KEY=%s %s runtime policy download >| temp.txt && cat temp.txt", appKey, apiKey, securityAgentPath))
 		assert.NotEmpty(c, policies, "should not be empty")
 		a.policies = policies
 	}, 5*time.Minute, 10*time.Second)
@@ -128,16 +143,16 @@ func (a *agentSuite) TestOpenSignal() {
 	assert.Contains(a.T(), a.policies, a.desc, "The policies should contain the created rule")
 
 	// Push policies
-	a.Env().RemoteHost.MustExecute(fmt.Sprintf("sudo cp temp.txt %s", cws.PoliciesPath))
+	a.Env().RemoteHost.MustExecute(fmt.Sprintf("sudo cp temp.txt %s", policiesPath))
 	a.Env().RemoteHost.MustExecute("rm temp.txt")
-	policiesFile := a.Env().RemoteHost.MustExecute(fmt.Sprintf("cat %s", cws.PoliciesPath))
+	policiesFile := a.Env().RemoteHost.MustExecute(fmt.Sprintf("cat %s", policiesPath))
 	assert.Contains(a.T(), policiesFile, a.desc, "The policies file should contain the created rule")
 
 	// Reload policies
-	a.Env().RemoteHost.MustExecute(fmt.Sprintf("sudo %s runtime policy reload", cws.SecurityAgentPath))
+	a.Env().RemoteHost.MustExecute(fmt.Sprintf("sudo %s runtime policy reload", securityAgentPath))
 
 	// Check `downloaded` ruleset_loaded
-	result, err := cws.WaitAppLogs(a.apiClient, "rule_id:ruleset_loaded")
+	result, err := api.WaitAppLogs(a.apiClient, "host:cws-new-e2e-test-host rule_id:ruleset_loaded")
 	require.NoError(a.T(), err, "could not get new ruleset")
 
 	agentContext := result.Attributes["agent"].(map[string]interface{})
@@ -151,9 +166,9 @@ func (a *agentSuite) TestOpenSignal() {
 	require.NoError(a.T(), err, "could not send payload")
 
 	// Check app signal
-	signal, err := cws.WaitAppSignal(a.apiClient, fmt.Sprintf("rule_id:%s", a.agentRuleName))
+	signal, err := api.WaitAppSignal(a.apiClient, fmt.Sprintf("host:cws-new-e2e-test-host @workflow.rule.id:%s", a.signalRuleID))
 	require.NoError(a.T(), err)
-	assert.Contains(a.T(), signal.Tags, fmt.Sprintf("rule_id:%s", a.agentRuleName), "unable to find agent_rule_name tag")
+	assert.Contains(a.T(), signal.Tags, fmt.Sprintf("rule_id:%s", strings.ToLower(a.agentRuleName)), "unable to find rule_id tag")
 	agentContext = signal.Attributes["agent"].(map[string]interface{})
 	assert.Contains(a.T(), agentContext["rule_id"], a.agentRuleName, "unable to find tag")
 }