Merge branch 'main' into brian/AMLII-2164-optimize-tailer-scheduling

.gitlab/binary_build/system_probe.yml

@@ -12,7 +12,7 @@
     - find "$CI_BUILDS_DIR" ! -path '*DataDog/datadog-agent*' -delete || true  # Allow failure, we can't remove parent folders of datadog-agent
   script:
     - inv check-go-version
-    - inv -e system-probe.build --strip-object-files --no-bundle
+    - inv -e system-probe.build --strip-object-files
     # fail if references to glibc >= 2.18
     - objdump -p $CI_PROJECT_DIR/$SYSTEM_PROBE_BINARIES_DIR/system-probe | egrep 'GLIBC_2\.(1[8-9]|[2-9][0-9])' && exit 1
     - inv -e system-probe.save-build-outputs $CI_PROJECT_DIR/sysprobe-build-outputs.tar.xz

.run/Build process-agent.run.xml

@@ -12,7 +12,7 @@
     <option name="EXECUTE_IN_TERMINAL" value="true" />
     <option name="EXECUTE_SCRIPT_FILE" value="true" />
     <envs>
-      <env name="BUILD_COMMAND" value="inv process-agent.build --no-bundle --build-exclude=systemd" />
+      <env name="BUILD_COMMAND" value="inv process-agent.build --build-exclude=systemd" />
       <env name="SCRIPT_TO_RUN" value=".run/build.sh" />
     </envs>
     <method v="2" />

.run/Build system-probe.run.xml

@@ -13,7 +13,7 @@
     <option name="EXECUTE_SCRIPT_FILE" value="true" />
     <envs>
       <env name="SCRIPT_TO_RUN" value=".run/build.sh" />
-      <env name="BUILD_COMMAND" value="invoke system-probe.build --no-bundle" />
+      <env name="BUILD_COMMAND" value="invoke system-probe.build" />
     </envs>
     <method v="2" />
   </configuration>

cmd/security-agent/subcommands/start/command.go

@@ -45,7 +45,7 @@ import (
 	"github.com/DataDog/datadog-agent/comp/core/sysprobeconfig"
 	"github.com/DataDog/datadog-agent/comp/core/sysprobeconfig/sysprobeconfigimpl"
 	tagger "github.com/DataDog/datadog-agent/comp/core/tagger/def"
-	dualTaggerfx "github.com/DataDog/datadog-agent/comp/core/tagger/fx-dual"
+	remoteTaggerfx "github.com/DataDog/datadog-agent/comp/core/tagger/fx-remote"
 	taggerTypes "github.com/DataDog/datadog-agent/comp/core/tagger/types"
 	"github.com/DataDog/datadog-agent/comp/core/telemetry"
 	wmcatalog "github.com/DataDog/datadog-agent/comp/core/workloadmeta/collectors/catalog"
@@ -113,11 +113,7 @@ func Commands(globalParams *command.GlobalParams) []*cobra.Command {
 						AgentType: catalog,
 					}
 				}),
-				dualTaggerfx.Module(tagger.DualParams{
-					UseRemote: func(c config.Component) bool {
-						return c.GetBool("security_agent.remote_tagger")
-					},
-				}, tagger.Params{}, tagger.RemoteParams{
+				remoteTaggerfx.Module(tagger.RemoteParams{
 					RemoteTarget: func(c config.Component) (string, error) {
 						return fmt.Sprintf(":%v", c.GetInt("cmd_port")), nil
 					},

cmd/trace-agent/subcommands/run/command.go

@@ -14,7 +14,6 @@ import (
 	"github.com/spf13/cobra"
 	"go.uber.org/fx"
 
-	"github.com/DataDog/datadog-agent/cmd/agent/common"
 	"github.com/DataDog/datadog-agent/cmd/trace-agent/subcommands"
 	"github.com/DataDog/datadog-agent/comp/agent/autoexit"
 	"github.com/DataDog/datadog-agent/comp/agent/autoexit/autoexitimpl"
@@ -27,12 +26,9 @@ import (
 	"github.com/DataDog/datadog-agent/comp/core/secrets"
 	"github.com/DataDog/datadog-agent/comp/core/secrets/secretsimpl"
 	tagger "github.com/DataDog/datadog-agent/comp/core/tagger/def"
-	dualTaggerfx "github.com/DataDog/datadog-agent/comp/core/tagger/fx-dual"
+	remoteTaggerfx "github.com/DataDog/datadog-agent/comp/core/tagger/fx-remote"
 	taggerTypes "github.com/DataDog/datadog-agent/comp/core/tagger/types"
 	"github.com/DataDog/datadog-agent/comp/core/telemetry/telemetryimpl"
-	wmcatalog "github.com/DataDog/datadog-agent/comp/core/workloadmeta/collectors/catalog"
-	workloadmeta "github.com/DataDog/datadog-agent/comp/core/workloadmeta/def"
-	workloadmetafx "github.com/DataDog/datadog-agent/comp/core/workloadmeta/fx"
 	"github.com/DataDog/datadog-agent/comp/dogstatsd/statsd"
 	"github.com/DataDog/datadog-agent/comp/trace"
 	traceagent "github.com/DataDog/datadog-agent/comp/trace/agent/def"
@@ -93,19 +89,9 @@ func runTraceAgentProcess(ctx context.Context, cliParams *Params, defaultConfPat
 			return log.ForDaemon("TRACE", "apm_config.log_file", config.DefaultLogFilePath)
 		}),
 		logtracefx.Module(),
-		// setup workloadmeta
-		wmcatalog.GetCatalog(),
-		workloadmetafx.Module(workloadmeta.Params{
-			AgentType:  workloadmeta.NodeAgent,
-			InitHelper: common.GetWorkloadmetaInit(),
-		}),
 		autoexitimpl.Module(),
 		statsd.Module(),
-		dualTaggerfx.Module(tagger.DualParams{
-			UseRemote: func(c coreconfig.Component) bool {
-				return c.GetBool("apm_config.remote_tagger")
-			},
-		}, tagger.Params{}, tagger.RemoteParams{
+		remoteTaggerfx.Module(tagger.RemoteParams{
 			RemoteTarget: func(c coreconfig.Component) (string, error) {
 				return fmt.Sprintf(":%v", c.GetInt("cmd_port")), nil
 			},

cmd/trace-agent/test/agent.go

@@ -7,10 +7,16 @@ package test
 
 import (
 	"bytes"
+	"crypto/rand"
+	"crypto/tls"
+	"crypto/x509"
+	"encoding/hex"
+	"encoding/pem"
 	"errors"
 	"fmt"
 	"io"
 	"log"
+	"net"
 	"os"
 	"os/exec"
 	"path/filepath"
@@ -21,24 +27,43 @@ import (
 	"github.com/DataDog/viper"
 	yaml "gopkg.in/yaml.v2"
 
+	"google.golang.org/grpc"
+	"google.golang.org/grpc/credentials"
+
+	grpc_auth "github.com/grpc-ecosystem/go-grpc-middleware/auth"
+
+	"github.com/DataDog/datadog-agent/pkg/api/security"
+	pb "github.com/DataDog/datadog-agent/pkg/proto/pbgo/core"
+	grpcutil "github.com/DataDog/datadog-agent/pkg/util/grpc"
+
 	"github.com/DataDog/datadog-agent/pkg/trace/testutil"
 )
 
 // ErrNotInstalled is returned when the trace-agent can not be found in $PATH.
 var ErrNotInstalled = errors.New("agent: trace-agent not found in $PATH")
 
+// SecretBackendBinary secret binary name
+var SecretBackendBinary = "secret-script.test"
+
+type grpcServer struct {
+	pb.UnimplementedAgentSecureServer
+}
+
 type agentRunner struct {
 	mu  sync.RWMutex // guards pid
 	pid int          // agent pid, if running
 
-	port    int         // agent receiver port
-	log     *safeBuffer // agent log output
-	ddAddr  string      // Datadog intake address (host:port)
-	bindir  string      // the temporary directory where the trace-agent binary is located
-	verbose bool
+	port                 int         // agent receiver port
+	log                  *safeBuffer // agent log output
+	ddAddr               string      // Datadog intake address (host:port)
+	bindir               string      // the temporary directory where the trace-agent binary is located
+	verbose              bool
+	agentServer          *grpc.Server
+	agentServerListerner net.Listener
+	authToken            string
 }
 
-func newAgentRunner(ddAddr string, verbose bool) (*agentRunner, error) {
+func newAgentRunner(ddAddr string, verbose bool, buildSecretBackend bool) (*agentRunner, error) {
 	bindir, err := os.MkdirTemp("", "trace-agent-integration-tests")
 	if err != nil {
 		return nil, err
@@ -57,17 +82,75 @@ func newAgentRunner(ddAddr string, verbose bool) (*agentRunner, error) {
 		}
 		return nil, ErrNotInstalled
 	}
+
+	if buildSecretBackend {
+		binSecrets := filepath.Join(bindir, SecretBackendBinary)
+		o, err := exec.Command("go", "build", "-o", binSecrets, "./testdata/secretscript.go").CombinedOutput()
+
+		if err != nil {
+			if verbose {
+				log.Printf("error installing secret-script: %v", err)
+				log.Print(string(o))
+			}
+			return nil, ErrNotInstalled
+		}
+
+		if err := os.Chmod(binSecrets, 0700); err != nil {
+			if verbose {
+				log.Printf("error changing permissions secret-script: %v", err)
+			}
+			return nil, ErrNotInstalled
+		}
+	}
+
+	tlsKeyPair, err := buildSelfSignedTLSCertificate("127.0.0.1")
+	if err != nil {
+		return nil, fmt.Errorf("unable to generate TLS certificate: %v", err)
+	}
+
+	// Generate an authentication token and set up our gRPC server to both serve over TLS and authenticate each RPC
+	// using the authentication token.
+	authToken, err := generateAuthenticationToken()
+	if err != nil {
+		return nil, fmt.Errorf("unable to generate authentication token: %v", err)
+	}
+
+	serverOpts := []grpc.ServerOption{
+		grpc.Creds(credentials.NewServerTLSFromCert(tlsKeyPair)),
+		grpc.UnaryInterceptor(grpc_auth.UnaryServerInterceptor(grpcutil.StaticAuthInterceptor(authToken))),
+	}
+
+	// Start dummy gRPc server mocking the core agent
+	serverListener, err := net.Listen("tcp", "127.0.0.1:5051")
+	if err != nil {
+		return nil, ErrNotInstalled
+	}
+	s := grpc.NewServer(serverOpts...)
+	pb.RegisterAgentSecureServer(s, &grpcServer{})
+
+	go func() {
+		err := s.Serve(serverListener)
+		if err != nil {
+			log.Fatalf("failed to serve: %v", err)
+		}
+	}()
+
 	return &agentRunner{
-		bindir:  bindir,
-		ddAddr:  ddAddr,
-		log:     newSafeBuffer(),
-		verbose: verbose,
+		bindir:               bindir,
+		ddAddr:               ddAddr,
+		log:                  newSafeBuffer(),
+		verbose:              verbose,
+		agentServer:          s,
+		agentServerListerner: serverListener,
+		authToken:            authToken,
 	}, nil
 }
 
 // cleanup removes the agent binary.
 func (s *agentRunner) cleanup() error {
 	s.Kill()
+	s.agentServer.Stop()
+	s.agentServerListerner.Close()
 	return os.RemoveAll(s.bindir)
 }
 
@@ -137,6 +220,7 @@ func (s *agentRunner) Kill() {
 		}
 		return
 	}
+
 	s.mu.Lock()
 	s.pid = 0
 	s.mu.Unlock()
@@ -196,22 +280,62 @@ func (s *agentRunner) createConfigFile(conf []byte) (string, error) {
 		v.Set("log_level", "debug")
 	}
 
-	// disable remote tagger to avoid running a core agent for testing
-	v.Set("apm_config.remote_tagger", false)
+	v.Set("cmd_port", s.agentServerListerner.Addr().(*net.TCPAddr).Port)
 
 	out, err := yaml.Marshal(v.AllSettings())
 	if err != nil {
 		return "", err
 	}
-	f, err := os.Create(filepath.Join(s.bindir, "datadog.yaml"))
+	confFile, err := os.Create(filepath.Join(s.bindir, "datadog.yaml"))
 	if err != nil {
 		return "", err
 	}
-	if _, err := f.Write(out); err != nil {
+	if _, err := confFile.Write(out); err != nil {
 		return "", err
 	}
-	if err := f.Close(); err != nil {
+	if err := confFile.Close(); err != nil {
 		return "", err
 	}
-	return f.Name(), nil
+	// create auth_token file
+	authTokenFile, err := os.Create(filepath.Join(s.bindir, "auth_token"))
+	if err != nil {
+		return "", err
+	}
+	if _, err := authTokenFile.Write([]byte(s.authToken)); err != nil {
+		return "", err
+	}
+	if err := authTokenFile.Close(); err != nil {
+		return "", err
+	}
+	return confFile.Name(), nil
+}
+
+func buildSelfSignedTLSCertificate(host string) (*tls.Certificate, error) {
+	hosts := []string{host}
+	_, certPEM, key, err := security.GenerateRootCert(hosts, 2048)
+	if err != nil {
+		return nil, errors.New("unable to generate certificate")
+	}
+
+	// PEM encode the private key
+	keyPEM := pem.EncodeToMemory(&pem.Block{
+		Type: "RSA PRIVATE KEY", Bytes: x509.MarshalPKCS1PrivateKey(key),
+	})
+
+	pair, err := tls.X509KeyPair(certPEM, keyPEM)
+	if err != nil {
+		return nil, fmt.Errorf("unable to generate TLS key pair: %v", err)
+	}
+
+	return &pair, nil
+}
+
+func generateAuthenticationToken() (string, error) {
+	rawToken := make([]byte, 32)
+	_, err := rand.Read(rawToken)
+	if err != nil {
+		return "", fmt.Errorf("can't create authentication token value: %s", err)
+	}
+
+	return hex.EncodeToString(rawToken), nil
 }

cmd/trace-agent/test/runner.go

@@ -48,7 +48,22 @@ func (s *Runner) Start() error {
 		// respect whatever the testing framework says
 		s.Verbose = testing.Verbose()
 	}
-	agent, err := newAgentRunner(s.backend.srv.Addr, s.Verbose)
+	agent, err := newAgentRunner(s.backend.srv.Addr, s.Verbose, false)
+	if err != nil {
+		return err
+	}
+	s.agent = agent
+	return s.backend.Start()
+}
+
+// StartAndBuildSecretBackend initializes the runner, creates the secret binary and starts the fake backend.
+func (s *Runner) StartAndBuildSecretBackend() error {
+	s.backend = newFakeBackend(s.ChannelSize)
+	if !s.Verbose {
+		// respect whatever the testing framework says
+		s.Verbose = testing.Verbose()
+	}
+	agent, err := newAgentRunner(s.backend.srv.Addr, s.Verbose, true)
 	if err != nil {
 		return err
 	}
@@ -105,6 +120,11 @@ func (s *Runner) Out() <-chan interface{} {
 	return s.backend.Out()
 }
 
+// BinDir return the binary directory where the binary, configuration and secret backend binary are stored.
+func (s *Runner) BinDir() string {
+	return s.agent.bindir
+}
+
 // PostMsgpack encodes data using msgpack and posts it to the given path. The agent
 // must be started using RunAgent.
 //