upgrade(fleet): Add new Fleet Policies layer & loader (#28096)

comp/core/config/params.go

@@ -15,6 +15,9 @@ type Params struct {
 	// Usually given by the --extracfgpath command-line flag.
 	ExtraConfFilePath []string
 
+	// FleetPoliciesDirPath is the path at which to look for remote configuration files
+	FleetPoliciesDirPath string
+
 	// configName is the root of the name of the configuration file.  The
 	// comp/core/config component will search for a file with this name
 	// in ConfFilePath, using a variety of extensions.  The default is
@@ -131,6 +134,13 @@ func WithExtraConfFiles(extraConfFilePath []string) func(*Params) {
 	}
 }
 
+// WithFleetPoliciesDirPath returns an option which sets FleetPoliciesDirPath
+func WithFleetPoliciesDirPath(fleetPoliciesDirPath string) func(*Params) {
+	return func(b *Params) {
+		b.FleetPoliciesDirPath = fleetPoliciesDirPath
+	}
+}
+
 // These functions are used in unit tests.
 
 // ConfigMissingOK determines whether it is a fatal error if the config

comp/core/config/setup.go

@@ -9,6 +9,7 @@ import (
 	"errors"
 	"fmt"
 	"io/fs"
+	"path"
 	"runtime"
 	"strings"
 
@@ -74,5 +75,21 @@ func setupConfig(config pkgconfigmodel.Config, deps configDependencies) (*pkgcon
 		}
 		return warnings, err
 	}
+
+	// Load the remote configuration
+	if p.FleetPoliciesDirPath != "" {
+		// Main config file
+		err := config.MergeFleetPolicy(path.Join(p.FleetPoliciesDirPath, "datadog.yaml"))
+		if err != nil {
+			return warnings, err
+		}
+		if p.configLoadSecurityAgent {
+			err := config.MergeFleetPolicy(path.Join(p.FleetPoliciesDirPath, "security-agent.yaml"))
+			if err != nil {
+				return warnings, err
+			}
+		}
+	}
+
 	return warnings, nil
 }

comp/core/settings/settingsimpl/settingsimpl_test.go

@@ -252,7 +252,7 @@ func TestRuntimeSettings(t *testing.T) {
 				// simply compare strings.
 				expected := map[string]interface{}{}
 				actual := map[string]interface{}{}
-				json.Unmarshal([]byte("{\"value\":{\"Value\":\"\",\"Source\":\"\"},\"sources_value\":[{\"Source\":\"default\",\"Value\":null},{\"Source\":\"unknown\",\"Value\":null},{\"Source\":\"file\",\"Value\":null},{\"Source\":\"environment-variable\",\"Value\":null},{\"Source\":\"agent-runtime\",\"Value\":null},{\"Source\":\"local-config-process\",\"Value\":null},{\"Source\":\"remote-config\",\"Value\":null},{\"Source\":\"cli\",\"Value\":null}]}"), &expected)
+				json.Unmarshal([]byte("{\"value\":{\"Value\":\"\",\"Source\":\"\"},\"sources_value\":[{\"Source\":\"default\",\"Value\":null},{\"Source\":\"unknown\",\"Value\":null},{\"Source\":\"file\",\"Value\":null},{\"Source\":\"environment-variable\",\"Value\":null},{\"Source\":\"fleet-policies\",\"Value\":null},{\"Source\":\"agent-runtime\",\"Value\":null},{\"Source\":\"local-config-process\",\"Value\":null},{\"Source\":\"remote-config\",\"Value\":null},{\"Source\":\"cli\",\"Value\":null}]}"), &expected)
 				err = json.Unmarshal(body, &actual)
 
 				require.NoError(t, err, fmt.Sprintf("error loading JSON body: %s", err))

comp/metadata/inventoryagent/README.md

@@ -118,6 +118,8 @@ The payload is a JSON dict with the following fields
     Only the settings set by the agent itself are included, and their value might not match what's applyed by the agent because they can be overriden by other sources.
   - `remote_configuration` - **string**: the Agent configuration specified by the Remote Configuration (scrubbed), as a YAML string.
     Only the settings currently used by Remote Configuration are included, and their value might not match what's applyed by the agent because they can be overriden by other sources.
+  - `fleet_policies_configuration` - **string**: the Agent configuration specified by the Fleet Automation Policies (scrubbed), as a YAML string.
+    Only the settings currently used by Fleet Automation Policies are included, and their value might not match what's applyed by the agent since they can be overriden by other sources.
   - `cli_configuration` - **string**: the Agent configuration specified by the CLI (scrubbed), as a YAML string.
     Only the settings set in the CLI are included, they cannot be overriden by any other sources.
   - `source_local_configuration` - **string**: the Agent configuration synchronized from the local Agent process, as a YAML string.

comp/metadata/inventoryagent/inventoryagentimpl/inventoryagent.go

@@ -422,6 +422,7 @@ func (ia *inventoryagent) getConfigs(data agentMetadata) {
 			model.SourceAgentRuntime:       "agent_runtime_configuration",
 			model.SourceLocalConfigProcess: "source_local_configuration",
 			model.SourceRC:                 "remote_configuration",
+			model.SourceFleetPolicies:      "fleet_policies_configuration",
 			model.SourceCLI:                "cli_configuration",
 			model.SourceProvided:           "provided_configuration",
 		}

comp/metadata/inventoryagent/inventoryagentimpl/inventoryagent_test.go

@@ -719,7 +719,7 @@ func TestGetProvidedConfigurationOnly(t *testing.T) {
 	}
 
 	sort.Strings(keys)
-	expected := []string{"provided_configuration", "full_configuration", "file_configuration", "environment_variable_configuration", "agent_runtime_configuration", "remote_configuration", "cli_configuration", "source_local_configuration"}
+	expected := []string{"provided_configuration", "full_configuration", "file_configuration", "environment_variable_configuration", "agent_runtime_configuration", "fleet_policies_configuration", "remote_configuration", "cli_configuration", "source_local_configuration"}
 	sort.Strings(expected)
 
 	assert.Equal(t, expected, keys)

comp/metadata/securityagent/README.md

@@ -34,6 +34,8 @@ The payload is a JSON dict with the following fields
     Only the settings set by the agent itself are included, and their value might not match what's applyed by the agent since they can be overriden by other sources.
   - `remote_configuration` - **string**: the Security-Agent configuration specified by the Remote Configuration (scrubbed), as a YAML string.
     Only the settings currently used by Remote Configuration are included, and their value might not match what's applyed by the agent since they can be overriden by other sources.
+  - `fleet_policies_configuration` - **string**: the Security-Agent configuration specified by the Fleet Automation Policies (scrubbed), as a YAML string.
+    Only the settings currently used by Fleet Automation Policies are included, and their value might not match what's applyed by the agent since they can be overriden by other sources.
   - `cli_configuration` - **string**: the Security-Agent configuration specified by the CLI (scrubbed), as a YAML string.
     Only the settings set in the CLI are included.
   - `source_local_configuration` - **string**: the Security-Agent configuration synchronized from the local Agent process, as a YAML string.

comp/metadata/securityagent/impl/security_agent.go

@@ -138,6 +138,7 @@ func (sa *secagent) getConfigLayers() map[string]interface{} {
 		model.SourceAgentRuntime:       "agent_runtime_configuration",
 		model.SourceLocalConfigProcess: "source_local_configuration",
 		model.SourceRC:                 "remote_configuration",
+		model.SourceFleetPolicies:      "fleet_policies_configuration",
 		model.SourceCLI:                "cli_configuration",
 		model.SourceProvided:           "provided_configuration",
 	}

pkg/config/model/types.go

@@ -137,6 +137,7 @@ type Compound interface {
 	ReadConfig(in io.Reader) error
 	MergeConfig(in io.Reader) error
 	MergeConfigMap(cfg map[string]any) error
+	MergeFleetPolicy(configPath string) error
 
 	AddConfigPath(in string)
 	AddExtraConfigPaths(in []string) error

pkg/config/model/viper.go

@@ -51,6 +51,8 @@ const (
 	SourceLocalConfigProcess Source = "local-config-process"
 	// SourceRC are the values loaded from remote-config (aka Datadog backend)
 	SourceRC Source = "remote-config"
+	// SourceFleetPolicies are the values loaded from remote-config file
+	SourceFleetPolicies Source = "fleet-policies"
 	// SourceCLI are the values set by the user at runtime through the CLI.
 	SourceCLI Source = "cli"
 	// SourceProvided are all values set by any source but default.
@@ -63,6 +65,7 @@ var sources = []Source{
 	SourceUnknown,
 	SourceFile,
 	SourceEnvVar,
+	SourceFleetPolicies,
 	SourceAgentRuntime,
 	SourceLocalConfigProcess,
 	SourceRC,
@@ -612,6 +615,37 @@ func (c *safeConfig) MergeConfig(in io.Reader) error {
 	return c.Viper.MergeConfig(in)
 }
 
+// MergeFleetPolicy merges the configuration from the reader given with an existing config
+// it overrides the existing values with the new ones in the FleetPolicies source, and updates the main config
+// according to sources priority order.
+func (c *safeConfig) MergeFleetPolicy(configPath string) error {
+	c.Lock()
+	defer c.Unlock()
+
+	// Check file existence & open it
+	_, err := os.Stat(configPath)
+	if err != nil && !os.IsNotExist(err) {
+		return fmt.Errorf("unable to open config file %s: %w", configPath, err)
+	} else if err != nil && os.IsNotExist(err) {
+		return nil
+	}
+	in, err := os.Open(configPath)
+	if err != nil {
+		return fmt.Errorf("unable to open config file %s: %w", configPath, err)
+	}
+	defer in.Close()
+
+	c.configSources[SourceFleetPolicies].SetConfigType("yaml")
+	err = c.configSources[SourceFleetPolicies].MergeConfigOverride(in)
+	if err != nil {
+		return err
+	}
+	for _, key := range c.configSources[SourceFleetPolicies].AllKeys() {
+		c.mergeViperInstances(key)
+	}
+	return nil
+}
+
 // MergeConfigMap merges the configuration from the map given with an existing config.
 // Note that the map given may be modified.
 func (c *safeConfig) MergeConfigMap(cfg map[string]any) error {
@@ -650,6 +684,7 @@ func (c *safeConfig) AllSettingsBySource() map[Source]interface{} {
 		SourceUnknown,
 		SourceFile,
 		SourceEnvVar,
+		SourceFleetPolicies,
 		SourceAgentRuntime,
 		SourceRC,
 		SourceCLI,

pkg/config/model/viper_test.go

@@ -403,3 +403,18 @@ proxy:
 	assert.NoError(t, err)
 	assert.True(t, reflect.DeepEqual(oldConf, config.AllSettings()))
 }
+
+func TestMergeFleetPolicy(t *testing.T) {
+	config := NewConfig("test", "DD", strings.NewReplacer(".", "_"))
+	config.SetConfigType("yaml")
+	config.Set("foo", "bar", SourceFile)
+
+	file, err := os.CreateTemp("", "datadog.yaml")
+	assert.NoError(t, err, "failed to create temporary file: %w", err)
+	file.Write([]byte("foo: baz"))
+	err = config.MergeFleetPolicy(file.Name())
+	assert.NoError(t, err)
+
+	assert.Equal(t, "baz", config.Get("foo"))
+	assert.Equal(t, SourceFleetPolicies, config.GetSource("foo"))
+}