[mq] working branch - merge 8fed744 on top of main at 5e200ff

{"baseBranch":"main","baseCommit":"5e200ffe39ad8d7adade5eae1092e7a10c8992ad","createdAt":"2023-12-18T16:09:15.692745Z","headSha":"8fed74423c12f7d11c64c8bc502f9042d7b5ff73","id":"18a91dba-053a-455c-9f0d-8c25f395fd2e","priority":"200","pullRequestNumber":"21575","queuedAt":"2023-12-18T16:09:15.691338Z","status":"STATUS_QUEUED"}

pkg/config/system_probe_cws.go

@@ -18,6 +18,7 @@ func initCWSSystemProbeConfig(cfg Config) {
 	cfg.BindEnvAndSetDefault("runtime_security_config.policies.watch_dir", false)
 	cfg.BindEnvAndSetDefault("runtime_security_config.policies.monitor.enabled", false)
 	cfg.BindEnvAndSetDefault("runtime_security_config.policies.monitor.per_rule_enabled", false)
+	cfg.BindEnvAndSetDefault("runtime_security_config.policies.monitor.report_internal_policies", false)
 	cfg.BindEnvAndSetDefault("runtime_security_config.event_server.burst", 40)
 	cfg.BindEnvAndSetDefault("runtime_security_config.event_server.retention", "6s")
 	cfg.BindEnvAndSetDefault("runtime_security_config.event_server.rate", 10)

pkg/security/config/config.go

@@ -47,6 +47,8 @@ type RuntimeSecurityConfig struct {
 	PolicyMonitorEnabled bool
 	// PolicyMonitorPerRuleEnabled enabled per-rule policy monitoring
 	PolicyMonitorPerRuleEnabled bool
+	// PolicyMonitorReportInternalPolicies enable internal policies monitoring
+	PolicyMonitorReportInternalPolicies bool
 	// SocketPath is the path to the socket that is used to communicate with the security agent
 	SocketPath string
 	// EventServerBurst defines the maximum burst of events that can be sent over the grpc server
@@ -257,10 +259,11 @@ func NewRuntimeSecurityConfig() (*RuntimeSecurityConfig, error) {
 		RemoteConfigurationEnabled: isRemoteConfigEnabled(),
 
 		// policy & ruleset
-		PoliciesDir:                 coreconfig.SystemProbe.GetString("runtime_security_config.policies.dir"),
-		WatchPoliciesDir:            coreconfig.SystemProbe.GetBool("runtime_security_config.policies.watch_dir"),
-		PolicyMonitorEnabled:        coreconfig.SystemProbe.GetBool("runtime_security_config.policies.monitor.enabled"),
-		PolicyMonitorPerRuleEnabled: coreconfig.SystemProbe.GetBool("runtime_security_config.policies.monitor.per_rule_enabled"),
+		PoliciesDir:                         coreconfig.SystemProbe.GetString("runtime_security_config.policies.dir"),
+		WatchPoliciesDir:                    coreconfig.SystemProbe.GetBool("runtime_security_config.policies.watch_dir"),
+		PolicyMonitorEnabled:                coreconfig.SystemProbe.GetBool("runtime_security_config.policies.monitor.enabled"),
+		PolicyMonitorPerRuleEnabled:         coreconfig.SystemProbe.GetBool("runtime_security_config.policies.monitor.per_rule_enabled"),
+		PolicyMonitorReportInternalPolicies: coreconfig.SystemProbe.GetBool("runtime_security_config.policies.monitor.report_internal_policies"),
 
 		LogPatterns: coreconfig.SystemProbe.GetStringSlice("runtime_security_config.log_patterns"),
 		LogTags:     coreconfig.SystemProbe.GetStringSlice("runtime_security_config.log_tags"),

pkg/security/probe/selftests/tester_linux.go

@@ -14,6 +14,8 @@ import (
 
 	"go.uber.org/atomic"
 
+	"github.com/hashicorp/go-multierror"
+
 	"github.com/DataDog/datadog-agent/pkg/security/config"
 	"github.com/DataDog/datadog-agent/pkg/security/probe"
 	"github.com/DataDog/datadog-agent/pkg/security/proto/api"
@@ -22,7 +24,6 @@ import (
 	"github.com/DataDog/datadog-agent/pkg/security/secl/rules"
 	"github.com/DataDog/datadog-agent/pkg/security/serializers"
 	"github.com/DataDog/datadog-agent/pkg/util/log"
-	"github.com/hashicorp/go-multierror"
 )
 
 // EventPredicate defines a self test event validation predicate
@@ -160,9 +161,10 @@ func (t *SelfTester) Close() error {
 // LoadPolicies implements the PolicyProvider interface
 func (t *SelfTester) LoadPolicies(_ []rules.MacroFilter, _ []rules.RuleFilter) ([]*rules.Policy, *multierror.Error) {
 	p := &rules.Policy{
-		Name:    policyName,
-		Source:  policySource,
-		Version: policyVersion,
+		Name:       policyName,
+		Source:     policySource,
+		Version:    policyVersion,
+		IsInternal: true,
 	}
 
 	for _, selftest := range FileSelfTests {

pkg/security/probe/selftests/tester_others.go

@@ -8,12 +8,13 @@
 package selftests
 
 import (
+	"github.com/hashicorp/go-multierror"
+
 	"github.com/DataDog/datadog-agent/pkg/security/config"
 	"github.com/DataDog/datadog-agent/pkg/security/probe"
 	"github.com/DataDog/datadog-agent/pkg/security/secl/compiler/eval"
 	"github.com/DataDog/datadog-agent/pkg/security/secl/rules"
 	"github.com/DataDog/datadog-agent/pkg/security/serializers"
-	"github.com/hashicorp/go-multierror"
 )
 
 // SelfTester represents all the state needed to conduct rule injection test at startup
@@ -44,9 +45,10 @@ func (t *SelfTester) Close() error {
 // LoadPolicies implements the PolicyProvider interface
 func (t *SelfTester) LoadPolicies(_ []rules.MacroFilter, _ []rules.RuleFilter) ([]*rules.Policy, *multierror.Error) {
 	p := &rules.Policy{
-		Name:    policyName,
-		Source:  policySource,
-		Version: policyVersion,
+		Name:       policyName,
+		Source:     policySource,
+		Version:    policyVersion,
+		IsInternal: true,
 	}
 
 	return []*rules.Policy{p}, nil

pkg/security/rules/bundled_policy_provider.go

@@ -7,9 +7,10 @@
 package rules
 
 import (
+	"github.com/hashicorp/go-multierror"
+
 	"github.com/DataDog/datadog-agent/pkg/security/secl/rules"
 	"github.com/DataDog/datadog-agent/pkg/version"
-	"github.com/hashicorp/go-multierror"
 )
 
 // BundledPolicyProvider specify the policy provider for bundled policies
@@ -23,6 +24,7 @@ func (p *BundledPolicyProvider) LoadPolicies([]rules.MacroFilter, []rules.RuleFi
 	policy.Source = "bundled"
 	policy.Version = version.AgentVersion
 	policy.Rules = bundledPolicyRules
+	policy.IsInternal = true
 
 	for _, rule := range bundledPolicyRules {
 		rule.Policy = policy

pkg/security/rules/engine.go

@@ -316,12 +316,12 @@ func (e *RuleEngine) LoadPolicies(providers []rules.PolicyProvider, sendLoadedRe
 
 	}
 
-	policies := monitor.NewPoliciesState(evaluationSet.RuleSets, loadErrs)
+	policies := monitor.NewPoliciesState(evaluationSet.RuleSets, loadErrs, e.config.PolicyMonitorReportInternalPolicies)
 	e.notifyAPIServer(ruleIDs, policies)
 
 	if sendLoadedReport {
 		monitor.ReportRuleSetLoaded(e.eventSender, e.statsdClient, policies)
-		e.policyMonitor.SetPolicies(evaluationSet.GetPolicies(), loadErrs)
+		e.policyMonitor.SetPolicies(policies)
 	}
 
 	return nil

pkg/security/rules/monitor/policy_monitor.go

@@ -35,63 +35,64 @@ const (
 	policyMetricRate = 30 * time.Second
 )
 
-// Policy describes policy related information
-type Policy struct {
-	Name    string
-	Source  string
-	Version string
+// policy describes policy related information
+type policy struct {
+	name    string
+	source  string
+	version string
 }
 
-// RuleStatus defines status of rules
-type RuleStatus = map[eval.RuleID]string
+// ruleStatus defines status of rules
+type ruleStatus = map[eval.RuleID]string
 
 // PolicyMonitor defines a policy monitor
 type PolicyMonitor struct {
 	sync.RWMutex
 
 	statsdClient         statsd.ClientInterface
-	policies             map[string]Policy
-	rules                RuleStatus
+	policies             []*policy
+	rules                ruleStatus
 	perRuleMetricEnabled bool
 }
 
-// SetPolicies add policies to the monitor
-func (p *PolicyMonitor) SetPolicies(policies []*rules.Policy, mErrs *multierror.Error) {
-	p.Lock()
-	defer p.Unlock()
+// SetPolicies sets the policies to monitor
+func (pm *PolicyMonitor) SetPolicies(policies []*PolicyState) {
+	pm.Lock()
+	defer pm.Unlock()
 
-	p.policies = map[string]Policy{}
-
-	for _, policy := range policies {
-		p.policies[policy.Name] = Policy{Name: policy.Name, Source: policy.Source, Version: policy.Version}
+	pm.policies = make([]*policy, 0, len(policies))
+	if pm.perRuleMetricEnabled {
+		pm.rules = make(ruleStatus)
+	}
 
-		for _, rule := range policy.Rules {
-			p.rules[rule.ID] = "loaded"
-		}
+	for _, p := range policies {
+		pm.policies = append(pm.policies, &policy{
+			name:    p.Name,
+			source:  p.Source,
+			version: p.Version,
+		})
 
-		if mErrs != nil && mErrs.Errors != nil {
-			for _, err := range mErrs.Errors {
-				if rerr, ok := err.(*rules.ErrRuleLoad); ok {
-					p.rules[rerr.Definition.ID] = string(rerr.Type())
-				}
+		if pm.perRuleMetricEnabled {
+			for _, rule := range p.Rules {
+				pm.rules[eval.RuleID(rule.ID)] = rule.Status
 			}
 		}
 	}
 }
 
 // ReportHeartbeatEvent sends HeartbeatEvents reporting the current set of policies
-func (p *PolicyMonitor) ReportHeartbeatEvent(sender events.EventSender) {
-	p.RLock()
-	rule, events := NewHeartbeatEvents(p.policies)
-	p.RUnlock()
+func (pm *PolicyMonitor) ReportHeartbeatEvent(sender events.EventSender) {
+	pm.RLock()
+	rule, events := newHeartbeatEvents(pm.policies)
+	pm.RUnlock()
 
 	for _, event := range events {
 		sender.SendEvent(rule, event, nil, "")
 	}
 }
 
 // Start the monitor
-func (p *PolicyMonitor) Start(ctx context.Context) {
+func (pm *PolicyMonitor) Start(ctx context.Context) {
 	go func() {
 		timerMetric := time.NewTicker(policyMetricRate)
 		defer timerMetric.Stop()
@@ -102,34 +103,34 @@ func (p *PolicyMonitor) Start(ctx context.Context) {
 				return
 
 			case <-timerMetric.C:
-				p.RLock()
-				for _, policy := range p.policies {
+				pm.RLock()
+				for _, p := range pm.policies {
 					tags := []string{
-						"policy_name:" + policy.Name,
-						"policy_source:" + policy.Source,
-						"policy_version:" + policy.Version,
+						"policy_name:" + p.name,
+						"policy_source:" + p.source,
+						"policy_version:" + p.version,
 						"agent_version:" + version.AgentVersion,
 					}
 
-					if err := p.statsdClient.Gauge(metrics.MetricPolicy, 1, tags, 1.0); err != nil {
+					if err := pm.statsdClient.Gauge(metrics.MetricPolicy, 1, tags, 1.0); err != nil {
 						log.Error(fmt.Errorf("failed to send policy metric: %w", err))
 					}
 				}
 
-				if p.perRuleMetricEnabled {
-					for id, status := range p.rules {
+				if pm.perRuleMetricEnabled {
+					for id, status := range pm.rules {
 						tags := []string{
 							"rule_id:" + id,
 							fmt.Sprintf("status:%v", status),
 							constants.CardinalityTagPrefix + collectors.LowCardinalityString,
 						}
 
-						if err := p.statsdClient.Gauge(metrics.MetricRulesStatus, 1, tags, 1.0); err != nil {
+						if err := pm.statsdClient.Gauge(metrics.MetricRulesStatus, 1, tags, 1.0); err != nil {
 							log.Error(fmt.Errorf("failed to send policy metric: %w", err))
 						}
 					}
 				}
-				p.RUnlock()
+				pm.RUnlock()
 			}
 		}
 	}()
@@ -139,8 +140,6 @@ func (p *PolicyMonitor) Start(ctx context.Context) {
 func NewPolicyMonitor(statsdClient statsd.ClientInterface, perRuleMetricEnabled bool) *PolicyMonitor {
 	return &PolicyMonitor{
 		statsdClient:         statsdClient,
-		policies:             make(map[string]Policy),
-		rules:                make(map[string]string),
 		perRuleMetricEnabled: perRuleMetricEnabled,
 	}
 }
@@ -151,9 +150,9 @@ type RuleSetLoadedReport struct {
 	Event *events.CustomEvent
 }
 
-// ReportRuleSetLoaded reports to Datadog that new ruleset was loaded
+// ReportRuleSetLoaded reports to Datadog that a new ruleset was loaded
 func ReportRuleSetLoaded(sender events.EventSender, statsdClient statsd.ClientInterface, policies []*PolicyState) {
-	rule, event := NewRuleSetLoadedEvent(policies)
+	rule, event := newRuleSetLoadedEvent(policies)
 
 	if err := statsdClient.Count(metrics.MetricRuleSetLoaded, 1, []string{}, 1.0); err != nil {
 		log.Error(fmt.Errorf("failed to send ruleset_loaded metric: %w", err))
@@ -228,14 +227,18 @@ func RuleStateFromDefinition(def *rules.RuleDefinition, status string, message s
 }
 
 // NewPoliciesState returns the states of policies and rules
-func NewPoliciesState(ruleSets map[string]*rules.RuleSet, err *multierror.Error) []*PolicyState {
+func NewPoliciesState(ruleSets map[string]*rules.RuleSet, err *multierror.Error, includeInternalPolicies bool) []*PolicyState {
 	mp := make(map[string]*PolicyState)
 
 	var policyState *PolicyState
 	var exists bool
 
 	for _, rs := range ruleSets {
 		for _, rule := range rs.GetRules() {
+			if rule.Definition.Policy.IsInternal && !includeInternalPolicies {
+				continue
+			}
+
 			ruleDef := rule.Definition
 			policyName := ruleDef.Policy.Name
 
@@ -251,6 +254,9 @@ func NewPoliciesState(ruleSets map[string]*rules.RuleSet, err *multierror.Error)
 	if err != nil && err.Errors != nil {
 		for _, err := range err.Errors {
 			if rerr, ok := err.(*rules.ErrRuleLoad); ok {
+				if rerr.Definition.Policy.IsInternal && !includeInternalPolicies {
+					continue
+				}
 				policyName := rerr.Definition.Policy.Name
 
 				if _, exists := mp[policyName]; !exists {
@@ -272,8 +278,8 @@ func NewPoliciesState(ruleSets map[string]*rules.RuleSet, err *multierror.Error)
 	return policies
 }
 
-// NewRuleSetLoadedEvent returns the rule (e.g. ruleset_loaded) and a populated custom event for a new_rules_loaded event
-func NewRuleSetLoadedEvent(policies []*PolicyState) (*rules.Rule, *events.CustomEvent) {
+// newRuleSetLoadedEvent returns the rule (e.g. ruleset_loaded) and a populated custom event for a new_rules_loaded event
+func newRuleSetLoadedEvent(policies []*PolicyState) (*rules.Rule, *events.CustomEvent) {
 	evt := RulesetLoadedEvent{
 		Policies: policies,
 	}
@@ -283,15 +289,15 @@ func NewRuleSetLoadedEvent(policies []*PolicyState) (*rules.Rule, *events.Custom
 		events.NewCustomEvent(model.CustomRulesetLoadedEventType, evt)
 }
 
-// NewHeartbeatEvents returns the rule (e.g. heartbeat) and a populated custom event for a heartbeat event
-func NewHeartbeatEvents(policies map[string]Policy) (*rules.Rule, []*events.CustomEvent) {
+// newHeartbeatEvents returns the rule (e.g. heartbeat) and a populated custom event for a heartbeat event
+func newHeartbeatEvents(policies []*policy) (*rules.Rule, []*events.CustomEvent) {
 	var evts []*events.CustomEvent
 
 	for _, policy := range policies {
 		var policyState = PolicyState{
-			Name:    policy.Name,
-			Version: policy.Version,
-			Source:  policy.Source,
+			Name:    policy.name,
+			Version: policy.version,
+			Source:  policy.source,
 			Rules:   nil, // The rules that have been loaded at startup are not reported in the heartbeat event
 		}
 

pkg/security/secl/rules/policy.go

@@ -11,9 +11,10 @@ import (
 	"fmt"
 	"io"
 
-	"github.com/DataDog/datadog-agent/pkg/security/secl/validators"
 	"github.com/hashicorp/go-multierror"
 	"gopkg.in/yaml.v2"
+
+	"github.com/DataDog/datadog-agent/pkg/security/secl/validators"
 )
 
 // PolicyDef represents a policy file definition
@@ -25,11 +26,12 @@ type PolicyDef struct {
 
 // Policy represents a policy file which is composed of a list of rules and macros
 type Policy struct {
-	Name    string
-	Source  string
-	Version string
-	Rules   []*RuleDefinition
-	Macros  []*MacroDefinition
+	Name       string
+	Source     string
+	Version    string
+	Rules      []*RuleDefinition
+	Macros     []*MacroDefinition
+	IsInternal bool
 }
 
 // AddMacro add a macro to the policy