From b833f62d23b31e37c78f2a54270da7115f3fa77e Mon Sep 17 00:00:00 2001 From: Andrew Glaude Date: Wed, 6 Nov 2024 16:20:41 -0500 Subject: [PATCH] APM: Support configuring skipped tag keys for the credit card obfuscator (APMSP-1492) (#30680) --- cmd/trace-agent/test/testsuite/cards_test.go | 11 +++++++++++ pkg/config/setup/apm.go | 3 ++- pkg/flare/envvars.go | 1 + pkg/obfuscate/credit_cards.go | 13 +++++++++++-- pkg/obfuscate/credit_cards_test.go | 8 ++++++++ pkg/obfuscate/obfuscate.go | 4 ++++ pkg/trace/agent/obfuscate.go | 1 + .../notes/CCObfSkipKeys-b639ee1a05455030.yaml | 14 ++++++++++++++ 8 files changed, 52 insertions(+), 3 deletions(-) create mode 100644 releasenotes/notes/CCObfSkipKeys-b639ee1a05455030.yaml diff --git a/cmd/trace-agent/test/testsuite/cards_test.go b/cmd/trace-agent/test/testsuite/cards_test.go index f82896e8abd59..af67a1815f985 100644 --- a/cmd/trace-agent/test/testsuite/cards_test.go +++ b/cmd/trace-agent/test/testsuite/cards_test.go @@ -86,6 +86,17 @@ apm_config: out: "?", version: "v0.7", }, + { + conf: []byte(` +apm_config: + env: my-env + obfuscation: + credit_cards: + enabled: false + keep_values: ["credit_card_number"]`), + out: "4166 6766 6766 6746", + version: "v0.5", + }, } { t.Run(string(tt.version)+"/"+tt.out, func(t *testing.T) { if err := r.RunAgent(tt.conf); err != nil { diff --git a/pkg/config/setup/apm.go b/pkg/config/setup/apm.go index 370bdcae39ff9..225aa0607b7a7 100644 --- a/pkg/config/setup/apm.go +++ b/pkg/config/setup/apm.go @@ -152,6 +152,7 @@ func setupAPM(config pkgconfigmodel.Setup) { config.BindEnv("apm_config.install_time", "DD_INSTRUMENTATION_INSTALL_TIME") config.BindEnv("apm_config.obfuscation.credit_cards.enabled", "DD_APM_OBFUSCATION_CREDIT_CARDS_ENABLED") config.BindEnv("apm_config.obfuscation.credit_cards.luhn", "DD_APM_OBFUSCATION_CREDIT_CARDS_LUHN") + config.BindEnv("apm_config.obfuscation.credit_cards.keep_values", "DD_APM_OBFUSCATION_CREDIT_CARDS_KEEP_VALUES") config.BindEnvAndSetDefault("apm_config.debug.port", 5012, "DD_APM_DEBUG_PORT") config.BindEnv("apm_config.features", "DD_APM_FEATURES") config.ParseEnvAsStringSlice("apm_config.features", func(s string) []string { @@ -183,7 +184,7 @@ func setupAPM(config pkgconfigmodel.Setup) { config.ParseEnvAsStringSlice("apm_config.filter_tags.reject", parseKVList("apm_config.filter_tags.reject")) config.ParseEnvAsStringSlice("apm_config.filter_tags_regex.require", parseKVList("apm_config.filter_tags_regex.require")) config.ParseEnvAsStringSlice("apm_config.filter_tags_regex.reject", parseKVList("apm_config.filter_tags_regex.reject")) - + config.ParseEnvAsStringSlice("apm_config.obfuscation.credit_cards.keep_values", parseKVList("apm_config.obfuscation.credit_cards.keep_values")) config.ParseEnvAsSliceMapString("apm_config.replace_tags", func(in string) []map[string]string { var out []map[string]string if err := json.Unmarshal([]byte(in), &out); err != nil { diff --git a/pkg/flare/envvars.go b/pkg/flare/envvars.go index 7f53fcbc2ceef..2d232cdbe5728 100644 --- a/pkg/flare/envvars.go +++ b/pkg/flare/envvars.go @@ -88,6 +88,7 @@ var allowedEnvvarNames = []string{ "DD_APM_SYMDB_DD_URL", "DD_APM_OBFUSCATION_CREDIT_CARDS_ENABLED", "DD_APM_OBFUSCATION_CREDIT_CARDS_LUHN", + "DD_APM_OBFUSCATION_CREDIT_CARDS_KEEP_VALUES", "DD_APM_OBFUSCATION_ELASTICSEARCH_ENABLED", "DD_APM_OBFUSCATION_ELASTICSEARCH_KEEP_VALUES", "DD_APM_OBFUSCATION_ELASTICSEARCH_OBFUSCATE_SQL_VALUES", diff --git a/pkg/obfuscate/credit_cards.go b/pkg/obfuscate/credit_cards.go index b172b2c173e97..3246d6b0a4fdd 100644 --- a/pkg/obfuscate/credit_cards.go +++ b/pkg/obfuscate/credit_cards.go @@ -11,12 +11,18 @@ import ( // creditCard maintains credit card obfuscation state and processing. type creditCard struct { - luhn bool + luhn bool + keepValues map[string]struct{} } func newCCObfuscator(config *CreditCardsConfig) *creditCard { + keepValues := make(map[string]struct{}, len(config.KeepValues)) + for _, sk := range config.KeepValues { + keepValues[sk] = struct{}{} + } return &creditCard{ - luhn: config.Luhn, + luhn: config.Luhn, + keepValues: keepValues, } } @@ -57,6 +63,9 @@ func (o *Obfuscator) ObfuscateCreditCardNumber(key, val string) string { if strings.HasPrefix(key, "_") { return val } + if _, ok := o.ccObfuscator.keepValues[key]; ok { + return val + } if o.ccObfuscator.IsCardNumber(val) { return "?" } diff --git a/pkg/obfuscate/credit_cards_test.go b/pkg/obfuscate/credit_cards_test.go index c9237b359543a..7024b3b737d6c 100644 --- a/pkg/obfuscate/credit_cards_test.go +++ b/pkg/obfuscate/credit_cards_test.go @@ -273,6 +273,14 @@ func TestIINIsSensitive(t *testing.T) { }) } +func TestCCKeepValues(t *testing.T) { + possibleCard := "378282246310005" + o := NewObfuscator(Config{CreditCard: CreditCardsConfig{Enabled: true, KeepValues: []string{"skip_me"}}}) + + assert.Equal(t, possibleCard, o.ObfuscateCreditCardNumber("skip_me", possibleCard)) + assert.Equal(t, "?", o.ObfuscateCreditCardNumber("obfuscate_me", possibleCard)) +} + func BenchmarkIsSensitive(b *testing.B) { run := func(str string, luhn bool) func(b *testing.B) { cco := &creditCard{luhn: luhn} diff --git a/pkg/obfuscate/obfuscate.go b/pkg/obfuscate/obfuscate.go index 3032fb54f38a4..c9de3906ed2a2 100644 --- a/pkg/obfuscate/obfuscate.go +++ b/pkg/obfuscate/obfuscate.go @@ -264,6 +264,10 @@ type CreditCardsConfig struct { // https://dev.to/shiraazm/goluhn-a-simple-library-for-generating-calculating-and-verifying-luhn-numbers-588j // It reduces false positives, but increases the CPU time X3. Luhn bool `mapstructure:"luhn"` + + // KeepValues specifies tag keys that are known to not ever contain credit cards + // and therefore their values can be kept. + KeepValues []string `mapstructure:"keep_values"` } // NewObfuscator creates a new obfuscator diff --git a/pkg/trace/agent/obfuscate.go b/pkg/trace/agent/obfuscate.go index 7d7d11ccaeb1b..00411697d49b7 100644 --- a/pkg/trace/agent/obfuscate.go +++ b/pkg/trace/agent/obfuscate.go @@ -32,6 +32,7 @@ func (a *Agent) obfuscateSpan(span *pb.Span) { for k, v := range span.Meta { newV := o.ObfuscateCreditCardNumber(k, v) if v != newV { + log.Debugf("obfuscating possible credit card under key %s from service %s", k, span.Service) span.Meta[k] = newV } } diff --git a/releasenotes/notes/CCObfSkipKeys-b639ee1a05455030.yaml b/releasenotes/notes/CCObfSkipKeys-b639ee1a05455030.yaml new file mode 100644 index 0000000000000..63f5f861d7621 --- /dev/null +++ b/releasenotes/notes/CCObfSkipKeys-b639ee1a05455030.yaml @@ -0,0 +1,14 @@ +# Each section from every release note are combined when the +# CHANGELOG.rst is rendered. So the text needs to be worded so that +# it does not depend on any information only available in another +# section. This may mean repeating some details, but each section +# must be readable independently of the other. +# +# Each section note must be formatted as reStructuredText. +--- + +features: + - | + APM: New configuration apm_config.obfuscation.credit_cards.keep_values (DD_APM_OBFUSCATION_CREDIT_CARDS_KEEP_VALUES) + can be used to skip specific tag keys that are known to never contain credit card numbers. This is especially useful + in cases where a span tag value is a number that triggers false positives from the credit card obfuscator.