From 7b5ccb2ab49e6cf0f039628ee28abc469a9f35f9 Mon Sep 17 00:00:00 2001 From: Thomas Watson Date: Fri, 13 Dec 2024 20:18:17 +0100 Subject: [PATCH 01/36] [DI] Improve sampling tests (#4999) To test that multiple probes doesn't interfere with each others sample rate, this commit also adds support for multiple breakpoints in a single file. --- integration-tests/debugger/basic.spec.js | 63 +++++++++++++++++-- .../debugger/target-app/basic.js | 8 ++- integration-tests/debugger/utils.js | 48 ++++++++++---- 3 files changed, 100 insertions(+), 19 deletions(-) diff --git a/integration-tests/debugger/basic.spec.js b/integration-tests/debugger/basic.spec.js index 22a8ec98ff1..57c0c4a67a8 100644 --- a/integration-tests/debugger/basic.spec.js +++ b/integration-tests/debugger/basic.spec.js @@ -14,7 +14,7 @@ describe('Dynamic Instrumentation', function () { it('base case: target app should work as expected if no test probe has been added', async function () { const response = await t.axios.get(t.breakpoint.url) assert.strictEqual(response.status, 200) - assert.deepStrictEqual(response.data, { hello: 'foo' }) + assert.deepStrictEqual(response.data, { hello: 'bar' }) }) describe('diagnostics messages', function () { @@ -54,7 +54,7 @@ describe('Dynamic Instrumentation', function () { t.axios.get(t.breakpoint.url) .then((response) => { assert.strictEqual(response.status, 200) - assert.deepStrictEqual(response.data, { hello: 'foo' }) + assert.deepStrictEqual(response.data, { hello: 'bar' }) }) .catch(done) } else { @@ -245,7 +245,7 @@ describe('Dynamic Instrumentation', function () { message: 'Hello World!', logger: { name: t.breakpoint.file, - method: 'handler', + method: 'fooHandler', version, thread_name: 'MainThread' }, @@ -279,7 +279,7 @@ describe('Dynamic Instrumentation', function () { const topFrame = payload['debugger.snapshot'].stack[0] // path seems to be prefeixed with `/private` on Mac assert.match(topFrame.fileName, new RegExp(`${t.appFile}$`)) - assert.strictEqual(topFrame.function, 'handler') + assert.strictEqual(topFrame.function, 'fooHandler') assert.strictEqual(topFrame.lineNumber, t.breakpoint.line) assert.strictEqual(topFrame.columnNumber, 3) @@ -375,6 +375,61 @@ describe('Dynamic Instrumentation', function () { t.agent.addRemoteConfig(rcConfig) }) + + it('should adhere to individual probes sample rate', function (done) { + const rcConfig1 = t.breakpoints[0].generateRemoteConfig({ sampling: { snapshotsPerSecond: 1 } }) + const rcConfig2 = t.breakpoints[1].generateRemoteConfig({ sampling: { snapshotsPerSecond: 1 } }) + const state = { + [rcConfig1.config.id]: { + payloadsReceived: 0, + tiggerBreakpointContinuously () { + t.axios.get(t.breakpoints[0].url).catch(done) + this.timer = setTimeout(this.tiggerBreakpointContinuously.bind(this), 10) + } + }, + [rcConfig2.config.id]: { + payloadsReceived: 0, + tiggerBreakpointContinuously () { + t.axios.get(t.breakpoints[1].url).catch(done) + this.timer = setTimeout(this.tiggerBreakpointContinuously.bind(this), 10) + } + } + } + + t.agent.on('debugger-diagnostics', ({ payload }) => { + const { probeId, status } = payload.debugger.diagnostics + if (status === 'INSTALLED') state[probeId].tiggerBreakpointContinuously() + }) + + t.agent.on('debugger-input', ({ payload }) => { + const _state = state[payload['debugger.snapshot'].probe.id] + _state.payloadsReceived++ + if (_state.payloadsReceived === 1) { + _state.start = Date.now() + } else if (_state.payloadsReceived === 2) { + const duration = Date.now() - _state.start + clearTimeout(_state.timer) + + // Allow for a variance of -5/+50ms (time will tell if this is enough) + assert.isAbove(duration, 995) + assert.isBelow(duration, 1050) + + // Wait at least a full sampling period, to see if we get any more payloads + _state.timer = setTimeout(doneWhenCalledTwice, 1250) + } else { + clearTimeout(_state.timer) + done(new Error('Too many payloads received!')) + } + }) + + t.agent.addRemoteConfig(rcConfig1) + t.agent.addRemoteConfig(rcConfig2) + + function doneWhenCalledTwice () { + if (doneWhenCalledTwice.calledOnce) return done() + doneWhenCalledTwice.calledOnce = true + } + }) }) describe('race conditions', function () { diff --git a/integration-tests/debugger/target-app/basic.js b/integration-tests/debugger/target-app/basic.js index 2fa9c16d221..d9d1e0e9185 100644 --- a/integration-tests/debugger/target-app/basic.js +++ b/integration-tests/debugger/target-app/basic.js @@ -5,8 +5,12 @@ const Fastify = require('fastify') const fastify = Fastify() -fastify.get('/:name', function handler (request) { - return { hello: request.params.name } // BREAKPOINT: /foo +fastify.get('/foo/:name', function fooHandler (request) { + return { hello: request.params.name } // BREAKPOINT: /foo/bar +}) + +fastify.get('/bar/:name', function barHandler (request) { + return { hello: request.params.name } // BREAKPOINT: /bar/baz }) fastify.listen({ port: process.env.APP_PORT }, (err) => { diff --git a/integration-tests/debugger/utils.js b/integration-tests/debugger/utils.js index 1ea6cb9b54c..bca970dea87 100644 --- a/integration-tests/debugger/utils.js +++ b/integration-tests/debugger/utils.js @@ -20,28 +20,43 @@ module.exports = { function setup () { let sandbox, cwd, appPort - const breakpoint = getBreakpointInfo(1) // `1` to disregard the `setup` function + const breakpoints = getBreakpointInfo(1) // `1` to disregard the `setup` function const t = { - breakpoint, + breakpoint: breakpoints[0], + breakpoints, + axios: null, appFile: null, agent: null, + + // Default to the first breakpoint in the file (normally there's only one) rcConfig: null, - triggerBreakpoint, - generateRemoteConfig, - generateProbeConfig + triggerBreakpoint: triggerBreakpoint.bind(null, breakpoints[0].url), + generateRemoteConfig: generateRemoteConfig.bind(null, breakpoints[0]), + generateProbeConfig: generateProbeConfig.bind(null, breakpoints[0]) } - function triggerBreakpoint () { + // Allow specific access to each breakpoint + for (let i = 0; i < breakpoints.length; i++) { + t.breakpoints[i] = { + rcConfig: null, + triggerBreakpoint: triggerBreakpoint.bind(null, breakpoints[i].url), + generateRemoteConfig: generateRemoteConfig.bind(null, breakpoints[i]), + generateProbeConfig: generateProbeConfig.bind(null, breakpoints[i]), + ...breakpoints[i] + } + } + + function triggerBreakpoint (url) { // Trigger the breakpoint once probe is successfully installed t.agent.on('debugger-diagnostics', ({ payload }) => { if (payload.debugger.diagnostics.status === 'INSTALLED') { - t.axios.get(breakpoint.url) + t.axios.get(url) } }) } - function generateRemoteConfig (overrides = {}) { + function generateRemoteConfig (breakpoint, overrides = {}) { overrides.id = overrides.id || randomUUID() return { product: 'LIVE_DEBUGGING', @@ -54,7 +69,7 @@ function setup () { sandbox = await createSandbox(['fastify']) // TODO: Make this dynamic cwd = sandbox.folder // The sandbox uses the `integration-tests` folder as its root - t.appFile = join(cwd, 'debugger', breakpoint.file) + t.appFile = join(cwd, 'debugger', breakpoints[0].file) }) after(async function () { @@ -62,7 +77,11 @@ function setup () { }) beforeEach(async function () { - t.rcConfig = generateRemoteConfig(breakpoint) + // Default to the first breakpoint in the file (normally there's only one) + t.rcConfig = generateRemoteConfig(breakpoints[0]) + // Allow specific access to each breakpoint + t.breakpoints.forEach((breakpoint) => { breakpoint.rcConfig = generateRemoteConfig(breakpoint) }) + appPort = await getPort() t.agent = await new FakeAgent().start() t.proc = await spawnProc(t.appFile, { @@ -96,16 +115,19 @@ function getBreakpointInfo (stackIndex = 0) { .slice(0, -1) .split(':')[0] - // Then, find the corresponding file in which the breakpoint exists + // Then, find the corresponding file in which the breakpoint(s) exists const file = join('target-app', basename(testFile).replace('.spec', '')) - // Finally, find the line number of the breakpoint + // Finally, find the line number(s) of the breakpoint(s) const lines = readFileSync(join(__dirname, file), 'utf8').split('\n') + const result = [] for (let i = 0; i < lines.length; i++) { const index = lines[i].indexOf(BREAKPOINT_TOKEN) if (index !== -1) { const url = lines[i].slice(index + BREAKPOINT_TOKEN.length + 1).trim() - return { file, line: i + 1, url } + result.push({ file, line: i + 1, url }) } } + + return result } From 880f15ae979493054cb3b3c74c639ac47641e408 Mon Sep 17 00:00:00 2001 From: Roch Devost Date: Fri, 13 Dec 2024 14:25:43 -0500 Subject: [PATCH 02/36] run benchmarks also on node 20 and 22 (#4975) --- .gitlab/benchmarks.yml | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/.gitlab/benchmarks.yml b/.gitlab/benchmarks.yml index 57eba976441..7461f88b98c 100644 --- a/.gitlab/benchmarks.yml +++ b/.gitlab/benchmarks.yml @@ -65,6 +65,18 @@ benchmark: GROUP: 2 - MAJOR_VERSION: 18 GROUP: 3 + - MAJOR_VERSION: 20 + GROUP: 1 + - MAJOR_VERSION: 20 + GROUP: 2 + - MAJOR_VERSION: 20 + GROUP: 3 + - MAJOR_VERSION: 22 + GROUP: 1 + - MAJOR_VERSION: 22 + GROUP: 2 + - MAJOR_VERSION: 22 + GROUP: 3 variables: SPLITS: 3 From 749b9a8949d6aba2f23b74af15f041e11a31791a Mon Sep 17 00:00:00 2001 From: Roch Devost Date: Fri, 13 Dec 2024 14:36:54 -0500 Subject: [PATCH 03/36] use gc observer for gc runtime metrics when available (#4961) --- packages/dd-trace/src/runtime_metrics.js | 109 ++++++++++++++++++++++- 1 file changed, 106 insertions(+), 3 deletions(-) diff --git a/packages/dd-trace/src/runtime_metrics.js b/packages/dd-trace/src/runtime_metrics.js index 49e724eb11c..a9036612a67 100644 --- a/packages/dd-trace/src/runtime_metrics.js +++ b/packages/dd-trace/src/runtime_metrics.js @@ -7,11 +7,19 @@ const os = require('os') const { DogStatsDClient } = require('./dogstatsd') const log = require('./log') const Histogram = require('./histogram') -const { performance } = require('perf_hooks') +const { performance, PerformanceObserver } = require('perf_hooks') +const { NODE_MAJOR, NODE_MINOR } = require('../../../version') const INTERVAL = 10 * 1000 +// Node >=16 has PerformanceObserver with `gc` type, but <16.7 had a critical bug. +// See: https://github.com/nodejs/node/issues/39548 +const hasGCObserver = NODE_MAJOR >= 18 || (NODE_MAJOR === 16 && NODE_MINOR >= 7) +const hasGCProfiler = NODE_MAJOR >= 20 || (NODE_MAJOR === 18 && NODE_MINOR >= 15) + let nativeMetrics = null +let gcObserver = null +let gcProfiler = null let interval let client @@ -24,13 +32,18 @@ let elu reset() -module.exports = { +const runtimeMetrics = module.exports = { start (config) { const clientConfig = DogStatsDClient.generateClientConfig(config) try { nativeMetrics = require('@datadog/native-metrics') - nativeMetrics.start() + + if (hasGCObserver) { + nativeMetrics.start('loop') // Only add event loop watcher and not GC. + } else { + nativeMetrics.start() + } } catch (e) { log.error('Error starting native metrics', e) nativeMetrics = null @@ -40,6 +53,9 @@ module.exports = { time = process.hrtime() + startGCObserver() + startGCProfiler() + if (nativeMetrics) { interval = setInterval(() => { captureCommonMetrics() @@ -138,6 +154,10 @@ function reset () { counters = {} histograms = {} nativeMetrics = null + gcObserver && gcObserver.disconnect() + gcObserver = null + gcProfiler && gcProfiler.stop() + gcProfiler = null } function captureCpuUsage () { @@ -202,6 +222,29 @@ function captureHeapSpace () { client.gauge('runtime.node.heap.physical_size.by.space', stats[i].physical_space_size, tags) } } +function captureGCMetrics () { + if (!gcProfiler) return + + const profile = gcProfiler.stop() + const pauseAll = new Histogram() + const pause = {} + + for (const stat of profile.statistics) { + const type = stat.gcType.replace(/([a-z])([A-Z])/g, '$1_$2').toLowerCase() + + pause[type] = pause[type] || new Histogram() + pause[type].record(stat.cost) + pauseAll.record(stat.cost) + } + + histogram('runtime.node.gc.pause', pauseAll) + + for (const type in pause) { + histogram('runtime.node.gc.pause.by.type', pause[type], [`gc_type:${type}`]) + } + + gcProfiler.start() +} function captureGauges () { Object.keys(gauges).forEach(name => { @@ -256,6 +299,7 @@ function captureCommonMetrics () { captureCounters() captureHistograms() captureELU() + captureGCMetrics() } function captureNativeMetrics () { @@ -297,6 +341,11 @@ function captureNativeMetrics () { function histogram (name, stats, tags) { tags = [].concat(tags) + // Stats can contain garbage data when a value was never recorded. + if (stats.count === 0) { + stats = { max: 0, min: 0, sum: 0, avg: 0, median: 0, p95: 0, count: 0 } + } + client.gauge(`${name}.min`, stats.min, tags) client.gauge(`${name}.max`, stats.max, tags) client.increment(`${name}.sum`, stats.sum, tags) @@ -306,3 +355,57 @@ function histogram (name, stats, tags) { client.gauge(`${name}.median`, stats.median, tags) client.gauge(`${name}.95percentile`, stats.p95, tags) } + +function startGCObserver () { + if (gcObserver || hasGCProfiler || !hasGCObserver) return + + gcObserver = new PerformanceObserver(list => { + for (const entry of list.getEntries()) { + const type = gcType(entry.kind) + + runtimeMetrics.histogram('runtime.node.gc.pause.by.type', entry.duration, `gc_type:${type}`) + runtimeMetrics.histogram('runtime.node.gc.pause', entry.duration) + } + }) + + gcObserver.observe({ type: 'gc' }) +} + +function startGCProfiler () { + if (gcProfiler || !hasGCProfiler) return + + gcProfiler = new v8.GCProfiler() + gcProfiler.start() +} + +function gcType (kind) { + if (NODE_MAJOR >= 22) { + switch (kind) { + case 1: return 'scavenge' + case 2: return 'minor_mark_sweep' + case 4: return 'mark_sweep_compact' // Deprecated, might be removed soon. + case 8: return 'incremental_marking' + case 16: return 'process_weak_callbacks' + case 31: return 'all' + } + } else if (NODE_MAJOR >= 18) { + switch (kind) { + case 1: return 'scavenge' + case 2: return 'minor_mark_compact' + case 4: return 'mark_sweep_compact' + case 8: return 'incremental_marking' + case 16: return 'process_weak_callbacks' + case 31: return 'all' + } + } else { + switch (kind) { + case 1: return 'scavenge' + case 2: return 'mark_sweep_compact' + case 4: return 'incremental_marking' + case 8: return 'process_weak_callbacks' + case 15: return 'all' + } + } + + return 'unknown' +} From 69b27b3c3d8488033dbfee79167674762bc69177 Mon Sep 17 00:00:00 2001 From: Thomas Hunter II Date: Fri, 13 Dec 2024 11:51:17 -0800 Subject: [PATCH 04/36] telemetry: make count logic faster (#5013) --- packages/dd-trace/src/telemetry/logs/index.js | 1 + .../dd-trace/src/telemetry/logs/log-collector.js | 15 +-------------- .../test/telemetry/logs/log-collector.spec.js | 6 ++---- 3 files changed, 4 insertions(+), 18 deletions(-) diff --git a/packages/dd-trace/src/telemetry/logs/index.js b/packages/dd-trace/src/telemetry/logs/index.js index d8fa1969e55..199b5fb7943 100644 --- a/packages/dd-trace/src/telemetry/logs/index.js +++ b/packages/dd-trace/src/telemetry/logs/index.js @@ -40,6 +40,7 @@ function onErrorLog (msg) { const telLog = { level: 'ERROR', + count: 1, // existing log.error(err) without message will be reported as 'Generic Error' message: message ?? 'Generic Error' diff --git a/packages/dd-trace/src/telemetry/logs/log-collector.js b/packages/dd-trace/src/telemetry/logs/log-collector.js index c43e25c8dc4..a2ee9d06f4a 100644 --- a/packages/dd-trace/src/telemetry/logs/log-collector.js +++ b/packages/dd-trace/src/telemetry/logs/log-collector.js @@ -79,7 +79,7 @@ const logCollector = { } const hash = createHash(logEntry) if (!logs.has(hash)) { - logs.set(hash, errorCopy(logEntry)) + logs.set(hash, logEntry) return true } else { logs.get(hash).count++ @@ -122,19 +122,6 @@ const logCollector = { } } -// clone an Error object to later serialize and transmit -// { ...error } doesn't work -// also users can add arbitrary fields to an error -function errorCopy (error) { - const keys = Object.getOwnPropertyNames(error) - const obj = {} - for (const key of keys) { - obj[key] = error[key] - } - obj.count = 1 - return obj -} - logCollector.reset() module.exports = logCollector diff --git a/packages/dd-trace/test/telemetry/logs/log-collector.spec.js b/packages/dd-trace/test/telemetry/logs/log-collector.spec.js index e3d4126c4c9..57600dcb441 100644 --- a/packages/dd-trace/test/telemetry/logs/log-collector.spec.js +++ b/packages/dd-trace/test/telemetry/logs/log-collector.spec.js @@ -128,11 +128,9 @@ describe('telemetry log collector', () => { }) it('duplicated errors should send incremented count values', () => { - const err1 = new Error('oh no') - err1.level = 'ERROR' + const err1 = { message: 'oh no', level: 'ERROR', count: 1 } - const err2 = new Error('foo buzz') - err2.level = 'ERROR' + const err2 = { message: 'foo buzz', level: 'ERROR', count: 1 } logCollector.add(err1) logCollector.add(err2) From baf22d9f4f68d74ddb7947fee3039e039dfe18e8 Mon Sep 17 00:00:00 2001 From: Bryan English Date: Fri, 13 Dec 2024 16:17:42 -0500 Subject: [PATCH 05/36] Verify yaml (#4639) * add script to verify plugin yaml * add github actions job to verify yaml * fix instrumentations * fix up aerospike * better version ranges for aerospike * fix ci script * make it pass hopefully * update license 3rdparty * fix it no longer assuming nodejs versions * fix aerospike * since node version is now ignored, run on only one version of node --- .github/workflows/appsec.yml | 2 +- .github/workflows/plugins.yml | 75 +++------ .github/workflows/project.yml | 7 + LICENSE-3rdparty.csv | 1 + package.json | 3 +- .../datadog-instrumentations/src/aerospike.js | 2 +- packages/datadog-instrumentations/src/next.js | 11 +- scripts/verify-ci-config.js | 156 ++++++++++++++++++ yarn.lock | 5 + 9 files changed, 200 insertions(+), 62 deletions(-) create mode 100644 scripts/verify-ci-config.js diff --git a/.github/workflows/appsec.yml b/.github/workflows/appsec.yml index 45edbde6ebc..17a4e66f15c 100644 --- a/.github/workflows/appsec.yml +++ b/.github/workflows/appsec.yml @@ -210,7 +210,7 @@ jobs: version: - 18 - latest - range: ['9.5.0', '11.1.4', '13.2.0', '>=14.0.0 <=14.2.6', '>=14.2.7 <15', '>=15.0.0'] + range: ['>=10.2.0 <11', '>=11.0.0 <13', '11.1.4', '>=13.0.0 <14', '13.2.0', '>=14.0.0 <=14.2.6', '>=14.2.7 <15', '>=15.0.0'] runs-on: ubuntu-latest env: PLUGINS: next diff --git a/.github/workflows/plugins.yml b/.github/workflows/plugins.yml index d25535e2aab..2a76db58145 100644 --- a/.github/workflows/plugins.yml +++ b/.github/workflows/plugins.yml @@ -15,54 +15,30 @@ concurrency: jobs: - aerospike-node-16: - runs-on: ubuntu-latest - services: - aerospike: - image: aerospike:ce-5.7.0.15 - ports: - - "127.0.0.1:3000-3002:3000-3002" - env: - PLUGINS: aerospike - SERVICES: aerospike - PACKAGE_VERSION_RANGE: '>=4.0.0 <5.2.0' - steps: - - uses: actions/checkout@v4 - - uses: ./.github/actions/testagent/start - - uses: ./.github/actions/node/setup - - id: pkg - run: | - content=`cat ./package.json | tr '\n' ' '` - echo "json=$content" >> $GITHUB_OUTPUT - - id: extract - run: | - version="${{fromJson(steps.pkg.outputs.json).version}}" - majorVersion=$(echo "$version" | cut -d '.' -f 1) - echo "Major Version: $majorVersion" - echo "MAJOR_VERSION=$majorVersion" >> $GITHUB_ENV - - uses: ./.github/actions/node/oldest - - name: Install dependencies - if: env.MAJOR_VERSION == '4' - uses: ./.github/actions/install - - name: Run tests - if: env.MAJOR_VERSION == '4' - run: yarn test:plugins:ci - - if: always() - uses: ./.github/actions/testagent/logs - - uses: codecov/codecov-action@v3 - - aerospike-node-18-20: + aerospike: strategy: matrix: - node-version: [18] - range: ['5.2.0 - 5.7.0'] + node-version: [16] + range: ['>=4.0.0 <5.2.0'] + aerospike-image: [ce-5.7.0.15] + test-image: [ubuntu-latest] include: + - node-version: 18 + range: '>=5.2.0' + aerospike-image: ce-6.4.0.3 + test-image: ubuntu-latest - node-version: 20 - range: '>=5.8.0' - runs-on: ubuntu-latest + range: '>=5.5.0' + aerospike-image: ce-6.4.0.3 + test-image: ubuntu-latest + - node-version: 22 + range: '>=5.12.1' + aerospike-image: ce-6.4.0.3 + test-image: ubuntu-latest + runs-on: ${{ matrix.test-image }} services: aerospike: - image: aerospike:ce-6.4.0.3 + image: aerospike:${{ matrix.aerospike-image }} ports: - "127.0.0.1:3000-3002:3000-3002" env: @@ -73,24 +49,13 @@ jobs: - uses: actions/checkout@v4 - uses: ./.github/actions/testagent/start - uses: ./.github/actions/node/setup - - id: pkg - run: | - content=`cat ./package.json | tr '\n' ' '` - echo "json=$content" >> $GITHUB_OUTPUT - - id: extract - run: | - version="${{fromJson(steps.pkg.outputs.json).version}}" - majorVersion=$(echo "$version" | cut -d '.' -f 1) - echo "Major Version: $majorVersion" - echo "MAJOR_VERSION=$majorVersion" >> $GITHUB_ENV - uses: actions/setup-node@v3 with: node-version: ${{ matrix.node-version }} + - run: yarn config set ignore-engines true - name: Install dependencies - if: env.MAJOR_VERSION == '5' uses: ./.github/actions/install - name: Run tests - if: env.MAJOR_VERSION == '5' run: yarn test:plugins:ci - if: always() uses: ./.github/actions/testagent/logs @@ -759,7 +724,7 @@ jobs: version: - 18 - latest - range: ['9.5.0', '11.1.4', '13.2.0', '>=14.0.0 <=14.2.6', '>=14.2.7 <15', '>=15.0.0'] + range: ['>=10.2.0 <11', '>=11.0.0 <13', '11.1.4', '>=13.0.0 <14', '13.2.0', '>=14.0.0 <=14.2.6', '>=14.2.7 <15', '>=15.0.0'] runs-on: ubuntu-latest env: PLUGINS: next diff --git a/.github/workflows/project.yml b/.github/workflows/project.yml index f7839ac941e..3dd8475811e 100644 --- a/.github/workflows/project.yml +++ b/.github/workflows/project.yml @@ -162,3 +162,10 @@ jobs: - run: yarn type:test - run: yarn type:doc + verify-yaml: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: ./.github/actions/node/setup + - uses: ./.github/actions/install + - run: node scripts/verify-ci-config.js diff --git a/LICENSE-3rdparty.csv b/LICENSE-3rdparty.csv index 23c1fcda420..4ba4775b73c 100644 --- a/LICENSE-3rdparty.csv +++ b/LICENSE-3rdparty.csv @@ -72,6 +72,7 @@ dev,sinon,BSD-3-Clause,Copyright 2010-2017 Christian Johansen dev,sinon-chai,WTFPL and BSD-2-Clause,Copyright 2004 Sam Hocevar 2012–2017 Domenic Denicola dev,tap,ISC,Copyright 2011-2022 Isaac Z. Schlueter and Contributors dev,tiktoken,MIT,Copyright (c) 2022 OpenAI, Shantanu Jain +dev,yaml,ISC,Copyright Eemeli Aro file,aws-lambda-nodejs-runtime-interface-client,Apache 2.0,Copyright 2019 Amazon.com Inc. or its affiliates. All Rights Reserved. file,profile.proto,Apache license 2.0,Copyright 2016 Google Inc. file,is-git-url,MIT,Copyright (c) 2017 Jon Schlinkert. diff --git a/package.json b/package.json index 008fd1f17d3..04d5c78e320 100644 --- a/package.json +++ b/package.json @@ -155,6 +155,7 @@ "sinon": "^16.1.3", "sinon-chai": "^3.7.0", "tap": "^16.3.7", - "tiktoken": "^1.0.15" + "tiktoken": "^1.0.15", + "yaml": "^2.5.0" } } diff --git a/packages/datadog-instrumentations/src/aerospike.js b/packages/datadog-instrumentations/src/aerospike.js index 724c518e050..497a64aaf80 100644 --- a/packages/datadog-instrumentations/src/aerospike.js +++ b/packages/datadog-instrumentations/src/aerospike.js @@ -40,7 +40,7 @@ function wrapProcess (process) { addHook({ name: 'aerospike', file: 'lib/commands/command.js', - versions: ['^3.16.2', '4', '5'] + versions: ['4', '5'] }, commandFactory => { return shimmer.wrapFunction(commandFactory, f => wrapCreateCommand(f)) diff --git a/packages/datadog-instrumentations/src/next.js b/packages/datadog-instrumentations/src/next.js index 56ce695fe76..770d340d567 100644 --- a/packages/datadog-instrumentations/src/next.js +++ b/packages/datadog-instrumentations/src/next.js @@ -2,7 +2,6 @@ const { channel, addHook } = require('./helpers/instrument') const shimmer = require('../../datadog-shimmer') -const { DD_MAJOR } = require('../../../version') const startChannel = channel('apm:next:request:start') const finishChannel = channel('apm:next:request:finish') @@ -221,7 +220,7 @@ addHook({ addHook({ name: 'next', - versions: DD_MAJOR >= 4 ? ['>=10.2 <11.1'] : ['>=9.5 <11.1'], + versions: ['>=10.2 <11.1'], file: 'dist/next-server/server/serve-static.js' }, serveStatic => shimmer.wrap(serveStatic, 'serveStatic', wrapServeStatic)) @@ -248,7 +247,11 @@ addHook({ name: 'next', versions: ['>=13.2'], file: 'dist/server/next-server.js' return nextServer }) -addHook({ name: 'next', versions: ['>=11.1 <13.2'], file: 'dist/server/next-server.js' }, nextServer => { +addHook({ + name: 'next', + versions: ['>=11.1 <13.2'], + file: 'dist/server/next-server.js' +}, nextServer => { const Server = nextServer.default shimmer.wrap(Server.prototype, 'handleApiRequest', wrapHandleApiRequest) return nextServer @@ -256,7 +259,7 @@ addHook({ name: 'next', versions: ['>=11.1 <13.2'], file: 'dist/server/next-serv addHook({ name: 'next', - versions: DD_MAJOR >= 4 ? ['>=10.2 <11.1'] : ['>=9.5 <11.1'], + versions: ['>=10.2 <11.1'], file: 'dist/next-server/server/next-server.js' }, nextServer => { const Server = nextServer.default diff --git a/scripts/verify-ci-config.js b/scripts/verify-ci-config.js new file mode 100644 index 00000000000..7a917132688 --- /dev/null +++ b/scripts/verify-ci-config.js @@ -0,0 +1,156 @@ +'use strict' +/* eslint-disable no-console */ + +const fs = require('fs') +const path = require('path') +const util = require('util') +const proxyquire = require('proxyquire') +const yaml = require('yaml') +const semver = require('semver') +const { execSync } = require('child_process') +const Module = require('module') +if (!Module.isBuiltin) { + Module.isBuiltin = mod => Module.builtinModules.includes(mod) +} + +const nodeMajor = Number(process.versions.node.split('.')[0]) + +const names = fs.readdirSync(path.join(__dirname, '..', 'packages', 'datadog-instrumentations', 'src')) + .filter(file => file.endsWith('.js')) + .map(file => file.slice(0, -3)) + +const instrumentations = names.reduce((acc, key) => { + let instrumentations = [] + const name = key + + try { + loadInstFile(`${name}/server.js`, instrumentations) + loadInstFile(`${name}/client.js`, instrumentations) + } catch (e) { + loadInstFile(`${name}.js`, instrumentations) + } + + instrumentations = instrumentations.filter(i => i.versions) + if (instrumentations.length) { + acc[key] = instrumentations + } + + return acc +}, {}) + +const versions = {} + +function checkYaml (yamlPath) { + const yamlContent = yaml.parse(fs.readFileSync(yamlPath, 'utf8')) + + const rangesPerPluginFromYaml = {} + const rangesPerPluginFromInst = {} + for (const jobName in yamlContent.jobs) { + const job = yamlContent.jobs[jobName] + if (!job.env || !job.env.PLUGINS) continue + + const pluginName = job.env.PLUGINS + if (Module.isBuiltin(pluginName)) continue + const rangesFromYaml = getRangesFromYaml(job) + if (rangesFromYaml) { + if (!rangesPerPluginFromYaml[pluginName]) { + rangesPerPluginFromYaml[pluginName] = new Set() + } + rangesFromYaml.forEach(range => rangesPerPluginFromYaml[pluginName].add(range)) + const plugin = instrumentations[pluginName] + const allRangesForPlugin = new Set(plugin.map(x => x.versions).flat()) + rangesPerPluginFromInst[pluginName] = allRangesForPlugin + } + } + for (const pluginName in rangesPerPluginFromYaml) { + const yamlRanges = Array.from(rangesPerPluginFromYaml[pluginName]) + const instRanges = Array.from(rangesPerPluginFromInst[pluginName]) + const yamlVersions = getMatchingVersions(pluginName, yamlRanges) + const instVersions = getMatchingVersions(pluginName, instRanges) + if (!util.isDeepStrictEqual(yamlVersions, instVersions)) { + const opts = { colors: true } + const colors = x => util.inspect(x, opts) + errorMsg(pluginName, 'Mismatch', ` +Valid version ranges from YAML: ${colors(yamlRanges)} +Valid version ranges from INST: ${colors(instRanges)} +${mismatching(yamlVersions, instVersions)} +Note that versions may be dependent on Node.js version. This is Node.js v${colors(nodeMajor)} + +> These don't match the same sets of versions in npm. +> +> Please check ${yamlPath} and the instrumentations +> for ${pluginName} to see that the version ranges match.`.trim()) + } + } +} + +function loadInstFile (file, instrumentations) { + const instrument = { + addHook (instrumentation) { + instrumentations.push(instrumentation) + } + } + + const instPath = path.join(__dirname, `../packages/datadog-instrumentations/src/${file}`) + + proxyquire.noPreserveCache()(instPath, { + './helpers/instrument': instrument, + '../helpers/instrument': instrument + }) +} + +function getRangesFromYaml (job) { + // eslint-disable-next-line no-template-curly-in-string + if (job.env && job.env.PACKAGE_VERSION_RANGE && job.env.PACKAGE_VERSION_RANGE !== '${{ matrix.range }}') { + errorMsg(job.env.PLUGINS, 'ERROR in YAML', 'You must use matrix.range instead of env.PACKAGE_VERSION_RANGE') + process.exitCode = 1 + } + if (job.strategy && job.strategy.matrix && job.strategy.matrix.range) { + const possibilities = [job.strategy.matrix] + if (job.strategy.matrix.include) { + possibilities.push(...job.strategy.matrix.include) + } + return possibilities.map(possibility => { + if (possibility.range) { + return [possibility.range].flat() + } else { + return undefined + } + }).flat() + } + + return null +} + +function getMatchingVersions (name, ranges) { + if (!versions[name]) { + versions[name] = JSON.parse(execSync('npm show ' + name + ' versions --json').toString()) + } + return versions[name].filter(version => ranges.some(range => semver.satisfies(version, range))) +} + +checkYaml(path.join(__dirname, '..', '.github', 'workflows', 'plugins.yml')) +checkYaml(path.join(__dirname, '..', '.github', 'workflows', 'appsec.yml')) + +function mismatching (yamlVersions, instVersions) { + const yamlSet = new Set(yamlVersions) + const instSet = new Set(instVersions) + + const onlyInYaml = yamlVersions.filter(v => !instSet.has(v)) + const onlyInInst = instVersions.filter(v => !yamlSet.has(v)) + + const opts = { colors: true } + return [ + `Versions only in YAML: ${util.inspect(onlyInYaml, opts)}`, + `Versions only in INST: ${util.inspect(onlyInInst, opts)}` + ].join('\n') +} + +function errorMsg (pluginName, title, message) { + console.log('===========================================') + console.log(title + ' for ' + pluginName) + console.log('-------------------------------------------') + console.log(message) + console.log('\n') + process.exitCode = 1 +} diff --git a/yarn.lock b/yarn.lock index 54222f765ba..107dfd70f1d 100644 --- a/yarn.lock +++ b/yarn.lock @@ -5252,6 +5252,11 @@ yaml@^1.10.2: resolved "https://registry.npmjs.org/yaml/-/yaml-1.10.2.tgz" integrity sha512-r3vXyErRCYJ7wg28yvBY5VSoAF8ZvlcW9/BwUzEtUsjvX/DKs24dIkuwjtuprwJJHsbyUbLApepYTR1BN4uHrg== +yaml@^2.5.0: + version "2.5.0" + resolved "https://registry.yarnpkg.com/yaml/-/yaml-2.5.0.tgz#c6165a721cf8000e91c36490a41d7be25176cf5d" + integrity sha512-2wWLbGbYDiSqqIKoPjar3MPgB94ErzCtrNE1FdqGuaO0pi2JGjmE8aW8TDZwzU7vuxcGRdL/4gPQwQ7hD5AMSw== + yargs-parser@20.2.4: version "20.2.4" resolved "https://registry.npmjs.org/yargs-parser/-/yargs-parser-20.2.4.tgz" From 75865b468568d2b54b2e959edeeb2e1e6b41077b Mon Sep 17 00:00:00 2001 From: Ugaitz Urien Date: Mon, 16 Dec 2024 11:35:20 +0100 Subject: [PATCH 06/36] Test aerospike node 16 with ubuntu-22.04 (#5017) --- .github/workflows/plugins.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/plugins.yml b/.github/workflows/plugins.yml index 2a76db58145..4822539ecab 100644 --- a/.github/workflows/plugins.yml +++ b/.github/workflows/plugins.yml @@ -21,7 +21,7 @@ jobs: node-version: [16] range: ['>=4.0.0 <5.2.0'] aerospike-image: [ce-5.7.0.15] - test-image: [ubuntu-latest] + test-image: [ubuntu-22.04] include: - node-version: 18 range: '>=5.2.0' From 23720bb6ef8ff900927ffe6a64f5f7c183ce18ec Mon Sep 17 00:00:00 2001 From: Igor Unanua Date: Mon, 16 Dec 2024 12:02:07 +0100 Subject: [PATCH 07/36] Upgrade iast rewriter version to 2.6.1 (#5010) * Upgrade iast rewriter version to 2.6.1 * fix nanoid version --- package.json | 2 +- yarn.lock | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/package.json b/package.json index 04d5c78e320..28c20dde6ed 100644 --- a/package.json +++ b/package.json @@ -83,7 +83,7 @@ "dependencies": { "@datadog/libdatadog": "^0.2.2", "@datadog/native-appsec": "8.3.0", - "@datadog/native-iast-rewriter": "2.6.0", + "@datadog/native-iast-rewriter": "2.6.1", "@datadog/native-iast-taint-tracking": "3.2.0", "@datadog/native-metrics": "^3.0.1", "@datadog/pprof": "5.4.1", diff --git a/yarn.lock b/yarn.lock index 107dfd70f1d..ebbc6922e3d 100644 --- a/yarn.lock +++ b/yarn.lock @@ -413,10 +413,10 @@ dependencies: node-gyp-build "^3.9.0" -"@datadog/native-iast-rewriter@2.6.0": - version "2.6.0" - resolved "https://registry.yarnpkg.com/@datadog/native-iast-rewriter/-/native-iast-rewriter-2.6.0.tgz#745148ac630cace48372fb3b3aaa50e32460b693" - integrity sha512-TCRe3QNm7hGWlfvW1RnE959sV/kBqDiSEGAHS+HlQYaIwG2y0WcxA5TjLxhcIJJsfmgou5ycIlknAvNkbaoDDQ== +"@datadog/native-iast-rewriter@2.6.1": + version "2.6.1" + resolved "https://registry.yarnpkg.com/@datadog/native-iast-rewriter/-/native-iast-rewriter-2.6.1.tgz#5e5393628c73c57dcf08256299c0e8cf71deb14f" + integrity sha512-zv7cr/MzHg560jhAnHcO7f9pLi4qaYrBEcB+Gla0xkVouYSDsp8cGXIGG4fiGdAMHdt7SpDNS6+NcEAqD/v8Ig== dependencies: lru-cache "^7.14.0" node-gyp-build "^4.5.0" From 02fba54df8cf8affdb088b4119d0b16bf61db8c0 Mon Sep 17 00:00:00 2001 From: Igor Unanua Date: Mon, 16 Dec 2024 12:09:46 +0100 Subject: [PATCH 08/36] Add some checks to avoid runtime errors (#4945) * Add some checks to avoid runtime errors * check span * linter --- packages/datadog-plugin-avsc/src/schema_iterator.js | 11 ++++++++--- packages/datadog-plugin-grpc/src/client.js | 4 ++-- .../dd-trace/src/opentracing/propagation/text_map.js | 2 +- packages/dd-trace/src/plugins/tracing.js | 2 +- .../src/profiling/profilers/event_plugins/event.js | 10 ++++++++-- 5 files changed, 20 insertions(+), 9 deletions(-) diff --git a/packages/datadog-plugin-avsc/src/schema_iterator.js b/packages/datadog-plugin-avsc/src/schema_iterator.js index 44fce95a765..0b4874ceea8 100644 --- a/packages/datadog-plugin-avsc/src/schema_iterator.js +++ b/packages/datadog-plugin-avsc/src/schema_iterator.js @@ -108,10 +108,15 @@ class SchemaExtractor { if (!builder.shouldExtractSchema(schemaName, depth)) { return false } - for (const field of schema.fields) { - if (!this.extractProperty(field, schemaName, field.name, builder, depth)) { - log.warn('DSM: Unable to extract field with name: %s from Avro schema with name: %s', field.name, schemaName) + if (schema.fields?.[Symbol.iterator]) { + for (const field of schema.fields) { + if (!this.extractProperty(field, schemaName, field.name, builder, depth)) { + log.warn('DSM: Unable to extract field with name: %s from Avro schema with name: %s', field.name, + schemaName) + } } + } else { + log.warn('DSM: schema.fields is not iterable from Avro schema with name: %s', schemaName) } } return true diff --git a/packages/datadog-plugin-grpc/src/client.js b/packages/datadog-plugin-grpc/src/client.js index 1b130a1f93e..db8dd89b9bf 100644 --- a/packages/datadog-plugin-grpc/src/client.js +++ b/packages/datadog-plugin-grpc/src/client.js @@ -62,7 +62,7 @@ class GrpcClientPlugin extends ClientPlugin { return parentStore } - error ({ span, error }) { + error ({ span = this.activeSpan, error }) { this.addCode(span, error.code) if (error.code && !this._tracerConfig.grpc.client.error.statuses.includes(error.code)) { return @@ -108,7 +108,7 @@ class GrpcClientPlugin extends ClientPlugin { } addCode (span, code) { - if (code !== undefined) { + if (code !== undefined && span) { span.setTag('grpc.status.code', code) } } diff --git a/packages/dd-trace/src/opentracing/propagation/text_map.js b/packages/dd-trace/src/opentracing/propagation/text_map.js index dcf8fb3fcc6..82bc9f2b30f 100644 --- a/packages/dd-trace/src/opentracing/propagation/text_map.js +++ b/packages/dd-trace/src/opentracing/propagation/text_map.js @@ -499,7 +499,7 @@ class TextMapPropagator { } _extractGenericContext (carrier, traceKey, spanKey, radix) { - if (carrier[traceKey] && carrier[spanKey]) { + if (carrier && carrier[traceKey] && carrier[spanKey]) { if (invalidSegment.test(carrier[traceKey])) return null return new DatadogSpanContext({ diff --git a/packages/dd-trace/src/plugins/tracing.js b/packages/dd-trace/src/plugins/tracing.js index 6f11b9bde6a..e384b8cb7a7 100644 --- a/packages/dd-trace/src/plugins/tracing.js +++ b/packages/dd-trace/src/plugins/tracing.js @@ -94,7 +94,7 @@ class TracingPlugin extends Plugin { } addError (error, span = this.activeSpan) { - if (!span._spanContext._tags.error) { + if (span && !span._spanContext._tags.error) { // Errors may be wrapped in a context. error = (error && error.error) || error span.setTag('error', error || 1) diff --git a/packages/dd-trace/src/profiling/profilers/event_plugins/event.js b/packages/dd-trace/src/profiling/profilers/event_plugins/event.js index 5d81e1d8a3f..48e430ba607 100644 --- a/packages/dd-trace/src/profiling/profilers/event_plugins/event.js +++ b/packages/dd-trace/src/profiling/profilers/event_plugins/event.js @@ -21,11 +21,17 @@ class EventPlugin extends TracingPlugin { } error () { - this.store.getStore().error = true + const store = this.store.getStore() + if (store) { + store.error = true + } } finish () { - const { startEvent, startTime, error } = this.store.getStore() + const store = this.store.getStore() + if (!store) return + + const { startEvent, startTime, error } = store if (error) { return // don't emit perf events for failed operations } From 048868e2f778423d75210528afe4226e3b10f954 Mon Sep 17 00:00:00 2001 From: simon-id Date: Mon, 16 Dec 2024 17:14:31 +0100 Subject: [PATCH 09/36] New automatic user event collection (#4674) --- docs/test.ts | 2 +- index.d.ts | 22 +- .../src/passport-http.js | 16 +- .../src/passport-local.js | 16 +- .../src/passport-utils.js | 62 +- .../test/passport-http.spec.js | 115 ++- .../test/passport-local.spec.js | 132 ++-- .../test/passport-utils.spec.js | 36 - packages/dd-trace/src/appsec/addresses.js | 3 + packages/dd-trace/src/appsec/index.js | 15 +- packages/dd-trace/src/appsec/passport.js | 110 --- .../src/appsec/remote_config/capabilities.js | 1 + .../src/appsec/remote_config/index.js | 26 +- .../dd-trace/src/appsec/sdk/track_event.js | 51 +- packages/dd-trace/src/appsec/telemetry.js | 10 + packages/dd-trace/src/appsec/user_tracking.js | 168 +++++ packages/dd-trace/src/config.js | 20 +- packages/dd-trace/test/appsec/index.spec.js | 154 ++-- .../dd-trace/test/appsec/passport.spec.js | 245 ------ .../test/appsec/remote_config/index.spec.js | 91 ++- .../test/appsec/sdk/track_event.spec.js | 145 ++-- .../dd-trace/test/appsec/telemetry.spec.js | 11 + .../test/appsec/user_tracking.spec.js | 696 ++++++++++++++++++ packages/dd-trace/test/config.spec.js | 26 +- 24 files changed, 1485 insertions(+), 688 deletions(-) delete mode 100644 packages/datadog-instrumentations/test/passport-utils.spec.js delete mode 100644 packages/dd-trace/src/appsec/passport.js create mode 100644 packages/dd-trace/src/appsec/user_tracking.js delete mode 100644 packages/dd-trace/test/appsec/passport.spec.js create mode 100644 packages/dd-trace/test/appsec/user_tracking.spec.js diff --git a/docs/test.ts b/docs/test.ts index 479b4620b4d..ce34a23d62b 100644 --- a/docs/test.ts +++ b/docs/test.ts @@ -111,7 +111,7 @@ tracer.init({ blockedTemplateJson: './blocked.json', blockedTemplateGraphql: './blockedgraphql.json', eventTracking: { - mode: 'safe' + mode: 'anon' }, apiSecurity: { enabled: true, diff --git a/index.d.ts b/index.d.ts index 9b4becec957..a41b4aee410 100644 --- a/index.d.ts +++ b/index.d.ts @@ -655,12 +655,24 @@ declare namespace tracer { */ eventTracking?: { /** - * Controls the automated user event tracking mode. Possible values are disabled, safe and extended. - * On safe mode, any detected Personally Identifiable Information (PII) about the user will be redacted from the event. - * On extended mode, no redaction will take place. - * @default 'safe' + * Controls the automated user tracking mode for user IDs and logins collections. Possible values: + * * 'anonymous': will hash user IDs and user logins before collecting them + * * 'anon': alias for 'anonymous' + * * 'safe': deprecated alias for 'anonymous' + * + * * 'identification': will collect user IDs and logins without redaction + * * 'ident': alias for 'identification' + * * 'extended': deprecated alias for 'identification' + * + * * 'disabled': will not collect user IDs and logins + * + * Unknown values will be considered as 'disabled' + * @default 'identification' */ - mode?: 'safe' | 'extended' | 'disabled' + mode?: + 'anonymous' | 'anon' | 'safe' | + 'identification' | 'ident' | 'extended' | + 'disabled' }, /** * Configuration for Api Security diff --git a/packages/datadog-instrumentations/src/passport-http.js b/packages/datadog-instrumentations/src/passport-http.js index 0969d2d3fc9..3b930a1a1cc 100644 --- a/packages/datadog-instrumentations/src/passport-http.js +++ b/packages/datadog-instrumentations/src/passport-http.js @@ -1,22 +1,10 @@ 'use strict' -const shimmer = require('../../datadog-shimmer') const { addHook } = require('./helpers/instrument') -const { wrapVerify } = require('./passport-utils') +const { strategyHook } = require('./passport-utils') addHook({ name: 'passport-http', file: 'lib/passport-http/strategies/basic.js', versions: ['>=0.3.0'] -}, BasicStrategy => { - return shimmer.wrapFunction(BasicStrategy, BasicStrategy => function () { - const type = 'http' - - if (typeof arguments[0] === 'function') { - arguments[0] = wrapVerify(arguments[0], false, type) - } else { - arguments[1] = wrapVerify(arguments[1], (arguments[0] && arguments[0].passReqToCallback), type) - } - return BasicStrategy.apply(this, arguments) - }) -}) +}, strategyHook) diff --git a/packages/datadog-instrumentations/src/passport-local.js b/packages/datadog-instrumentations/src/passport-local.js index dab74eb470e..c6dcec9a48d 100644 --- a/packages/datadog-instrumentations/src/passport-local.js +++ b/packages/datadog-instrumentations/src/passport-local.js @@ -1,22 +1,10 @@ 'use strict' -const shimmer = require('../../datadog-shimmer') const { addHook } = require('./helpers/instrument') -const { wrapVerify } = require('./passport-utils') +const { strategyHook } = require('./passport-utils') addHook({ name: 'passport-local', file: 'lib/strategy.js', versions: ['>=1.0.0'] -}, Strategy => { - return shimmer.wrapFunction(Strategy, Strategy => function () { - const type = 'local' - - if (typeof arguments[0] === 'function') { - arguments[0] = wrapVerify(arguments[0], false, type) - } else { - arguments[1] = wrapVerify(arguments[1], (arguments[0] && arguments[0].passReqToCallback), type) - } - return Strategy.apply(this, arguments) - }) -}) +}, strategyHook) diff --git a/packages/datadog-instrumentations/src/passport-utils.js b/packages/datadog-instrumentations/src/passport-utils.js index 7969ab486b4..de1cd090a71 100644 --- a/packages/datadog-instrumentations/src/passport-utils.js +++ b/packages/datadog-instrumentations/src/passport-utils.js @@ -5,33 +5,57 @@ const { channel } = require('./helpers/instrument') const passportVerifyChannel = channel('datadog:passport:verify:finish') -function wrapVerifiedAndPublish (username, password, verified, type) { - if (!passportVerifyChannel.hasSubscribers) { - return verified - } +function wrapVerifiedAndPublish (framework, username, verified) { + return shimmer.wrapFunction(verified, function wrapVerified (verified) { + return function wrappedVerified (err, user) { + // if there is an error, it's neither an auth success nor a failure + if (!err) { + const abortController = new AbortController() + + passportVerifyChannel.publish({ framework, login: username, user, success: !!user, abortController }) + + if (abortController.signal.aborted) return + } - // eslint-disable-next-line n/handle-callback-err - return shimmer.wrapFunction(verified, verified => function (err, user, info) { - const credentials = { type, username } - passportVerifyChannel.publish({ credentials, user }) - return verified.apply(this, arguments) + return verified.apply(this, arguments) + } }) } -function wrapVerify (verify, passReq, type) { - if (passReq) { - return function (req, username, password, verified) { - arguments[3] = wrapVerifiedAndPublish(username, password, verified, type) - return verify.apply(this, arguments) +function wrapVerify (verify) { + return function wrappedVerify (req, username, password, verified) { + if (passportVerifyChannel.hasSubscribers) { + const framework = `passport-${this.name}` + + // replace the callback with our own wrapper to get the result + if (this._passReqToCallback) { + arguments[3] = wrapVerifiedAndPublish(framework, arguments[1], arguments[3]) + } else { + arguments[2] = wrapVerifiedAndPublish(framework, arguments[0], arguments[2]) + } } - } else { - return function (username, password, verified) { - arguments[2] = wrapVerifiedAndPublish(username, password, verified, type) - return verify.apply(this, arguments) + + return verify.apply(this, arguments) + } +} + +function wrapStrategy (Strategy) { + return function wrappedStrategy () { + // verify function can be either the first or second argument + if (typeof arguments[0] === 'function') { + arguments[0] = wrapVerify(arguments[0]) + } else { + arguments[1] = wrapVerify(arguments[1]) } + + return Strategy.apply(this, arguments) } } +function strategyHook (Strategy) { + return shimmer.wrapFunction(Strategy, wrapStrategy) +} + module.exports = { - wrapVerify + strategyHook } diff --git a/packages/datadog-instrumentations/test/passport-http.spec.js b/packages/datadog-instrumentations/test/passport-http.spec.js index 2918c935e20..5cb0282ec2f 100644 --- a/packages/datadog-instrumentations/test/passport-http.spec.js +++ b/packages/datadog-instrumentations/test/passport-http.spec.js @@ -1,8 +1,9 @@ 'use strict' const agent = require('../../dd-trace/test/plugins/agent') -const axios = require('axios') +const axios = require('axios').create({ validateStatus: null }) const dc = require('dc-polyfill') +const { storage } = require('../../datadog-core') withVersions('passport-http', 'passport-http', version => { describe('passport-http instrumentation', () => { @@ -10,7 +11,7 @@ withVersions('passport-http', 'passport-http', version => { let port, server, subscriberStub before(() => { - return agent.load(['express', 'passport', 'passport-http'], { client: false }) + return agent.load(['http', 'express', 'passport', 'passport-http'], { client: false }) }) before((done) => { @@ -19,7 +20,17 @@ withVersions('passport-http', 'passport-http', version => { const BasicStrategy = require(`../../../versions/passport-http@${version}`).get().BasicStrategy const app = express() - passport.use(new BasicStrategy((username, password, done) => { + function validateUser (req, username, password, done) { + // support with or without passReqToCallback + if (typeof done !== 'function') { + done = password + password = username + username = req + } + + // simulate db error + if (username === 'error') return done('error') + const users = [{ _id: 1, username: 'test', @@ -35,7 +46,18 @@ withVersions('passport-http', 'passport-http', version => { return done(null, user) } } - )) + + passport.use('basic', new BasicStrategy({ + usernameField: 'username', + passwordField: 'password', + passReqToCallback: false + }, validateUser)) + + passport.use('basic-withreq', new BasicStrategy({ + usernameField: 'username', + passwordField: 'password', + passReqToCallback: true + }, validateUser)) app.use(passport.initialize()) app.use(express.json()) @@ -44,16 +66,14 @@ withVersions('passport-http', 'passport-http', version => { passport.authenticate('basic', { successRedirect: '/grant', failureRedirect: '/deny', - passReqToCallback: false, session: false }) ) - app.post('/req', - passport.authenticate('basic', { + app.get('/req', + passport.authenticate('basic-withreq', { successRedirect: '/grant', failureRedirect: '/deny', - passReqToCallback: true, session: false }) ) @@ -66,9 +86,7 @@ withVersions('passport-http', 'passport-http', version => { res.send('Denied') }) - passportVerifyChannel.subscribe(function ({ credentials, user, err, info }) { - subscriberStub(arguments[0]) - }) + passportVerifyChannel.subscribe((data) => subscriberStub(data)) server = app.listen(0, () => { port = server.address().port @@ -85,6 +103,18 @@ withVersions('passport-http', 'passport-http', version => { return agent.close({ ritmReset: false }) }) + it('should not call subscriber when an error occurs', async () => { + const res = await axios.get(`http://localhost:${port}/`, { + headers: { + // error:1234 + Authorization: 'Basic ZXJyb3I6MTIzNA==' + } + }) + + expect(res.status).to.equal(500) + expect(subscriberStub).to.not.be.called + }) + it('should call subscriber with proper arguments on success', async () => { const res = await axios.get(`http://localhost:${port}/`, { headers: { @@ -95,16 +125,17 @@ withVersions('passport-http', 'passport-http', version => { expect(res.status).to.equal(200) expect(res.data).to.equal('Granted') - expect(subscriberStub).to.be.calledOnceWithExactly( - { - credentials: { type: 'http', username: 'test' }, - user: { _id: 1, username: 'test', password: '1234', email: 'testuser@ddog.com' } - } - ) + expect(subscriberStub).to.be.calledOnceWithExactly({ + framework: 'passport-basic', + login: 'test', + user: { _id: 1, username: 'test', password: '1234', email: 'testuser@ddog.com' }, + success: true, + abortController: new AbortController() + }) }) it('should call subscriber with proper arguments on success with passReqToCallback set to true', async () => { - const res = await axios.get(`http://localhost:${port}/`, { + const res = await axios.get(`http://localhost:${port}/req`, { headers: { // test:1234 Authorization: 'Basic dGVzdDoxMjM0' @@ -113,12 +144,13 @@ withVersions('passport-http', 'passport-http', version => { expect(res.status).to.equal(200) expect(res.data).to.equal('Granted') - expect(subscriberStub).to.be.calledOnceWithExactly( - { - credentials: { type: 'http', username: 'test' }, - user: { _id: 1, username: 'test', password: '1234', email: 'testuser@ddog.com' } - } - ) + expect(subscriberStub).to.be.calledOnceWithExactly({ + framework: 'passport-basic', + login: 'test', + user: { _id: 1, username: 'test', password: '1234', email: 'testuser@ddog.com' }, + success: true, + abortController: new AbortController() + }) }) it('should call subscriber with proper arguments on failure', async () => { @@ -131,12 +163,37 @@ withVersions('passport-http', 'passport-http', version => { expect(res.status).to.equal(200) expect(res.data).to.equal('Denied') - expect(subscriberStub).to.be.calledOnceWithExactly( - { - credentials: { type: 'http', username: 'test' }, - user: false + expect(subscriberStub).to.be.calledOnceWithExactly({ + framework: 'passport-basic', + login: 'test', + user: false, + success: false, + abortController: new AbortController() + }) + }) + + it('should block when subscriber aborts', async () => { + subscriberStub = sinon.spy(({ abortController }) => { + storage.getStore().req.res.writeHead(403).end('Blocked') + abortController.abort() + }) + + const res = await axios.get(`http://localhost:${port}/`, { + headers: { + // test:1234 + Authorization: 'Basic dGVzdDoxMjM0' } - ) + }) + + expect(res.status).to.equal(403) + expect(res.data).to.equal('Blocked') + expect(subscriberStub).to.be.calledOnceWithExactly({ + framework: 'passport-basic', + login: 'test', + user: { _id: 1, username: 'test', password: '1234', email: 'testuser@ddog.com' }, + success: true, + abortController: new AbortController() + }) }) }) }) diff --git a/packages/datadog-instrumentations/test/passport-local.spec.js b/packages/datadog-instrumentations/test/passport-local.spec.js index d54f02b289f..bcfc2e56dc9 100644 --- a/packages/datadog-instrumentations/test/passport-local.spec.js +++ b/packages/datadog-instrumentations/test/passport-local.spec.js @@ -1,8 +1,9 @@ 'use strict' const agent = require('../../dd-trace/test/plugins/agent') -const axios = require('axios') +const axios = require('axios').create({ validateStatus: null }) const dc = require('dc-polyfill') +const { storage } = require('../../datadog-core') withVersions('passport-local', 'passport-local', version => { describe('passport-local instrumentation', () => { @@ -10,7 +11,7 @@ withVersions('passport-local', 'passport-local', version => { let port, server, subscriberStub before(() => { - return agent.load(['express', 'passport', 'passport-local'], { client: false }) + return agent.load(['http', 'express', 'passport', 'passport-local'], { client: false }) }) before((done) => { @@ -19,24 +20,44 @@ withVersions('passport-local', 'passport-local', version => { const LocalStrategy = require(`../../../versions/passport-local@${version}`).get().Strategy const app = express() - passport.use(new LocalStrategy({ usernameField: 'username', passwordField: 'password' }, - (username, password, done) => { - const users = [{ - _id: 1, - username: 'test', - password: '1234', - email: 'testuser@ddog.com' - }] - - const user = users.find(user => (user.username === username) && (user.password === password)) - - if (!user) { - return done(null, false) - } else { - return done(null, user) - } + function validateUser (req, username, password, done) { + // support with or without passReqToCallback + if (typeof done !== 'function') { + done = password + password = username + username = req } - )) + + // simulate db error + if (username === 'error') return done('error') + + const users = [{ + _id: 1, + username: 'test', + password: '1234', + email: 'testuser@ddog.com' + }] + + const user = users.find(user => (user.username === username) && (user.password === password)) + + if (!user) { + return done(null, false) + } else { + return done(null, user) + } + } + + passport.use('local', new LocalStrategy({ + usernameField: 'username', + passwordField: 'password', + passReqToCallback: false + }, validateUser)) + + passport.use('local-withreq', new LocalStrategy({ + usernameField: 'username', + passwordField: 'password', + passReqToCallback: true + }, validateUser)) app.use(passport.initialize()) app.use(express.json()) @@ -45,16 +66,14 @@ withVersions('passport-local', 'passport-local', version => { passport.authenticate('local', { successRedirect: '/grant', failureRedirect: '/deny', - passReqToCallback: false, session: false }) ) app.post('/req', - passport.authenticate('local', { + passport.authenticate('local-withreq', { successRedirect: '/grant', failureRedirect: '/deny', - passReqToCallback: true, session: false }) ) @@ -67,9 +86,7 @@ withVersions('passport-local', 'passport-local', version => { res.send('Denied') }) - passportVerifyChannel.subscribe(function ({ credentials, user, err, info }) { - subscriberStub(arguments[0]) - }) + passportVerifyChannel.subscribe((data) => subscriberStub(data)) server = app.listen(0, () => { port = server.address().port @@ -86,17 +103,25 @@ withVersions('passport-local', 'passport-local', version => { return agent.close({ ritmReset: false }) }) + it('should not call subscriber when an error occurs', async () => { + const res = await axios.post(`http://localhost:${port}/`, { username: 'error', password: '1234' }) + + expect(res.status).to.equal(500) + expect(subscriberStub).to.not.be.called + }) + it('should call subscriber with proper arguments on success', async () => { const res = await axios.post(`http://localhost:${port}/`, { username: 'test', password: '1234' }) expect(res.status).to.equal(200) expect(res.data).to.equal('Granted') - expect(subscriberStub).to.be.calledOnceWithExactly( - { - credentials: { type: 'local', username: 'test' }, - user: { _id: 1, username: 'test', password: '1234', email: 'testuser@ddog.com' } - } - ) + expect(subscriberStub).to.be.calledOnceWithExactly({ + framework: 'passport-local', + login: 'test', + user: { _id: 1, username: 'test', password: '1234', email: 'testuser@ddog.com' }, + success: true, + abortController: new AbortController() + }) }) it('should call subscriber with proper arguments on success with passReqToCallback set to true', async () => { @@ -104,12 +129,13 @@ withVersions('passport-local', 'passport-local', version => { expect(res.status).to.equal(200) expect(res.data).to.equal('Granted') - expect(subscriberStub).to.be.calledOnceWithExactly( - { - credentials: { type: 'local', username: 'test' }, - user: { _id: 1, username: 'test', password: '1234', email: 'testuser@ddog.com' } - } - ) + expect(subscriberStub).to.be.calledOnceWithExactly({ + framework: 'passport-local', + login: 'test', + user: { _id: 1, username: 'test', password: '1234', email: 'testuser@ddog.com' }, + success: true, + abortController: new AbortController() + }) }) it('should call subscriber with proper arguments on failure', async () => { @@ -117,12 +143,32 @@ withVersions('passport-local', 'passport-local', version => { expect(res.status).to.equal(200) expect(res.data).to.equal('Denied') - expect(subscriberStub).to.be.calledOnceWithExactly( - { - credentials: { type: 'local', username: 'test' }, - user: false - } - ) + expect(subscriberStub).to.be.calledOnceWithExactly({ + framework: 'passport-local', + login: 'test', + user: false, + success: false, + abortController: new AbortController() + }) + }) + + it('should block when subscriber aborts', async () => { + subscriberStub = sinon.spy(({ abortController }) => { + storage.getStore().req.res.writeHead(403).end('Blocked') + abortController.abort() + }) + + const res = await axios.post(`http://localhost:${port}/`, { username: 'test', password: '1234' }) + + expect(res.status).to.equal(403) + expect(res.data).to.equal('Blocked') + expect(subscriberStub).to.be.calledOnceWithExactly({ + framework: 'passport-local', + login: 'test', + user: { _id: 1, username: 'test', password: '1234', email: 'testuser@ddog.com' }, + success: true, + abortController: new AbortController() + }) }) }) }) diff --git a/packages/datadog-instrumentations/test/passport-utils.spec.js b/packages/datadog-instrumentations/test/passport-utils.spec.js deleted file mode 100644 index 3cf6a64a60a..00000000000 --- a/packages/datadog-instrumentations/test/passport-utils.spec.js +++ /dev/null @@ -1,36 +0,0 @@ -'use strict' - -const proxyquire = require('proxyquire') -const { channel } = require('../src/helpers/instrument') - -const passportVerifyChannel = channel('datadog:passport:verify:finish') - -describe('passport-utils', () => { - const shimmer = { - wrap: sinon.stub() - } - - let passportUtils - - beforeEach(() => { - passportUtils = proxyquire('../src/passport-utils', { - '../../datadog-shimmer': shimmer - }) - }) - - it('should not call wrap when there is no subscribers', () => { - const wrap = passportUtils.wrapVerify(() => {}, false, 'type') - - wrap() - expect(shimmer.wrap).not.to.have.been.called - }) - - it('should call wrap when there is subscribers', () => { - const wrap = passportUtils.wrapVerify(() => {}, false, 'type') - - passportVerifyChannel.subscribe(() => {}) - - wrap() - expect(shimmer.wrap).to.have.been.called - }) -}) diff --git a/packages/dd-trace/src/appsec/addresses.js b/packages/dd-trace/src/appsec/addresses.js index cb540bc4e6f..a492a5e454f 100644 --- a/packages/dd-trace/src/appsec/addresses.js +++ b/packages/dd-trace/src/appsec/addresses.js @@ -1,5 +1,6 @@ 'use strict' +// TODO: reorder all this, it's a mess module.exports = { HTTP_INCOMING_BODY: 'server.request.body', HTTP_INCOMING_QUERY: 'server.request.query', @@ -20,6 +21,8 @@ module.exports = { HTTP_CLIENT_IP: 'http.client_ip', USER_ID: 'usr.id', + USER_LOGIN: 'usr.login', + WAF_CONTEXT_PROCESSOR: 'waf.context.processor', HTTP_OUTGOING_URL: 'server.io.net.url', diff --git a/packages/dd-trace/src/appsec/index.js b/packages/dd-trace/src/appsec/index.js index d4f4adc6554..db089a61dca 100644 --- a/packages/dd-trace/src/appsec/index.js +++ b/packages/dd-trace/src/appsec/index.js @@ -28,7 +28,7 @@ const web = require('../plugins/util/web') const { extractIp } = require('../plugins/util/ip_extractor') const { HTTP_CLIENT_IP } = require('../../../../ext/tags') const { isBlocked, block, setTemplates, getBlockingAction } = require('./blocking') -const { passportTrackEvent } = require('./passport') +const UserTracking = require('./user_tracking') const { storage } = require('../../../datadog-core') const graphql = require('./graphql') const rasp = require('./rasp') @@ -59,11 +59,14 @@ function enable (_config) { apiSecuritySampler.configure(_config.appsec) + UserTracking.setCollectionMode(_config.appsec.eventTracking.mode, false) + bodyParser.subscribe(onRequestBodyParsed) multerParser.subscribe(onRequestBodyParsed) cookieParser.subscribe(onRequestCookieParser) incomingHttpRequestStart.subscribe(incomingHttpStartTranslator) incomingHttpRequestEnd.subscribe(incomingHttpEndTranslator) + passportVerify.subscribe(onPassportVerify) // possible optimization: only subscribe if collection mode is enabled queryParser.subscribe(onRequestQueryParsed) nextBodyParsed.subscribe(onRequestBodyParsed) nextQueryParsed.subscribe(onRequestQueryParsed) @@ -73,10 +76,6 @@ function enable (_config) { responseWriteHead.subscribe(onResponseWriteHead) responseSetHeader.subscribe(onResponseSetHeader) - if (_config.appsec.eventTracking.enabled) { - passportVerify.subscribe(onPassportVerify) - } - isEnabled = true config = _config } catch (err) { @@ -184,7 +183,7 @@ function incomingHttpEndTranslator ({ req, res }) { Reporter.finishRequest(req, res) } -function onPassportVerify ({ credentials, user }) { +function onPassportVerify ({ framework, login, user, success, abortController }) { const store = storage.getStore() const rootSpan = store?.req && web.root(store.req) @@ -193,7 +192,9 @@ function onPassportVerify ({ credentials, user }) { return } - passportTrackEvent(credentials, user, rootSpan, config.appsec.eventTracking.mode) + const results = UserTracking.trackLogin(framework, login, user, success, rootSpan) + + handleResults(results, store.req, store.req.res, rootSpan, abortController) } function onRequestQueryParsed ({ req, res, query, abortController }) { diff --git a/packages/dd-trace/src/appsec/passport.js b/packages/dd-trace/src/appsec/passport.js deleted file mode 100644 index 2093b7b1fdc..00000000000 --- a/packages/dd-trace/src/appsec/passport.js +++ /dev/null @@ -1,110 +0,0 @@ -'use strict' - -const log = require('../log') -const { trackEvent } = require('./sdk/track_event') -const { setUserTags } = require('./sdk/set_user') - -const UUID_PATTERN = '^[0-9A-F]{8}-[0-9A-F]{4}-[1-5][0-9A-F]{3}-[89AB][0-9A-F]{3}-[0-9A-F]{12}$' -const regexUsername = new RegExp(UUID_PATTERN, 'i') - -const SDK_USER_EVENT_PATTERN = '^_dd\\.appsec\\.events\\.users\\.[\\W\\w+]+\\.sdk$' -const regexSdkEvent = new RegExp(SDK_USER_EVENT_PATTERN, 'i') - -function isSdkCalled (tags) { - let called = false - - if (tags !== null && typeof tags === 'object') { - called = Object.entries(tags).some(([key, value]) => regexSdkEvent.test(key) && value === 'true') - } - - return called -} - -// delete this function later if we know it's always credential.username -function getLogin (credentials) { - const type = credentials && credentials.type - let login - if (type === 'local' || type === 'http') { - login = credentials.username - } - - return login -} - -function parseUser (login, passportUser, mode) { - const user = { - 'usr.id': login - } - - if (!user['usr.id']) { - return user - } - - if (passportUser) { - // Guess id - if (passportUser.id) { - user['usr.id'] = passportUser.id - } else if (passportUser._id) { - user['usr.id'] = passportUser._id - } - - if (mode === 'extended') { - if (login) { - user['usr.login'] = login - } - - if (passportUser.email) { - user['usr.email'] = passportUser.email - } - - // Guess username - if (passportUser.username) { - user['usr.username'] = passportUser.username - } else if (passportUser.name) { - user['usr.username'] = passportUser.name - } - } - } - - if (mode === 'safe') { - // Remove PII in safe mode - if (!regexUsername.test(user['usr.id'])) { - user['usr.id'] = '' - } - } - - return user -} - -function passportTrackEvent (credentials, passportUser, rootSpan, mode) { - const tags = rootSpan && rootSpan.context() && rootSpan.context()._tags - - if (isSdkCalled(tags)) { - // Don't overwrite tags set by SDK callings - return - } - const user = parseUser(getLogin(credentials), passportUser, mode) - - if (user['usr.id'] === undefined) { - log.warn('No user ID found in authentication instrumentation') - return - } - - if (passportUser) { - // If a passportUser object is published then the login succeded - const userTags = {} - Object.entries(user).forEach(([k, v]) => { - const attr = k.split('.', 2)[1] - userTags[attr] = v - }) - - setUserTags(userTags, rootSpan) - trackEvent('users.login.success', null, 'passportTrackEvent', rootSpan, mode) - } else { - trackEvent('users.login.failure', user, 'passportTrackEvent', rootSpan, mode) - } -} - -module.exports = { - passportTrackEvent -} diff --git a/packages/dd-trace/src/appsec/remote_config/capabilities.js b/packages/dd-trace/src/appsec/remote_config/capabilities.js index bd729cc39cc..16034f5f9ee 100644 --- a/packages/dd-trace/src/appsec/remote_config/capabilities.js +++ b/packages/dd-trace/src/appsec/remote_config/capabilities.js @@ -22,6 +22,7 @@ module.exports = { ASM_RASP_SSRF: 1n << 23n, ASM_RASP_SHI: 1n << 24n, APM_TRACING_SAMPLE_RULES: 1n << 29n, + ASM_AUTO_USER_INSTRUM_MODE: 1n << 31n, ASM_ENDPOINT_FINGERPRINT: 1n << 32n, ASM_NETWORK_FINGERPRINT: 1n << 34n, ASM_HEADER_FINGERPRINT: 1n << 35n diff --git a/packages/dd-trace/src/appsec/remote_config/index.js b/packages/dd-trace/src/appsec/remote_config/index.js index 90cda5c6f61..7884175abb0 100644 --- a/packages/dd-trace/src/appsec/remote_config/index.js +++ b/packages/dd-trace/src/appsec/remote_config/index.js @@ -4,6 +4,8 @@ const Activation = require('../activation') const RemoteConfigManager = require('./manager') const RemoteConfigCapabilities = require('./capabilities') +const { setCollectionMode } = require('../user_tracking') +const log = require('../../log') let rc @@ -23,9 +25,31 @@ function enable (config, appsec) { rc.updateCapabilities(RemoteConfigCapabilities.ASM_ACTIVATION, true) } - rc.setProductHandler('ASM_FEATURES', (action, rcConfig) => { + rc.updateCapabilities(RemoteConfigCapabilities.ASM_AUTO_USER_INSTRUM_MODE, true) + + let autoUserInstrumModeId + + rc.setProductHandler('ASM_FEATURES', (action, rcConfig, configId) => { if (!rcConfig) return + // this is put before other handlers because it can reject the config + if (typeof rcConfig.auto_user_instrum?.mode === 'string') { + if (action === 'apply' || action === 'modify') { + // check if there is already a config applied with this field + if (autoUserInstrumModeId && configId !== autoUserInstrumModeId) { + log.error('[RC] Multiple auto_user_instrum received in ASM_FEATURES. Discarding config') + // eslint-disable-next-line no-throw-literal + throw 'Multiple auto_user_instrum.mode received in ASM_FEATURES' + } + + setCollectionMode(rcConfig.auto_user_instrum.mode) + autoUserInstrumModeId = configId + } else if (configId === autoUserInstrumModeId) { + setCollectionMode(config.appsec.eventTracking.mode) + autoUserInstrumModeId = null + } + } + if (activation === Activation.ONECLICK) { enableOrDisableAppsec(action, rcConfig, config, appsec) } diff --git a/packages/dd-trace/src/appsec/sdk/track_event.js b/packages/dd-trace/src/appsec/sdk/track_event.js index 0c1ef9c2bd9..a04f596bbc3 100644 --- a/packages/dd-trace/src/appsec/sdk/track_event.js +++ b/packages/dd-trace/src/appsec/sdk/track_event.js @@ -7,6 +7,7 @@ const standalone = require('../standalone') const waf = require('../waf') const { SAMPLING_MECHANISM_APPSEC } = require('../../constants') const { keepTrace } = require('../../priority_sampler') +const addresses = require('../addresses') function trackUserLoginSuccessEvent (tracer, user, metadata) { // TODO: better user check here and in _setUser() ? @@ -23,7 +24,13 @@ function trackUserLoginSuccessEvent (tracer, user, metadata) { setUserTags(user, rootSpan) - trackEvent('users.login.success', metadata, 'trackUserLoginSuccessEvent', rootSpan, 'sdk') + const login = user.login ?? user.id + + metadata = { 'usr.login': login, ...metadata } + + trackEvent('users.login.success', metadata, 'trackUserLoginSuccessEvent', rootSpan) + + runWaf('users.login.success', { id: user.id, login }) } function trackUserLoginFailureEvent (tracer, userId, exists, metadata) { @@ -34,11 +41,14 @@ function trackUserLoginFailureEvent (tracer, userId, exists, metadata) { const fields = { 'usr.id': userId, + 'usr.login': userId, 'usr.exists': exists ? 'true' : 'false', ...metadata } - trackEvent('users.login.failure', fields, 'trackUserLoginFailureEvent', getRootSpan(tracer), 'sdk') + trackEvent('users.login.failure', fields, 'trackUserLoginFailureEvent', getRootSpan(tracer)) + + runWaf('users.login.failure', { login: userId }) } function trackCustomEvent (tracer, eventName, metadata) { @@ -47,27 +57,18 @@ function trackCustomEvent (tracer, eventName, metadata) { return } - trackEvent(eventName, metadata, 'trackCustomEvent', getRootSpan(tracer), 'sdk') + trackEvent(eventName, metadata, 'trackCustomEvent', getRootSpan(tracer)) } -function trackEvent (eventName, fields, sdkMethodName, rootSpan, mode) { +function trackEvent (eventName, fields, sdkMethodName, rootSpan) { if (!rootSpan) { log.warn('[ASM] Root span not available in %s', sdkMethodName) return } - keepTrace(rootSpan, SAMPLING_MECHANISM_APPSEC) - const tags = { - [`appsec.events.${eventName}.track`]: 'true' - } - - if (mode === 'sdk') { - tags[`_dd.appsec.events.${eventName}.sdk`] = 'true' - } - - if (mode === 'safe' || mode === 'extended') { - tags[`_dd.appsec.events.${eventName}.auto.mode`] = mode + [`appsec.events.${eventName}.track`]: 'true', + [`_dd.appsec.events.${eventName}.sdk`]: 'true' } if (fields) { @@ -78,16 +79,28 @@ function trackEvent (eventName, fields, sdkMethodName, rootSpan, mode) { rootSpan.addTags(tags) + keepTrace(rootSpan, SAMPLING_MECHANISM_APPSEC) standalone.sample(rootSpan) +} + +function runWaf (eventName, user) { + const persistent = { + [`server.business_logic.${eventName}`]: null + } + + if (user.id) { + persistent[addresses.USER_ID] = '' + user.id + } - if (['users.login.success', 'users.login.failure'].includes(eventName)) { - waf.run({ persistent: { [`server.business_logic.${eventName}`]: null } }) + if (user.login) { + persistent[addresses.USER_LOGIN] = '' + user.login } + + waf.run({ persistent }) } module.exports = { trackUserLoginSuccessEvent, trackUserLoginFailureEvent, - trackCustomEvent, - trackEvent + trackCustomEvent } diff --git a/packages/dd-trace/src/appsec/telemetry.js b/packages/dd-trace/src/appsec/telemetry.js index d96ca77601f..8e9a2518f80 100644 --- a/packages/dd-trace/src/appsec/telemetry.js +++ b/packages/dd-trace/src/appsec/telemetry.js @@ -172,6 +172,15 @@ function addRaspRequestMetrics (store, { duration, durationExt }) { store[DD_TELEMETRY_REQUEST_METRICS].raspEvalCount++ } +function incrementMissingUserLoginMetric (framework, eventType) { + if (!enabled) return + + appsecMetrics.count('instrum.user_auth.missing_user_login', { + framework, + event_type: eventType + }).inc() +} + function getRequestMetrics (req) { if (req) { const store = getStore(req) @@ -188,6 +197,7 @@ module.exports = { incrementWafInitMetric, incrementWafUpdatesMetric, incrementWafRequestsMetric, + incrementMissingUserLoginMetric, getRequestMetrics } diff --git a/packages/dd-trace/src/appsec/user_tracking.js b/packages/dd-trace/src/appsec/user_tracking.js new file mode 100644 index 00000000000..5b92f80d642 --- /dev/null +++ b/packages/dd-trace/src/appsec/user_tracking.js @@ -0,0 +1,168 @@ +'use strict' + +const crypto = require('crypto') +const log = require('../log') +const telemetry = require('./telemetry') +const addresses = require('./addresses') +const { keepTrace } = require('../priority_sampler') +const { SAMPLING_MECHANISM_APPSEC } = require('../constants') +const standalone = require('./standalone') +const waf = require('./waf') + +// the RFC doesn't include '_id', but it's common in MongoDB +const USER_ID_FIELDS = ['id', '_id', 'email', 'username', 'login', 'user'] + +let collectionMode + +function setCollectionMode (mode, overwrite = true) { + // don't overwrite if already set, only used in appsec/index.js to not overwrite RC values + if (!overwrite && collectionMode) return + + /* eslint-disable no-fallthrough */ + switch (mode) { + case 'safe': + log.warn('[ASM] Using deprecated value "safe" in config.appsec.eventTracking.mode') + case 'anon': + case 'anonymization': + collectionMode = 'anonymization' + break + + case 'extended': + log.warn('[ASM] Using deprecated value "extended" in config.appsec.eventTracking.mode') + case 'ident': + case 'identification': + collectionMode = 'identification' + break + + default: + collectionMode = 'disabled' + } + /* eslint-enable no-fallthrough */ +} + +function obfuscateIfNeeded (str) { + if (collectionMode === 'anonymization') { + // get first 16 bytes of sha256 hash in lowercase hex + return 'anon_' + crypto.createHash('sha256').update(str).digest().toString('hex', 0, 16).toLowerCase() + } else { + return str + } +} + +// TODO: should we find other ways to get the user ID ? +function getUserId (user) { + if (!user) return + + for (const field of USER_ID_FIELDS) { + let id = user[field] + + // try to find a field that can be stringified + if (id && typeof id.toString === 'function') { + id = id.toString() + + if (typeof id !== 'string' || id.startsWith('[object ')) { + // probably not a usable ID ? + continue + } + + return obfuscateIfNeeded(id) + } + } +} + +function trackLogin (framework, login, user, success, rootSpan) { + if (!collectionMode || collectionMode === 'disabled') return + + if (!rootSpan) { + log.error('[ASM] No rootSpan found in AppSec trackLogin') + return + } + + if (typeof login !== 'string') { + log.error('[ASM] Invalid login provided to AppSec trackLogin') + + telemetry.incrementMissingUserLoginMetric(framework, success ? 'login_success' : 'login_failure') + // note: + // if we start supporting using userId if login is missing, we need to only give up if both are missing, and + // implement 'appsec.instrum.user_auth.missing_user_id' telemetry too + return + } + + login = obfuscateIfNeeded(login) + const userId = getUserId(user) + + let newTags + + const persistent = { + [addresses.USER_LOGIN]: login + } + + const currentTags = rootSpan.context()._tags + const isSdkCalled = currentTags[`_dd.appsec.events.users.login.${success ? 'success' : 'failure'}.sdk`] === 'true' + + // used to not overwrite tags set by SDK + function shouldSetTag (tag) { + return !(isSdkCalled && currentTags[tag]) + } + + if (success) { + newTags = { + 'appsec.events.users.login.success.track': 'true', + '_dd.appsec.events.users.login.success.auto.mode': collectionMode, + '_dd.appsec.usr.login': login + } + + if (shouldSetTag('appsec.events.users.login.success.usr.login')) { + newTags['appsec.events.users.login.success.usr.login'] = login + } + + if (userId) { + newTags['_dd.appsec.usr.id'] = userId + + if (shouldSetTag('usr.id')) { + newTags['usr.id'] = userId + persistent[addresses.USER_ID] = userId + } + } + + persistent[addresses.LOGIN_SUCCESS] = null + } else { + newTags = { + 'appsec.events.users.login.failure.track': 'true', + '_dd.appsec.events.users.login.failure.auto.mode': collectionMode, + '_dd.appsec.usr.login': login + } + + if (shouldSetTag('appsec.events.users.login.failure.usr.login')) { + newTags['appsec.events.users.login.failure.usr.login'] = login + } + + if (userId) { + newTags['_dd.appsec.usr.id'] = userId + + if (shouldSetTag('appsec.events.users.login.failure.usr.id')) { + newTags['appsec.events.users.login.failure.usr.id'] = userId + } + } + + /* TODO: if one day we have this info + if (exists != null && shouldSetTag('appsec.events.users.login.failure.usr.exists')) { + newTags['appsec.events.users.login.failure.usr.exists'] = exists + } + */ + + persistent[addresses.LOGIN_FAILURE] = null + } + + keepTrace(rootSpan, SAMPLING_MECHANISM_APPSEC) + standalone.sample(rootSpan) + + rootSpan.addTags(newTags) + + return waf.run({ persistent }) +} + +module.exports = { + setCollectionMode, + trackLogin +} diff --git a/packages/dd-trace/src/config.js b/packages/dd-trace/src/config.js index 808704bd7e4..6f630212799 100644 --- a/packages/dd-trace/src/config.js +++ b/packages/dd-trace/src/config.js @@ -449,8 +449,7 @@ class Config { this._setValue(defaults, 'appsec.blockedTemplateHtml', undefined) this._setValue(defaults, 'appsec.blockedTemplateJson', undefined) this._setValue(defaults, 'appsec.enabled', undefined) - this._setValue(defaults, 'appsec.eventTracking.enabled', true) - this._setValue(defaults, 'appsec.eventTracking.mode', 'safe') + this._setValue(defaults, 'appsec.eventTracking.mode', 'identification') this._setValue(defaults, 'appsec.obfuscatorKeyRegex', defaultWafObfuscatorKeyRegex) this._setValue(defaults, 'appsec.obfuscatorValueRegex', defaultWafObfuscatorValueRegex) this._setValue(defaults, 'appsec.rasp.enabled', true) @@ -574,6 +573,7 @@ class Config { DD_AGENT_HOST, DD_API_SECURITY_ENABLED, DD_API_SECURITY_SAMPLE_DELAY, + DD_APPSEC_AUTO_USER_INSTRUMENTATION_MODE, DD_APPSEC_AUTOMATED_USER_EVENTS_TRACKING, DD_APPSEC_ENABLED, DD_APPSEC_GRAPHQL_BLOCKED_TEMPLATE_JSON, @@ -712,11 +712,10 @@ class Config { this._setValue(env, 'appsec.blockedTemplateJson', maybeFile(DD_APPSEC_HTTP_BLOCKED_TEMPLATE_JSON)) this._envUnprocessed['appsec.blockedTemplateJson'] = DD_APPSEC_HTTP_BLOCKED_TEMPLATE_JSON this._setBoolean(env, 'appsec.enabled', DD_APPSEC_ENABLED) - if (DD_APPSEC_AUTOMATED_USER_EVENTS_TRACKING) { - this._setValue(env, 'appsec.eventTracking.enabled', - ['extended', 'safe'].includes(DD_APPSEC_AUTOMATED_USER_EVENTS_TRACKING.toLowerCase())) - this._setValue(env, 'appsec.eventTracking.mode', DD_APPSEC_AUTOMATED_USER_EVENTS_TRACKING.toLowerCase()) - } + this._setString(env, 'appsec.eventTracking.mode', coalesce( + DD_APPSEC_AUTO_USER_INSTRUMENTATION_MODE, + DD_APPSEC_AUTOMATED_USER_EVENTS_TRACKING // TODO: remove in next major + )) this._setString(env, 'appsec.obfuscatorKeyRegex', DD_APPSEC_OBFUSCATION_PARAMETER_KEY_REGEXP) this._setString(env, 'appsec.obfuscatorValueRegex', DD_APPSEC_OBFUSCATION_PARAMETER_VALUE_REGEXP) this._setBoolean(env, 'appsec.rasp.enabled', DD_APPSEC_RASP_ENABLED) @@ -895,12 +894,7 @@ class Config { this._setValue(opts, 'appsec.blockedTemplateJson', maybeFile(options.appsec.blockedTemplateJson)) this._optsUnprocessed['appsec.blockedTemplateJson'] = options.appsec.blockedTemplateJson this._setBoolean(opts, 'appsec.enabled', options.appsec.enabled) - let eventTracking = options.appsec.eventTracking?.mode - if (eventTracking) { - eventTracking = eventTracking.toLowerCase() - this._setValue(opts, 'appsec.eventTracking.enabled', ['extended', 'safe'].includes(eventTracking)) - this._setValue(opts, 'appsec.eventTracking.mode', eventTracking) - } + this._setString(opts, 'appsec.eventTracking.mode', options.appsec.eventTracking?.mode) this._setString(opts, 'appsec.obfuscatorKeyRegex', options.appsec.obfuscatorKeyRegex) this._setString(opts, 'appsec.obfuscatorValueRegex', options.appsec.obfuscatorValueRegex) this._setBoolean(opts, 'appsec.rasp.enabled', options.appsec.rasp?.enabled) diff --git a/packages/dd-trace/test/appsec/index.spec.js b/packages/dd-trace/test/appsec/index.spec.js index 4ec92f7b0e6..7ca54e9241b 100644 --- a/packages/dd-trace/test/appsec/index.spec.js +++ b/packages/dd-trace/test/appsec/index.spec.js @@ -44,7 +44,7 @@ describe('AppSec Index', function () { let AppSec let web let blocking - let passport + let UserTracking let log let appsecTelemetry let graphql @@ -65,8 +65,7 @@ describe('AppSec Index', function () { blockedTemplateHtml: blockedTemplate.html, blockedTemplateJson: blockedTemplate.json, eventTracking: { - enabled: true, - mode: 'safe' + mode: 'anon' }, apiSecurity: { enabled: false, @@ -90,8 +89,9 @@ describe('AppSec Index', function () { setTemplates: sinon.stub() } - passport = { - passportTrackEvent: sinon.stub() + UserTracking = { + setCollectionMode: sinon.stub(), + trackLogin: sinon.stub() } log = { @@ -124,7 +124,7 @@ describe('AppSec Index', function () { '../log': log, '../plugins/util/web': web, './blocking': blocking, - './passport': passport, + './user_tracking': UserTracking, './telemetry': appsecTelemetry, './graphql': graphql, './api_security_sampler': apiSecuritySampler, @@ -152,6 +152,7 @@ describe('AppSec Index', function () { expect(blocking.setTemplates).to.have.been.calledOnceWithExactly(config) expect(RuleManager.loadRules).to.have.been.calledOnceWithExactly(config.appsec) expect(Reporter.setRateLimit).to.have.been.calledOnceWithExactly(42) + expect(UserTracking.setCollectionMode).to.have.been.calledOnceWithExactly('anon', false) expect(incomingHttpRequestStart.subscribe) .to.have.been.calledOnceWithExactly(AppSec.incomingHttpStartTranslator) expect(incomingHttpRequestEnd.subscribe).to.have.been.calledOnceWithExactly(AppSec.incomingHttpEndTranslator) @@ -197,13 +198,13 @@ describe('AppSec Index', function () { expect(responseSetHeader.hasSubscribers).to.be.true }) - it('should not subscribe to passportVerify if eventTracking is disabled', () => { - config.appsec.eventTracking.enabled = false + it('should still subscribe to passportVerify if eventTracking is disabled', () => { + config.appsec.eventTracking.mode = 'disabled' AppSec.disable() AppSec.enable(config) - expect(passportVerify.hasSubscribers).to.be.false + expect(passportVerify.hasSubscribers).to.be.true }) it('should call appsec telemetry enable', () => { @@ -365,7 +366,7 @@ describe('AppSec Index', function () { const res = { getHeaders: () => ({ 'content-type': 'application/json', - 'content-lenght': 42 + 'content-length': 42 }), statusCode: 201 } @@ -403,7 +404,7 @@ describe('AppSec Index', function () { const res = { getHeaders: () => ({ 'content-type': 'application/json', - 'content-lenght': 42 + 'content-length': 42 }), statusCode: 201 } @@ -449,7 +450,7 @@ describe('AppSec Index', function () { const res = { getHeaders: () => ({ 'content-type': 'application/json', - 'content-lenght': 42 + 'content-length': 42 }), statusCode: 201 } @@ -515,7 +516,7 @@ describe('AppSec Index', function () { const res = { getHeaders: () => ({ 'content-type': 'application/json', - 'content-lenght': 42 + 'content-length': 42 }), statusCode: 201 } @@ -561,7 +562,7 @@ describe('AppSec Index', function () { const res = { getHeaders: () => ({ 'content-type': 'application/json', - 'content-lenght': 42 + 'content-length': 42 }), statusCode: 201 } @@ -649,6 +650,17 @@ describe('AppSec Index', function () { abortController = { abort: sinon.stub() } + res = { + getHeaders: () => ({ + 'content-type': 'application/json', + 'content-length': 42 + }), + writeHead: sinon.stub(), + end: sinon.stub(), + getHeaderNames: sinon.stub().returns([]) + } + res.writeHead.returns(res) + req = { url: '/path', headers: { @@ -659,18 +671,9 @@ describe('AppSec Index', function () { socket: { remoteAddress: '127.0.0.1', remotePort: 8080 - } - } - res = { - getHeaders: () => ({ - 'content-type': 'application/json', - 'content-lenght': 42 - }), - writeHead: sinon.stub(), - end: sinon.stub(), - getHeaderNames: sinon.stub().returns([]) + }, + res } - res.writeHead.returns(res) AppSec.enable(config) AppSec.incomingHttpStartTranslator({ req, res }) @@ -807,31 +810,84 @@ describe('AppSec Index', function () { }) describe('onPassportVerify', () => { - it('Should call passportTrackEvent', () => { - const credentials = { type: 'local', username: 'test' } - const user = { id: '1234', username: 'Test' } + beforeEach(() => { + web.root.resetHistory() + sinon.stub(storage, 'getStore').returns({ req }) + }) - sinon.stub(storage, 'getStore').returns({ req: {} }) + it('should block when UserTracking.login() returns action', () => { + UserTracking.trackLogin.returns(resultActions) - passportVerify.publish({ credentials, user }) + const abortController = new AbortController() + const payload = { + framework: 'passport-local', + login: 'test', + user: { _id: 1, username: 'test', password: '1234' }, + success: true, + abortController + } + + passportVerify.publish(payload) + + expect(storage.getStore).to.have.been.calledOnce + expect(web.root).to.have.been.calledOnceWithExactly(req) + expect(UserTracking.trackLogin).to.have.been.calledOnceWithExactly( + payload.framework, + payload.login, + payload.user, + payload.success, + rootSpan + ) + expect(abortController.signal.aborted).to.be.true + expect(res.end).to.have.been.called + }) - expect(passport.passportTrackEvent).to.have.been.calledOnceWithExactly( - credentials, - user, - rootSpan, - config.appsec.eventTracking.mode) + it('should not block when UserTracking.login() returns nothing', () => { + UserTracking.trackLogin.returns(undefined) + + const abortController = new AbortController() + const payload = { + framework: 'passport-local', + login: 'test', + user: { _id: 1, username: 'test', password: '1234' }, + success: true, + abortController + } + + passportVerify.publish(payload) + + expect(storage.getStore).to.have.been.calledOnce + expect(web.root).to.have.been.calledOnceWithExactly(req) + expect(UserTracking.trackLogin).to.have.been.calledOnceWithExactly( + payload.framework, + payload.login, + payload.user, + payload.success, + rootSpan + ) + expect(abortController.signal.aborted).to.be.false + expect(res.end).to.not.have.been.called }) - it('Should call log if no rootSpan is found', () => { - const credentials = { type: 'local', username: 'test' } - const user = { id: '1234', username: 'Test' } + it('should not block and call log if no rootSpan is found', () => { + storage.getStore.returns(undefined) - sinon.stub(storage, 'getStore').returns(undefined) + const abortController = new AbortController() + const payload = { + framework: 'passport-local', + login: 'test', + user: { _id: 1, username: 'test', password: '1234' }, + success: true, + abortController + } - passportVerify.publish({ credentials, user }) + passportVerify.publish(payload) + expect(storage.getStore).to.have.been.calledOnce expect(log.warn).to.have.been.calledOnceWithExactly('[ASM] No rootSpan found in onPassportVerify') - expect(passport.passportTrackEvent).not.to.have.been.called + expect(UserTracking.trackLogin).to.not.have.been.called + expect(abortController.signal.aborted).to.be.false + expect(res.end).to.not.have.been.called }) }) @@ -841,7 +897,7 @@ describe('AppSec Index', function () { const responseHeaders = { 'content-type': 'application/json', - 'content-lenght': 42, + 'content-length': 42, 'set-cookie': 'a=1;b=2' } @@ -852,7 +908,7 @@ describe('AppSec Index', function () { 'server.response.status': '404', 'server.response.headers.no_cookies': { 'content-type': 'application/json', - 'content-lenght': 42 + 'content-length': 42 } } }, req) @@ -873,7 +929,7 @@ describe('AppSec Index', function () { const responseHeaders = { 'content-type': 'application/json', - 'content-lenght': 42, + 'content-length': 42, 'set-cookie': 'a=1;b=2' } @@ -884,7 +940,7 @@ describe('AppSec Index', function () { 'server.response.status': '404', 'server.response.headers.no_cookies': { 'content-type': 'application/json', - 'content-lenght': 42 + 'content-length': 42 } } }, req) @@ -904,7 +960,7 @@ describe('AppSec Index', function () { const responseHeaders = { 'content-type': 'application/json', - 'content-lenght': 42, + 'content-length': 42, 'set-cookie': 'a=1;b=2' } @@ -920,7 +976,7 @@ describe('AppSec Index', function () { const responseHeaders = { 'content-type': 'application/json', - 'content-lenght': 42, + 'content-length': 42, 'set-cookie': 'a=1;b=2' } @@ -931,7 +987,7 @@ describe('AppSec Index', function () { 'server.response.status': '404', 'server.response.headers.no_cookies': { 'content-type': 'application/json', - 'content-lenght': 42 + 'content-length': 42 } } }, req) @@ -947,7 +1003,7 @@ describe('AppSec Index', function () { const responseHeaders = { 'content-type': 'application/json', - 'content-lenght': 42, + 'content-length': 42, 'set-cookie': 'a=1;b=2' } responseWriteHead.publish({ req, res, abortController, statusCode: 404, responseHeaders }) diff --git a/packages/dd-trace/test/appsec/passport.spec.js b/packages/dd-trace/test/appsec/passport.spec.js deleted file mode 100644 index 7a3db36798c..00000000000 --- a/packages/dd-trace/test/appsec/passport.spec.js +++ /dev/null @@ -1,245 +0,0 @@ -'use strict' - -const proxyquire = require('proxyquire') - -describe('Passport', () => { - const rootSpan = { - context: () => { return {} } - } - const loginLocal = { type: 'local', username: 'test' } - const userUuid = { - id: '591dc126-8431-4d0f-9509-b23318d3dce4', - email: 'testUser@test.com', - username: 'Test User' - } - - let passportModule, log, events, setUser - - beforeEach(() => { - rootSpan.context = () => { return {} } - - log = { - warn: sinon.stub() - } - - events = { - trackEvent: sinon.stub() - } - - setUser = { - setUserTags: sinon.stub() - } - - passportModule = proxyquire('../../src/appsec/passport', { - '../log': log, - './sdk/track_event': events, - './sdk/set_user': setUser - }) - }) - - describe('passportTrackEvent', () => { - it('should call log when credentials is undefined', () => { - passportModule.passportTrackEvent(undefined, undefined, undefined, 'safe') - - expect(log.warn).to.have.been.calledOnceWithExactly('No user ID found in authentication instrumentation') - }) - - it('should call log when type is not known', () => { - const credentials = { type: 'unknown', username: 'test' } - - passportModule.passportTrackEvent(credentials, undefined, undefined, 'safe') - - expect(log.warn).to.have.been.calledOnceWithExactly('No user ID found in authentication instrumentation') - }) - - it('should call log when type is known but username not present', () => { - const credentials = { type: 'http' } - - passportModule.passportTrackEvent(credentials, undefined, undefined, 'safe') - - expect(log.warn).to.have.been.calledOnceWithExactly('No user ID found in authentication instrumentation') - }) - - it('should report login failure when passportUser is not present', () => { - passportModule.passportTrackEvent(loginLocal, undefined, undefined, 'safe') - - expect(setUser.setUserTags).not.to.have.been.called - expect(events.trackEvent).to.have.been.calledOnceWithExactly( - 'users.login.failure', - { 'usr.id': '' }, - 'passportTrackEvent', - undefined, - 'safe' - ) - }) - - it('should report login success when passportUser is present', () => { - passportModule.passportTrackEvent(loginLocal, userUuid, rootSpan, 'safe') - - expect(setUser.setUserTags).to.have.been.calledOnceWithExactly({ id: userUuid.id }, rootSpan) - expect(events.trackEvent).to.have.been.calledOnceWithExactly( - 'users.login.success', - null, - 'passportTrackEvent', - rootSpan, - 'safe' - ) - }) - - it('should report login success and blank id in safe mode when id is not a uuid', () => { - const user = { - id: 'publicName', - email: 'testUser@test.com', - username: 'Test User' - } - passportModule.passportTrackEvent(loginLocal, user, rootSpan, 'safe') - - expect(setUser.setUserTags).to.have.been.calledOnceWithExactly({ id: '' }, rootSpan) - expect(events.trackEvent).to.have.been.calledOnceWithExactly( - 'users.login.success', - null, - 'passportTrackEvent', - rootSpan, - 'safe' - ) - }) - - it('should report login success and the extended fields in extended mode', () => { - const user = { - id: 'publicName', - email: 'testUser@test.com', - username: 'Test User' - } - - passportModule.passportTrackEvent(loginLocal, user, rootSpan, 'extended') - expect(setUser.setUserTags).to.have.been.calledOnceWithExactly( - { - id: 'publicName', - login: 'test', - email: 'testUser@test.com', - username: 'Test User' - }, - rootSpan - ) - expect(events.trackEvent).to.have.been.calledOnceWithExactly( - 'users.login.success', - null, - 'passportTrackEvent', - rootSpan, - 'extended' - ) - }) - - it('should not call trackEvent in safe mode if sdk user event functions are already called', () => { - rootSpan.context = () => { - return { - _tags: { - '_dd.appsec.events.users.login.success.sdk': 'true' - } - } - } - - passportModule.passportTrackEvent(loginLocal, userUuid, rootSpan, 'safe') - expect(setUser.setUserTags).not.to.have.been.called - expect(events.trackEvent).not.to.have.been.called - }) - - it('should not call trackEvent in extended mode if trackUserLoginSuccessEvent is already called', () => { - rootSpan.context = () => { - return { - _tags: { - '_dd.appsec.events.users.login.success.sdk': 'true' - } - } - } - - passportModule.passportTrackEvent(loginLocal, userUuid, rootSpan, 'extended') - expect(setUser.setUserTags).not.to.have.been.called - expect(events.trackEvent).not.to.have.been.called - }) - - it('should call trackEvent in extended mode if trackCustomEvent function is already called', () => { - rootSpan.context = () => { - return { - _tags: { - '_dd.appsec.events.custom.event.sdk': 'true' - } - } - } - - passportModule.passportTrackEvent(loginLocal, userUuid, rootSpan, 'extended') - expect(events.trackEvent).to.have.been.calledOnceWithExactly( - 'users.login.success', - null, - 'passportTrackEvent', - rootSpan, - 'extended' - ) - }) - - it('should not call trackEvent in extended mode if trackUserLoginFailureEvent is already called', () => { - rootSpan.context = () => { - return { - _tags: { - '_dd.appsec.events.users.login.failure.sdk': 'true' - } - } - } - - passportModule.passportTrackEvent(loginLocal, userUuid, rootSpan, 'extended') - expect(setUser.setUserTags).not.to.have.been.called - expect(events.trackEvent).not.to.have.been.called - }) - - it('should report login success with the _id field', () => { - const user = { - _id: '591dc126-8431-4d0f-9509-b23318d3dce4', - email: 'testUser@test.com', - username: 'Test User' - } - - passportModule.passportTrackEvent(loginLocal, user, rootSpan, 'extended') - expect(setUser.setUserTags).to.have.been.calledOnceWithExactly( - { - id: '591dc126-8431-4d0f-9509-b23318d3dce4', - login: 'test', - email: 'testUser@test.com', - username: 'Test User' - }, - rootSpan - ) - expect(events.trackEvent).to.have.been.calledOnceWithExactly( - 'users.login.success', - null, - 'passportTrackEvent', - rootSpan, - 'extended' - ) - }) - - it('should report login success with the username field passport name', () => { - const user = { - email: 'testUser@test.com', - name: 'Test User' - } - - rootSpan.context = () => { return {} } - - passportModule.passportTrackEvent(loginLocal, user, rootSpan, 'extended') - expect(setUser.setUserTags).to.have.been.calledOnceWithExactly( - { - id: 'test', - login: 'test', - email: 'testUser@test.com', - username: 'Test User' - }, rootSpan) - expect(events.trackEvent).to.have.been.calledOnceWithExactly( - 'users.login.success', - null, - 'passportTrackEvent', - rootSpan, - 'extended' - ) - }) - }) -}) diff --git a/packages/dd-trace/test/appsec/remote_config/index.spec.js b/packages/dd-trace/test/appsec/remote_config/index.spec.js index 67447cf7a69..f3cc6a32dac 100644 --- a/packages/dd-trace/test/appsec/remote_config/index.spec.js +++ b/packages/dd-trace/test/appsec/remote_config/index.spec.js @@ -7,6 +7,8 @@ let config let rc let RemoteConfigManager let RuleManager +let UserTracking +let log let appsec let remoteConfig @@ -14,7 +16,10 @@ describe('Remote Config index', () => { beforeEach(() => { config = { appsec: { - enabled: undefined + enabled: undefined, + eventTracking: { + mode: 'identification' + } } } @@ -32,6 +37,14 @@ describe('Remote Config index', () => { updateWafFromRC: sinon.stub() } + UserTracking = { + setCollectionMode: sinon.stub() + } + + log = { + error: sinon.stub() + } + appsec = { enable: sinon.spy(), disable: sinon.spy() @@ -40,40 +53,48 @@ describe('Remote Config index', () => { remoteConfig = proxyquire('../src/appsec/remote_config', { './manager': RemoteConfigManager, '../rule_manager': RuleManager, + '../user_tracking': UserTracking, + '../../log': log, '..': appsec }) }) describe('enable', () => { it('should listen to remote config when appsec is not explicitly configured', () => { - config.appsec = { enabled: undefined } + config.appsec.enabled = undefined remoteConfig.enable(config) expect(RemoteConfigManager).to.have.been.calledOnceWithExactly(config) expect(rc.updateCapabilities).to.have.been.calledWithExactly(RemoteConfigCapabilities.ASM_ACTIVATION, true) + expect(rc.updateCapabilities) + .to.have.been.calledWithExactly(RemoteConfigCapabilities.ASM_AUTO_USER_INSTRUM_MODE, true) expect(rc.setProductHandler).to.have.been.calledWith('ASM_FEATURES') expect(rc.setProductHandler.firstCall.args[1]).to.be.a('function') }) it('should listen to remote config when appsec is explicitly configured as enabled=true', () => { - config.appsec = { enabled: true } + config.appsec.enabled = true remoteConfig.enable(config) expect(RemoteConfigManager).to.have.been.calledOnceWithExactly(config) - expect(rc.updateCapabilities).to.not.have.been.calledWith('ASM_ACTIVATION') + expect(rc.updateCapabilities).to.not.have.been.calledWith(RemoteConfigCapabilities.ASM_ACTIVATION) + expect(rc.updateCapabilities) + .to.have.been.calledWithExactly(RemoteConfigCapabilities.ASM_AUTO_USER_INSTRUM_MODE, true) expect(rc.setProductHandler).to.have.been.calledOnceWith('ASM_FEATURES') expect(rc.setProductHandler.firstCall.args[1]).to.be.a('function') }) it('should not listen to remote config when appsec is explicitly configured as enabled=false', () => { - config.appsec = { enabled: false } + config.appsec.enabled = false remoteConfig.enable(config) expect(RemoteConfigManager).to.have.been.calledOnceWithExactly(config) expect(rc.updateCapabilities).to.not.have.been.calledWith(RemoteConfigCapabilities.ASM_ACTIVATION, true) + expect(rc.updateCapabilities) + .to.not.have.been.calledWith(RemoteConfigCapabilities.ASM_AUTO_USER_INSTRUM_MODE, true) expect(rc.setProductHandler).to.not.have.been.called }) @@ -81,8 +102,6 @@ describe('Remote Config index', () => { let listener beforeEach(() => { - config.appsec = { enabled: undefined } - remoteConfig.enable(config, appsec) listener = rc.setProductHandler.firstCall.args[1] @@ -100,8 +119,8 @@ describe('Remote Config index', () => { expect(appsec.enable).to.have.been.called }) - it('should disable appsec when listener is called with unnaply and enabled', () => { - listener('unnaply', { asm: { enabled: true } }) + it('should disable appsec when listener is called with unapply and enabled', () => { + listener('unapply', { asm: { enabled: true } }) expect(appsec.disable).to.have.been.calledOnce }) @@ -112,6 +131,60 @@ describe('Remote Config index', () => { expect(appsec.enable).to.not.have.been.called expect(appsec.disable).to.not.have.been.called }) + + describe('auto_user_instrum', () => { + const rcConfig = { auto_user_instrum: { mode: 'anonymous' } } + const configId = 'collectionModeId' + + afterEach(() => { + listener('unapply', rcConfig, configId) + }) + + it('should not update collection mode when not a string', () => { + listener('apply', { auto_user_instrum: { mode: 123 } }, configId) + + expect(UserTracking.setCollectionMode).to.not.have.been.called + }) + + it('should throw when called two times with different config ids', () => { + listener('apply', rcConfig, configId) + + expect(() => listener('apply', rcConfig, 'anotherId')).to.throw() + expect(log.error).to.have.been.calledOnceWithExactly( + '[RC] Multiple auto_user_instrum received in ASM_FEATURES. Discarding config' + ) + }) + + it('should update collection mode when called with apply', () => { + listener('apply', rcConfig, configId) + + expect(UserTracking.setCollectionMode).to.have.been.calledOnceWithExactly(rcConfig.auto_user_instrum.mode) + }) + + it('should update collection mode when called with modify', () => { + listener('modify', rcConfig, configId) + + expect(UserTracking.setCollectionMode).to.have.been.calledOnceWithExactly(rcConfig.auto_user_instrum.mode) + }) + + it('should revert collection mode when called with unapply', () => { + listener('apply', rcConfig, configId) + UserTracking.setCollectionMode.resetHistory() + + listener('unapply', rcConfig, configId) + + expect(UserTracking.setCollectionMode).to.have.been.calledOnceWithExactly(config.appsec.eventTracking.mode) + }) + + it('should not revert collection mode when called with unapply and unknown id', () => { + listener('apply', rcConfig, configId) + UserTracking.setCollectionMode.resetHistory() + + listener('unapply', rcConfig, 'unknownId') + + expect(UserTracking.setCollectionMode).to.not.have.been.called + }) + }) }) }) diff --git a/packages/dd-trace/test/appsec/sdk/track_event.spec.js b/packages/dd-trace/test/appsec/sdk/track_event.spec.js index 97f1ac07bd7..8e3c1a177bd 100644 --- a/packages/dd-trace/test/appsec/sdk/track_event.spec.js +++ b/packages/dd-trace/test/appsec/sdk/track_event.spec.js @@ -4,7 +4,7 @@ const proxyquire = require('proxyquire') const agent = require('../../plugins/agent') const axios = require('axios') const tracer = require('../../../../../index') -const { LOGIN_SUCCESS, LOGIN_FAILURE } = require('../../../src/appsec/addresses') +const { LOGIN_SUCCESS, LOGIN_FAILURE, USER_ID, USER_LOGIN } = require('../../../src/appsec/addresses') const { SAMPLING_MECHANISM_APPSEC } = require('../../../src/constants') const { USER_KEEP } = require('../../../../../ext/priority') @@ -12,13 +12,13 @@ describe('track_event', () => { describe('Internal API', () => { const tracer = {} let log + let prioritySampler let rootSpan let getRootSpan let setUserTags - let trackUserLoginSuccessEvent, trackUserLoginFailureEvent, trackCustomEvent, trackEvent let sample let waf - let prioritySampler + let trackUserLoginSuccessEvent, trackUserLoginFailureEvent, trackCustomEvent beforeEach(() => { log = { @@ -62,11 +62,6 @@ describe('track_event', () => { trackUserLoginSuccessEvent = trackEvents.trackUserLoginSuccessEvent trackUserLoginFailureEvent = trackEvents.trackUserLoginFailureEvent trackCustomEvent = trackEvents.trackCustomEvent - trackEvent = trackEvents.trackEvent - }) - - afterEach(() => { - sinon.restore() }) describe('trackUserLoginSuccessEvent', () => { @@ -108,12 +103,21 @@ describe('track_event', () => { { 'appsec.events.users.login.success.track': 'true', '_dd.appsec.events.users.login.success.sdk': 'true', + 'appsec.events.users.login.success.usr.login': 'user_id', 'appsec.events.users.login.success.metakey1': 'metaValue1', 'appsec.events.users.login.success.metakey2': 'metaValue2', 'appsec.events.users.login.success.metakey3': 'metaValue3' }) expect(prioritySampler.setPriority) .to.have.been.calledOnceWithExactly(rootSpan, USER_KEEP, SAMPLING_MECHANISM_APPSEC) + expect(sample).to.have.been.calledOnceWithExactly(rootSpan) + expect(waf.run).to.have.been.calledOnceWithExactly({ + persistent: { + [LOGIN_SUCCESS]: null, + [USER_ID]: 'user_id', + [USER_LOGIN]: 'user_id' + } + }) }) it('should call setUser and addTags without metadata', () => { @@ -125,27 +129,50 @@ describe('track_event', () => { expect(setUserTags).to.have.been.calledOnceWithExactly(user, rootSpan) expect(rootSpan.addTags).to.have.been.calledOnceWithExactly({ 'appsec.events.users.login.success.track': 'true', - '_dd.appsec.events.users.login.success.sdk': 'true' + '_dd.appsec.events.users.login.success.sdk': 'true', + 'appsec.events.users.login.success.usr.login': 'user_id' }) expect(prioritySampler.setPriority) .to.have.been.calledOnceWithExactly(rootSpan, USER_KEEP, SAMPLING_MECHANISM_APPSEC) + expect(sample).to.have.been.calledOnceWithExactly(rootSpan) + expect(waf.run).to.have.been.calledOnceWithExactly({ + persistent: { + [LOGIN_SUCCESS]: null, + [USER_ID]: 'user_id', + [USER_LOGIN]: 'user_id' + } + }) }) - it('should call waf run with login success address', () => { - const user = { id: 'user_id' } + it('should call waf with user login', () => { + const user = { id: 'user_id', login: 'user_login' } trackUserLoginSuccessEvent(tracer, user) - sinon.assert.calledOnceWithExactly( - waf.run, - { persistent: { [LOGIN_SUCCESS]: null } } - ) + + expect(log.warn).to.not.have.been.called + expect(setUserTags).to.have.been.calledOnceWithExactly(user, rootSpan) + expect(rootSpan.addTags).to.have.been.calledOnceWithExactly({ + 'appsec.events.users.login.success.track': 'true', + '_dd.appsec.events.users.login.success.sdk': 'true', + 'appsec.events.users.login.success.usr.login': 'user_login' + }) + expect(prioritySampler.setPriority) + .to.have.been.calledOnceWithExactly(rootSpan, USER_KEEP, SAMPLING_MECHANISM_APPSEC) + expect(sample).to.have.been.calledOnceWithExactly(rootSpan) + expect(waf.run).to.have.been.calledOnceWithExactly({ + persistent: { + [LOGIN_SUCCESS]: null, + [USER_ID]: 'user_id', + [USER_LOGIN]: 'user_login' + } + }) }) }) describe('trackUserLoginFailureEvent', () => { it('should log warning when passed invalid userId', () => { - trackUserLoginFailureEvent(tracer, null, false) - trackUserLoginFailureEvent(tracer, [], false) + trackUserLoginFailureEvent(tracer, null, false, { key: 'value' }) + trackUserLoginFailureEvent(tracer, [], false, { key: 'value' }) expect(log.warn).to.have.been.calledTwice expect(log.warn.firstCall) @@ -159,7 +186,7 @@ describe('track_event', () => { it('should log warning when root span is not available', () => { rootSpan = undefined - trackUserLoginFailureEvent(tracer, 'user_id', false) + trackUserLoginFailureEvent(tracer, 'user_id', false, { key: 'value' }) expect(log.warn) .to.have.been.calledOnceWithExactly('[ASM] Root span not available in %s', 'trackUserLoginFailureEvent') @@ -168,7 +195,9 @@ describe('track_event', () => { it('should call addTags with metadata', () => { trackUserLoginFailureEvent(tracer, 'user_id', true, { - metakey1: 'metaValue1', metakey2: 'metaValue2', metakey3: 'metaValue3' + metakey1: 'metaValue1', + metakey2: 'metaValue2', + metakey3: 'metaValue3' }) expect(log.warn).to.not.have.been.called @@ -177,6 +206,7 @@ describe('track_event', () => { 'appsec.events.users.login.failure.track': 'true', '_dd.appsec.events.users.login.failure.sdk': 'true', 'appsec.events.users.login.failure.usr.id': 'user_id', + 'appsec.events.users.login.failure.usr.login': 'user_id', 'appsec.events.users.login.failure.usr.exists': 'true', 'appsec.events.users.login.failure.metakey1': 'metaValue1', 'appsec.events.users.login.failure.metakey2': 'metaValue2', @@ -184,11 +214,20 @@ describe('track_event', () => { }) expect(prioritySampler.setPriority) .to.have.been.calledOnceWithExactly(rootSpan, USER_KEEP, SAMPLING_MECHANISM_APPSEC) + expect(sample).to.have.been.calledOnceWithExactly(rootSpan) + expect(waf.run).to.have.been.calledOnceWithExactly({ + persistent: { + [LOGIN_FAILURE]: null, + [USER_LOGIN]: 'user_id' + } + }) }) it('should send false `usr.exists` property when the user does not exist', () => { trackUserLoginFailureEvent(tracer, 'user_id', false, { - metakey1: 'metaValue1', metakey2: 'metaValue2', metakey3: 'metaValue3' + metakey1: 'metaValue1', + metakey2: 'metaValue2', + metakey3: 'metaValue3' }) expect(log.warn).to.not.have.been.called @@ -197,6 +236,7 @@ describe('track_event', () => { 'appsec.events.users.login.failure.track': 'true', '_dd.appsec.events.users.login.failure.sdk': 'true', 'appsec.events.users.login.failure.usr.id': 'user_id', + 'appsec.events.users.login.failure.usr.login': 'user_id', 'appsec.events.users.login.failure.usr.exists': 'false', 'appsec.events.users.login.failure.metakey1': 'metaValue1', 'appsec.events.users.login.failure.metakey2': 'metaValue2', @@ -204,6 +244,13 @@ describe('track_event', () => { }) expect(prioritySampler.setPriority) .to.have.been.calledOnceWithExactly(rootSpan, USER_KEEP, SAMPLING_MECHANISM_APPSEC) + expect(sample).to.have.been.calledOnceWithExactly(rootSpan) + expect(waf.run).to.have.been.calledOnceWithExactly({ + persistent: { + [LOGIN_FAILURE]: null, + [USER_LOGIN]: 'user_id' + } + }) }) it('should call addTags without metadata', () => { @@ -215,18 +262,18 @@ describe('track_event', () => { 'appsec.events.users.login.failure.track': 'true', '_dd.appsec.events.users.login.failure.sdk': 'true', 'appsec.events.users.login.failure.usr.id': 'user_id', + 'appsec.events.users.login.failure.usr.login': 'user_id', 'appsec.events.users.login.failure.usr.exists': 'true' }) expect(prioritySampler.setPriority) .to.have.been.calledOnceWithExactly(rootSpan, USER_KEEP, SAMPLING_MECHANISM_APPSEC) - }) - - it('should call waf run with login failure address', () => { - trackUserLoginFailureEvent(tracer, 'user_id') - sinon.assert.calledOnceWithExactly( - waf.run, - { persistent: { [LOGIN_FAILURE]: null } } - ) + expect(sample).to.have.been.calledOnceWithExactly(rootSpan) + expect(waf.run).to.have.been.calledOnceWithExactly({ + persistent: { + [LOGIN_FAILURE]: null, + [USER_LOGIN]: 'user_id' + } + }) }) }) @@ -255,7 +302,10 @@ describe('track_event', () => { }) it('should call addTags with metadata', () => { - trackCustomEvent(tracer, 'custom_event', { metaKey1: 'metaValue1', metakey2: 'metaValue2' }) + trackCustomEvent(tracer, 'custom_event', { + metaKey1: 'metaValue1', + metakey2: 'metaValue2' + }) expect(log.warn).to.not.have.been.called expect(setUserTags).to.not.have.been.called @@ -267,6 +317,7 @@ describe('track_event', () => { }) expect(prioritySampler.setPriority) .to.have.been.calledOnceWithExactly(rootSpan, USER_KEEP, SAMPLING_MECHANISM_APPSEC) + expect(sample).to.have.been.calledOnceWithExactly(rootSpan) }) it('should call addTags without metadata', () => { @@ -280,42 +331,6 @@ describe('track_event', () => { }) expect(prioritySampler.setPriority) .to.have.been.calledOnceWithExactly(rootSpan, USER_KEEP, SAMPLING_MECHANISM_APPSEC) - }) - }) - - describe('trackEvent', () => { - it('should call addTags with safe mode', () => { - trackEvent('event', { metaKey1: 'metaValue1', metakey2: 'metaValue2' }, 'trackEvent', rootSpan, 'safe') - expect(rootSpan.addTags).to.have.been.calledOnceWithExactly({ - 'appsec.events.event.track': 'true', - '_dd.appsec.events.event.auto.mode': 'safe', - 'appsec.events.event.metaKey1': 'metaValue1', - 'appsec.events.event.metakey2': 'metaValue2' - }) - expect(prioritySampler.setPriority) - .to.have.been.calledOnceWithExactly(rootSpan, USER_KEEP, SAMPLING_MECHANISM_APPSEC) - }) - - it('should call addTags with extended mode', () => { - trackEvent('event', { metaKey1: 'metaValue1', metakey2: 'metaValue2' }, 'trackEvent', rootSpan, 'extended') - expect(rootSpan.addTags).to.have.been.calledOnceWithExactly({ - 'appsec.events.event.track': 'true', - '_dd.appsec.events.event.auto.mode': 'extended', - 'appsec.events.event.metaKey1': 'metaValue1', - 'appsec.events.event.metakey2': 'metaValue2' - }) - expect(prioritySampler.setPriority) - .to.have.been.calledOnceWithExactly(rootSpan, USER_KEEP, SAMPLING_MECHANISM_APPSEC) - }) - - it('should call standalone sample', () => { - trackEvent('event', undefined, 'trackEvent', rootSpan, undefined) - - expect(rootSpan.addTags).to.have.been.calledOnceWithExactly({ - 'appsec.events.event.track': 'true' - }) - expect(prioritySampler.setPriority) - .to.have.been.calledOnceWithExactly(rootSpan, USER_KEEP, SAMPLING_MECHANISM_APPSEC) expect(sample).to.have.been.calledOnceWithExactly(rootSpan) }) }) diff --git a/packages/dd-trace/test/appsec/telemetry.spec.js b/packages/dd-trace/test/appsec/telemetry.spec.js index a297ede3280..3eb3b8521b4 100644 --- a/packages/dd-trace/test/appsec/telemetry.spec.js +++ b/packages/dd-trace/test/appsec/telemetry.spec.js @@ -339,6 +339,17 @@ describe('Appsec Telemetry metrics', () => { expect(count).to.not.have.been.called }) }) + + describe('incrementMissingUserLoginMetric', () => { + it('should increment instrum.user_auth.missing_user_login metric', () => { + appsecTelemetry.incrementMissingUserLoginMetric('passport-local', 'login_success') + + expect(count).to.have.been.calledOnceWithExactly('instrum.user_auth.missing_user_login', { + framework: 'passport-local', + event_type: 'login_success' + }) + }) + }) }) describe('if disabled', () => { diff --git a/packages/dd-trace/test/appsec/user_tracking.spec.js b/packages/dd-trace/test/appsec/user_tracking.spec.js new file mode 100644 index 00000000000..651048d5515 --- /dev/null +++ b/packages/dd-trace/test/appsec/user_tracking.spec.js @@ -0,0 +1,696 @@ +'use strict' + +const assert = require('assert') + +const log = require('../../src/log') +const telemetry = require('../../src/appsec/telemetry') +const { SAMPLING_MECHANISM_APPSEC } = require('../../src/constants') +const standalone = require('../../src/appsec/standalone') +const waf = require('../../src/appsec/waf') + +describe('User Tracking', () => { + let currentTags + let rootSpan + let keepTrace + + let setCollectionMode + let trackLogin + + beforeEach(() => { + sinon.stub(log, 'warn') + sinon.stub(log, 'error') + sinon.stub(telemetry, 'incrementMissingUserLoginMetric') + sinon.stub(standalone, 'sample') + sinon.stub(waf, 'run').returns(['action1']) + + currentTags = {} + + rootSpan = { + context: () => ({ _tags: currentTags }), + addTags: sinon.stub() + } + + keepTrace = sinon.stub() + + const UserTracking = proxyquire('../src/appsec/user_tracking', { + '../priority_sampler': { keepTrace } + }) + + setCollectionMode = UserTracking.setCollectionMode + trackLogin = UserTracking.trackLogin + }) + + afterEach(() => { + sinon.restore() + }) + + describe('getUserId', () => { + beforeEach(() => { + setCollectionMode('identification') + }) + + it('should find an id field in user object', () => { + const user = { + notId: 'no', + id: '123', + email: 'a@b.c' + } + + const results = trackLogin('passport-local', 'login', user, true, rootSpan) + + assert.deepStrictEqual(results, ['action1']) + + sinon.assert.notCalled(log.error) + sinon.assert.notCalled(telemetry.incrementMissingUserLoginMetric) + + sinon.assert.calledOnceWithExactly(keepTrace, rootSpan, SAMPLING_MECHANISM_APPSEC) + sinon.assert.calledOnceWithExactly(standalone.sample, rootSpan) + sinon.assert.calledOnceWithExactly(rootSpan.addTags, { + 'appsec.events.users.login.success.track': 'true', + '_dd.appsec.events.users.login.success.auto.mode': 'identification', + '_dd.appsec.usr.login': 'login', + 'appsec.events.users.login.success.usr.login': 'login', + '_dd.appsec.usr.id': '123', + 'usr.id': '123' + }) + sinon.assert.calledOnceWithExactly(waf.run, { + persistent: { + 'usr.login': 'login', + 'usr.id': '123', + 'server.business_logic.users.login.success': null + } + }) + }) + + it('should find an id-like field in user object when no id field is present', () => { + const user = { + notId: 'no', + email: 'a@b.c', + username: 'azerty' + } + + const results = trackLogin('passport-local', 'login', user, true, rootSpan) + + assert.deepStrictEqual(results, ['action1']) + + sinon.assert.notCalled(log.error) + sinon.assert.notCalled(telemetry.incrementMissingUserLoginMetric) + + sinon.assert.calledOnceWithExactly(keepTrace, rootSpan, SAMPLING_MECHANISM_APPSEC) + sinon.assert.calledOnceWithExactly(standalone.sample, rootSpan) + sinon.assert.calledOnceWithExactly(rootSpan.addTags, { + 'appsec.events.users.login.success.track': 'true', + '_dd.appsec.events.users.login.success.auto.mode': 'identification', + '_dd.appsec.usr.login': 'login', + 'appsec.events.users.login.success.usr.login': 'login', + '_dd.appsec.usr.id': 'a@b.c', + 'usr.id': 'a@b.c' + }) + sinon.assert.calledOnceWithExactly(waf.run, { + persistent: { + 'usr.login': 'login', + 'usr.id': 'a@b.c', + 'server.business_logic.users.login.success': null + } + }) + }) + + it('should find a stringifiable id in user object', () => { + const stringifiableObject = { + a: 1, + toString: () => '123' + } + + const user = { + notId: 'no', + id: { a: 1 }, + _id: stringifiableObject, + email: 'a@b.c' + } + + const results = trackLogin('passport-local', 'login', user, true, rootSpan) + + assert.deepStrictEqual(results, ['action1']) + + sinon.assert.notCalled(log.error) + sinon.assert.notCalled(telemetry.incrementMissingUserLoginMetric) + + sinon.assert.calledOnceWithExactly(keepTrace, rootSpan, SAMPLING_MECHANISM_APPSEC) + sinon.assert.calledOnceWithExactly(standalone.sample, rootSpan) + sinon.assert.calledOnceWithExactly(rootSpan.addTags, { + 'appsec.events.users.login.success.track': 'true', + '_dd.appsec.events.users.login.success.auto.mode': 'identification', + '_dd.appsec.usr.login': 'login', + 'appsec.events.users.login.success.usr.login': 'login', + '_dd.appsec.usr.id': '123', + 'usr.id': '123' + }) + sinon.assert.calledOnceWithExactly(waf.run, { + persistent: { + 'usr.login': 'login', + 'usr.id': '123', + 'server.business_logic.users.login.success': null + } + }) + }) + }) + + describe('trackLogin', () => { + it('should not do anything if collectionMode is empty or disabled', () => { + setCollectionMode('disabled') + + const results = trackLogin('passport-local', 'login', { id: '123', email: 'a@b.c' }, true, rootSpan) + + assert.deepStrictEqual(results, undefined) + + sinon.assert.notCalled(log.error) + sinon.assert.notCalled(telemetry.incrementMissingUserLoginMetric) + sinon.assert.notCalled(keepTrace) + sinon.assert.notCalled(standalone.sample) + sinon.assert.notCalled(rootSpan.addTags) + sinon.assert.notCalled(waf.run) + }) + + it('should log error when rootSpan is not found', () => { + setCollectionMode('identification') + + const results = trackLogin('passport-local', 'login', { id: '123', email: 'a@b.c' }, true) + + assert.deepStrictEqual(results, undefined) + + sinon.assert.calledOnceWithExactly(log.error, '[ASM] No rootSpan found in AppSec trackLogin') + sinon.assert.notCalled(telemetry.incrementMissingUserLoginMetric) + sinon.assert.notCalled(keepTrace) + sinon.assert.notCalled(standalone.sample) + sinon.assert.notCalled(rootSpan.addTags) + sinon.assert.notCalled(waf.run) + }) + + it('should log error and send telemetry when login success is not a string', () => { + setCollectionMode('identification') + + const results = trackLogin('passport-local', {}, { id: '123', email: 'a@b.c' }, true, rootSpan) + + assert.deepStrictEqual(results, undefined) + + sinon.assert.calledOnceWithExactly(log.error, '[ASM] Invalid login provided to AppSec trackLogin') + sinon.assert.calledOnceWithExactly(telemetry.incrementMissingUserLoginMetric, 'passport-local', 'login_success') + sinon.assert.notCalled(keepTrace) + sinon.assert.notCalled(standalone.sample) + sinon.assert.notCalled(rootSpan.addTags) + sinon.assert.notCalled(waf.run) + }) + + it('should log error and send telemetry when login failure is not a string', () => { + setCollectionMode('identification') + + const results = trackLogin('passport-local', {}, { id: '123', email: 'a@b.c' }, false, rootSpan) + + assert.deepStrictEqual(results, undefined) + + sinon.assert.calledOnceWithExactly(log.error, '[ASM] Invalid login provided to AppSec trackLogin') + sinon.assert.calledOnceWithExactly(telemetry.incrementMissingUserLoginMetric, 'passport-local', 'login_failure') + sinon.assert.notCalled(keepTrace) + sinon.assert.notCalled(standalone.sample) + sinon.assert.notCalled(rootSpan.addTags) + sinon.assert.notCalled(waf.run) + }) + + describe('when collectionMode is indentification', () => { + beforeEach(() => { + setCollectionMode('identification') + }) + + it('should write tags and call waf when success is true', () => { + const results = trackLogin('passport-local', 'login', { id: '123', email: 'a@b.c' }, true, rootSpan) + + assert.deepStrictEqual(results, ['action1']) + + sinon.assert.notCalled(log.error) + sinon.assert.notCalled(telemetry.incrementMissingUserLoginMetric) + + sinon.assert.calledOnceWithExactly(keepTrace, rootSpan, SAMPLING_MECHANISM_APPSEC) + sinon.assert.calledOnceWithExactly(standalone.sample, rootSpan) + sinon.assert.calledOnceWithExactly(rootSpan.addTags, { + 'appsec.events.users.login.success.track': 'true', + '_dd.appsec.events.users.login.success.auto.mode': 'identification', + '_dd.appsec.usr.login': 'login', + 'appsec.events.users.login.success.usr.login': 'login', + '_dd.appsec.usr.id': '123', + 'usr.id': '123' + }) + sinon.assert.calledOnceWithExactly(waf.run, { + persistent: { + 'usr.login': 'login', + 'usr.id': '123', + 'server.business_logic.users.login.success': null + } + }) + }) + + it('should write tags and call waf when success is false', () => { + const results = trackLogin('passport-local', 'login', { id: '123', email: 'a@b.c' }, false, rootSpan) + + assert.deepStrictEqual(results, ['action1']) + + sinon.assert.notCalled(log.error) + sinon.assert.notCalled(telemetry.incrementMissingUserLoginMetric) + + sinon.assert.calledOnceWithExactly(keepTrace, rootSpan, SAMPLING_MECHANISM_APPSEC) + sinon.assert.calledOnceWithExactly(standalone.sample, rootSpan) + sinon.assert.calledOnceWithExactly(rootSpan.addTags, { + 'appsec.events.users.login.failure.track': 'true', + '_dd.appsec.events.users.login.failure.auto.mode': 'identification', + '_dd.appsec.usr.login': 'login', + 'appsec.events.users.login.failure.usr.login': 'login', + '_dd.appsec.usr.id': '123', + 'appsec.events.users.login.failure.usr.id': '123' + }) + sinon.assert.calledOnceWithExactly(waf.run, { + persistent: { + 'usr.login': 'login', + 'server.business_logic.users.login.failure': null + } + }) + }) + + it('should not overwrite tags set by SDK when success is true', () => { + currentTags = { + '_dd.appsec.events.users.login.success.sdk': 'true', + 'appsec.events.users.login.success.usr.login': 'sdk_login', + 'usr.id': 'sdk_id' + } + + const results = trackLogin('passport-local', 'login', { id: '123', email: 'a@b.c' }, true, rootSpan) + + assert.deepStrictEqual(results, ['action1']) + + sinon.assert.notCalled(log.error) + sinon.assert.notCalled(telemetry.incrementMissingUserLoginMetric) + + sinon.assert.calledOnceWithExactly(keepTrace, rootSpan, SAMPLING_MECHANISM_APPSEC) + sinon.assert.calledOnceWithExactly(standalone.sample, rootSpan) + sinon.assert.calledOnceWithExactly(rootSpan.addTags, { + 'appsec.events.users.login.success.track': 'true', + '_dd.appsec.events.users.login.success.auto.mode': 'identification', + '_dd.appsec.usr.login': 'login', + '_dd.appsec.usr.id': '123' + }) + sinon.assert.calledOnceWithExactly(waf.run, { + persistent: { + 'usr.login': 'login', + 'server.business_logic.users.login.success': null + } + }) + }) + + it('should not overwwrite tags set by SDK when success is false', () => { + currentTags = { + '_dd.appsec.events.users.login.failure.sdk': 'true', + 'appsec.events.users.login.failure.usr.login': 'sdk_login', + 'appsec.events.users.login.failure.usr.id': 'sdk_id' + } + + const results = trackLogin('passport-local', 'login', { id: '123', email: 'a@b.c' }, false, rootSpan) + + assert.deepStrictEqual(results, ['action1']) + + sinon.assert.notCalled(log.error) + sinon.assert.notCalled(telemetry.incrementMissingUserLoginMetric) + + sinon.assert.calledOnceWithExactly(keepTrace, rootSpan, SAMPLING_MECHANISM_APPSEC) + sinon.assert.calledOnceWithExactly(standalone.sample, rootSpan) + sinon.assert.calledOnceWithExactly(rootSpan.addTags, { + 'appsec.events.users.login.failure.track': 'true', + '_dd.appsec.events.users.login.failure.auto.mode': 'identification', + '_dd.appsec.usr.login': 'login', + '_dd.appsec.usr.id': '123' + }) + sinon.assert.calledOnceWithExactly(waf.run, { + persistent: { + 'usr.login': 'login', + 'server.business_logic.users.login.failure': null + } + }) + }) + + it('should write tags and call waf without user object when success is true', () => { + const results = trackLogin('passport-local', 'login', null, true, rootSpan) + + assert.deepStrictEqual(results, ['action1']) + + sinon.assert.notCalled(log.error) + sinon.assert.notCalled(telemetry.incrementMissingUserLoginMetric) + + sinon.assert.calledOnceWithExactly(keepTrace, rootSpan, SAMPLING_MECHANISM_APPSEC) + sinon.assert.calledOnceWithExactly(standalone.sample, rootSpan) + sinon.assert.calledOnceWithExactly(rootSpan.addTags, { + 'appsec.events.users.login.success.track': 'true', + '_dd.appsec.events.users.login.success.auto.mode': 'identification', + '_dd.appsec.usr.login': 'login', + 'appsec.events.users.login.success.usr.login': 'login' + }) + sinon.assert.calledOnceWithExactly(waf.run, { + persistent: { + 'usr.login': 'login', + 'server.business_logic.users.login.success': null + } + }) + }) + + it('should write tags and call waf without user object when success is false', () => { + const results = trackLogin('passport-local', 'login', null, false, rootSpan) + + assert.deepStrictEqual(results, ['action1']) + + sinon.assert.notCalled(log.error) + sinon.assert.notCalled(telemetry.incrementMissingUserLoginMetric) + + sinon.assert.calledOnceWithExactly(keepTrace, rootSpan, SAMPLING_MECHANISM_APPSEC) + sinon.assert.calledOnceWithExactly(standalone.sample, rootSpan) + sinon.assert.calledOnceWithExactly(rootSpan.addTags, { + 'appsec.events.users.login.failure.track': 'true', + '_dd.appsec.events.users.login.failure.auto.mode': 'identification', + '_dd.appsec.usr.login': 'login', + 'appsec.events.users.login.failure.usr.login': 'login' + }) + sinon.assert.calledOnceWithExactly(waf.run, { + persistent: { + 'usr.login': 'login', + 'server.business_logic.users.login.failure': null + } + }) + }) + }) + + describe('when collectionMode is anonymization', () => { + beforeEach(() => { + setCollectionMode('anonymization') + }) + + it('should write tags and call waf when success is true', () => { + const results = trackLogin('passport-local', 'login', { id: '123', email: 'a@b.c' }, true, rootSpan) + + assert.deepStrictEqual(results, ['action1']) + + sinon.assert.notCalled(log.error) + sinon.assert.notCalled(telemetry.incrementMissingUserLoginMetric) + + sinon.assert.calledOnceWithExactly(keepTrace, rootSpan, SAMPLING_MECHANISM_APPSEC) + sinon.assert.calledOnceWithExactly(standalone.sample, rootSpan) + sinon.assert.calledOnceWithExactly(rootSpan.addTags, { + 'appsec.events.users.login.success.track': 'true', + '_dd.appsec.events.users.login.success.auto.mode': 'anonymization', + '_dd.appsec.usr.login': 'anon_428821350e9691491f616b754cd8315f', + 'appsec.events.users.login.success.usr.login': 'anon_428821350e9691491f616b754cd8315f', + '_dd.appsec.usr.id': 'anon_a665a45920422f9d417e4867efdc4fb8', + 'usr.id': 'anon_a665a45920422f9d417e4867efdc4fb8' + }) + sinon.assert.calledOnceWithExactly(waf.run, { + persistent: { + 'usr.login': 'anon_428821350e9691491f616b754cd8315f', + 'usr.id': 'anon_a665a45920422f9d417e4867efdc4fb8', + 'server.business_logic.users.login.success': null + } + }) + }) + + it('should write tags and call waf when success is false', () => { + const results = trackLogin('passport-local', 'login', { id: '123', email: 'a@b.c' }, false, rootSpan) + + assert.deepStrictEqual(results, ['action1']) + + sinon.assert.notCalled(log.error) + sinon.assert.notCalled(telemetry.incrementMissingUserLoginMetric) + + sinon.assert.calledOnceWithExactly(keepTrace, rootSpan, SAMPLING_MECHANISM_APPSEC) + sinon.assert.calledOnceWithExactly(standalone.sample, rootSpan) + sinon.assert.calledOnceWithExactly(rootSpan.addTags, { + 'appsec.events.users.login.failure.track': 'true', + '_dd.appsec.events.users.login.failure.auto.mode': 'anonymization', + '_dd.appsec.usr.login': 'anon_428821350e9691491f616b754cd8315f', + 'appsec.events.users.login.failure.usr.login': 'anon_428821350e9691491f616b754cd8315f', + '_dd.appsec.usr.id': 'anon_a665a45920422f9d417e4867efdc4fb8', + 'appsec.events.users.login.failure.usr.id': 'anon_a665a45920422f9d417e4867efdc4fb8' + }) + sinon.assert.calledOnceWithExactly(waf.run, { + persistent: { + 'usr.login': 'anon_428821350e9691491f616b754cd8315f', + 'server.business_logic.users.login.failure': null + } + }) + }) + + it('should not overwrite tags set by SDK when success is true', () => { + currentTags = { + '_dd.appsec.events.users.login.success.sdk': 'true', + 'appsec.events.users.login.success.usr.login': 'sdk_login', + 'usr.id': 'sdk_id' + } + + const results = trackLogin('passport-local', 'login', { id: '123', email: 'a@b.c' }, true, rootSpan) + + assert.deepStrictEqual(results, ['action1']) + + sinon.assert.notCalled(log.error) + sinon.assert.notCalled(telemetry.incrementMissingUserLoginMetric) + + sinon.assert.calledOnceWithExactly(keepTrace, rootSpan, SAMPLING_MECHANISM_APPSEC) + sinon.assert.calledOnceWithExactly(standalone.sample, rootSpan) + sinon.assert.calledOnceWithExactly(rootSpan.addTags, { + 'appsec.events.users.login.success.track': 'true', + '_dd.appsec.events.users.login.success.auto.mode': 'anonymization', + '_dd.appsec.usr.login': 'anon_428821350e9691491f616b754cd8315f', + '_dd.appsec.usr.id': 'anon_a665a45920422f9d417e4867efdc4fb8' + }) + sinon.assert.calledOnceWithExactly(waf.run, { + persistent: { + 'usr.login': 'anon_428821350e9691491f616b754cd8315f', + 'server.business_logic.users.login.success': null + } + }) + }) + + it('should not overwwrite tags set by SDK when success is false', () => { + currentTags = { + '_dd.appsec.events.users.login.failure.sdk': 'true', + 'appsec.events.users.login.failure.usr.login': 'sdk_login', + 'appsec.events.users.login.failure.usr.id': 'sdk_id' + } + + const results = trackLogin('passport-local', 'login', { id: '123', email: 'a@b.c' }, false, rootSpan) + + assert.deepStrictEqual(results, ['action1']) + + sinon.assert.notCalled(log.error) + sinon.assert.notCalled(telemetry.incrementMissingUserLoginMetric) + + sinon.assert.calledOnceWithExactly(keepTrace, rootSpan, SAMPLING_MECHANISM_APPSEC) + sinon.assert.calledOnceWithExactly(standalone.sample, rootSpan) + sinon.assert.calledOnceWithExactly(rootSpan.addTags, { + 'appsec.events.users.login.failure.track': 'true', + '_dd.appsec.events.users.login.failure.auto.mode': 'anonymization', + '_dd.appsec.usr.login': 'anon_428821350e9691491f616b754cd8315f', + '_dd.appsec.usr.id': 'anon_a665a45920422f9d417e4867efdc4fb8' + }) + sinon.assert.calledOnceWithExactly(waf.run, { + persistent: { + 'usr.login': 'anon_428821350e9691491f616b754cd8315f', + 'server.business_logic.users.login.failure': null + } + }) + }) + + it('should write tags and call waf without user object when success is true', () => { + const results = trackLogin('passport-local', 'login', null, true, rootSpan) + + assert.deepStrictEqual(results, ['action1']) + + sinon.assert.notCalled(log.error) + sinon.assert.notCalled(telemetry.incrementMissingUserLoginMetric) + + sinon.assert.calledOnceWithExactly(keepTrace, rootSpan, SAMPLING_MECHANISM_APPSEC) + sinon.assert.calledOnceWithExactly(standalone.sample, rootSpan) + sinon.assert.calledOnceWithExactly(rootSpan.addTags, { + 'appsec.events.users.login.success.track': 'true', + '_dd.appsec.events.users.login.success.auto.mode': 'anonymization', + '_dd.appsec.usr.login': 'anon_428821350e9691491f616b754cd8315f', + 'appsec.events.users.login.success.usr.login': 'anon_428821350e9691491f616b754cd8315f' + }) + sinon.assert.calledOnceWithExactly(waf.run, { + persistent: { + 'usr.login': 'anon_428821350e9691491f616b754cd8315f', + 'server.business_logic.users.login.success': null + } + }) + }) + + it('should write tags and call waf without user object when success is false', () => { + const results = trackLogin('passport-local', 'login', null, false, rootSpan) + + assert.deepStrictEqual(results, ['action1']) + + sinon.assert.notCalled(log.error) + sinon.assert.notCalled(telemetry.incrementMissingUserLoginMetric) + + sinon.assert.calledOnceWithExactly(keepTrace, rootSpan, SAMPLING_MECHANISM_APPSEC) + sinon.assert.calledOnceWithExactly(standalone.sample, rootSpan) + sinon.assert.calledOnceWithExactly(rootSpan.addTags, { + 'appsec.events.users.login.failure.track': 'true', + '_dd.appsec.events.users.login.failure.auto.mode': 'anonymization', + '_dd.appsec.usr.login': 'anon_428821350e9691491f616b754cd8315f', + 'appsec.events.users.login.failure.usr.login': 'anon_428821350e9691491f616b754cd8315f' + }) + sinon.assert.calledOnceWithExactly(waf.run, { + persistent: { + 'usr.login': 'anon_428821350e9691491f616b754cd8315f', + 'server.business_logic.users.login.failure': null + } + }) + }) + }) + + describe('collectionMode aliases', () => { + it('should log warning and use anonymization mode when collectionMode is safe', () => { + setCollectionMode('safe') + + sinon.assert.calledOnceWithExactly( + log.warn, + '[ASM] Using deprecated value "safe" in config.appsec.eventTracking.mode' + ) + + const results = trackLogin('passport-local', 'login', { id: '123', email: 'a@b.c' }, true, rootSpan) + + assert.deepStrictEqual(results, ['action1']) + + sinon.assert.notCalled(log.error) + sinon.assert.notCalled(telemetry.incrementMissingUserLoginMetric) + + sinon.assert.calledOnceWithExactly(keepTrace, rootSpan, SAMPLING_MECHANISM_APPSEC) + sinon.assert.calledOnceWithExactly(standalone.sample, rootSpan) + sinon.assert.calledOnceWithExactly(rootSpan.addTags, { + 'appsec.events.users.login.success.track': 'true', + '_dd.appsec.events.users.login.success.auto.mode': 'anonymization', + '_dd.appsec.usr.login': 'anon_428821350e9691491f616b754cd8315f', + 'appsec.events.users.login.success.usr.login': 'anon_428821350e9691491f616b754cd8315f', + '_dd.appsec.usr.id': 'anon_a665a45920422f9d417e4867efdc4fb8', + 'usr.id': 'anon_a665a45920422f9d417e4867efdc4fb8' + }) + sinon.assert.calledOnceWithExactly(waf.run, { + persistent: { + 'usr.login': 'anon_428821350e9691491f616b754cd8315f', + 'usr.id': 'anon_a665a45920422f9d417e4867efdc4fb8', + 'server.business_logic.users.login.success': null + } + }) + }) + + it('should use anonymization mode when collectionMode is anon', () => { + setCollectionMode('anon') + + const results = trackLogin('passport-local', 'login', { id: '123', email: 'a@b.c' }, true, rootSpan) + + assert.deepStrictEqual(results, ['action1']) + + sinon.assert.notCalled(log.error) + sinon.assert.notCalled(telemetry.incrementMissingUserLoginMetric) + + sinon.assert.calledOnceWithExactly(keepTrace, rootSpan, SAMPLING_MECHANISM_APPSEC) + sinon.assert.calledOnceWithExactly(standalone.sample, rootSpan) + sinon.assert.calledOnceWithExactly(rootSpan.addTags, { + 'appsec.events.users.login.success.track': 'true', + '_dd.appsec.events.users.login.success.auto.mode': 'anonymization', + '_dd.appsec.usr.login': 'anon_428821350e9691491f616b754cd8315f', + 'appsec.events.users.login.success.usr.login': 'anon_428821350e9691491f616b754cd8315f', + '_dd.appsec.usr.id': 'anon_a665a45920422f9d417e4867efdc4fb8', + 'usr.id': 'anon_a665a45920422f9d417e4867efdc4fb8' + }) + sinon.assert.calledOnceWithExactly(waf.run, { + persistent: { + 'usr.login': 'anon_428821350e9691491f616b754cd8315f', + 'usr.id': 'anon_a665a45920422f9d417e4867efdc4fb8', + 'server.business_logic.users.login.success': null + } + }) + }) + + it('should log warning and use identification mode when collectionMode is extended', () => { + setCollectionMode('extended') + + sinon.assert.calledOnceWithExactly( + log.warn, + '[ASM] Using deprecated value "extended" in config.appsec.eventTracking.mode' + ) + + const results = trackLogin('passport-local', 'login', { id: '123', email: 'a@b.c' }, true, rootSpan) + + assert.deepStrictEqual(results, ['action1']) + + sinon.assert.notCalled(log.error) + sinon.assert.notCalled(telemetry.incrementMissingUserLoginMetric) + + sinon.assert.calledOnceWithExactly(keepTrace, rootSpan, SAMPLING_MECHANISM_APPSEC) + sinon.assert.calledOnceWithExactly(standalone.sample, rootSpan) + sinon.assert.calledOnceWithExactly(rootSpan.addTags, { + 'appsec.events.users.login.success.track': 'true', + '_dd.appsec.events.users.login.success.auto.mode': 'identification', + '_dd.appsec.usr.login': 'login', + 'appsec.events.users.login.success.usr.login': 'login', + '_dd.appsec.usr.id': '123', + 'usr.id': '123' + }) + sinon.assert.calledOnceWithExactly(waf.run, { + persistent: { + 'usr.login': 'login', + 'usr.id': '123', + 'server.business_logic.users.login.success': null + } + }) + }) + + it('should use identification mode when collectionMode is ident', () => { + setCollectionMode('ident') + + const results = trackLogin('passport-local', 'login', { id: '123', email: 'a@b.c' }, true, rootSpan) + + assert.deepStrictEqual(results, ['action1']) + + sinon.assert.notCalled(log.error) + sinon.assert.notCalled(telemetry.incrementMissingUserLoginMetric) + + sinon.assert.calledOnceWithExactly(keepTrace, rootSpan, SAMPLING_MECHANISM_APPSEC) + sinon.assert.calledOnceWithExactly(standalone.sample, rootSpan) + sinon.assert.calledOnceWithExactly(rootSpan.addTags, { + 'appsec.events.users.login.success.track': 'true', + '_dd.appsec.events.users.login.success.auto.mode': 'identification', + '_dd.appsec.usr.login': 'login', + 'appsec.events.users.login.success.usr.login': 'login', + '_dd.appsec.usr.id': '123', + 'usr.id': '123' + }) + sinon.assert.calledOnceWithExactly(waf.run, { + persistent: { + 'usr.login': 'login', + 'usr.id': '123', + 'server.business_logic.users.login.success': null + } + }) + }) + + it('should use disabled mode when collectionMode is not recognized', () => { + setCollectionMode('saperlipopette') + + const results = trackLogin('passport-local', 'login', { id: '123', email: 'a@b.c' }, true, rootSpan) + + assert.deepStrictEqual(results, undefined) + + sinon.assert.notCalled(log.error) + sinon.assert.notCalled(telemetry.incrementMissingUserLoginMetric) + sinon.assert.notCalled(keepTrace) + sinon.assert.notCalled(standalone.sample) + sinon.assert.notCalled(rootSpan.addTags) + sinon.assert.notCalled(waf.run) + }) + }) + }) +}) diff --git a/packages/dd-trace/test/config.spec.js b/packages/dd-trace/test/config.spec.js index 1eb711dbd2c..32afdf7c8f7 100644 --- a/packages/dd-trace/test/config.spec.js +++ b/packages/dd-trace/test/config.spec.js @@ -170,7 +170,7 @@ describe('Config', () => { it('should correctly map OTEL_RESOURCE_ATTRIBUTES', () => { process.env.OTEL_RESOURCE_ATTRIBUTES = - 'deployment.environment=test1,service.name=test2,service.version=5,foo=bar1,baz=qux1' + 'deployment.environment=test1,service.name=test2,service.version=5,foo=bar1,baz=qux1' const config = new Config() expect(config).to.have.property('env', 'test1') @@ -259,8 +259,7 @@ describe('Config', () => { expect(config).to.have.nested.property('appsec.blockedTemplateHtml', undefined) expect(config).to.have.nested.property('appsec.blockedTemplateJson', undefined) expect(config).to.have.nested.property('appsec.blockedTemplateGraphql', undefined) - expect(config).to.have.nested.property('appsec.eventTracking.enabled', true) - expect(config).to.have.nested.property('appsec.eventTracking.mode', 'safe') + expect(config).to.have.nested.property('appsec.eventTracking.mode', 'identification') expect(config).to.have.nested.property('appsec.apiSecurity.enabled', true) expect(config).to.have.nested.property('appsec.apiSecurity.sampleDelay', 30) expect(config).to.have.nested.property('appsec.sca.enabled', null) @@ -285,6 +284,7 @@ describe('Config', () => { { name: 'appsec.blockedTemplateHtml', value: undefined, origin: 'default' }, { name: 'appsec.blockedTemplateJson', value: undefined, origin: 'default' }, { name: 'appsec.enabled', value: undefined, origin: 'default' }, + { name: 'appsec.eventTracking.mode', value: 'identification', origin: 'default' }, { name: 'appsec.obfuscatorKeyRegex', // eslint-disable-next-line @stylistic/js/max-len @@ -603,7 +603,6 @@ describe('Config', () => { expect(config).to.have.nested.property('appsec.blockedTemplateHtml', BLOCKED_TEMPLATE_HTML) expect(config).to.have.nested.property('appsec.blockedTemplateJson', BLOCKED_TEMPLATE_JSON) expect(config).to.have.nested.property('appsec.blockedTemplateGraphql', BLOCKED_TEMPLATE_GRAPHQL) - expect(config).to.have.nested.property('appsec.eventTracking.enabled', true) expect(config).to.have.nested.property('appsec.eventTracking.mode', 'extended') expect(config).to.have.nested.property('appsec.apiSecurity.enabled', true) expect(config).to.have.nested.property('appsec.apiSecurity.sampleDelay', 25) @@ -635,6 +634,7 @@ describe('Config', () => { { name: 'appsec.blockedTemplateHtml', value: BLOCKED_TEMPLATE_HTML_PATH, origin: 'env_var' }, { name: 'appsec.blockedTemplateJson', value: BLOCKED_TEMPLATE_JSON_PATH, origin: 'env_var' }, { name: 'appsec.enabled', value: true, origin: 'env_var' }, + { name: 'appsec.eventTracking.mode', value: 'extended', origin: 'env_var' }, { name: 'appsec.obfuscatorKeyRegex', value: '.*', origin: 'env_var' }, { name: 'appsec.obfuscatorValueRegex', value: '.*', origin: 'env_var' }, { name: 'appsec.rateLimit', value: '42', origin: 'env_var' }, @@ -773,6 +773,15 @@ describe('Config', () => { expect(config).to.have.nested.deep.property('crashtracking.enabled', false) }) + it('should prioritize DD_APPSEC_AUTO_USER_INSTRUMENTATION_MODE over DD_APPSEC_AUTOMATED_USER_EVENTS_TRACKING', () => { + process.env.DD_APPSEC_AUTO_USER_INSTRUMENTATION_MODE = 'anonymous' + process.env.DD_APPSEC_AUTOMATED_USER_EVENTS_TRACKING = 'extended' + + const config = new Config() + + expect(config).to.have.nested.property('appsec.eventTracking.mode', 'anonymous') + }) + it('should initialize from the options', () => { const logger = {} const tags = { @@ -1187,6 +1196,7 @@ describe('Config', () => { process.env.DD_APPSEC_HTTP_BLOCKED_TEMPLATE_HTML = BLOCKED_TEMPLATE_JSON_PATH // note the inversion between process.env.DD_APPSEC_HTTP_BLOCKED_TEMPLATE_JSON = BLOCKED_TEMPLATE_HTML_PATH // json and html here process.env.DD_APPSEC_GRAPHQL_BLOCKED_TEMPLATE_JSON = BLOCKED_TEMPLATE_JSON_PATH // json and html here + process.env.DD_APPSEC_AUTO_USER_INSTRUMENTATION_MODE = 'disabled' process.env.DD_APPSEC_AUTOMATED_USER_EVENTS_TRACKING = 'disabled' process.env.DD_API_SECURITY_ENABLED = 'false' process.env.DD_REMOTE_CONFIG_POLL_INTERVAL_SECONDS = 11 @@ -1251,7 +1261,7 @@ describe('Config', () => { blockedTemplateJson: BLOCKED_TEMPLATE_JSON_PATH, blockedTemplateGraphql: BLOCKED_TEMPLATE_GRAPHQL_PATH, eventTracking: { - mode: 'safe' + mode: 'anonymous' }, apiSecurity: { enabled: true @@ -1329,8 +1339,7 @@ describe('Config', () => { expect(config).to.have.nested.property('appsec.blockedTemplateHtml', BLOCKED_TEMPLATE_HTML) expect(config).to.have.nested.property('appsec.blockedTemplateJson', BLOCKED_TEMPLATE_JSON) expect(config).to.have.nested.property('appsec.blockedTemplateGraphql', BLOCKED_TEMPLATE_GRAPHQL) - expect(config).to.have.nested.property('appsec.eventTracking.enabled', true) - expect(config).to.have.nested.property('appsec.eventTracking.mode', 'safe') + expect(config).to.have.nested.property('appsec.eventTracking.mode', 'anonymous') expect(config).to.have.nested.property('appsec.apiSecurity.enabled', true) expect(config).to.have.nested.property('remoteConfig.pollInterval', 42) expect(config).to.have.nested.property('iast.enabled', true) @@ -1392,7 +1401,7 @@ describe('Config', () => { blockedTemplateJson: BLOCKED_TEMPLATE_JSON_PATH, blockedTemplateGraphql: BLOCKED_TEMPLATE_GRAPHQL_PATH, eventTracking: { - mode: 'safe' + mode: 'anonymous' }, apiSecurity: { enabled: false @@ -1427,7 +1436,6 @@ describe('Config', () => { blockedTemplateJson: undefined, blockedTemplateGraphql: undefined, eventTracking: { - enabled: false, mode: 'disabled' }, apiSecurity: { From e4d4cc3456c6f192082d3884010384c0d39b640c Mon Sep 17 00:00:00 2001 From: Bryan English Date: Mon, 16 Dec 2024 14:50:46 -0500 Subject: [PATCH 10/36] consolidate instances of `loadInst`, so code isn't repeated (#5020) --- .../dd-trace/test/setup/helpers/load-inst.js | 62 +++++++++++++++++++ packages/dd-trace/test/setup/mocha.js | 35 +---------- scripts/install_plugin_modules.js | 37 ++--------- scripts/verify-ci-config.js | 41 +----------- 4 files changed, 72 insertions(+), 103 deletions(-) create mode 100644 packages/dd-trace/test/setup/helpers/load-inst.js diff --git a/packages/dd-trace/test/setup/helpers/load-inst.js b/packages/dd-trace/test/setup/helpers/load-inst.js new file mode 100644 index 00000000000..91abd8baa77 --- /dev/null +++ b/packages/dd-trace/test/setup/helpers/load-inst.js @@ -0,0 +1,62 @@ +'use strict' + +const fs = require('fs') +const path = require('path') +const proxyquire = require('proxyquire') + +function loadInstFile (file, instrumentations) { + const instrument = { + addHook (instrumentation) { + instrumentations.push(instrumentation) + } + } + + const instPath = path.join(__dirname, `../../../../datadog-instrumentations/src/${file}`) + + proxyquire.noPreserveCache()(instPath, { + './helpers/instrument': instrument, + '../helpers/instrument': instrument + }) +} + +function loadOneInst (name) { + const instrumentations = [] + + try { + loadInstFile(`${name}/server.js`, instrumentations) + loadInstFile(`${name}/client.js`, instrumentations) + } catch (e) { + try { + loadInstFile(`${name}/main.js`, instrumentations) + } catch (e) { + loadInstFile(`${name}.js`, instrumentations) + } + } + + return instrumentations +} + +function getAllInstrumentations () { + const names = fs.readdirSync(path.join(__dirname, '../../../../', 'datadog-instrumentations', 'src')) + .filter(file => file.endsWith('.js')) + .map(file => file.slice(0, -3)) + + const instrumentations = names.reduce((acc, key) => { + const name = key + let instrumentations = loadOneInst(name) + + instrumentations = instrumentations.filter(i => i.versions) + if (instrumentations.length) { + acc[key] = instrumentations + } + + return acc + }, {}) + + return instrumentations +} + +module.exports = { + getInstrumentation: loadOneInst, + getAllInstrumentations +} diff --git a/packages/dd-trace/test/setup/mocha.js b/packages/dd-trace/test/setup/mocha.js index d3520c3fe1c..53a2c95897a 100644 --- a/packages/dd-trace/test/setup/mocha.js +++ b/packages/dd-trace/test/setup/mocha.js @@ -11,6 +11,7 @@ const agent = require('../plugins/agent') const Nomenclature = require('../../src/service-naming') const { storage } = require('../../../datadog-core') const { schemaDefinitions } = require('../../src/service-naming/schemas') +const { getInstrumentation } = require('./helpers/load-inst') global.withVersions = withVersions global.withExports = withExports @@ -19,38 +20,6 @@ global.withPeerService = withPeerService const testedPlugins = agent.testedPlugins -function loadInst (plugin) { - const instrumentations = [] - - try { - loadInstFile(`${plugin}/server.js`, instrumentations) - loadInstFile(`${plugin}/client.js`, instrumentations) - } catch (e) { - try { - loadInstFile(`${plugin}/main.js`, instrumentations) - } catch (e) { - loadInstFile(`${plugin}.js`, instrumentations) - } - } - - return instrumentations -} - -function loadInstFile (file, instrumentations) { - const instrument = { - addHook (instrumentation) { - instrumentations.push(instrumentation) - } - } - - const instPath = path.join(__dirname, `../../../datadog-instrumentations/src/${file}`) - - proxyquire.noPreserveCache()(instPath, { - './helpers/instrument': instrument, - '../helpers/instrument': instrument - }) -} - function withNamingSchema ( spanProducerFn, expected, @@ -174,7 +143,7 @@ function withPeerService (tracer, pluginName, spanGenerationFn, service, service } function withVersions (plugin, modules, range, cb) { - const instrumentations = typeof plugin === 'string' ? loadInst(plugin) : [].concat(plugin) + const instrumentations = typeof plugin === 'string' ? getInstrumentation(plugin) : [].concat(plugin) const names = instrumentations.map(instrumentation => instrumentation.name) modules = [].concat(modules) diff --git a/scripts/install_plugin_modules.js b/scripts/install_plugin_modules.js index 608fe71a992..212dc5928ed 100644 --- a/scripts/install_plugin_modules.js +++ b/scripts/install_plugin_modules.js @@ -5,10 +5,10 @@ const os = require('os') const path = require('path') const crypto = require('crypto') const semver = require('semver') -const proxyquire = require('proxyquire') const exec = require('./helpers/exec') const childProcess = require('child_process') const externals = require('../packages/dd-trace/test/plugins/externals') +const { getInstrumentation } = require('../packages/dd-trace/test/setup/helpers/load-inst') const requirePackageJsonPath = require.resolve('../packages/dd-trace/src/require-package-json') @@ -47,19 +47,7 @@ async function run () { async function assertVersions () { const internals = names - .map(key => { - const instrumentations = [] - const name = key - - try { - loadInstFile(`${name}/server.js`, instrumentations) - loadInstFile(`${name}/client.js`, instrumentations) - } catch (e) { - loadInstFile(`${name}.js`, instrumentations) - } - - return instrumentations - }) + .map(getInstrumentation) .reduce((prev, next) => prev.concat(next), []) for (const inst of internals) { @@ -117,10 +105,10 @@ function assertFolder (name, version) { } } -async function assertPackage (name, version, dependency, external) { - const dependencies = { [name]: dependency } +async function assertPackage (name, version, dependencyVersionRange, external) { + const dependencies = { [name]: dependencyVersionRange } if (deps[name]) { - await addDependencies(dependencies, name, dependency) + await addDependencies(dependencies, name, dependencyVersionRange) } const pkg = { name: [name, sha1(name).substr(0, 8), sha1(version)].filter(val => val).join('-'), @@ -240,18 +228,3 @@ function sha1 (str) { shasum.update(str) return shasum.digest('hex') } - -function loadInstFile (file, instrumentations) { - const instrument = { - addHook (instrumentation) { - instrumentations.push(instrumentation) - } - } - - const instPath = path.join(__dirname, `../packages/datadog-instrumentations/src/${file}`) - - proxyquire.noPreserveCache()(instPath, { - './helpers/instrument': instrument, - '../helpers/instrument': instrument - }) -} diff --git a/scripts/verify-ci-config.js b/scripts/verify-ci-config.js index 7a917132688..2e16ac0f7c3 100644 --- a/scripts/verify-ci-config.js +++ b/scripts/verify-ci-config.js @@ -4,39 +4,19 @@ const fs = require('fs') const path = require('path') const util = require('util') -const proxyquire = require('proxyquire') const yaml = require('yaml') const semver = require('semver') const { execSync } = require('child_process') const Module = require('module') +const { getAllInstrumentations } = require('../packages/dd-trace/test/setup/helpers/load-inst') + if (!Module.isBuiltin) { Module.isBuiltin = mod => Module.builtinModules.includes(mod) } const nodeMajor = Number(process.versions.node.split('.')[0]) -const names = fs.readdirSync(path.join(__dirname, '..', 'packages', 'datadog-instrumentations', 'src')) - .filter(file => file.endsWith('.js')) - .map(file => file.slice(0, -3)) - -const instrumentations = names.reduce((acc, key) => { - let instrumentations = [] - const name = key - - try { - loadInstFile(`${name}/server.js`, instrumentations) - loadInstFile(`${name}/client.js`, instrumentations) - } catch (e) { - loadInstFile(`${name}.js`, instrumentations) - } - - instrumentations = instrumentations.filter(i => i.versions) - if (instrumentations.length) { - acc[key] = instrumentations - } - - return acc -}, {}) +const instrumentations = getAllInstrumentations() const versions = {} @@ -84,21 +64,6 @@ Note that versions may be dependent on Node.js version. This is Node.js v${color } } -function loadInstFile (file, instrumentations) { - const instrument = { - addHook (instrumentation) { - instrumentations.push(instrumentation) - } - } - - const instPath = path.join(__dirname, `../packages/datadog-instrumentations/src/${file}`) - - proxyquire.noPreserveCache()(instPath, { - './helpers/instrument': instrument, - '../helpers/instrument': instrument - }) -} - function getRangesFromYaml (job) { // eslint-disable-next-line no-template-curly-in-string if (job.env && job.env.PACKAGE_VERSION_RANGE && job.env.PACKAGE_VERSION_RANGE !== '${{ matrix.range }}') { From a17c93f64f44e189e2d0eac45dbf1c9fcf00b747 Mon Sep 17 00:00:00 2001 From: Thomas Hunter II Date: Tue, 17 Dec 2024 00:40:28 -0800 Subject: [PATCH 11/36] repo: mandatory issue templates (#5023) --- .github/ISSUE_TEMPLATE/bug_report.yaml | 64 +++++++++++++++++++++ .github/ISSUE_TEMPLATE/config.yml | 11 ++-- .github/ISSUE_TEMPLATE/feature_request.yaml | 50 ++++++++++++++++ 3 files changed, 119 insertions(+), 6 deletions(-) create mode 100644 .github/ISSUE_TEMPLATE/bug_report.yaml create mode 100644 .github/ISSUE_TEMPLATE/feature_request.yaml diff --git a/.github/ISSUE_TEMPLATE/bug_report.yaml b/.github/ISSUE_TEMPLATE/bug_report.yaml new file mode 100644 index 00000000000..8fb53ba14fa --- /dev/null +++ b/.github/ISSUE_TEMPLATE/bug_report.yaml @@ -0,0 +1,64 @@ +name: "Bug Report (Low Priority)" +description: "Create a public Bug Report. Note that these may not be addressed as quickly as the helpdesk and that looking up account information will be difficult." +title: "[BUG]: " +labels: bug +body: + - type: input + attributes: + label: Tracer Version(s) + description: "Version(s) of the tracer affected by this bug" + placeholder: 1.2.3, 4.5.6 + validations: + required: true + + - type: input + attributes: + label: Node.js Version(s) + description: "Version(s) of Node.js (`node --version`) that you've encountered this bug with" + placeholder: 20.1.1 + validations: + required: true + + - type: textarea + attributes: + label: Bug Report + description: Please add a clear and concise description of the bug here + validations: + required: true + + - type: textarea + attributes: + label: Reproduction Code + description: Please add code here to help us reproduce the problem + validations: + required: false + + - type: textarea + attributes: + label: Error Logs + description: "Please provide any error logs from the tracer (`DD_TRACE_DEBUG=true` can help)" + validations: + required: false + + - type: input + attributes: + label: Operating System + description: "Provide your operating system and version (e.g. `uname -a`)" + placeholder: Darwin Kernel Version 23.6.0 + validations: + required: false + + - type: dropdown + attributes: + label: Bundling + description: "How is your application being bundled" + options: + - Unsure + - No Bundling + - ESBuild + - Webpack + - Next.js + - Vite + - Rollup + validations: + required: true diff --git a/.github/ISSUE_TEMPLATE/config.yml b/.github/ISSUE_TEMPLATE/config.yml index b5a5eb1d199..5f822733ea5 100644 --- a/.github/ISSUE_TEMPLATE/config.yml +++ b/.github/ISSUE_TEMPLATE/config.yml @@ -1,9 +1,8 @@ -blank_issues_enabled: true +blank_issues_enabled: false contact_links: - - name: Bug Report + - name: Bug Report (High Priority) url: https://help.datadoghq.com/hc/en-us/requests/new?tf_1260824651490=pt_product_type:apm&tf_1900004146284=pt_apm_language:node - about: This option creates an expedited Bug Report via the helpdesk (no login required). This will allow us to look up your account and allows you to provide additional information in private. Please do not create a GitHub issue to report a bug. - - name: Feature Request + about: Create an expedited Bug Report via the helpdesk (no login required). This will allow us to look up your account and allows you to provide additional information in private. Please do not create a GitHub issue to report a bug. + - name: Feature Request (High Priority) url: https://help.datadoghq.com/hc/en-us/requests/new?tf_1260824651490=pt_product_type:apm&tf_1900004146284=pt_apm_language:node&tf_1260825272270=pt_apm_category_feature_request - about: This option creates an expedited Feature Request via the helpdesk (no login required). This helps with prioritization and allows you to provide additional information in private. Please do not create a GitHub issue to request a feature. - + about: Create an expedited Feature Request via the helpdesk (no login required). This helps with prioritization and allows you to provide additional information in private. Please do not create a GitHub issue to request a feature. diff --git a/.github/ISSUE_TEMPLATE/feature_request.yaml b/.github/ISSUE_TEMPLATE/feature_request.yaml new file mode 100644 index 00000000000..9d26ea1dd33 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/feature_request.yaml @@ -0,0 +1,50 @@ +name: Feature Request (Low Priority) +description: Create a public Feature Request. Note that these may not be addressed as quickly as the helpdesk and that looking up account information will be difficult. +title: "[FEATURE]: " +labels: feature-request +body: + - type: input + attributes: + label: Package Name + description: "If your feature request is to add instrumentation support for an npm package please provide the name here" + placeholder: left-pad + validations: + required: false + + - type: input + attributes: + label: Package Version(s) + description: "If your feature request is to add instrumentation support for an npm package please provide the version you use" + placeholder: 1.2.3 + validations: + required: false + + - type: textarea + attributes: + label: Describe the feature you'd like + description: A clear and concise description of what you want to happen. + validations: + required: true + + - type: textarea + attributes: + label: Is your feature request related to a problem? + description: | + Please add a clear and concise description of your problem. + E.g. I'm unable to instrument my database queries... + validations: + required: false + + - type: textarea + attributes: + label: Describe alternatives you've considered + description: A clear and concise description of any alternative solutions or features you've considered + validations: + required: false + + - type: textarea + attributes: + label: Additional context + description: Add any other context or screenshots about the feature request here + validations: + required: false From fb9ccca58325820a2acfbd9e6862d65a0543aa14 Mon Sep 17 00:00:00 2001 From: Roch Devost Date: Tue, 17 Dec 2024 12:07:07 -0500 Subject: [PATCH 12/36] update native-metrics to 3.1.0 (#5022) --- package.json | 2 +- yarn.lock | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/package.json b/package.json index 28c20dde6ed..3b8ff598b98 100644 --- a/package.json +++ b/package.json @@ -85,7 +85,7 @@ "@datadog/native-appsec": "8.3.0", "@datadog/native-iast-rewriter": "2.6.1", "@datadog/native-iast-taint-tracking": "3.2.0", - "@datadog/native-metrics": "^3.0.1", + "@datadog/native-metrics": "^3.1.0", "@datadog/pprof": "5.4.1", "@datadog/sketches-js": "^2.1.0", "@isaacs/ttlcache": "^1.4.1", diff --git a/yarn.lock b/yarn.lock index ebbc6922e3d..375177cb17b 100644 --- a/yarn.lock +++ b/yarn.lock @@ -428,10 +428,10 @@ dependencies: node-gyp-build "^3.9.0" -"@datadog/native-metrics@^3.0.1": - version "3.0.1" - resolved "https://registry.yarnpkg.com/@datadog/native-metrics/-/native-metrics-3.0.1.tgz#dc276c93785c0377a048e316f23b7c8ff3acfa84" - integrity sha512-0GuMyYyXf+Qpb/F+Fcekz58f2mO37lit9U3jMbWY/m8kac44gCPABzL5q3gWbdH+hWgqYfQoEYsdNDGSrKfwoQ== +"@datadog/native-metrics@^3.1.0": + version "3.1.0" + resolved "https://registry.yarnpkg.com/@datadog/native-metrics/-/native-metrics-3.1.0.tgz#c2378841accd9fdd6866d0e49bdf6e3d76e79f22" + integrity sha512-yOBi4x0OQRaGNPZ2bx9TGvDIgEdQ8fkudLTFAe7gEM1nAlvFmbE5YfpH8WenEtTSEBwojSau06m2q7axtEEmCg== dependencies: node-addon-api "^6.1.0" node-gyp-build "^3.9.0" From a38aaddd8b11a359b8153830b634b028a563db4e Mon Sep 17 00:00:00 2001 From: Roch Devost Date: Tue, 17 Dec 2024 13:24:25 -0500 Subject: [PATCH 13/36] enable crashtracking by default outside of ssi (#5026) * enable crashtracking by default outside of ssi * update libdatadog --- package.json | 2 +- packages/dd-trace/src/config.js | 6 +----- packages/dd-trace/test/config.spec.js | 8 ++++---- yarn.lock | 8 ++++---- 4 files changed, 10 insertions(+), 14 deletions(-) diff --git a/package.json b/package.json index 3b8ff598b98..cd540cb08a0 100644 --- a/package.json +++ b/package.json @@ -81,7 +81,7 @@ "node": ">=18" }, "dependencies": { - "@datadog/libdatadog": "^0.2.2", + "@datadog/libdatadog": "^0.3.0", "@datadog/native-appsec": "8.3.0", "@datadog/native-iast-rewriter": "2.6.1", "@datadog/native-iast-taint-tracking": "3.2.0", diff --git a/packages/dd-trace/src/config.js b/packages/dd-trace/src/config.js index 6f630212799..beb15ebc010 100644 --- a/packages/dd-trace/src/config.js +++ b/packages/dd-trace/src/config.js @@ -466,7 +466,7 @@ class Config { this._setValue(defaults, 'ciVisibilityTestSessionName', '') this._setValue(defaults, 'clientIpEnabled', false) this._setValue(defaults, 'clientIpHeader', null) - this._setValue(defaults, 'crashtracking.enabled', false) + this._setValue(defaults, 'crashtracking.enabled', true) this._setValue(defaults, 'codeOriginForSpans.enabled', false) this._setValue(defaults, 'dbmPropagationMode', 'disabled') this._setValue(defaults, 'dogstatsd.hostname', '127.0.0.1') @@ -1136,10 +1136,6 @@ class Config { calc['tracePropagationStyle.inject'] = calc['tracePropagationStyle.inject'] || defaultPropagationStyle calc['tracePropagationStyle.extract'] = calc['tracePropagationStyle.extract'] || defaultPropagationStyle } - - if (this._env.injectionEnabled?.length > 0) { - this._setBoolean(calc, 'crashtracking.enabled', true) - } } _applyRemote (options) { diff --git a/packages/dd-trace/test/config.spec.js b/packages/dd-trace/test/config.spec.js index 32afdf7c8f7..ca1a8bcb575 100644 --- a/packages/dd-trace/test/config.spec.js +++ b/packages/dd-trace/test/config.spec.js @@ -220,7 +220,7 @@ describe('Config', () => { expect(config).to.have.property('queryStringObfuscation').with.length(626) expect(config).to.have.property('clientIpEnabled', false) expect(config).to.have.property('clientIpHeader', null) - expect(config).to.have.nested.property('crashtracking.enabled', false) + expect(config).to.have.nested.property('crashtracking.enabled', true) expect(config).to.have.property('sampleRate', undefined) expect(config).to.have.property('runtimeMetrics', false) expect(config.tags).to.have.property('service', 'node') @@ -451,7 +451,7 @@ describe('Config', () => { process.env.DD_TRACE_OBFUSCATION_QUERY_STRING_REGEXP = '.*' process.env.DD_TRACE_CLIENT_IP_ENABLED = 'true' process.env.DD_TRACE_CLIENT_IP_HEADER = 'x-true-client-ip' - process.env.DD_CRASHTRACKING_ENABLED = 'true' + process.env.DD_CRASHTRACKING_ENABLED = 'false' process.env.DD_RUNTIME_METRICS_ENABLED = 'true' process.env.DD_TRACE_REPORT_HOSTNAME = 'true' process.env.DD_ENV = 'test' @@ -543,7 +543,7 @@ describe('Config', () => { expect(config).to.have.property('queryStringObfuscation', '.*') expect(config).to.have.property('clientIpEnabled', true) expect(config).to.have.property('clientIpHeader', 'x-true-client-ip') - expect(config).to.have.nested.property('crashtracking.enabled', true) + expect(config).to.have.nested.property('crashtracking.enabled', false) expect(config.grpc.client.error.statuses).to.deep.equal([3, 13, 400, 401, 402, 403]) expect(config.grpc.server.error.statuses).to.deep.equal([3, 13, 400, 401, 402, 403]) expect(config).to.have.property('runtimeMetrics', true) @@ -648,7 +648,7 @@ describe('Config', () => { { name: 'appsec.wafTimeout', value: '42', origin: 'env_var' }, { name: 'clientIpEnabled', value: true, origin: 'env_var' }, { name: 'clientIpHeader', value: 'x-true-client-ip', origin: 'env_var' }, - { name: 'crashtracking.enabled', value: true, origin: 'env_var' }, + { name: 'crashtracking.enabled', value: false, origin: 'env_var' }, { name: 'codeOriginForSpans.enabled', value: true, origin: 'env_var' }, { name: 'dogstatsd.hostname', value: 'dsd-agent', origin: 'env_var' }, { name: 'dogstatsd.port', value: '5218', origin: 'env_var' }, diff --git a/yarn.lock b/yarn.lock index 375177cb17b..49411da5f2f 100644 --- a/yarn.lock +++ b/yarn.lock @@ -401,10 +401,10 @@ resolved "https://registry.npmjs.org/@colors/colors/-/colors-1.5.0.tgz" integrity "sha1-u1BFecHK6SPmV2pPXaQ9Jfl729k= sha512-ooWCrlZP11i8GImSjTHYHLkvFDP48nS4+204nGb1RiX/WXYHmJA2III9/e2DWVabCESdW7hBAEzHRqUn9OUVvQ==" -"@datadog/libdatadog@^0.2.2": - version "0.2.2" - resolved "https://registry.yarnpkg.com/@datadog/libdatadog/-/libdatadog-0.2.2.tgz#ac02c76ac9a38250dca740727c7cdf00244ce3d3" - integrity sha512-rTWo96mEPTY5UbtGoFj8/wY0uKSViJhsPg/Z6aoFWBFXQ8b45Ix2e/yvf92AAwrhG+gPLTxEqTXh3kef2dP8Ow== +"@datadog/libdatadog@^0.3.0": + version "0.3.0" + resolved "https://registry.yarnpkg.com/@datadog/libdatadog/-/libdatadog-0.3.0.tgz#2fc1e2695872840bc8c356f66acf675da428d6f0" + integrity sha512-TbP8+WyXfh285T17FnLeLUOPl4SbkRYMqKgcmknID2mXHNrbt5XJgW9bnDgsrrtu31Q7FjWWw2WolgRLWyzLRA== "@datadog/native-appsec@8.3.0": version "8.3.0" From 7d53c2674615b2d6009229f5cef2fc4eecf2d7b7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Juan=20Antonio=20Fern=C3=A1ndez=20de=20Alba?= Date: Wed, 18 Dec 2024 11:26:25 +0100 Subject: [PATCH 14/36] =?UTF-8?q?[test=20optimization]=20[SDTEST-1332]?= =?UTF-8?q?=C2=A0Fetch=20`di=5Fenabled`=20flag=20(#5006)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- integration-tests/cucumber/cucumber.spec.js | 71 ++++++++++++---- integration-tests/jest/jest.spec.js | 77 +++++++++++------ integration-tests/mocha/mocha.spec.js | 82 +++++++++++++------ integration-tests/vitest/vitest.spec.js | 71 +++++++++++++--- packages/datadog-instrumentations/src/jest.js | 9 +- .../datadog-instrumentations/src/vitest.js | 24 +++++- packages/datadog-plugin-cucumber/src/index.js | 2 +- packages/datadog-plugin-jest/src/index.js | 5 +- packages/datadog-plugin-mocha/src/index.js | 2 +- packages/datadog-plugin-vitest/src/index.js | 4 +- .../exporters/ci-visibility-exporter.js | 6 +- .../requests/get-library-configuration.js | 6 +- 12 files changed, 272 insertions(+), 87 deletions(-) diff --git a/integration-tests/cucumber/cucumber.spec.js b/integration-tests/cucumber/cucumber.spec.js index 8f21b3a688f..f7925210a87 100644 --- a/integration-tests/cucumber/cucumber.spec.js +++ b/integration-tests/cucumber/cucumber.spec.js @@ -1544,6 +1544,11 @@ versions.forEach(version => { // Dynamic instrumentation only supported from >=8.0.0 context('dynamic instrumentation', () => { it('does not activate if DD_TEST_DYNAMIC_INSTRUMENTATION_ENABLED is not set', (done) => { + receiver.setSettings({ + flaky_test_retries_enabled: true, + di_enabled: true + }) + const eventsPromise = receiver .gatherPayloadsMaxTimeout(({ url }) => url.endsWith('/api/v2/citestcycle'), (payloads) => { const events = payloads.flatMap(({ payload }) => payload.events) @@ -1582,16 +1587,59 @@ versions.forEach(version => { }) }) + it('does not activate dynamic instrumentation if remote settings are disabled', (done) => { + receiver.setSettings({ + flaky_test_retries_enabled: true, + di_enabled: false + }) + + const eventsPromise = receiver + .gatherPayloadsMaxTimeout(({ url }) => url.endsWith('/api/v2/citestcycle'), (payloads) => { + const events = payloads.flatMap(({ payload }) => payload.events) + + const tests = events.filter(event => event.type === 'test').map(event => event.content) + const retriedTests = tests.filter(test => test.meta[TEST_IS_RETRY] === 'true') + + assert.equal(retriedTests.length, 1) + const [retriedTest] = retriedTests + + assert.notProperty(retriedTest.meta, DI_ERROR_DEBUG_INFO_CAPTURED) + assert.notProperty(retriedTest.meta, DI_DEBUG_ERROR_FILE) + assert.notProperty(retriedTest.metrics, DI_DEBUG_ERROR_LINE) + assert.notProperty(retriedTest.meta, DI_DEBUG_ERROR_SNAPSHOT_ID) + }) + const logsPromise = receiver + .gatherPayloadsMaxTimeout(({ url }) => url === logsEndpoint, (payloads) => { + if (payloads.length > 0) { + throw new Error('Unexpected logs') + } + }, 5000) + + childProcess = exec( + './node_modules/.bin/cucumber-js ci-visibility/features-di/test-hit-breakpoint.feature --retry 1', + { + cwd, + env: { + ...envVars, + DD_TEST_DYNAMIC_INSTRUMENTATION_ENABLED: 'true' + }, + stdio: 'pipe' + } + ) + + childProcess.on('exit', () => { + Promise.all([eventsPromise, logsPromise]).then(() => { + done() + }).catch(done) + }) + }) + it('runs retries with dynamic instrumentation', (done) => { receiver.setSettings({ - itr_enabled: false, - code_coverage: false, - tests_skipping: false, - early_flake_detection: { - enabled: false - }, - flaky_test_retries_enabled: false + flaky_test_retries_enabled: true, + di_enabled: true }) + let snapshotIdByTest, snapshotIdByLog let spanIdByTest, spanIdByLog, traceIdByTest, traceIdByLog @@ -1671,13 +1719,8 @@ versions.forEach(version => { it('does not crash if the retry does not hit the breakpoint', (done) => { receiver.setSettings({ - itr_enabled: false, - code_coverage: false, - tests_skipping: false, - early_flake_detection: { - enabled: false - }, - flaky_test_retries_enabled: false + flaky_test_retries_enabled: true, + di_enabled: true }) const eventsPromise = receiver diff --git a/integration-tests/jest/jest.spec.js b/integration-tests/jest/jest.spec.js index 7bdf04ec071..d8d9f8231a6 100644 --- a/integration-tests/jest/jest.spec.js +++ b/integration-tests/jest/jest.spec.js @@ -2413,14 +2413,8 @@ describe('jest CommonJS', () => { context('dynamic instrumentation', () => { it('does not activate dynamic instrumentation if DD_TEST_DYNAMIC_INSTRUMENTATION_ENABLED is not set', (done) => { receiver.setSettings({ - itr_enabled: false, - code_coverage: false, - tests_skipping: false, - flaky_test_retries_enabled: false, - early_flake_detection: { - enabled: false - } - // di_enabled: true // TODO + flaky_test_retries_enabled: true, + di_enabled: true }) const eventsPromise = receiver .gatherPayloadsMaxTimeout(({ url }) => url.endsWith('/api/v2/citestcycle'), (payloads) => { @@ -2463,16 +2457,57 @@ describe('jest CommonJS', () => { }) }) - it('runs retries with dynamic instrumentation', (done) => { + it('does not activate dynamic instrumentation if remote settings are disabled', (done) => { receiver.setSettings({ - itr_enabled: false, - code_coverage: false, - tests_skipping: false, - flaky_test_retries_enabled: false, - early_flake_detection: { - enabled: false + flaky_test_retries_enabled: true, + di_enabled: false + }) + const eventsPromise = receiver + .gatherPayloadsMaxTimeout(({ url }) => url.endsWith('/api/v2/citestcycle'), (payloads) => { + const events = payloads.flatMap(({ payload }) => payload.events) + + const tests = events.filter(event => event.type === 'test').map(event => event.content) + const retriedTests = tests.filter(test => test.meta[TEST_IS_RETRY] === 'true') + + assert.equal(retriedTests.length, 1) + const [retriedTest] = retriedTests + + assert.notProperty(retriedTest.meta, DI_ERROR_DEBUG_INFO_CAPTURED) + assert.notProperty(retriedTest.meta, DI_DEBUG_ERROR_FILE) + assert.notProperty(retriedTest.metrics, DI_DEBUG_ERROR_LINE) + assert.notProperty(retriedTest.meta, DI_DEBUG_ERROR_SNAPSHOT_ID) + }) + const logsPromise = receiver + .gatherPayloadsMaxTimeout(({ url }) => url.endsWith('/api/v2/logs'), (payloads) => { + if (payloads.length > 0) { + throw new Error('Unexpected logs') + } + }, 5000) + + childProcess = exec(runTestsWithCoverageCommand, + { + cwd, + env: { + ...getCiVisAgentlessConfig(receiver.port), + TESTS_TO_RUN: 'dynamic-instrumentation/test-hit-breakpoint', + DD_TEST_DYNAMIC_INSTRUMENTATION_ENABLED: 'true' + }, + stdio: 'inherit' } - // di_enabled: true // TODO + ) + + childProcess.on('exit', (code) => { + Promise.all([eventsPromise, logsPromise]).then(() => { + assert.equal(code, 0) + done() + }).catch(done) + }) + }) + + it('runs retries with dynamic instrumentation', (done) => { + receiver.setSettings({ + flaky_test_retries_enabled: true, + di_enabled: true }) let snapshotIdByTest, snapshotIdByLog let spanIdByTest, spanIdByLog, traceIdByTest, traceIdByLog @@ -2555,14 +2590,8 @@ describe('jest CommonJS', () => { it('does not crash if the retry does not hit the breakpoint', (done) => { receiver.setSettings({ - itr_enabled: false, - code_coverage: false, - tests_skipping: false, - flaky_test_retries_enabled: false, - early_flake_detection: { - enabled: false - } - // di_enabled: true // TODO + flaky_test_retries_enabled: true, + di_enabled: true }) const eventsPromise = receiver .gatherPayloadsMaxTimeout(({ url }) => url.endsWith('/api/v2/citestcycle'), (payloads) => { diff --git a/integration-tests/mocha/mocha.spec.js b/integration-tests/mocha/mocha.spec.js index f777792c44b..d6d13673485 100644 --- a/integration-tests/mocha/mocha.spec.js +++ b/integration-tests/mocha/mocha.spec.js @@ -2152,14 +2152,8 @@ describe('mocha CommonJS', function () { context('dynamic instrumentation', () => { it('does not activate dynamic instrumentation if DD_TEST_DYNAMIC_INSTRUMENTATION_ENABLED is not set', (done) => { receiver.setSettings({ - itr_enabled: false, - code_coverage: false, - tests_skipping: false, - flaky_test_retries_enabled: false, - early_flake_detection: { - enabled: false - } - // di_enabled: true // TODO + flaky_test_retries_enabled: true, + di_enabled: true }) const eventsPromise = receiver @@ -2207,16 +2201,62 @@ describe('mocha CommonJS', function () { }) }) - it('runs retries with dynamic instrumentation', (done) => { + it('does not activate dynamic instrumentation if remote settings are disabled', (done) => { receiver.setSettings({ - itr_enabled: false, - code_coverage: false, - tests_skipping: false, - flaky_test_retries_enabled: false, - early_flake_detection: { - enabled: false + flaky_test_retries_enabled: true, + di_enabled: false + }) + + const eventsPromise = receiver + .gatherPayloadsMaxTimeout(({ url }) => url.endsWith('/api/v2/citestcycle'), (payloads) => { + const events = payloads.flatMap(({ payload }) => payload.events) + + const tests = events.filter(event => event.type === 'test').map(event => event.content) + const retriedTests = tests.filter(test => test.meta[TEST_IS_RETRY] === 'true') + + assert.equal(retriedTests.length, 1) + const [retriedTest] = retriedTests + + assert.notProperty(retriedTest.meta, DI_ERROR_DEBUG_INFO_CAPTURED) + assert.notProperty(retriedTest.meta, DI_DEBUG_ERROR_FILE) + assert.notProperty(retriedTest.metrics, DI_DEBUG_ERROR_LINE) + assert.notProperty(retriedTest.meta, DI_DEBUG_ERROR_SNAPSHOT_ID) + }) + + const logsPromise = receiver + .gatherPayloadsMaxTimeout(({ url }) => url.endsWith('/api/v2/logs'), (payloads) => { + if (payloads.length > 0) { + throw new Error('Unexpected logs') + } + }, 5000) + + childProcess = exec( + runTestsWithCoverageCommand, + { + cwd, + env: { + ...getCiVisAgentlessConfig(receiver.port), + TESTS_TO_RUN: JSON.stringify([ + './dynamic-instrumentation/test-hit-breakpoint' + ]), + DD_TEST_DYNAMIC_INSTRUMENTATION_ENABLED: 'true' + }, + stdio: 'inherit' } - // di_enabled: true // TODO + ) + + childProcess.on('exit', (code) => { + Promise.all([eventsPromise, logsPromise]).then(() => { + assert.equal(code, 0) + done() + }).catch(done) + }) + }) + + it('runs retries with dynamic instrumentation', (done) => { + receiver.setSettings({ + flaky_test_retries_enabled: true, + di_enabled: true }) let snapshotIdByTest, snapshotIdByLog @@ -2304,14 +2344,8 @@ describe('mocha CommonJS', function () { it('does not crash if the retry does not hit the breakpoint', (done) => { receiver.setSettings({ - itr_enabled: false, - code_coverage: false, - tests_skipping: false, - flaky_test_retries_enabled: false, - early_flake_detection: { - enabled: false - } - // di_enabled: true // TODO + flaky_test_retries_enabled: true, + di_enabled: true }) const eventsPromise = receiver diff --git a/integration-tests/vitest/vitest.spec.js b/integration-tests/vitest/vitest.spec.js index 0489db04b44..2007baefd52 100644 --- a/integration-tests/vitest/vitest.spec.js +++ b/integration-tests/vitest/vitest.spec.js @@ -906,10 +906,8 @@ versions.forEach((version) => { context('dynamic instrumentation', () => { it('does not activate it if DD_TEST_DYNAMIC_INSTRUMENTATION_ENABLED is not set', (done) => { receiver.setSettings({ - itr_enabled: false, - code_coverage: false, - tests_skipping: false, - flaky_test_retries_enabled: false + flaky_test_retries_enabled: true, + di_enabled: true }) const eventsPromise = receiver @@ -955,16 +953,60 @@ versions.forEach((version) => { }) }) - it('runs retries with dynamic instrumentation', (done) => { + it('does not activate dynamic instrumentation if remote settings are disabled', (done) => { receiver.setSettings({ - itr_enabled: false, - code_coverage: false, - tests_skipping: false, - flaky_test_retries_enabled: false, - early_flake_detection: { - enabled: false + flaky_test_retries_enabled: true, + di_enabled: false + }) + + const eventsPromise = receiver + .gatherPayloadsMaxTimeout(({ url }) => url.endsWith('/api/v2/citestcycle'), (payloads) => { + const events = payloads.flatMap(({ payload }) => payload.events) + + const tests = events.filter(event => event.type === 'test').map(event => event.content) + const retriedTests = tests.filter(test => test.meta[TEST_IS_RETRY] === 'true') + + assert.equal(retriedTests.length, 1) + const [retriedTest] = retriedTests + + assert.notProperty(retriedTest.meta, DI_ERROR_DEBUG_INFO_CAPTURED) + assert.notProperty(retriedTest.meta, DI_DEBUG_ERROR_FILE) + assert.notProperty(retriedTest.metrics, DI_DEBUG_ERROR_LINE) + assert.notProperty(retriedTest.meta, DI_DEBUG_ERROR_SNAPSHOT_ID) + }) + + const logsPromise = receiver + .gatherPayloadsMaxTimeout(({ url }) => url.endsWith('/api/v2/logs'), (payloads) => { + if (payloads.length > 0) { + throw new Error('Unexpected logs') + } + }, 5000) + + childProcess = exec( + './node_modules/.bin/vitest run --retry=1', + { + cwd, + env: { + ...getCiVisAgentlessConfig(receiver.port), + TEST_DIR: 'ci-visibility/vitest-tests/dynamic-instrumentation*', + NODE_OPTIONS: '--import dd-trace/register.js -r dd-trace/ci/init', + DD_TEST_DYNAMIC_INSTRUMENTATION_ENABLED: '1' + }, + stdio: 'pipe' } - // di_enabled: true // TODO + ) + + childProcess.on('exit', () => { + Promise.all([eventsPromise, logsPromise]).then(() => { + done() + }).catch(done) + }) + }) + + it('runs retries with dynamic instrumentation', (done) => { + receiver.setSettings({ + flaky_test_retries_enabled: true, + di_enabled: true }) let snapshotIdByTest, snapshotIdByLog @@ -1050,6 +1092,11 @@ versions.forEach((version) => { }) it('does not crash if the retry does not hit the breakpoint', (done) => { + receiver.setSettings({ + flaky_test_retries_enabled: true, + di_enabled: true + }) + const eventsPromise = receiver .gatherPayloadsMaxTimeout(({ url }) => url.endsWith('/api/v2/citestcycle'), (payloads) => { const events = payloads.flatMap(({ payload }) => payload.events) diff --git a/packages/datadog-instrumentations/src/jest.js b/packages/datadog-instrumentations/src/jest.js index fd13d2fc805..2d27fdc0acb 100644 --- a/packages/datadog-instrumentations/src/jest.js +++ b/packages/datadog-instrumentations/src/jest.js @@ -133,6 +133,7 @@ function getWrappedEnvironment (BaseEnvironment, jestVersion) { this.isEarlyFlakeDetectionEnabled = this.testEnvironmentOptions._ddIsEarlyFlakeDetectionEnabled this.isFlakyTestRetriesEnabled = this.testEnvironmentOptions._ddIsFlakyTestRetriesEnabled this.flakyTestRetriesCount = this.testEnvironmentOptions._ddFlakyTestRetriesCount + this.isDiEnabled = this.testEnvironmentOptions._ddIsDiEnabled if (this.isEarlyFlakeDetectionEnabled) { const hasKnownTests = !!knownTests.jest @@ -284,7 +285,12 @@ function getWrappedEnvironment (BaseEnvironment, jestVersion) { const willBeRetried = numRetries > 0 && numTestExecutions - 1 < numRetries const error = formatJestError(event.test.errors[0]) - testErrCh.publish({ error, willBeRetried, probe, numTestExecutions }) + testErrCh.publish({ + error, + willBeRetried, + probe, + isDiEnabled: this.isDiEnabled + }) } testRunFinishCh.publish({ status, @@ -786,6 +792,7 @@ addHook({ _ddRepositoryRoot, _ddIsFlakyTestRetriesEnabled, _ddFlakyTestRetriesCount, + _ddIsDiEnabled, ...restOfTestEnvironmentOptions } = testEnvironmentOptions diff --git a/packages/datadog-instrumentations/src/vitest.js b/packages/datadog-instrumentations/src/vitest.js index 6e2d1d6e048..de7c6d2dc30 100644 --- a/packages/datadog-instrumentations/src/vitest.js +++ b/packages/datadog-instrumentations/src/vitest.js @@ -117,6 +117,7 @@ function getSortWrapper (sort) { let isEarlyFlakeDetectionEnabled = false let earlyFlakeDetectionNumRetries = 0 let isEarlyFlakeDetectionFaulty = false + let isDiEnabled = false let knownTests = {} try { @@ -126,10 +127,12 @@ function getSortWrapper (sort) { flakyTestRetriesCount = libraryConfig.flakyTestRetriesCount isEarlyFlakeDetectionEnabled = libraryConfig.isEarlyFlakeDetectionEnabled earlyFlakeDetectionNumRetries = libraryConfig.earlyFlakeDetectionNumRetries + isDiEnabled = libraryConfig.isDiEnabled } } catch (e) { isFlakyTestRetriesEnabled = false isEarlyFlakeDetectionEnabled = false + isDiEnabled = false } if (isFlakyTestRetriesEnabled && !this.ctx.config.retry && flakyTestRetriesCount > 0) { @@ -169,6 +172,15 @@ function getSortWrapper (sort) { } } + if (isDiEnabled) { + try { + const workspaceProject = this.ctx.getCoreWorkspaceProject() + workspaceProject._provided._ddIsDiEnabled = isDiEnabled + } catch (e) { + log.warn('Could not send Dynamic Instrumentation configuration to workers.') + } + } + let testCodeCoverageLinesTotal if (this.ctx.coverageProvider?.generateCoverage) { @@ -298,13 +310,16 @@ addHook({ const testName = getTestName(task) let isNew = false let isEarlyFlakeDetectionEnabled = false + let isDiEnabled = false try { const { - _ddIsEarlyFlakeDetectionEnabled + _ddIsEarlyFlakeDetectionEnabled, + _ddIsDiEnabled } = globalThis.__vitest_worker__.providedContext isEarlyFlakeDetectionEnabled = _ddIsEarlyFlakeDetectionEnabled + isDiEnabled = _ddIsDiEnabled if (isEarlyFlakeDetectionEnabled) { isNew = newTasks.has(task) @@ -321,7 +336,12 @@ addHook({ const testError = task.result?.errors?.[0] if (asyncResource) { asyncResource.runInAsyncScope(() => { - testErrorCh.publish({ error: testError, willBeRetried: true, probe }) + testErrorCh.publish({ + error: testError, + willBeRetried: true, + probe, + isDiEnabled + }) }) // We wait for the probe to be set if (probe.setProbePromise) { diff --git a/packages/datadog-plugin-cucumber/src/index.js b/packages/datadog-plugin-cucumber/src/index.js index e674131d639..1c4403b7ce6 100644 --- a/packages/datadog-plugin-cucumber/src/index.js +++ b/packages/datadog-plugin-cucumber/src/index.js @@ -255,7 +255,7 @@ class CucumberPlugin extends CiPlugin { span.setTag(TEST_IS_RETRY, 'true') } span.setTag('error', error) - if (this.di && error) { + if (this.di && error && this.libraryConfig?.isDiEnabled) { const testName = span.context()._tags[TEST_NAME] const debuggerParameters = this.addDiProbe(error) debuggerParameterPerTest.set(testName, debuggerParameters) diff --git a/packages/datadog-plugin-jest/src/index.js b/packages/datadog-plugin-jest/src/index.js index f2494da264d..0287f837653 100644 --- a/packages/datadog-plugin-jest/src/index.js +++ b/packages/datadog-plugin-jest/src/index.js @@ -161,6 +161,7 @@ class JestPlugin extends CiPlugin { config._ddRepositoryRoot = this.repositoryRoot config._ddIsFlakyTestRetriesEnabled = this.libraryConfig?.isFlakyTestRetriesEnabled ?? false config._ddFlakyTestRetriesCount = this.libraryConfig?.flakyTestRetriesCount + config._ddIsDiEnabled = this.libraryConfig?.isDiEnabled ?? false }) }) @@ -355,14 +356,14 @@ class JestPlugin extends CiPlugin { finishAllTraceSpans(span) }) - this.addSub('ci:jest:test:err', ({ error, willBeRetried, probe }) => { + this.addSub('ci:jest:test:err', ({ error, willBeRetried, probe, isDiEnabled }) => { if (error) { const store = storage.getStore() if (store && store.span) { const span = store.span span.setTag(TEST_STATUS, 'fail') span.setTag('error', error) - if (willBeRetried && this.di) { + if (willBeRetried && this.di && isDiEnabled) { // if we use numTestExecutions, we have to remove the breakpoint after each execution const testName = span.context()._tags[TEST_NAME] const debuggerParameters = this.addDiProbe(error, probe) diff --git a/packages/datadog-plugin-mocha/src/index.js b/packages/datadog-plugin-mocha/src/index.js index 302f52ccfb3..1b40b9c5a1c 100644 --- a/packages/datadog-plugin-mocha/src/index.js +++ b/packages/datadog-plugin-mocha/src/index.js @@ -294,7 +294,7 @@ class MochaPlugin extends CiPlugin { browserDriver: spanTags[TEST_BROWSER_DRIVER] } ) - if (willBeRetried && this.di) { + if (willBeRetried && this.di && this.libraryConfig?.isDiEnabled) { const testName = span.context()._tags[TEST_NAME] const debuggerParameters = this.addDiProbe(err) debuggerParameterPerTest.set(testName, debuggerParameters) diff --git a/packages/datadog-plugin-vitest/src/index.js b/packages/datadog-plugin-vitest/src/index.js index d0a2984ac74..ba2554bf9f9 100644 --- a/packages/datadog-plugin-vitest/src/index.js +++ b/packages/datadog-plugin-vitest/src/index.js @@ -137,12 +137,12 @@ class VitestPlugin extends CiPlugin { } }) - this.addSub('ci:vitest:test:error', ({ duration, error, willBeRetried, probe }) => { + this.addSub('ci:vitest:test:error', ({ duration, error, willBeRetried, probe, isDiEnabled }) => { const store = storage.getStore() const span = store?.span if (span) { - if (willBeRetried && this.di) { + if (willBeRetried && this.di && isDiEnabled) { const testName = span.context()._tags[TEST_NAME] const debuggerParameters = this.addDiProbe(error, probe) debuggerParameterPerTest.set(testName, debuggerParameters) diff --git a/packages/dd-trace/src/ci-visibility/exporters/ci-visibility-exporter.js b/packages/dd-trace/src/ci-visibility/exporters/ci-visibility-exporter.js index dde5955bc75..3ad1a11e027 100644 --- a/packages/dd-trace/src/ci-visibility/exporters/ci-visibility-exporter.js +++ b/packages/dd-trace/src/ci-visibility/exporters/ci-visibility-exporter.js @@ -196,7 +196,8 @@ class CiVisibilityExporter extends AgentInfoExporter { isEarlyFlakeDetectionEnabled, earlyFlakeDetectionNumRetries, earlyFlakeDetectionFaultyThreshold, - isFlakyTestRetriesEnabled + isFlakyTestRetriesEnabled, + isDiEnabled } = remoteConfiguration return { isCodeCoverageEnabled, @@ -207,7 +208,8 @@ class CiVisibilityExporter extends AgentInfoExporter { earlyFlakeDetectionNumRetries, earlyFlakeDetectionFaultyThreshold, isFlakyTestRetriesEnabled: isFlakyTestRetriesEnabled && this._config.isFlakyTestRetriesEnabled, - flakyTestRetriesCount: this._config.flakyTestRetriesCount + flakyTestRetriesCount: this._config.flakyTestRetriesCount, + isDiEnabled: isDiEnabled && this._config.isTestDynamicInstrumentationEnabled } } diff --git a/packages/dd-trace/src/ci-visibility/requests/get-library-configuration.js b/packages/dd-trace/src/ci-visibility/requests/get-library-configuration.js index 9a32efad05e..e39770dea82 100644 --- a/packages/dd-trace/src/ci-visibility/requests/get-library-configuration.js +++ b/packages/dd-trace/src/ci-visibility/requests/get-library-configuration.js @@ -92,7 +92,8 @@ function getLibraryConfiguration ({ itr_enabled: isItrEnabled, require_git: requireGit, early_flake_detection: earlyFlakeDetectionConfig, - flaky_test_retries_enabled: isFlakyTestRetriesEnabled + flaky_test_retries_enabled: isFlakyTestRetriesEnabled, + di_enabled: isDiEnabled } } } = JSON.parse(res) @@ -107,7 +108,8 @@ function getLibraryConfiguration ({ earlyFlakeDetectionConfig?.slow_test_retries?.['5s'] || DEFAULT_EARLY_FLAKE_DETECTION_NUM_RETRIES, earlyFlakeDetectionFaultyThreshold: earlyFlakeDetectionConfig?.faulty_session_threshold ?? DEFAULT_EARLY_FLAKE_DETECTION_ERROR_THRESHOLD, - isFlakyTestRetriesEnabled + isFlakyTestRetriesEnabled, + isDiEnabled: isDiEnabled && isFlakyTestRetriesEnabled } log.debug(() => `Remote settings: ${JSON.stringify(settings)}`) From 50619f7408f27056b0153a1c16712e1e40cdd90f Mon Sep 17 00:00:00 2001 From: Thomas Watson Date: Wed, 18 Dec 2024 13:45:20 +0100 Subject: [PATCH 15/36] [DI] Associate probe results with active span (#5035) --- integration-tests/debugger/basic.spec.js | 29 ++++++++++++++++++- .../src/debugger/devtools_client/index.js | 27 ++++++++++++++++- .../src/debugger/devtools_client/send.js | 3 +- 3 files changed, 56 insertions(+), 3 deletions(-) diff --git a/integration-tests/debugger/basic.spec.js b/integration-tests/debugger/basic.spec.js index 57c0c4a67a8..275e2765270 100644 --- a/integration-tests/debugger/basic.spec.js +++ b/integration-tests/debugger/basic.spec.js @@ -235,8 +235,18 @@ describe('Dynamic Instrumentation', function () { describe('input messages', function () { it('should capture and send expected payload when a log line probe is triggered', function (done) { + let traceId, spanId, dd + t.triggerBreakpoint() + t.agent.on('message', ({ payload }) => { + const span = payload.find((arr) => arr[0].name === 'fastify.request')[0] + traceId = span.trace_id.toString() + spanId = span.span_id.toString() + + assertDD() + }) + t.agent.on('debugger-input', ({ payload }) => { const expected = { ddsource: 'dd_debugger', @@ -260,7 +270,17 @@ describe('Dynamic Instrumentation', function () { } assertObjectContains(payload, expected) + assert.match(payload.logger.thread_id, /^pid:\d+$/) + + assert.isObject(payload.dd) + assert.hasAllKeys(payload.dd, ['trace_id', 'span_id']) + assert.typeOf(payload.dd.trace_id, 'string') + assert.typeOf(payload.dd.span_id, 'string') + assert.isAbove(payload.dd.trace_id.length, 0) + assert.isAbove(payload.dd.span_id.length, 0) + dd = payload.dd + assertUUID(payload['debugger.snapshot'].id) assert.isNumber(payload['debugger.snapshot'].timestamp) assert.isTrue(payload['debugger.snapshot'].timestamp > Date.now() - 1000 * 60) @@ -283,10 +303,17 @@ describe('Dynamic Instrumentation', function () { assert.strictEqual(topFrame.lineNumber, t.breakpoint.line) assert.strictEqual(topFrame.columnNumber, 3) - done() + assertDD() }) t.agent.addRemoteConfig(t.rcConfig) + + function assertDD () { + if (!traceId || !spanId || !dd) return + assert.strictEqual(dd.trace_id, traceId) + assert.strictEqual(dd.span_id, spanId) + done() + } }) it('should respond with updated message if probe message is updated', function (done) { diff --git a/packages/dd-trace/src/debugger/devtools_client/index.js b/packages/dd-trace/src/debugger/devtools_client/index.js index 7ca828786ac..9634003bf61 100644 --- a/packages/dd-trace/src/debugger/devtools_client/index.js +++ b/packages/dd-trace/src/debugger/devtools_client/index.js @@ -13,6 +13,12 @@ const { version } = require('../../../../../package.json') require('./remote_config') +// Expression to run on a call frame of the paused thread to get its active trace and span id. +const expression = ` + const context = global.require('dd-trace').scope().active()?.context(); + ({ trace_id: context?.toTraceId(), span_id: context?.toSpanId() }) +` + // There doesn't seem to be an official standard for the content of these fields, so we're just populating them with // something that should be useful to a Node.js developer. const threadId = parentThreadId === 0 ? `pid:${process.pid}` : `pid:${process.pid};tid:${parentThreadId}` @@ -59,6 +65,7 @@ session.on('Debugger.paused', async ({ params }) => { } const timestamp = Date.now() + const dd = await getDD(params.callFrames[0].callFrameId) let processLocalState if (captureSnapshotForProbe !== null) { @@ -122,7 +129,7 @@ session.on('Debugger.paused', async ({ params }) => { } // TODO: Process template (DEBUG-2628) - send(probe.template, logger, snapshot, (err) => { + send(probe.template, logger, dd, snapshot, (err) => { if (err) log.error('Debugger error', err) else ackEmitting(probe) }) @@ -132,3 +139,21 @@ session.on('Debugger.paused', async ({ params }) => { function highestOrUndefined (num, max) { return num === undefined ? max : Math.max(num, max ?? 0) } + +async function getDD (callFrameId) { + const { result } = await session.post('Debugger.evaluateOnCallFrame', { + callFrameId, + expression, + returnByValue: true, + includeCommandLineAPI: true + }) + + if (result?.value?.trace_id === undefined) { + if (result?.subtype === 'error') { + log.error('[debugger:devtools_client] Error getting trace/span id:', result.description) + } + return + } + + return result.value +} diff --git a/packages/dd-trace/src/debugger/devtools_client/send.js b/packages/dd-trace/src/debugger/devtools_client/send.js index f2ba5befd46..9d607b1ad1c 100644 --- a/packages/dd-trace/src/debugger/devtools_client/send.js +++ b/packages/dd-trace/src/debugger/devtools_client/send.js @@ -22,7 +22,7 @@ const ddtags = [ const path = `/debugger/v1/input?${stringify({ ddtags })}` -function send (message, logger, snapshot, cb) { +function send (message, logger, dd, snapshot, cb) { const opts = { method: 'POST', url: config.url, @@ -36,6 +36,7 @@ function send (message, logger, snapshot, cb) { service, message, logger, + dd, 'debugger.snapshot': snapshot } From 28bca839ec6b600f74aecab1f549e2980fd09763 Mon Sep 17 00:00:00 2001 From: Thomas Watson Date: Wed, 18 Dec 2024 14:21:44 +0100 Subject: [PATCH 16/36] [DI] Improve trace/span-id probe results tests (#5036) Add test that checks if everything works as expected even if tracing is disabled. --- integration-tests/debugger/basic.spec.js | 38 +++++++++++++++++------- integration-tests/debugger/utils.js | 5 ++-- 2 files changed, 31 insertions(+), 12 deletions(-) diff --git a/integration-tests/debugger/basic.spec.js b/integration-tests/debugger/basic.spec.js index 275e2765270..6db68d0607d 100644 --- a/integration-tests/debugger/basic.spec.js +++ b/integration-tests/debugger/basic.spec.js @@ -9,7 +9,17 @@ const { ACKNOWLEDGED, ERROR } = require('../../packages/dd-trace/src/appsec/remo const { version } = require('../../package.json') describe('Dynamic Instrumentation', function () { - const t = setup() + describe('DD_TRACING_ENABLED=true', function () { + testWithTracingEnabled() + }) + + describe('DD_TRACING_ENABLED=false', function () { + testWithTracingEnabled(false) + }) +}) + +function testWithTracingEnabled (tracingEnabled = true) { + const t = setup({ DD_TRACING_ENABLED: tracingEnabled }) it('base case: target app should work as expected if no test probe has been added', async function () { const response = await t.axios.get(t.breakpoint.url) @@ -273,13 +283,17 @@ describe('Dynamic Instrumentation', function () { assert.match(payload.logger.thread_id, /^pid:\d+$/) - assert.isObject(payload.dd) - assert.hasAllKeys(payload.dd, ['trace_id', 'span_id']) - assert.typeOf(payload.dd.trace_id, 'string') - assert.typeOf(payload.dd.span_id, 'string') - assert.isAbove(payload.dd.trace_id.length, 0) - assert.isAbove(payload.dd.span_id.length, 0) - dd = payload.dd + if (tracingEnabled) { + assert.isObject(payload.dd) + assert.hasAllKeys(payload.dd, ['trace_id', 'span_id']) + assert.typeOf(payload.dd.trace_id, 'string') + assert.typeOf(payload.dd.span_id, 'string') + assert.isAbove(payload.dd.trace_id.length, 0) + assert.isAbove(payload.dd.span_id.length, 0) + dd = payload.dd + } else { + assert.doesNotHaveAnyKeys(payload, ['dd']) + } assertUUID(payload['debugger.snapshot'].id) assert.isNumber(payload['debugger.snapshot'].timestamp) @@ -303,7 +317,11 @@ describe('Dynamic Instrumentation', function () { assert.strictEqual(topFrame.lineNumber, t.breakpoint.line) assert.strictEqual(topFrame.columnNumber, 3) - assertDD() + if (tracingEnabled) { + assertDD() + } else { + done() + } }) t.agent.addRemoteConfig(t.rcConfig) @@ -501,4 +519,4 @@ describe('Dynamic Instrumentation', function () { t.agent.addRemoteConfig(t.rcConfig) }) }) -}) +} diff --git a/integration-tests/debugger/utils.js b/integration-tests/debugger/utils.js index bca970dea87..b260e5eabe5 100644 --- a/integration-tests/debugger/utils.js +++ b/integration-tests/debugger/utils.js @@ -18,7 +18,7 @@ module.exports = { setup } -function setup () { +function setup (env) { let sandbox, cwd, appPort const breakpoints = getBreakpointInfo(1) // `1` to disregard the `setup` function const t = { @@ -91,7 +91,8 @@ function setup () { DD_DYNAMIC_INSTRUMENTATION_ENABLED: true, DD_TRACE_AGENT_PORT: t.agent.port, DD_TRACE_DEBUG: process.env.DD_TRACE_DEBUG, // inherit to make debugging the sandbox easier - DD_REMOTE_CONFIG_POLL_INTERVAL_SECONDS: pollInterval + DD_REMOTE_CONFIG_POLL_INTERVAL_SECONDS: pollInterval, + ...env } }) t.axios = Axios.create({ From 275bb7ef9dd5a755c88e826cd5bd6f14346fe0ae Mon Sep 17 00:00:00 2001 From: Ugaitz Urien Date: Wed, 18 Dec 2024 15:14:34 +0100 Subject: [PATCH 17/36] Support tainted strings coming from database for SQLi, SSTi and Code injection (#4904) --- docs/test.ts | 2 + index.d.ts | 22 +- packages/datadog-instrumentations/src/pg.js | 10 +- .../datadog-instrumentations/src/sequelize.js | 13 +- .../iast/analyzers/code-injection-analyzer.js | 4 + .../iast/analyzers/injection-analyzer.js | 13 +- .../iast/analyzers/sql-injection-analyzer.js | 4 + .../analyzers/template-injection-analyzer.js | 4 + .../dd-trace/src/appsec/iast/iast-plugin.js | 3 +- .../src/appsec/iast/taint-tracking/index.js | 6 +- .../src/appsec/iast/taint-tracking/plugin.js | 49 +- .../iast/taint-tracking/source-types.js | 3 +- packages/dd-trace/src/config.js | 4 + ...-injection-analyzer.express.plugin.spec.js | 18 +- .../analyzers/ldap-injection-analyzer.spec.js | 15 +- .../analyzers/path-traversal-analyzer.spec.js | 81 +- .../analyzers/sql-injection-analyzer.spec.js | 15 +- ...jection-analyzer.handlebars.plugin.spec.js | 26 + ...late-injection-analyzer.pug.plugin.spec.js | 33 + .../appsec/iast/taint-tracking/plugin.spec.js | 276 +++- .../sources/sql_row.pg.plugin.spec.js | 113 ++ .../sources/sql_row.sequelize.plugin.spec.js | 106 ++ packages/dd-trace/test/appsec/iast/utils.js | 4 +- packages/dd-trace/test/config.spec.js | 13 + .../fixtures/telemetry/config_norm_rules.json | 1469 +++++++++-------- packages/dd-trace/test/plugins/externals.json | 4 + 26 files changed, 1537 insertions(+), 773 deletions(-) create mode 100644 packages/dd-trace/test/appsec/iast/taint-tracking/sources/sql_row.pg.plugin.spec.js create mode 100644 packages/dd-trace/test/appsec/iast/taint-tracking/sources/sql_row.sequelize.plugin.spec.js diff --git a/docs/test.ts b/docs/test.ts index ce34a23d62b..2c2cbea332e 100644 --- a/docs/test.ts +++ b/docs/test.ts @@ -131,6 +131,7 @@ tracer.init({ requestSampling: 50, maxConcurrentRequests: 4, maxContextOperations: 30, + dbRowsToTaint: 12, deduplicationEnabled: true, redactionEnabled: true, redactionNamePattern: 'password', @@ -147,6 +148,7 @@ tracer.init({ requestSampling: 50, maxConcurrentRequests: 4, maxContextOperations: 30, + dbRowsToTaint: 6, deduplicationEnabled: true, redactionEnabled: true, redactionNamePattern: 'password', diff --git a/index.d.ts b/index.d.ts index a41b4aee410..8984d02f81a 100644 --- a/index.d.ts +++ b/index.d.ts @@ -764,7 +764,7 @@ declare namespace tracer { */ maxDepth?: number } - + /** * Configuration enabling LLM Observability. Enablement is superceded by the DD_LLMOBS_ENABLED environment variable. */ @@ -2203,6 +2203,12 @@ declare namespace tracer { */ cookieFilterPattern?: string, + /** + * Defines the number of rows to taint in data coming from databases + * @default 1 + */ + dbRowsToTaint?: number, + /** * Whether to enable vulnerability deduplication */ @@ -2247,7 +2253,7 @@ declare namespace tracer { * Disable LLM Observability tracing. */ disable (): void, - + /** * Instruments a function by automatically creating a span activated on its * scope. @@ -2289,10 +2295,10 @@ declare namespace tracer { /** * Decorate a function in a javascript runtime that supports function decorators. * Note that this is **not** supported in the Node.js runtime, but is in TypeScript. - * + * * In TypeScript, this decorator is only supported in contexts where general TypeScript * function decorators are supported. - * + * * @param options Optional LLM Observability span options. */ decorate (options: llmobs.LLMObsNamelessSpanOptions): any @@ -2309,7 +2315,7 @@ declare namespace tracer { /** * Sets inputs, outputs, tags, metadata, and metrics as provided for a given LLM Observability span. * Note that with the exception of tags, this method will override any existing values for the provided fields. - * + * * For example: * ```javascript * llmobs.trace({ kind: 'llm', name: 'myLLM', modelName: 'gpt-4o', modelProvider: 'openai' }, () => { @@ -2322,7 +2328,7 @@ declare namespace tracer { * }) * }) * ``` - * + * * @param span The span to annotate (defaults to the current LLM Observability span if not provided) * @param options An object containing the inputs, outputs, tags, metadata, and metrics to set on the span. */ @@ -2498,14 +2504,14 @@ declare namespace tracer { * LLM Observability span kind. One of `agent`, `workflow`, `task`, `tool`, `retrieval`, `embedding`, or `llm`. */ kind: llmobs.spanKind, - + /** * The ID of the underlying user session. Required for tracking sessions. */ sessionId?: string, /** - * The name of the ML application that the agent is orchestrating. + * The name of the ML application that the agent is orchestrating. * If not provided, the default value will be set to mlApp provided during initalization, or `DD_LLMOBS_ML_APP`. */ mlApp?: string, diff --git a/packages/datadog-instrumentations/src/pg.js b/packages/datadog-instrumentations/src/pg.js index 6c3d621ad00..331557cd239 100644 --- a/packages/datadog-instrumentations/src/pg.js +++ b/packages/datadog-instrumentations/src/pg.js @@ -62,11 +62,11 @@ function wrapQuery (query) { abortController }) - const finish = asyncResource.bind(function (error) { + const finish = asyncResource.bind(function (error, res) { if (error) { errorCh.publish(error) } - finishCh.publish() + finishCh.publish({ result: res?.rows }) }) if (abortController.signal.aborted) { @@ -119,15 +119,15 @@ function wrapQuery (query) { if (newQuery.callback) { const originalCallback = callbackResource.bind(newQuery.callback) newQuery.callback = function (err, res) { - finish(err) + finish(err, res) return originalCallback.apply(this, arguments) } } else if (newQuery.once) { newQuery .once('error', finish) - .once('end', () => finish()) + .once('end', (res) => finish(null, res)) } else { - newQuery.then(() => finish(), finish) + newQuery.then((res) => finish(null, res), finish) } try { diff --git a/packages/datadog-instrumentations/src/sequelize.js b/packages/datadog-instrumentations/src/sequelize.js index 8ba56ee8909..d8e41b17704 100644 --- a/packages/datadog-instrumentations/src/sequelize.js +++ b/packages/datadog-instrumentations/src/sequelize.js @@ -13,7 +13,7 @@ addHook({ name: 'sequelize', versions: ['>=4'] }, Sequelize => { const finishCh = channel('datadog:sequelize:query:finish') shimmer.wrap(Sequelize.prototype, 'query', query => { - return function (sql) { + return function (sql, options) { if (!startCh.hasSubscribers) { return query.apply(this, arguments) } @@ -27,9 +27,14 @@ addHook({ name: 'sequelize', versions: ['>=4'] }, Sequelize => { dialect = this.dialect.name } - function onFinish () { + function onFinish (result) { + const type = options?.type || 'RAW' + if (type === 'RAW' && result?.length > 1) { + result = result[0] + } + asyncResource.bind(function () { - finishCh.publish() + finishCh.publish({ result }) }, this).apply(this) } @@ -40,7 +45,7 @@ addHook({ name: 'sequelize', versions: ['>=4'] }, Sequelize => { }) const promise = query.apply(this, arguments) - promise.then(onFinish, onFinish) + promise.then(onFinish, () => { onFinish() }) return promise }, this).apply(this, arguments) diff --git a/packages/dd-trace/src/appsec/iast/analyzers/code-injection-analyzer.js b/packages/dd-trace/src/appsec/iast/analyzers/code-injection-analyzer.js index f8937417e42..3741c12ef8f 100644 --- a/packages/dd-trace/src/appsec/iast/analyzers/code-injection-analyzer.js +++ b/packages/dd-trace/src/appsec/iast/analyzers/code-injection-analyzer.js @@ -11,6 +11,10 @@ class CodeInjectionAnalyzer extends InjectionAnalyzer { onConfigure () { this.addSub('datadog:eval:call', ({ script }) => this.analyze(script)) } + + _areRangesVulnerable () { + return true + } } module.exports = new CodeInjectionAnalyzer() diff --git a/packages/dd-trace/src/appsec/iast/analyzers/injection-analyzer.js b/packages/dd-trace/src/appsec/iast/analyzers/injection-analyzer.js index cb4bc2866b0..f0d42bf95ae 100644 --- a/packages/dd-trace/src/appsec/iast/analyzers/injection-analyzer.js +++ b/packages/dd-trace/src/appsec/iast/analyzers/injection-analyzer.js @@ -1,12 +1,15 @@ 'use strict' const Analyzer = require('./vulnerability-analyzer') -const { isTainted, getRanges } = require('../taint-tracking/operations') +const { getRanges } = require('../taint-tracking/operations') +const { SQL_ROW_VALUE } = require('../taint-tracking/source-types') class InjectionAnalyzer extends Analyzer { _isVulnerable (value, iastContext) { - if (value) { - return isTainted(iastContext, value) + const ranges = value && getRanges(iastContext, value) + if (ranges?.length > 0) { + return this._areRangesVulnerable(ranges) } + return false } @@ -14,6 +17,10 @@ class InjectionAnalyzer extends Analyzer { const ranges = getRanges(iastContext, value) return { value, ranges } } + + _areRangesVulnerable (ranges) { + return ranges?.some(range => range.iinfo.type !== SQL_ROW_VALUE) + } } module.exports = InjectionAnalyzer diff --git a/packages/dd-trace/src/appsec/iast/analyzers/sql-injection-analyzer.js b/packages/dd-trace/src/appsec/iast/analyzers/sql-injection-analyzer.js index 4d302ece1b6..8f7ca5a39ed 100644 --- a/packages/dd-trace/src/appsec/iast/analyzers/sql-injection-analyzer.js +++ b/packages/dd-trace/src/appsec/iast/analyzers/sql-injection-analyzer.js @@ -82,6 +82,10 @@ class SqlInjectionAnalyzer extends InjectionAnalyzer { return knexDialect.toUpperCase() } } + + _areRangesVulnerable () { + return true + } } module.exports = new SqlInjectionAnalyzer() diff --git a/packages/dd-trace/src/appsec/iast/analyzers/template-injection-analyzer.js b/packages/dd-trace/src/appsec/iast/analyzers/template-injection-analyzer.js index 1be35933223..8a5af919b2d 100644 --- a/packages/dd-trace/src/appsec/iast/analyzers/template-injection-analyzer.js +++ b/packages/dd-trace/src/appsec/iast/analyzers/template-injection-analyzer.js @@ -13,6 +13,10 @@ class TemplateInjectionAnalyzer extends InjectionAnalyzer { this.addSub('datadog:handlebars:register-partial:start', ({ partial }) => this.analyze(partial)) this.addSub('datadog:pug:compile:start', ({ source }) => this.analyze(source)) } + + _areRangesVulnerable () { + return true + } } module.exports = new TemplateInjectionAnalyzer() diff --git a/packages/dd-trace/src/appsec/iast/iast-plugin.js b/packages/dd-trace/src/appsec/iast/iast-plugin.js index 10dcde340c3..42dab0a4af1 100644 --- a/packages/dd-trace/src/appsec/iast/iast-plugin.js +++ b/packages/dd-trace/src/appsec/iast/iast-plugin.js @@ -98,7 +98,8 @@ class IastPlugin extends Plugin { } } - enable () { + enable (iastConfig) { + this.iastConfig = iastConfig this.configure(true) } diff --git a/packages/dd-trace/src/appsec/iast/taint-tracking/index.js b/packages/dd-trace/src/appsec/iast/taint-tracking/index.js index 5c7109c4cda..b541629f3b7 100644 --- a/packages/dd-trace/src/appsec/iast/taint-tracking/index.js +++ b/packages/dd-trace/src/appsec/iast/taint-tracking/index.js @@ -18,10 +18,10 @@ module.exports = { enableTaintTracking (config, telemetryVerbosity) { enableRewriter(telemetryVerbosity) enableTaintOperations(telemetryVerbosity) - taintTrackingPlugin.enable() + taintTrackingPlugin.enable(config) - kafkaContextPlugin.enable() - kafkaConsumerPlugin.enable() + kafkaContextPlugin.enable(config) + kafkaConsumerPlugin.enable(config) setMaxTransactions(config.maxConcurrentRequests) }, diff --git a/packages/dd-trace/src/appsec/iast/taint-tracking/plugin.js b/packages/dd-trace/src/appsec/iast/taint-tracking/plugin.js index 62fdd46d027..9e236666619 100644 --- a/packages/dd-trace/src/appsec/iast/taint-tracking/plugin.js +++ b/packages/dd-trace/src/appsec/iast/taint-tracking/plugin.js @@ -12,7 +12,8 @@ const { HTTP_REQUEST_HEADER_NAME, HTTP_REQUEST_PARAMETER, HTTP_REQUEST_PATH_PARAM, - HTTP_REQUEST_URI + HTTP_REQUEST_URI, + SQL_ROW_VALUE } = require('./source-types') const { EXECUTED_SOURCE } = require('../telemetry/iast-metric') @@ -26,6 +27,16 @@ class TaintTrackingPlugin extends SourceIastPlugin { this._taintedURLs = new WeakMap() } + configure (config) { + super.configure(config) + + let rowsToTaint = this.iastConfig?.dbRowsToTaint + if (typeof rowsToTaint !== 'number') { + rowsToTaint = 1 + } + this._rowsToTaint = rowsToTaint + } + onConfigure () { const onRequestBody = ({ req }) => { const iastContext = getIastContext(storage.getStore()) @@ -73,6 +84,16 @@ class TaintTrackingPlugin extends SourceIastPlugin { ({ cookies }) => this._cookiesTaintTrackingHandler(cookies) ) + this.addSub( + { channelName: 'datadog:sequelize:query:finish', tag: SQL_ROW_VALUE }, + ({ result }) => this._taintDatabaseResult(result, 'sequelize') + ) + + this.addSub( + { channelName: 'apm:pg:query:finish', tag: SQL_ROW_VALUE }, + ({ result }) => this._taintDatabaseResult(result, 'pg') + ) + this.addSub( { channelName: 'datadog:express:process_params:start', tag: HTTP_REQUEST_PATH_PARAM }, ({ req }) => { @@ -184,6 +205,32 @@ class TaintTrackingPlugin extends SourceIastPlugin { this.taintHeaders(req.headers, iastContext) this.taintUrl(req, iastContext) } + + _taintDatabaseResult (result, dbOrigin, iastContext = getIastContext(storage.getStore()), name) { + if (!iastContext) return result + + if (this._rowsToTaint === 0) return result + + if (Array.isArray(result)) { + for (let i = 0; i < result.length && i < this._rowsToTaint; i++) { + const nextName = name ? `${name}.${i}` : '' + i + result[i] = this._taintDatabaseResult(result[i], dbOrigin, iastContext, nextName) + } + } else if (result && typeof result === 'object') { + if (dbOrigin === 'sequelize' && result.dataValues) { + result.dataValues = this._taintDatabaseResult(result.dataValues, dbOrigin, iastContext, name) + } else { + for (const key in result) { + const nextName = name ? `${name}.${key}` : key + result[key] = this._taintDatabaseResult(result[key], dbOrigin, iastContext, nextName) + } + } + } else if (typeof result === 'string') { + result = newTaintedString(iastContext, result, name, SQL_ROW_VALUE) + } + + return result + } } module.exports = new TaintTrackingPlugin() diff --git a/packages/dd-trace/src/appsec/iast/taint-tracking/source-types.js b/packages/dd-trace/src/appsec/iast/taint-tracking/source-types.js index f5c2ca2e8b0..f3ccf0505c3 100644 --- a/packages/dd-trace/src/appsec/iast/taint-tracking/source-types.js +++ b/packages/dd-trace/src/appsec/iast/taint-tracking/source-types.js @@ -11,5 +11,6 @@ module.exports = { HTTP_REQUEST_PATH_PARAM: 'http.request.path.parameter', HTTP_REQUEST_URI: 'http.request.uri', KAFKA_MESSAGE_KEY: 'kafka.message.key', - KAFKA_MESSAGE_VALUE: 'kafka.message.value' + KAFKA_MESSAGE_VALUE: 'kafka.message.value', + SQL_ROW_VALUE: 'sql.row.value' } diff --git a/packages/dd-trace/src/config.js b/packages/dd-trace/src/config.js index beb15ebc010..a46cc3153fc 100644 --- a/packages/dd-trace/src/config.js +++ b/packages/dd-trace/src/config.js @@ -485,6 +485,7 @@ class Config { this._setValue(defaults, 'headerTags', []) this._setValue(defaults, 'hostname', '127.0.0.1') this._setValue(defaults, 'iast.cookieFilterPattern', '.{32,}') + this._setValue(defaults, 'iast.dbRowsToTaint', 1) this._setValue(defaults, 'iast.deduplicationEnabled', true) this._setValue(defaults, 'iast.enabled', false) this._setValue(defaults, 'iast.maxConcurrentRequests', 2) @@ -605,6 +606,7 @@ class Config { DD_GRPC_SERVER_ERROR_STATUSES, JEST_WORKER_ID, DD_IAST_COOKIE_FILTER_PATTERN, + DD_IAST_DB_ROWS_TO_TAINT, DD_IAST_DEDUPLICATION_ENABLED, DD_IAST_ENABLED, DD_IAST_MAX_CONCURRENT_REQUESTS, @@ -757,6 +759,7 @@ class Config { this._setArray(env, 'headerTags', DD_TRACE_HEADER_TAGS) this._setString(env, 'hostname', coalesce(DD_AGENT_HOST, DD_TRACE_AGENT_HOSTNAME)) this._setString(env, 'iast.cookieFilterPattern', DD_IAST_COOKIE_FILTER_PATTERN) + this._setValue(env, 'iast.dbRowsToTaint', maybeInt(DD_IAST_DB_ROWS_TO_TAINT)) this._setBoolean(env, 'iast.deduplicationEnabled', DD_IAST_DEDUPLICATION_ENABLED) this._setBoolean(env, 'iast.enabled', DD_IAST_ENABLED) this._setValue(env, 'iast.maxConcurrentRequests', maybeInt(DD_IAST_MAX_CONCURRENT_REQUESTS)) @@ -932,6 +935,7 @@ class Config { this._setArray(opts, 'headerTags', options.headerTags) this._setString(opts, 'hostname', options.hostname) this._setString(opts, 'iast.cookieFilterPattern', options.iast?.cookieFilterPattern) + this._setValue(opts, 'iast.dbRowsToTaint', maybeInt(options.iast?.dbRowsToTaint)) this._setBoolean(opts, 'iast.deduplicationEnabled', options.iast && options.iast.deduplicationEnabled) this._setBoolean(opts, 'iast.enabled', options.iast && (options.iast === true || options.iast.enabled === true)) diff --git a/packages/dd-trace/test/appsec/iast/analyzers/code-injection-analyzer.express.plugin.spec.js b/packages/dd-trace/test/appsec/iast/analyzers/code-injection-analyzer.express.plugin.spec.js index 4177dc78aba..64e15b9161b 100644 --- a/packages/dd-trace/test/appsec/iast/analyzers/code-injection-analyzer.express.plugin.spec.js +++ b/packages/dd-trace/test/appsec/iast/analyzers/code-injection-analyzer.express.plugin.spec.js @@ -6,6 +6,10 @@ const path = require('path') const os = require('os') const fs = require('fs') const { clearCache } = require('../../../../src/appsec/iast/vulnerability-reporter') +const { newTaintedString } = require('../../../../src/appsec/iast/taint-tracking/operations') +const { SQL_ROW_VALUE } = require('../../../../src/appsec/iast/taint-tracking/source-types') +const { storage } = require('../../../../../datadog-core') +const iastContextFunctions = require('../../../../src/appsec/iast/iast-context') describe('Code injection vulnerability', () => { withVersions('express', 'express', '>4.18.0', version => { @@ -29,7 +33,6 @@ describe('Code injection vulnerability', () => { (testThatRequestHasVulnerability, testThatRequestHasNoVulnerability) => { testThatRequestHasVulnerability({ fn: (req, res) => { - // eslint-disable-next-line no-eval res.send(require(evalFunctionsPath).runEval(req.query.script, 'test-result')) }, vulnerability: 'CODE_INJECTION', @@ -42,6 +45,19 @@ describe('Code injection vulnerability', () => { } }) + testThatRequestHasVulnerability({ + fn: (req, res) => { + const source = '1 + 2' + const store = storage.getStore() + const iastContext = iastContextFunctions.getIastContext(store) + const str = newTaintedString(iastContext, source, 'param', SQL_ROW_VALUE) + + res.send(require(evalFunctionsPath).runEval(str, 'test-result')) + }, + vulnerability: 'CODE_INJECTION', + testDescription: 'Should detect CODE_INJECTION vulnerability with DB source' + }) + testThatRequestHasNoVulnerability({ fn: (req, res) => { res.send('' + require(evalFunctionsPath).runFakeEval(req.query.script)) diff --git a/packages/dd-trace/test/appsec/iast/analyzers/ldap-injection-analyzer.spec.js b/packages/dd-trace/test/appsec/iast/analyzers/ldap-injection-analyzer.spec.js index 59413db0a4f..c8af2de6846 100644 --- a/packages/dd-trace/test/appsec/iast/analyzers/ldap-injection-analyzer.spec.js +++ b/packages/dd-trace/test/appsec/iast/analyzers/ldap-injection-analyzer.spec.js @@ -1,14 +1,27 @@ 'use strict' const proxyquire = require('proxyquire') +const { HTTP_REQUEST_PARAMETER } = require('../../../../src/appsec/iast/taint-tracking/source-types') describe('ldap-injection-analyzer', () => { const NOT_TAINTED_QUERY = 'no vulnerable query' const TAINTED_QUERY = 'vulnerable query' const TaintTrackingMock = { - isTainted: (iastContext, string) => { + getRanges: (iastContext, string) => { return string === TAINTED_QUERY + ? [ + { + start: 0, + end: string.length, + iinfo: { + parameterName: 'param', + parameterValue: string, + type: HTTP_REQUEST_PARAMETER + } + } + ] + : [] } } diff --git a/packages/dd-trace/test/appsec/iast/analyzers/path-traversal-analyzer.spec.js b/packages/dd-trace/test/appsec/iast/analyzers/path-traversal-analyzer.spec.js index 6c39799f916..3fe86dacd8d 100644 --- a/packages/dd-trace/test/appsec/iast/analyzers/path-traversal-analyzer.spec.js +++ b/packages/dd-trace/test/appsec/iast/analyzers/path-traversal-analyzer.spec.js @@ -12,6 +12,7 @@ const { newTaintedString } = require('../../../../src/appsec/iast/taint-tracking const { prepareTestServerForIast } = require('../utils') const fs = require('fs') +const { HTTP_REQUEST_PARAMETER } = require('../../../../src/appsec/iast/taint-tracking/source-types') const iastContext = { rootSpan: { @@ -25,26 +26,23 @@ const iastContext = { } } -const TaintTrackingMock = { - isTainted: sinon.stub() +const getRanges = (ctx, val) => { + return [ + { + start: 0, + end: val.length, + iinfo: { + parameterName: 'param', + parameterValue: val, + type: HTTP_REQUEST_PARAMETER + } + } + ] } -const getIastContext = sinon.stub() -const hasQuota = sinon.stub() -const addVulnerability = sinon.stub() - -const ProxyAnalyzer = proxyquire('../../../../src/appsec/iast/analyzers/vulnerability-analyzer', { - '../iast-context': { getIastContext }, - '../overhead-controller': { hasQuota }, - '../vulnerability-reporter': { addVulnerability } -}) - -const InjectionAnalyzer = proxyquire('../../../../src/appsec/iast/analyzers/injection-analyzer', { - './vulnerability-analyzer': ProxyAnalyzer, - '../taint-tracking/operations': TaintTrackingMock -}) - describe('path-traversal-analyzer', () => { + let TaintTrackingMock, getIastContext, hasQuota, addVulnerability, ProxyAnalyzer, InjectionAnalyzer + before(() => { pathTraversalAnalyzer.enable() }) @@ -53,6 +51,28 @@ describe('path-traversal-analyzer', () => { pathTraversalAnalyzer.disable() }) + beforeEach(() => { + TaintTrackingMock = { + isTainted: sinon.stub(), + getRanges: sinon.stub() + } + + getIastContext = sinon.stub() + hasQuota = sinon.stub() + addVulnerability = sinon.stub() + + ProxyAnalyzer = proxyquire('../../../../src/appsec/iast/analyzers/vulnerability-analyzer', { + '../iast-context': { getIastContext }, + '../overhead-controller': { hasQuota }, + '../vulnerability-reporter': { addVulnerability } + }) + + InjectionAnalyzer = proxyquire('../../../../src/appsec/iast/analyzers/injection-analyzer', { + './vulnerability-analyzer': ProxyAnalyzer, + '../taint-tracking/operations': TaintTrackingMock + }) + }) + it('Analyzer should be subscribed to proper channel', () => { expect(pathTraversalAnalyzer._subscriptions).to.have.lengthOf(1) expect(pathTraversalAnalyzer._subscriptions[0]._channel.name).to.equals('apm:fs:operation:start') @@ -72,26 +92,25 @@ describe('path-traversal-analyzer', () => { }) it('if context exists but value is not a string it should not call isTainted', () => { - const isTainted = sinon.stub() + const getRanges = sinon.stub() const iastContext = {} const proxyPathAnalyzer = proxyquire('../../../../src/appsec/iast/analyzers/path-traversal-analyzer', { - '../taint-tracking': { isTainted } + '../taint-tracking': { getRanges } }) proxyPathAnalyzer._isVulnerable(undefined, iastContext) - expect(isTainted).not.to.have.been.called + expect(getRanges).not.to.have.been.called }) it('if context and value are valid it should call isTainted', () => { - // const isTainted = sinon.stub() const iastContext = {} const proxyPathAnalyzer = proxyquire('../../../../src/appsec/iast/analyzers/path-traversal-analyzer', { './injection-analyzer': InjectionAnalyzer }) - TaintTrackingMock.isTainted.returns(false) + TaintTrackingMock.getRanges.returns([]) const result = proxyPathAnalyzer._isVulnerable('test', iastContext) expect(result).to.be.false - expect(TaintTrackingMock.isTainted).to.have.been.calledOnce + expect(TaintTrackingMock.getRanges).to.have.been.calledOnce }) it('Should report proper vulnerability type', () => { @@ -102,7 +121,7 @@ describe('path-traversal-analyzer', () => { getIastContext.returns(iastContext) hasQuota.returns(true) - TaintTrackingMock.isTainted.returns(true) + TaintTrackingMock.getRanges.callsFake(getRanges) proxyPathAnalyzer.analyze(['test']) expect(addVulnerability).to.have.been.calledOnce @@ -116,9 +135,8 @@ describe('path-traversal-analyzer', () => { '../iast-context': { getIastContext: () => iastContext } }) - addVulnerability.reset() getIastContext.returns(iastContext) - TaintTrackingMock.isTainted.returns(true) + TaintTrackingMock.getRanges.callsFake(getRanges) hasQuota.returns(true) proxyPathAnalyzer.analyze(['taintedArg1', 'taintedArg2']) @@ -132,11 +150,10 @@ describe('path-traversal-analyzer', () => { '../iast-context': { getIastContext: () => iastContext } }) - addVulnerability.reset() - TaintTrackingMock.isTainted.reset() getIastContext.returns(iastContext) - TaintTrackingMock.isTainted.onFirstCall().returns(false) - TaintTrackingMock.isTainted.onSecondCall().returns(true) + + TaintTrackingMock.getRanges.onFirstCall().returns([]) + TaintTrackingMock.getRanges.onSecondCall().callsFake(getRanges) hasQuota.returns(true) proxyPathAnalyzer.analyze(['arg1', 'taintedArg2']) @@ -155,10 +172,8 @@ describe('path-traversal-analyzer', () => { return { path: mockPath, line: 3 } } - addVulnerability.reset() - TaintTrackingMock.isTainted.reset() getIastContext.returns(iastContext) - TaintTrackingMock.isTainted.returns(true) + TaintTrackingMock.getRanges.callsFake(getRanges) hasQuota.returns(true) proxyPathAnalyzer.analyze(['arg1']) diff --git a/packages/dd-trace/test/appsec/iast/analyzers/sql-injection-analyzer.spec.js b/packages/dd-trace/test/appsec/iast/analyzers/sql-injection-analyzer.spec.js index de662075cf3..8c4d26103d3 100644 --- a/packages/dd-trace/test/appsec/iast/analyzers/sql-injection-analyzer.spec.js +++ b/packages/dd-trace/test/appsec/iast/analyzers/sql-injection-analyzer.spec.js @@ -4,14 +4,27 @@ const proxyquire = require('proxyquire') const log = require('../../../../src/log') const dc = require('dc-polyfill') +const { HTTP_REQUEST_PARAMETER } = require('../../../../src/appsec/iast/taint-tracking/source-types') describe('sql-injection-analyzer', () => { const NOT_TAINTED_QUERY = 'no vulnerable query' const TAINTED_QUERY = 'vulnerable query' const TaintTrackingMock = { - isTainted: (iastContext, string) => { + getRanges: (iastContext, string) => { return string === TAINTED_QUERY + ? [ + { + start: 0, + end: string.length, + iinfo: { + parameterName: 'param', + parameterValue: string, + type: HTTP_REQUEST_PARAMETER + } + } + ] + : [] } } diff --git a/packages/dd-trace/test/appsec/iast/analyzers/template-injection-analyzer.handlebars.plugin.spec.js b/packages/dd-trace/test/appsec/iast/analyzers/template-injection-analyzer.handlebars.plugin.spec.js index 4152f4ab6e9..b3398543a04 100644 --- a/packages/dd-trace/test/appsec/iast/analyzers/template-injection-analyzer.handlebars.plugin.spec.js +++ b/packages/dd-trace/test/appsec/iast/analyzers/template-injection-analyzer.handlebars.plugin.spec.js @@ -4,6 +4,7 @@ const { prepareTestServerForIast } = require('../utils') const { storage } = require('../../../../../datadog-core') const iastContextFunctions = require('../../../../src/appsec/iast/iast-context') const { newTaintedString } = require('../../../../src/appsec/iast/taint-tracking/operations') +const { SQL_ROW_VALUE } = require('../../../../src/appsec/iast/taint-tracking/source-types') describe('template-injection-analyzer with handlebars', () => { withVersions('handlebars', 'handlebars', version => { @@ -27,6 +28,14 @@ describe('template-injection-analyzer with handlebars', () => { lib.compile(template) }, 'TEMPLATE_INJECTION') + testThatRequestHasVulnerability(() => { + const store = storage.getStore() + const iastContext = iastContextFunctions.getIastContext(store) + const template = newTaintedString(iastContext, source, 'param', SQL_ROW_VALUE) + lib.compile(template) + }, 'TEMPLATE_INJECTION', undefined, undefined, undefined, + 'Should detect TEMPLATE_INJECTION vulnerability with DB source') + testThatRequestHasNoVulnerability(() => { lib.compile(source) }, 'TEMPLATE_INJECTION') @@ -48,6 +57,14 @@ describe('template-injection-analyzer with handlebars', () => { lib.precompile(template) }, 'TEMPLATE_INJECTION') + testThatRequestHasVulnerability(() => { + const store = storage.getStore() + const iastContext = iastContextFunctions.getIastContext(store) + const template = newTaintedString(iastContext, source, 'param', SQL_ROW_VALUE) + lib.precompile(template) + }, 'TEMPLATE_INJECTION', undefined, undefined, undefined, + 'Should detect TEMPLATE_INJECTION vulnerability with DB source') + testThatRequestHasNoVulnerability(() => { lib.precompile(source) }, 'TEMPLATE_INJECTION') @@ -70,6 +87,15 @@ describe('template-injection-analyzer with handlebars', () => { lib.registerPartial('vulnerablePartial', partial) }, 'TEMPLATE_INJECTION') + testThatRequestHasVulnerability(() => { + const store = storage.getStore() + const iastContext = iastContextFunctions.getIastContext(store) + const partial = newTaintedString(iastContext, source, 'param', SQL_ROW_VALUE) + + lib.registerPartial('vulnerablePartial', partial) + }, 'TEMPLATE_INJECTION', undefined, undefined, undefined, + 'Should detect TEMPLATE_INJECTION vulnerability with DB source') + testThatRequestHasNoVulnerability(() => { lib.registerPartial('vulnerablePartial', source) }, 'TEMPLATE_INJECTION') diff --git a/packages/dd-trace/test/appsec/iast/analyzers/template-injection-analyzer.pug.plugin.spec.js b/packages/dd-trace/test/appsec/iast/analyzers/template-injection-analyzer.pug.plugin.spec.js index 412da3a62f0..574f256fd53 100644 --- a/packages/dd-trace/test/appsec/iast/analyzers/template-injection-analyzer.pug.plugin.spec.js +++ b/packages/dd-trace/test/appsec/iast/analyzers/template-injection-analyzer.pug.plugin.spec.js @@ -4,6 +4,7 @@ const { prepareTestServerForIast } = require('../utils') const { storage } = require('../../../../../datadog-core') const iastContextFunctions = require('../../../../src/appsec/iast/iast-context') const { newTaintedString } = require('../../../../src/appsec/iast/taint-tracking/operations') +const { SQL_ROW_VALUE } = require('../../../../src/appsec/iast/taint-tracking/source-types') describe('template-injection-analyzer with pug', () => { withVersions('pug', 'pug', version => { @@ -27,6 +28,14 @@ describe('template-injection-analyzer with pug', () => { lib.compile(template) }, 'TEMPLATE_INJECTION') + testThatRequestHasVulnerability(() => { + const store = storage.getStore() + const iastContext = iastContextFunctions.getIastContext(store) + const template = newTaintedString(iastContext, source, 'param', SQL_ROW_VALUE) + lib.compile(template) + }, 'TEMPLATE_INJECTION', undefined, undefined, undefined, + 'Should detect TEMPLATE_INJECTION vulnerability with DB source') + testThatRequestHasNoVulnerability(() => { const template = lib.compile(source) template() @@ -49,6 +58,14 @@ describe('template-injection-analyzer with pug', () => { lib.compileClient(template) }, 'TEMPLATE_INJECTION') + testThatRequestHasVulnerability(() => { + const store = storage.getStore() + const iastContext = iastContextFunctions.getIastContext(store) + const template = newTaintedString(iastContext, source, 'param', SQL_ROW_VALUE) + lib.compileClient(template) + }, 'TEMPLATE_INJECTION', undefined, undefined, undefined, + 'Should detect TEMPLATE_INJECTION vulnerability with DB source') + testThatRequestHasNoVulnerability(() => { lib.compileClient(source) }, 'TEMPLATE_INJECTION') @@ -70,6 +87,14 @@ describe('template-injection-analyzer with pug', () => { lib.compileClientWithDependenciesTracked(template, {}) }, 'TEMPLATE_INJECTION') + testThatRequestHasVulnerability(() => { + const store = storage.getStore() + const iastContext = iastContextFunctions.getIastContext(store) + const template = newTaintedString(iastContext, source, 'param', SQL_ROW_VALUE) + lib.compileClientWithDependenciesTracked(template, {}) + }, 'TEMPLATE_INJECTION', undefined, undefined, undefined, + 'Should detect TEMPLATE_INJECTION vulnerability with DB source') + testThatRequestHasNoVulnerability(() => { lib.compileClient(source) }, 'TEMPLATE_INJECTION') @@ -91,6 +116,14 @@ describe('template-injection-analyzer with pug', () => { lib.render(str) }, 'TEMPLATE_INJECTION') + testThatRequestHasVulnerability(() => { + const store = storage.getStore() + const iastContext = iastContextFunctions.getIastContext(store) + const str = newTaintedString(iastContext, source, 'param', SQL_ROW_VALUE) + lib.render(str) + }, 'TEMPLATE_INJECTION', undefined, undefined, undefined, + 'Should detect TEMPLATE_INJECTION vulnerability with DB source') + testThatRequestHasNoVulnerability(() => { lib.render(source) }, 'TEMPLATE_INJECTION') diff --git a/packages/dd-trace/test/appsec/iast/taint-tracking/plugin.spec.js b/packages/dd-trace/test/appsec/iast/taint-tracking/plugin.spec.js index 5f9c4f4860f..af575ce9652 100644 --- a/packages/dd-trace/test/appsec/iast/taint-tracking/plugin.spec.js +++ b/packages/dd-trace/test/appsec/iast/taint-tracking/plugin.spec.js @@ -8,8 +8,10 @@ const { HTTP_REQUEST_COOKIE_VALUE, HTTP_REQUEST_HEADER_VALUE, HTTP_REQUEST_PATH_PARAM, - HTTP_REQUEST_URI + HTTP_REQUEST_URI, + SQL_ROW_VALUE } = require('../../../../src/appsec/iast/taint-tracking/source-types') +const Config = require('../../../../src/config') const middlewareNextChannel = dc.channel('apm:express:middleware:next') const queryReadFinishChannel = dc.channel('datadog:query:read:finish') @@ -17,6 +19,7 @@ const bodyParserFinishChannel = dc.channel('datadog:body-parser:read:finish') const cookieParseFinishCh = dc.channel('datadog:cookie:parse:finish') const processParamsStartCh = dc.channel('datadog:express:process_params:start') const routerParamStartCh = dc.channel('datadog:router:param:start') +const sequelizeFinish = dc.channel('datadog:sequelize:query:finish') describe('IAST Taint tracking plugin', () => { let taintTrackingPlugin @@ -34,7 +37,8 @@ describe('IAST Taint tracking plugin', () => { './operations': sinon.spy(taintTrackingOperations), '../../../../../datadog-core': datadogCore }) - taintTrackingPlugin.enable() + const config = new Config() + taintTrackingPlugin.enable(config.iast) }) afterEach(() => { @@ -43,18 +47,20 @@ describe('IAST Taint tracking plugin', () => { }) it('Should subscribe to body parser, qs, cookie and process_params channel', () => { - expect(taintTrackingPlugin._subscriptions).to.have.lengthOf(11) + expect(taintTrackingPlugin._subscriptions).to.have.lengthOf(13) expect(taintTrackingPlugin._subscriptions[0]._channel.name).to.equals('datadog:body-parser:read:finish') expect(taintTrackingPlugin._subscriptions[1]._channel.name).to.equals('datadog:multer:read:finish') expect(taintTrackingPlugin._subscriptions[2]._channel.name).to.equals('datadog:query:read:finish') expect(taintTrackingPlugin._subscriptions[3]._channel.name).to.equals('datadog:express:query:finish') expect(taintTrackingPlugin._subscriptions[4]._channel.name).to.equals('apm:express:middleware:next') expect(taintTrackingPlugin._subscriptions[5]._channel.name).to.equals('datadog:cookie:parse:finish') - expect(taintTrackingPlugin._subscriptions[6]._channel.name).to.equals('datadog:express:process_params:start') - expect(taintTrackingPlugin._subscriptions[7]._channel.name).to.equals('datadog:router:param:start') - expect(taintTrackingPlugin._subscriptions[8]._channel.name).to.equals('apm:graphql:resolve:start') - expect(taintTrackingPlugin._subscriptions[9]._channel.name).to.equals('datadog:url:parse:finish') - expect(taintTrackingPlugin._subscriptions[10]._channel.name).to.equals('datadog:url:getter:finish') + expect(taintTrackingPlugin._subscriptions[6]._channel.name).to.equals('datadog:sequelize:query:finish') + expect(taintTrackingPlugin._subscriptions[7]._channel.name).to.equals('apm:pg:query:finish') + expect(taintTrackingPlugin._subscriptions[8]._channel.name).to.equals('datadog:express:process_params:start') + expect(taintTrackingPlugin._subscriptions[9]._channel.name).to.equals('datadog:router:param:start') + expect(taintTrackingPlugin._subscriptions[10]._channel.name).to.equals('apm:graphql:resolve:start') + expect(taintTrackingPlugin._subscriptions[11]._channel.name).to.equals('datadog:url:parse:finish') + expect(taintTrackingPlugin._subscriptions[12]._channel.name).to.equals('datadog:url:getter:finish') }) describe('taint sources', () => { @@ -271,5 +277,259 @@ describe('IAST Taint tracking plugin', () => { HTTP_REQUEST_URI ) }) + + describe('taint database sources', () => { + it('Should not taint if config is set to 0', () => { + taintTrackingPlugin.disable() + const config = new Config() + config.dbRowsToTaint = 0 + taintTrackingPlugin.enable(config) + + const result = [ + { + id: 1, + name: 'string value 1' + }, + { + id: 2, + name: 'string value 2' + }] + sequelizeFinish.publish({ result }) + + expect(taintTrackingOperations.newTaintedString).to.not.have.been.called + }) + + describe('with default config', () => { + it('Should taint first database row coming from sequelize', () => { + const result = [ + { + id: 1, + name: 'string value 1' + }, + { + id: 2, + name: 'string value 2' + }] + sequelizeFinish.publish({ result }) + + expect(taintTrackingOperations.newTaintedString).to.be.calledOnceWith( + iastContext, + 'string value 1', + '0.name', + SQL_ROW_VALUE + ) + }) + + it('Should taint whole object', () => { + const result = { id: 1, description: 'value' } + sequelizeFinish.publish({ result }) + + expect(taintTrackingOperations.newTaintedString).to.be.calledOnceWith( + iastContext, + 'value', + 'description', + SQL_ROW_VALUE + ) + }) + + it('Should taint first row in nested objects', () => { + const result = [ + { + id: 1, + description: 'value', + children: [ + { + id: 11, + name: 'child1' + }, + { + id: 12, + name: 'child2' + } + ] + }, + { + id: 2, + description: 'value', + children: [ + { + id: 21, + name: 'child3' + }, + { + id: 22, + name: 'child4' + } + ] + } + ] + sequelizeFinish.publish({ result }) + + expect(taintTrackingOperations.newTaintedString).to.be.calledTwice + expect(taintTrackingOperations.newTaintedString).to.be.calledWith( + iastContext, + 'value', + '0.description', + SQL_ROW_VALUE + ) + expect(taintTrackingOperations.newTaintedString).to.be.calledWith( + iastContext, + 'child1', + '0.children.0.name', + SQL_ROW_VALUE + ) + }) + }) + + describe('with config set to 2', () => { + beforeEach(() => { + taintTrackingPlugin.disable() + const config = new Config() + config.dbRowsToTaint = 2 + taintTrackingPlugin.enable(config) + }) + + it('Should taint first database row coming from sequelize', () => { + const result = [ + { + id: 1, + name: 'string value 1' + }, + { + id: 2, + name: 'string value 2' + }, + { + id: 3, + name: 'string value 2' + }] + sequelizeFinish.publish({ result }) + + expect(taintTrackingOperations.newTaintedString).to.be.calledTwice + expect(taintTrackingOperations.newTaintedString).to.be.calledWith( + iastContext, + 'string value 1', + '0.name', + SQL_ROW_VALUE + ) + expect(taintTrackingOperations.newTaintedString).to.be.calledWith( + iastContext, + 'string value 2', + '1.name', + SQL_ROW_VALUE + ) + }) + + it('Should taint whole object', () => { + const result = { id: 1, description: 'value' } + sequelizeFinish.publish({ result }) + + expect(taintTrackingOperations.newTaintedString).to.be.calledOnceWith( + iastContext, + 'value', + 'description', + SQL_ROW_VALUE + ) + }) + + it('Should taint first row in nested objects', () => { + const result = [ + { + id: 1, + description: 'value', + children: [ + { + id: 11, + name: 'child1' + }, + { + id: 12, + name: 'child2' + }, + { + id: 13, + name: 'child3' + } + ] + }, + { + id: 2, + description: 'value2', + children: [ + { + id: 21, + name: 'child4' + }, + { + id: 22, + name: 'child5' + }, + { + id: 23, + name: 'child6' + } + ] + }, + { + id: 3, + description: 'value3', + children: [ + { + id: 31, + name: 'child7' + }, + { + id: 32, + name: 'child8' + }, + { + id: 33, + name: 'child9' + } + ] + } + ] + sequelizeFinish.publish({ result }) + + expect(taintTrackingOperations.newTaintedString).to.callCount(6) + expect(taintTrackingOperations.newTaintedString).to.be.calledWith( + iastContext, + 'value', + '0.description', + SQL_ROW_VALUE + ) + expect(taintTrackingOperations.newTaintedString).to.be.calledWith( + iastContext, + 'child1', + '0.children.0.name', + SQL_ROW_VALUE + ) + expect(taintTrackingOperations.newTaintedString).to.be.calledWith( + iastContext, + 'child2', + '0.children.1.name', + SQL_ROW_VALUE + ) + expect(taintTrackingOperations.newTaintedString).to.be.calledWith( + iastContext, + 'value2', + '1.description', + SQL_ROW_VALUE + ) + expect(taintTrackingOperations.newTaintedString).to.be.calledWith( + iastContext, + 'child4', + '1.children.0.name', + SQL_ROW_VALUE + ) + expect(taintTrackingOperations.newTaintedString).to.be.calledWith( + iastContext, + 'child5', + '1.children.1.name', + SQL_ROW_VALUE + ) + }) + }) + }) }) }) diff --git a/packages/dd-trace/test/appsec/iast/taint-tracking/sources/sql_row.pg.plugin.spec.js b/packages/dd-trace/test/appsec/iast/taint-tracking/sources/sql_row.pg.plugin.spec.js new file mode 100644 index 00000000000..69e73b0ccb0 --- /dev/null +++ b/packages/dd-trace/test/appsec/iast/taint-tracking/sources/sql_row.pg.plugin.spec.js @@ -0,0 +1,113 @@ +'use strict' + +const { prepareTestServerForIast } = require('../../utils') + +const connectionData = { + host: '127.0.0.1', + user: 'postgres', + password: 'postgres', + database: 'postgres', + application_name: 'test' +} + +describe('db sources with pg', () => { + let pg + withVersions('pg', 'pg', '>=8.0.3', version => { + let client + beforeEach(async () => { + pg = require(`../../../../../../../versions/pg@${version}`).get() + client = new pg.Client(connectionData) + await client.connect() + + await client.query(`CREATE TABLE IF NOT EXISTS examples ( + id INT, + name VARCHAR(50), + query VARCHAR(100), + command VARCHAR(50))`) + + await client.query(`INSERT INTO examples (id, name, query, command) + VALUES (1, 'Item1', 'SELECT 1', 'ls'), + (2, 'Item2', 'SELECT 1', 'ls'), + (3, 'Item3', 'SELECT 1', 'ls')`) + }) + + afterEach(async () => { + await client.query('DROP TABLE examples') + client.end() + }) + + prepareTestServerForIast('sequelize', (testThatRequestHasVulnerability, testThatRequestHasNoVulnerability) => { + describe('using pg.Client', () => { + testThatRequestHasVulnerability(async (req, res) => { + const result = await client.query('SELECT * FROM examples') + + const firstItem = result.rows[0] + + await client.query(firstItem.query) + + res.end() + }, 'SQL_INJECTION', { occurrences: 1 }, null, null, + 'Should have SQL_INJECTION using the first row of the result') + + testThatRequestHasNoVulnerability(async (req, res) => { + const result = await client.query('SELECT * FROM examples') + + const secondItem = result.rows[1] + + await client.query(secondItem.query) + + res.end() + }, 'SQL_INJECTION', null, 'Should not taint the second row of a query with default configuration') + + testThatRequestHasNoVulnerability(async (req, res) => { + const result = await client.query('SELECT * from examples') + const firstItem = result.rows[0] + + const childProcess = require('child_process') + childProcess.execSync(firstItem.command) + + res.end('OK') + }, 'COMMAND_INJECTION', null, 'Should not detect COMMAND_INJECTION with database source') + }) + + describe('using pg.Pool', () => { + let pool + + beforeEach(() => { + pool = new pg.Pool(connectionData) + }) + + testThatRequestHasVulnerability(async (req, res) => { + const result = await pool.query('SELECT * FROM examples') + + const firstItem = result.rows[0] + + await client.query(firstItem.query) + + res.end() + }, 'SQL_INJECTION', { occurrences: 1 }, null, null, + 'Should have SQL_INJECTION using the first row of the result') + + testThatRequestHasNoVulnerability(async (req, res) => { + const result = await pool.query('SELECT * FROM examples') + + const secondItem = result.rows[1] + + await client.query(secondItem.query) + + res.end() + }, 'SQL_INJECTION', null, 'Should not taint the second row of a query with default configuration') + + testThatRequestHasNoVulnerability(async (req, res) => { + const result = await pool.query('SELECT * from examples') + const firstItem = result.rows[0] + + const childProcess = require('child_process') + childProcess.execSync(firstItem.command) + + res.end('OK') + }, 'COMMAND_INJECTION', null, 'Should not detect COMMAND_INJECTION with database source') + }) + }) + }) +}) diff --git a/packages/dd-trace/test/appsec/iast/taint-tracking/sources/sql_row.sequelize.plugin.spec.js b/packages/dd-trace/test/appsec/iast/taint-tracking/sources/sql_row.sequelize.plugin.spec.js new file mode 100644 index 00000000000..0e1e84888c7 --- /dev/null +++ b/packages/dd-trace/test/appsec/iast/taint-tracking/sources/sql_row.sequelize.plugin.spec.js @@ -0,0 +1,106 @@ +'use strict' + +const { prepareTestServerForIast } = require('../../utils') + +describe('db sources with sequelize', () => { + withVersions('sequelize', 'sequelize', sequelizeVersion => { + prepareTestServerForIast('sequelize', (testThatRequestHasVulnerability, testThatRequestHasNoVulnerability) => { + let Sequelize, sequelize + + beforeEach(async () => { + Sequelize = require(`../../../../../../../versions/sequelize@${sequelizeVersion}`).get() + sequelize = new Sequelize('database', 'username', 'password', { + dialect: 'sqlite', + logging: false + }) + await sequelize.query(`CREATE TABLE examples ( + id INT, + name VARCHAR(50), + query VARCHAR(100), + command VARCHAR(50), + createdAt DATETIME DEFAULT CURRENT_TIMESTAMP, + updatedAt DATETIME DEFAULT CURRENT_TIMESTAMP )`) + + await sequelize.query(`INSERT INTO examples (id, name, query, command) + VALUES (1, 'Item1', 'SELECT 1', 'ls'), + (2, 'Item2', 'SELECT 1', 'ls'), + (3, 'Item3', 'SELECT 1', 'ls')`) + }) + + afterEach(() => { + return sequelize.close() + }) + + describe('using query method', () => { + testThatRequestHasVulnerability(async (req, res) => { + const result = await sequelize.query('SELECT * from examples') + + await sequelize.query(result[0][0].query) + + res.end('OK') + }, 'SQL_INJECTION', { occurrences: 1 }, null, null, + 'Should have SQL_INJECTION using the first row of the result') + + testThatRequestHasNoVulnerability(async (req, res) => { + const result = await sequelize.query('SELECT * from examples') + + await sequelize.query(result[0][1].query) + + res.end('OK') + }, 'SQL_INJECTION', null, 'Should not taint the second row of a query with default configuration') + + testThatRequestHasNoVulnerability(async (req, res) => { + const result = await sequelize.query('SELECT * from examples') + + const childProcess = require('child_process') + childProcess.execSync(result[0][0].command) + + res.end('OK') + }, 'COMMAND_INJECTION', null, 'Should not detect COMMAND_INJECTION with database source') + }) + + describe('using Model', () => { + // let Model + let Example + + beforeEach(() => { + Example = sequelize.define('example', { + id: { + type: Sequelize.DataTypes.INTEGER, + primaryKey: true + }, + name: Sequelize.DataTypes.STRING, + query: Sequelize.DataTypes.STRING, + command: Sequelize.DataTypes.STRING + }) + }) + + testThatRequestHasVulnerability(async (req, res) => { + const examples = await Example.findAll() + + await sequelize.query(examples[0].query) + + res.end('OK') + }, 'SQL_INJECTION', { occurrences: 1 }, null, null, + 'Should have SQL_INJECTION using the first row of the result') + + testThatRequestHasNoVulnerability(async (req, res) => { + const examples = await Example.findAll() + + await sequelize.query(examples[1].query) + + res.end('OK') + }, 'SQL_INJECTION', null, 'Should not taint the second row of a query with default configuration') + + testThatRequestHasNoVulnerability(async (req, res) => { + const examples = await Example.findAll() + + const childProcess = require('child_process') + childProcess.execSync(examples[0].command) + + res.end('OK') + }, 'COMMAND_INJECTION', null, 'Should not detect COMMAND_INJECTION with database source') + }) + }) + }) +}) diff --git a/packages/dd-trace/test/appsec/iast/utils.js b/packages/dd-trace/test/appsec/iast/utils.js index 6e427bcb629..01274dd954e 100644 --- a/packages/dd-trace/test/appsec/iast/utils.js +++ b/packages/dd-trace/test/appsec/iast/utils.js @@ -256,8 +256,8 @@ function prepareTestServerForIast (description, tests, iastConfig) { }) } - function testThatRequestHasNoVulnerability (fn, vulnerability, makeRequest) { - it(`should not have ${vulnerability} vulnerability`, function (done) { + function testThatRequestHasNoVulnerability (fn, vulnerability, makeRequest, description) { + it(description || `should not have ${vulnerability} vulnerability`, function (done) { app = fn checkNoVulnerabilityInRequest(vulnerability, config, done, makeRequest) }) diff --git a/packages/dd-trace/test/config.spec.js b/packages/dd-trace/test/config.spec.js index ca1a8bcb575..8e87b6fa855 100644 --- a/packages/dd-trace/test/config.spec.js +++ b/packages/dd-trace/test/config.spec.js @@ -324,6 +324,7 @@ describe('Config', () => { { name: 'headerTags', value: [], origin: 'default' }, { name: 'hostname', value: '127.0.0.1', origin: 'default' }, { name: 'iast.cookieFilterPattern', value: '.{32,}', origin: 'default' }, + { name: 'iast.dbRowsToTaint', value: 1, origin: 'default' }, { name: 'iast.deduplicationEnabled', value: true, origin: 'default' }, { name: 'iast.enabled', value: false, origin: 'default' }, { name: 'iast.maxConcurrentRequests', value: 2, origin: 'default' }, @@ -504,6 +505,7 @@ describe('Config', () => { process.env.DD_IAST_MAX_CONCURRENT_REQUESTS = '3' process.env.DD_IAST_MAX_CONTEXT_OPERATIONS = '4' process.env.DD_IAST_COOKIE_FILTER_PATTERN = '.*' + process.env.DD_IAST_DB_ROWS_TO_TAINT = 2 process.env.DD_IAST_DEDUPLICATION_ENABLED = false process.env.DD_IAST_REDACTION_ENABLED = false process.env.DD_IAST_REDACTION_NAME_PATTERN = 'REDACTION_NAME_PATTERN' @@ -615,6 +617,7 @@ describe('Config', () => { expect(config).to.have.nested.property('iast.maxConcurrentRequests', 3) expect(config).to.have.nested.property('iast.maxContextOperations', 4) expect(config).to.have.nested.property('iast.cookieFilterPattern', '.*') + expect(config).to.have.nested.property('iast.dbRowsToTaint', 2) expect(config).to.have.nested.property('iast.deduplicationEnabled', false) expect(config).to.have.nested.property('iast.redactionEnabled', false) expect(config).to.have.nested.property('iast.redactionNamePattern', 'REDACTION_NAME_PATTERN') @@ -659,6 +662,7 @@ describe('Config', () => { { name: 'experimental.runtimeId', value: true, origin: 'env_var' }, { name: 'hostname', value: 'agent', origin: 'env_var' }, { name: 'iast.cookieFilterPattern', value: '.*', origin: 'env_var' }, + { name: 'iast.dbRowsToTaint', value: 2, origin: 'env_var' }, { name: 'iast.deduplicationEnabled', value: false, origin: 'env_var' }, { name: 'iast.enabled', value: true, origin: 'env_var' }, { name: 'iast.maxConcurrentRequests', value: '3', origin: 'env_var' }, @@ -857,6 +861,7 @@ describe('Config', () => { maxConcurrentRequests: 4, maxContextOperations: 5, cookieFilterPattern: '.*', + dbRowsToTaint: 2, deduplicationEnabled: false, redactionEnabled: false, redactionNamePattern: 'REDACTION_NAME_PATTERN', @@ -929,6 +934,7 @@ describe('Config', () => { expect(config).to.have.nested.property('iast.maxConcurrentRequests', 4) expect(config).to.have.nested.property('iast.maxContextOperations', 5) expect(config).to.have.nested.property('iast.cookieFilterPattern', '.*') + expect(config).to.have.nested.property('iast.dbRowsToTaint', 2) expect(config).to.have.nested.property('iast.deduplicationEnabled', false) expect(config).to.have.nested.property('iast.redactionEnabled', false) expect(config).to.have.nested.property('iast.redactionNamePattern', 'REDACTION_NAME_PATTERN') @@ -976,6 +982,7 @@ describe('Config', () => { { name: 'flushMinSpans', value: 500, origin: 'code' }, { name: 'hostname', value: 'agent', origin: 'code' }, { name: 'iast.cookieFilterPattern', value: '.*', origin: 'code' }, + { name: 'iast.dbRowsToTaint', value: 2, origin: 'code' }, { name: 'iast.deduplicationEnabled', value: false, origin: 'code' }, { name: 'iast.enabled', value: true, origin: 'code' }, { name: 'iast.maxConcurrentRequests', value: 4, origin: 'code' }, @@ -1201,6 +1208,7 @@ describe('Config', () => { process.env.DD_API_SECURITY_ENABLED = 'false' process.env.DD_REMOTE_CONFIG_POLL_INTERVAL_SECONDS = 11 process.env.DD_IAST_ENABLED = 'false' + process.env.DD_IAST_DB_ROWS_TO_TAINT = '2' process.env.DD_IAST_COOKIE_FILTER_PATTERN = '.*' process.env.DD_IAST_REDACTION_NAME_PATTERN = 'name_pattern_to_be_overriden_by_options' process.env.DD_IAST_REDACTION_VALUE_PATTERN = 'value_pattern_to_be_overriden_by_options' @@ -1278,6 +1286,7 @@ describe('Config', () => { iast: { enabled: true, cookieFilterPattern: '.{10,}', + dbRowsToTaint: 3, redactionNamePattern: 'REDACTION_NAME_PATTERN', redactionValuePattern: 'REDACTION_VALUE_PATTERN' }, @@ -1346,6 +1355,7 @@ describe('Config', () => { expect(config).to.have.nested.property('iast.requestSampling', 30) expect(config).to.have.nested.property('iast.maxConcurrentRequests', 2) expect(config).to.have.nested.property('iast.maxContextOperations', 2) + expect(config).to.have.nested.property('iast.dbRowsToTaint', 3) expect(config).to.have.nested.property('iast.deduplicationEnabled', true) expect(config).to.have.nested.property('iast.cookieFilterPattern', '.{10,}') expect(config).to.have.nested.property('iast.redactionEnabled', true) @@ -1383,6 +1393,7 @@ describe('Config', () => { maxConcurrentRequests: 3, maxContextOperations: 4, cookieFilterPattern: '.*', + dbRowsToTaint: 3, deduplicationEnabled: false, redactionEnabled: false, redactionNamePattern: 'REDACTION_NAME_PATTERN', @@ -1416,6 +1427,7 @@ describe('Config', () => { maxConcurrentRequests: 6, maxContextOperations: 7, cookieFilterPattern: '.{10,}', + dbRowsToTaint: 2, deduplicationEnabled: true, redactionEnabled: true, redactionNamePattern: 'IGNORED_REDACTION_NAME_PATTERN', @@ -1464,6 +1476,7 @@ describe('Config', () => { maxConcurrentRequests: 3, maxContextOperations: 4, cookieFilterPattern: '.*', + dbRowsToTaint: 3, deduplicationEnabled: false, redactionEnabled: false, redactionNamePattern: 'REDACTION_NAME_PATTERN', diff --git a/packages/dd-trace/test/fixtures/telemetry/config_norm_rules.json b/packages/dd-trace/test/fixtures/telemetry/config_norm_rules.json index f00fbc27dcb..d4014e8b839 100644 --- a/packages/dd-trace/test/fixtures/telemetry/config_norm_rules.json +++ b/packages/dd-trace/test/fixtures/telemetry/config_norm_rules.json @@ -1,741 +1,808 @@ { - "aas_app_type": "aas_app_type", - "aas_configuration_error": "aas_configuration_error", - "aas_functions_runtime_version": "aas_functions_runtime_version", - "aas_siteextensions_version": "aas_site_extensions_version", - "activity_listener_enabled": "activity_listener_enabled", - "agent_transport": "agent_transport", - "DD_AGENT_TRANSPORT": "agent_transport", - "agent_url": "trace_agent_url", - "analytics_enabled": "analytics_enabled", - "autoload_no_compile": "autoload_no_compile", - "cloud_hosting": "cloud_hosting_provider", - "code_hotspots_enabled": "code_hotspots_enabled", - "data_streams_enabled": "data_streams_enabled", - "dsmEnabled": "data_streams_enabled", - "enabled": "trace_enabled", - "environment_fulltrust_appdomain": "environment_fulltrust_appdomain_enabled", - "logInjection_enabled": "logs_injection_enabled", - "partialflush_enabled": "trace_partial_flush_enabled", - "partialflush_minspans": "trace_partial_flush_min_spans", - "platform": "platform", - "profiler_loaded": "profiler_loaded", - "routetemplate_expansion_enabled": "trace_route_template_expansion_enabled", - "routetemplate_resourcenames_enabled": "trace_route_template_resource_names_enabled", - "runtimemetrics_enabled": "runtime_metrics_enabled", - "runtime.metrics.enabled": "runtime_metrics_enabled", - "sample_rate": "trace_sample_rate", - "sampling_rules": "trace_sample_rules", - "span_sampling_rules": "span_sample_rules", - "spanattributeschema": "trace_span_attribute_schema", - "security_enabled": "appsec_enabled", - "stats_computation_enabled": "trace_stats_computation_enabled", - "native_tracer_version": "native_tracer_version", - "managed_tracer_framework": "managed_tracer_framework", - "wcf_obfuscation_enabled": "trace_wcf_obfuscation_enabled", - "data.streams.enabled": "data_streams_enabled", - "dynamic.instrumentation.enabled": "dynamic_instrumentation_enabled", - "dynamic_instrumentation.enabled": "dynamic_instrumentation_enabled", - "HOSTNAME": "agent_hostname", - "dd_agent_host": "agent_host", - "instrumentation.telemetry.enabled": "instrumentation_telemetry_enabled", - "integrations.enabled": "trace_integrations_enabled", - "logs.injection": "logs_injection_enabled", - "logs.mdc.tags.injection": "logs_mdc_tags_injection_enabled", - "os.name": "os_name", - "openai_service": "open_ai_service", - "openai_logs_enabled": "open_ai_logs_enabled", - "openAiLogsEnabled": "open_ai_logs_enabled", - "openai_span_char_limit": "open_ai_span_char_limit", - "openaiSpanCharLimit": "open_ai_span_char_limit", - "openai_span_prompt_completion_sample_rate": "open_ai_span_prompt_completion_sample_rate", - "openai_log_prompt_completion_sample_rate": "open_ai_log_prompt_completion_sample_rate", - "openai_metrics_enabled": "open_ai_metrics_enabled", - "priority.sampling": "trace_priority_sample_enabled", - "profiling.allocation.enabled": "profiling_allocation_enabled", - "profiling.enabled": "profiling_enabled", - "profiling.start-force-first": "profiling_start_force_first", - "remote_config.enabled": "remote_config_enabled", - "remoteConfig.enabled": "remote_config_enabled", - "remoteConfig.pollInterval": "remote_config_poll_interval", - "trace.agent.port": "trace_agent_port", - "trace.agent.v0.5.enabled": "trace_agent_v0.5_enabled", - "trace.analytics.enabled": "trace_analytics_enabled", - "trace.enabled": "trace_enabled", - "trace.client-ip.enabled": "trace_client_ip_enabled", - "trace.jms.propagation.enabled": "trace_jms_propagation_enabled", - "trace.x-datadog-tags.max.length": "trace_x_datadog_tags_max_length", - "trace.kafka.client.propagation.enabled": "trace_kafka_client_propagation_enabled", - "trace.laravel_queue_distributed_tracing": "trace_laravel_queue_distributed_tracing", - "trace.symfony_messenger_distributed_tracing": "trace_symfony_messenger_distributed_tracing", - "trace.symfony_messenger_middlewares": "trace_symfony_messenger_middlewares", - "trace.sources_path": "trace_sources_path", - "trace.log_file": "trace_log_file", - "trace.log_level": "trace_log_level", - "kafka.client.base64.decoding.enabled": "trace_kafka_client_base64_decoding_enabled", - "trace.aws-sdk.propagation.enabled": "trace_aws_sdk_propagation_enabled", - "trace.aws-sdk.legacy.tracing.enabled": "trace_aws_sdk_legacy_tracing_enabled", - "trace.servlet.principal.enabled": "trace_servlet_principal_enabled", - "trace.servlet.async-timeout.error": "trace_servlet_async_timeout_error_enabled", - "trace.rabbit.propagation.enabled": "trace_rabbit_propagation_enabled", - "trace.partial.flush.min.spans": "trace_partial_flush_min_spans", - "trace.sample.rate": "trace_sample_rate", - "trace.sqs.propagation.enabled": "trace_sqs_propagation_enabled", - "trace.peerservicetaginterceptor.enabled": "trace_peer_service_tag_interceptor_enabled", - "dd_trace_sample_rate": "trace_sample_rate", - "trace_methods": "trace_methods", - "tracer_instance_count": "trace_instance_count", - "trace.db.client.split-by-instance": "trace_db_client_split_by_instance", - "trace.db.client.split-by-instance.type.suffix": "trace_db_client_split_by_instance_type_suffix", - "trace.http.client.split-by-domain" : "trace_http_client_split_by_domain", - "trace.agent.timeout": "trace_agent_timeout", - "trace.header.tags.legacy.parsing.enabled": "trace_header_tags_legacy_parsing_enabled", - "trace.client-ip.resolver.enabled": "trace_client_ip_resolver_enabled", - "trace.play.report-http-status": "trace_play_report_http_status", - "trace.jmxfetch.tomcat.enabled": "trace_jmxfetch_tomcat_enabled", - "trace.jmxfetch.kafka.enabled": "trace_jmxfetch_kafka_enabled", - "trace.scope.depth.limit": "trace_scope_depth_limit", - "inferredProxyServicesEnabled": "inferred_proxy_services_enabled", - "resolver.use.loadclass": "resolver_use_loadclass", - "resolver.outline.pool.enabled": "resolver_outline_pool_enabled", - "appsec.apiSecurity.enabled": "api_security_enabled", - "appsec.apiSecurity.requestSampling": "api_security_request_sample_rate", - "appsec.enabled": "appsec_enabled", - "appsec.eventTracking": "appsec_auto_user_events_tracking", - "appsec.eventTracking.mode": "appsec_auto_user_events_tracking", - "appsec.testing": "appsec_testing", - "appsec.trace.rate.limit": "appsec_trace_rate_limit", - "appsec.obfuscatorKeyRegex": "appsec_obfuscation_parameter_key_regexp", - "appsec.obfuscatorValueRegex": "appsec_obfuscation_parameter_value_regexp", - "appsec.rasp.enabled": "appsec_rasp_enabled", - "appsec.rateLimit": "appsec_rate_limit", - "appsec.rules": "appsec_rules", - "appsec.sca_enabled": "appsec_sca_enabled", - "appsec.wafTimeout": "appsec_waf_timeout", - "appsec.sca.enabled": "appsec_sca_enabled", - "clientIpHeader": "trace_client_ip_header", - "clientIpEnabled": "trace_client_ip_enabled", - "clientIpHeaderDisabled": "client_ip_header_disabled", - "debug": "trace_debug_enabled", - "dd.trace.debug": "trace_debug_enabled", - "dogstatsd.hostname": "dogstatsd_hostname", - "dogstatsd.port": "dogstatsd_port", - "dogstatsd.start-delay": "dogstatsd_start_delay", - "env": "env", - "experimental.b3": "experimental_b3", - "experimental.enableGetRumData": "experimental_enable_get_rum_data", - "experimental.exporter": "experimental_exporter", - "experimental.runtimeId": "experimental_runtime_id", - "experimental.sampler.rateLimit": "experimental_sampler_rate_limit", - "experimental.sampler.sampleRate": "experimental_sampler_sample_rate", - "experimental.traceparent": "experimental_traceparent", - "flushInterval": "flush_interval", - "flushMinSpans": "flush_min_spans", - "hostname": "agent_hostname", - "iast.enabled": "iast_enabled", - "iast.cookieFilterPattern": "iast_cookie_filter_pattern", - "iast.deduplication.enabled": "iast_deduplication_enabled", - "iast.maxConcurrentRequests": "iast_max_concurrent_requests", - "iast.max-concurrent-requests": "iast_max_concurrent_requests", - "iast.maxContextOperations": "iast_max_context_operations", - "iast.requestSampling": "iast_request_sampling", - "iast.request-sampling": "iast_request_sampling", - "iast.debug.enabled": "iast_debug_enabled", - "iast.vulnerabilities-per-request": "iast_vulnerability_per_request", - "iast.deduplicationEnabled": "iast_deduplication_enabled", - "iast.redactionEnabled": "iast_redaction_enabled", - "iast.redactionNamePattern": "iast_redaction_name_pattern", - "iast.redactionValuePattern": "iast_redaction_value_pattern", - "iast.telemetryVerbosity": "iast_telemetry_verbosity", - "isAzureFunction": "azure_function", - "isGitUploadEnabled": "git_upload_enabled", - "isIntelligentTestRunnerEnabled": "intelligent_test_runner_enabled", - "logger": "logger", - "logInjection": "logs_injection_enabled", - "logLevel": "trace_log_level", - "memcachedCommandEnabled": "memchached_command_enabled", - "lookup": "lookup", - "plugins": "plugins", - "port": "trace_agent_port", - "profiling.exporters": "profiling_exporters", - "profiling.sourceMap": "profiling_source_map_enabled", - "protocolVersion": "trace_agent_protocol_version", - "querystringObfuscation": "trace_obfuscation_query_string_regexp", - "reportHostname": "trace_report_hostname", - "trace.report-hostname": "trace_report_hostname", - "runtimeMetrics": "runtime_metrics_enabled", - "sampler.rateLimit": "trace_rate_limit", - "trace.rate.limit": "trace_rate_limit", - "sampler.sampleRate": "trace_sample_rate", - "sampleRate": "trace_sample_rate", - "scope": "scope", - "service": "service", - "serviceMapping": "dd_service_mapping", - "site": "site", - "startupLogs": "trace_startup_logs_enabled", - "stats.enabled": "stats_enabled", - "DD_TRACE_HEADER_TAGS": "trace_header_tags", - "tagsHeaderMaxLength": "trace_header_tags_max_length", - "telemetryEnabled": "instrumentation_telemetry_enabled", - "otel_enabled": "trace_otel_enabled", - "trace.otel.enabled": "trace_otel_enabled", - "trace.otel_enabled": "trace_otel_enabled", - "tracing": "trace_enabled", - "url": "trace_agent_url", - "version": "application_version", - "trace.tracer.metrics.enabled": "trace_metrics_enabled", - "trace.perf.metrics.enabled": "trace_perf_metrics_enabled", - "trace.health.metrics.enabled": "trace_health_metrics_enabled", - "trace.health.metrics.statsd.port": "trace_health_metrics_statsd_port", - "trace.grpc.server.trim-package-resource": "trace_grpc_server_trim_package_resource_enabled", - "DD_TRACE_DEBUG": "trace_debug_enabled", - "profiling.start-delay": "profiling_start_delay", - "profiling.upload.period": "profiling_upload_period", - "profiling.async.enabled": "profiling_async_enabled", - "profiling.async.alloc.enabled": "profiling_async_alloc_enabled", - "profiling.directallocation.enabled": "profiling_direct_allocation_enabled", - "profiling.hotspots.enabled": "profiling_hotspots_enabled", - "profiling.async.cpu.enabled": "profiling_async_cpu_enabled", - "profiling.async.memleak.enabled": "profiling_async_memleak_enabled", - "profiling.async.wall.enabled": "profiling_async_wall_enabled", - "profiling.ddprof.enabled": "profiling_ddprof_enabled", - "profiling.heap.enabled": "profiling_heap_enabled", - "profiling.legacy.tracing.integration": "profiling_legacy_tracing_integration_enabled", - "queryStringObfuscation": "trace_obfuscation_query_string_regexp", - "dbmPropagationMode": "dbm_propagation_mode", - "rcPollingInterval": "rc_polling_interval", - "jmxfetch.initial-refresh-beans-period": "jmxfetch_initial_refresh_beans_period", - "jmxfetch.refresh-beans-period": "jmxfetch_initial_refresh_beans_period", - "jmxfetch.multiple-runtime-services.enabled": "jmxfetch_multiple_runtime_services_enabled", - "jmxfetch.enabled": "jmxfetch_enabled", - "jmxfetch.statsd.port": "jmxfetch_statsd_port", - "jmxfetch.check-period": "jmxfetch_check_period", - "appsec.blockedTemplateGraphql": "appsec_blocked_template_graphql", - "appsec.blockedTemplateHtml": "appsec_blocked_template_html", - "appsec.blockedTemplateJson": "appsec_blocked_template_json", - "appsec.waf.timeout": "appsec_waf_timeout", - "civisibility.enabled": "ci_visibility_enabled", - "civisibility.agentless.enabled": "ci_visibility_agentless_enabled", - "isCiVisibility": "ci_visibility_enabled", - "cws.enabled": "cws_enabled", - "AWS_LAMBDA_INITIALIZATION_TYPE": "aws_lambda_initialization_type", - "http.server.tag.query-string": "trace_http_server_tag_query_string", - "http.server.route-based-naming": "trace_http_server_route_based_naming_enabled", - "http.client.tag.query-string": "trace_http_client_tag_query_string", - "DD_TRACE_HTTP_CLIENT_TAG_QUERY_STRING": "trace_http_client_tag_query_string", - "hystrix.tags.enabled": "hystrix_tags_enabled", - "hystrix.measured.enabled": "hystrix_measured_enabled", - "ignite.cache.include_keys": "ignite_cache_include_keys_enabled", - "dynamic.instrumentation.classfile.dump.enabled": "dynamic_instrumentation_classfile_dump_enabled", - "dynamic.instrumentation.metrics.enabled": "dynamic_instrumentation_metrics_enabled", - "message.broker.split-by-destination": "message_broker_split_by_destination", - "agent_feature_drop_p0s": "agent_feature_drop_p0s", - "appsec.rules.metadata.rules_version": "appsec_rules_metadata_rules_version", - "appsec.rules.version": "appsec_rules_version", - "appsec.customRulesProvided": "appsec_rules_custom_provided", - "dogstatsd_addr": "dogstatsd_url", - "lambda_mode": "lambda_mode", - "profiling.ddprof.alloc.enabled": "profiling_ddprof_alloc_enabled", - "profiling.ddprof.cpu.enabled": "profiling_ddprof_cpu_enabled", - "profiling.ddprof.memleak.enabled": "profiling_ddprof_memleak_enabled", - "profiling.ddprof.wall.enabled": "profiling_ddprof_wall_enabled", - "profiling_endpoints_enabled": "profiling_endpoints_enabled", - "send_retries": "trace_send_retries", - "telemetry.enabled": "instrumentation_telemetry_enabled", - "telemetry.debug": "instrumentation_telemetry_debug_enabled", - "telemetry.logCollection": "instrumentation_telemetry_log_collection_enabled", - "telemetry.metrics": "instrumentation_telemetry_metrics_enabled", - "telemetry.metricsInterval": "instrumentation_telemetry_metrics_interval", - "telemetry.heartbeat.interval": "instrumentation_telemetry_heartbeat_interval", - "telemetry_heartbeat_interval": "instrumentation_telemetry_heartbeat_interval", - "universal_version": "universal_version_enabled", - "global_tag_version": "version", - "traceId128BitGenerationEnabled": "trace_128_bits_id_enabled", - "traceId128BitLoggingEnabled": "trace_128_bits_id_logging_enabled", - "trace.status404decorator.enabled": "trace_status_404_decorator_enabled", - "trace.status404rule.enabled": "trace_status_404_rule_enabled", - "discovery": "agent_discovery_enabled", - "repositoryurl": "repository_url", - "gitmetadataenabled": "git_metadata_enabled", - "commitsha": "commit_sha", - "isgcpfunction": "is_gcp_function", - "isGCPFunction": "is_gcp_function", - "legacy.installer.enabled": "legacy_installer_enabled", - "trace.request_init_hook": "trace_request_init_hook", - "dogstatsd_url": "dogstatsd_url", - "distributed_tracing": "trace_distributed_trace_enabled", - "autofinish_spans": "trace_auto_finish_spans_enabled", - "trace.url_as_resource_names_enabled": "trace_url_as_resource_names_enabled", - "integrations_disabled": "trace_disabled_integrations", - "priority_sampling": "trace_priority_sampling_enabled", - "trace.auto_flush_enabled": "trace_auto_flush_enabled", - "trace.measure_compile_time": "trace_measure_compile_time_enabled", - "trace.measure_peak_memory_usage": "trace_measure_peak_memory_usage_enabled", - "trace.health_metrics_heartbeat_sample_rate": "trace_health_metrics_heartbeat_sample_rate", - "trace.redis_client_split_by_host": "trace_redis_client_split_by_host_enabled", - "trace.memory_limit": "trace_memory_limit", - "trace.flush_collect_cycles": "trace_flush_collect_cycles_enabled", - "trace.resource_uri_fragment_regex": "trace_resource_uri_fragment_regex", - "trace.resource_uri_mapping_incoming": "trace_resource_uri_mapping_incoming", - "trace.resource_uri_mapping_outgoing": "trace_resource_uri_mapping_outgoing", - "trace.resource_uri_query_param_allowed": "trace_resource_uri_query_param_allowed", - "trace.http_url_query_param_allowed": "trace_http_url_query_param_allowed", - "trace.http_post_data_param_allowed": "trace_http_post_data_param_allowed", - "trace.sampling_rules": "trace_sample_rules", - "span_sampling_rules_file": "span_sample_rules_file", - "trace.propagation_style_extract": "trace_propagation_style_extract", - "trace.propagation_style_inject": "trace_propagation_style_inject", - "trace.propagation_style": "trace_propagation_style", - "trace.propagation_extract_first": "trace_propagation_extract_first", - "tracePropagationExtractFirst": "trace_propagation_extract_first", - "tracePropagationStyle.extract": "trace_propagation_style_extract", - "tracePropagationStyle.inject": "trace_propagation_style_inject", - "tracePropagationStyle,otelPropagators": "trace_propagation_style_otel_propagators", - "tracing.distributed_tracing.propagation_extract_style": "trace_propagation_style_extract", - "tracing.distributed_tracing.propagation_inject_style": "trace_propagation_style_inject", - "trace.traced_internal_functions": "trace_traced_internal_functions", - "trace.agent_connect_timeout": "trace_agent_connect_timeout", - "trace.debug_prng_seed": "trace_debug_prng_seed", - "log_backtrace": "trace_log_backtrace_enabled", - "trace.generate_root_span": "trace_generate_root_span_enabled", - "trace.spans_limit": "trace_spans_limit", - "trace.128_bit_traceid_generation_enabled": "trace_128_bits_id_enabled", - "trace.agent_max_consecutive_failures": "trace_send_retries", - "trace.agent_attempt_retry_time_msec": "trace_agent_attempt_retry_time_msec", - "trace.bgs_connect_timeout": "trace_bgs_connect_timeout", - "trace.bgs_timeout": "trace_bgs_timeout", - "trace.agent_flush_interval": "trace_agent_flush_interval", - "trace.agent_flush_after_n_requests": "trace_agent_flush_after_n_requests", - "trace.shutdown_timeout": "trace_shutdown_timeout", - "trace.agent_debug_verbose_curl": "trace_agent_debug_verbose_curl_enabled", - "trace.debug_curl_output": "trace_debug_curl_output_enabled", - "trace.beta_high_memory_pressure_percent": "trace_beta_high_memory_pressure_percent", - "trace.warn_legacy_dd_trace": "trace_warn_legacy_dd_trace_enabled", - "trace.retain_thread_capabilities": "trace_retain_thread_capabilities_enabled", - "trace.client_ip_header": "client_ip_header", - "trace.forked_process": "trace_forked_process_enabled", - "trace.hook_limit": "trace_hook_limit", - "trace.agent_max_payload_size": "trace_agent_max_payload_size", - "trace.agent_stack_initial_size": "trace_agent_stack_initial_size", - "trace.agent_stack_backlog": "trace_agent_stack_backlog", - "trace.agent_retries": "trace_send_retries", - "trace.agent_test_session_token": "trace_agent_test_session_token", - "trace.propagate_user_id_default": "trace_propagate_user_id_default_enabled", - "dbm_propagation_mode": "dbm_propagation_mode", - "trace.remove_root_span_laravel_queue": "trace_remove_root_span_laravel_queue_enabled", - "trace.remove_root_span_symfony_messenger": "trace_remove_root_span_symfony_messenger_enabled", - "trace.remove_autoinstrumentation_orphans": "trace_remove_auto_instrumentation_orphans_enabled", - "trace.memcached_obfuscation": "trace_memcached_obfuscation_enabled", - "DD_TRACE_CONFIG_FILE": "trace_config_file", - "DD_DOTNET_TRACER_CONFIG_FILE": "trace_config_file", - "DD_ENV": "env", - "DD_SERVICE": "service", - "DD_SERVICE_NAME": "service", - "DD_VERSION": "application_version", - "DD_GIT_REPOSITORY_URL": "repository_url", - "git_repository_url": "repository_url", - "DD_GIT_COMMIT_SHA": "commit_sha", - "DD_TRACE_GIT_METADATA_ENABLED": "git_metadata_enabled", - "trace.git_metadata_enabled": "git_metadata_enabled", - "git_commit_sha": "commit_sha", - "DD_TRACE_ENABLED": "trace_enabled", - "DD_EXPERIMENTAL_APPSEC_STANDALONE_ENABLED": "experimental_appsec_standalone_enabled", - "DD_INTERNAL_WAIT_FOR_DEBUGGER_ATTACH": "internal_wait_for_debugger_attach_enabled", - "DD_INTERNAL_WAIT_FOR_NATIVE_DEBUGGER_ATTACH": "internal_wait_for_native_debugger_attach_enabled", - "DD_DISABLED_INTEGRATIONS": "trace_disabled_integrations", - "DD_TRACE_ANALYTICS_ENABLED": "trace_analytics_enabled", - "DD_TRACE_BUFFER_SIZE": "trace_serialization_buffer_size", - "trace.buffer_size": "trace_serialization_buffer_size", - "DD_TRACE_BATCH_INTERVAL": "trace_serialization_batch_interval", - "DD_LOG_INJECTION": "logs_injection_enabled", - "DD_LOGS_INJECTION": "logs_injection_enabled", - "DD_TRACE_RATE_LIMIT": "trace_rate_limit", - "DD_MAX_TRACES_PER_SECOND": "trace_rate_limit", - "DD_TRACE_SAMPLING_RULES": "trace_sample_rules", - "DD_SPAN_SAMPLING_RULES": "span_sample_rules", - "DD_TRACE_SAMPLE_RATE": "trace_sample_rate", + "AWS_LAMBDA_INITIALIZATION_TYPE": "aws_lambda_initialization_type", + "COMPUTERNAME": "aas_instance_name", + "DATADOG_TRACE_AGENT_HOSTNAME": "agent_host", + "DATADOG_TRACE_AGENT_PORT": "trace_agent_port", + "DD_AAS_DOTNET_EXTENSION_VERSION": "aas_site_extensions_version", + "DD_AAS_ENABLE_CUSTOM_METRICS": "aas_custom_metrics_enabled", + "DD_AAS_ENABLE_CUSTOM_TRACING": "aas_custom_tracing_enabled", + "DD_AGENT_TRANSPORT": "agent_transport", + "DD_API_SECURITY_ENABLED": "api_security_enabled", + "DD_API_SECURITY_MAX_CONCURRENT_REQUESTS": "api_security_max_concurrent_requests", + "DD_API_SECURITY_REQUEST_SAMPLE_RATE": "api_security_request_sample_rate", + "DD_API_SECURITY_SAMPLE_DELAY": "api_security_sample_delay", "DD_APM_ENABLE_RARE_SAMPLER": "trace_rare_sampler_enabled", - "DD_TRACE_METRICS_ENABLED": "trace_metrics_enabled", - "DD_RUNTIME_METRICS_ENABLED": "runtime_metrics_enabled", - "DD_TRACE_AGENT_PATH": "agent_trace_agent_excecutable_path", - "DD_TRACE_AGENT_ARGS": "agent_trace_agent_excecutable_args", - "DD_DOGSTATSD_PATH": "agent_dogstatsd_executable_path", - "DD_DOGSTATSD_ARGS": "agent_dogstatsd_executable_args", - "DD_DIAGNOSTIC_SOURCE_ENABLED": "trace_diagnostic_source_enabled", - "DD_SITE": "site", - "DD_TRACE_HTTP_CLIENT_EXCLUDED_URL_SUBSTRINGS": "trace_http_client_excluded_urls", - "DD_HTTP_SERVER_ERROR_STATUSES": "trace_http_server_error_statuses", - "DD_HTTP_CLIENT_ERROR_STATUSES": "trace_http_client_error_statuses", - "DD_TRACE_HTTP_SERVER_ERROR_STATUSES": "trace_http_server_error_statuses", - "DD_TRACE_HTTP_CLIENT_ERROR_STATUSES": "trace_http_client_error_statuses", - "DD_TRACE_CLIENT_IP_HEADER": "trace_client_ip_header", - "DD_TRACE_CLIENT_IP_ENABLED": "trace_client_ip_enabled", - "DD_TRACE_KAFKA_CREATE_CONSUMER_SCOPE_ENABLED": "trace_kafka_create_consumer_scope_enabled", - "DD_TRACE_EXPAND_ROUTE_TEMPLATES_ENABLED": "trace_route_template_expansion_enabled", - "DD_TRACE_STATS_COMPUTATION_ENABLED": "trace_stats_computation_enabled", - "_DD_TRACE_STATS_COMPUTATION_INTERVAL": "trace_stats_computation_interval", - "DD_TRACE_PROPAGATION_STYLE_INJECT": "trace_propagation_style_inject", - "DD_PROPAGATION_STYLE_INJECT": "trace_propagation_style_inject", - "DD_TRACE_PROPAGATION_STYLE_EXTRACT": "trace_propagation_style_extract", - "DD_PROPAGATION_STYLE_EXTRACT": "trace_propagation_style_extract", - "DD_TRACE_PROPAGATION_STYLE": "trace_propagation_style", - "DD_TRACE_PROPAGATION_EXTRACT_FIRST": "trace_propagation_extract_first", - "DD_TRACE_METHODS": "trace_methods", - "DD_TRACE_OBFUSCATION_QUERY_STRING_REGEXP": "trace_obfuscation_query_string_regexp", - "DD_TRACE_OBFUSCATION_QUERY_STRING_REGEXP_TIMEOUT": "trace_obfuscation_query_string_regexp_timeout", - "DD_HTTP_SERVER_TAG_QUERY_STRING_SIZE": "trace_http_server_tag_query_string_size", - "DD_HTTP_SERVER_TAG_QUERY_STRING": "trace_http_server_tag_query_string_enabled", - "DD_DBM_PROPAGATION_MODE": "dbm_propagation_mode", - "DD_TRACE_SPAN_ATTRIBUTE_SCHEMA": "trace_span_attribute_schema", - "DD_TRACE_PEER_SERVICE_DEFAULTS_ENABLED": "trace_peer_service_defaults_enabled", - "DD_TRACE_REMOVE_INTEGRATION_SERVICE_NAMES_ENABLED": "trace_remove_integration_service_names_enabled", - "DD_TRACE_X_DATADOG_TAGS_MAX_LENGTH": "trace_x_datadog_tags_max_length", - "DD_DATA_STREAMS_ENABLED": "data_streams_enabled", - "DD_DATA_STREAMS_LEGACY_HEADERS": "data_streams_legacy_headers", - "DD_CIVISIBILITY_ENABLED": "ci_visibility_enabled", + "DD_APM_RECEIVER_PORT": "trace_agent_port", + "DD_APM_RECEIVER_SOCKET": "trace_agent_socket", + "DD_APPSEC_AUTOMATED_USER_EVENTS_TRACKING": "appsec_auto_user_events_tracking", + "DD_APPSEC_AUTO_USER_INSTRUMENTATION_MODE": "appsec_auto_user_instrumentation_mode", + "DD_APPSEC_ENABLED": "appsec_enabled", + "DD_APPSEC_EXTRA_HEADERS": "appsec_extra_headers", + "DD_APPSEC_HTTP_BLOCKED_TEMPLATE_HTML": "appsec_blocked_template_html", + "DD_APPSEC_HTTP_BLOCKED_TEMPLATE_JSON": "appsec_blocked_template_json", + "DD_APPSEC_IPHEADER": "appsec_ip_header", + "DD_APPSEC_KEEP_TRACES": "appsec_force_keep_traces_enabled", + "DD_APPSEC_MAX_STACK_TRACES": "appsec_max_stack_traces", + "DD_APPSEC_MAX_STACK_TRACE_DEPTH": "appsec_max_stack_trace_depth", + "DD_APPSEC_MAX_STACK_TRACE_DEPTH_TOP_PERCENT": "appsec_max_stack_trace_depth_top_percent", + "DD_APPSEC_OBFUSCATION_PARAMETER_KEY_REGEXP": "appsec_obfuscation_parameter_key_regexp", + "DD_APPSEC_OBFUSCATION_PARAMETER_VALUE_REGEXP": "appsec_obfuscation_parameter_value_regexp", + "DD_APPSEC_RASP_ENABLED": "appsec_rasp_enabled", + "DD_APPSEC_RULES": "appsec_rules", + "DD_APPSEC_SCA_ENABLED": "appsec_sca_enabled", + "DD_APPSEC_STACK_TRACE_ENABLED": "appsec_stack_trace_enabled", + "DD_APPSEC_TRACE_RATE_LIMIT": "appsec_trace_rate_limit", + "DD_APPSEC_WAF_DEBUG": "appsec_waf_debug_enabled", + "DD_APPSEC_WAF_TIMEOUT": "appsec_waf_timeout", + "DD_AZURE_APP_SERVICES": "aas_enabled", + "DD_CALL_BASIC_CONFIG": "dd_call_basic_config", "DD_CIVISIBILITY_AGENTLESS_ENABLED": "ci_visibility_agentless_enabled", "DD_CIVISIBILITY_AGENTLESS_URL": "ci_visibility_agentless_url", - "DD_CIVISIBILITY_LOGS_ENABLED": "ci_visibility_logs_enabled", + "DD_CIVISIBILITY_CODE_COVERAGE_COLLECTORPATH": "ci_visibility_code_coverage_collectorpath", "DD_CIVISIBILITY_CODE_COVERAGE_ENABLED": "ci_visibility_code_coverage_enabled", - "DD_CIVISIBILITY_CODE_COVERAGE_MODE": "ci_visibility_code_coverage_mode", - "DD_CIVISIBILITY_CODE_COVERAGE_SNK_FILEPATH": "ci_visibility_code_coverage_snk_path", "DD_CIVISIBILITY_CODE_COVERAGE_ENABLE_JIT_OPTIMIZATIONS": "ci_visibility_code_coverage_jit_optimisations_enabled", + "DD_CIVISIBILITY_CODE_COVERAGE_MODE": "ci_visibility_code_coverage_mode", "DD_CIVISIBILITY_CODE_COVERAGE_PATH": "ci_visibility_code_coverage_path", - "DD_CIVISIBILITY_GIT_UPLOAD_ENABLED": "ci_visibility_git_upload_enabled", - "DD_CIVISIBILITY_TESTSSKIPPING_ENABLED": "ci_visibility_test_skipping_enabled", - "DD_CIVISIBILITY_ITR_ENABLED": "ci_visibility_intelligent_test_runner_enabled", - "DD_CIVISIBILITY_FORCE_AGENT_EVP_PROXY": "ci_visibility_force_agent_evp_proxy_enabled", + "DD_CIVISIBILITY_CODE_COVERAGE_SNK_FILEPATH": "ci_visibility_code_coverage_snk_path", + "DD_CIVISIBILITY_EARLY_FLAKE_DETECTION_ENABLED": "ci_visibility_early_flake_detection_enabled", + "DD_CIVISIBILITY_ENABLED": "ci_visibility_enabled", "DD_CIVISIBILITY_EXTERNAL_CODE_COVERAGE_PATH": "ci_visibility_code_coverage_external_path", + "DD_CIVISIBILITY_FLAKY_RETRY_COUNT": "ci_visibility_flaky_retry_count", + "DD_CIVISIBILITY_FLAKY_RETRY_ENABLED": "ci_visibility_flaky_retry_enabled", + "DD_CIVISIBILITY_FORCE_AGENT_EVP_PROXY": "ci_visibility_force_agent_evp_proxy_enabled", "DD_CIVISIBILITY_GAC_INSTALL_ENABLED": "ci_visibility_gac_install_enabled", - "DD_CIVISIBILITY_EARLY_FLAKE_DETECTION_ENABLED": "ci_visibility_early_flake_detection_enabled", - "DD_CIVISIBILITY_CODE_COVERAGE_COLLECTORPATH": "ci_visibility_code_coverage_collectorpath", + "DD_CIVISIBILITY_GIT_UPLOAD_ENABLED": "ci_visibility_git_upload_enabled", + "DD_CIVISIBILITY_ITR_ENABLED": "ci_visibility_intelligent_test_runner_enabled", + "DD_CIVISIBILITY_LOGS_ENABLED": "ci_visibility_logs_enabled", "DD_CIVISIBILITY_RUM_FLUSH_WAIT_MILLIS": "ci_visibility_rum_flush_wait_millis", - "DD_CIVISIBILITY_FLAKY_RETRY_ENABLED": "ci_visibility_flaky_retry_enabled", - "DD_CIVISIBILITY_FLAKY_RETRY_COUNT": "ci_visibility_flaky_retry_count", + "DD_CIVISIBILITY_TESTSSKIPPING_ENABLED": "ci_visibility_test_skipping_enabled", "DD_CIVISIBILITY_TOTAL_FLAKY_RETRY_COUNT": "ci_visibility_total_flaky_retry_count", - "DD_TEST_SESSION_NAME": "test_session_name", + "DD_CODE_ORIGIN_FOR_SPANS_ENABLED": "code_origin_for_spans_enabled", + "DD_CODE_ORIGIN_FOR_SPANS_MAX_USER_FRAMES": "code_origin_for_spans_max_user_frames", + "DD_DATA_STREAMS_ENABLED": "data_streams_enabled", + "DD_DATA_STREAMS_LEGACY_HEADERS": "data_streams_legacy_headers", + "DD_DBM_PROPAGATION_MODE": "dbm_propagation_mode", + "DD_DEBUGGER_DIAGNOSTICS_INTERVAL": "dynamic_instrumentation_diagnostics_interval", + "DD_DEBUGGER_MAX_DEPTH_TO_SERIALIZE": "dynamic_instrumentation_serialization_max_depth", + "DD_DEBUGGER_MAX_TIME_TO_SERIALIZE": "dynamic_instrumentation_serialization_max_duration", + "DD_DEBUGGER_UPLOAD_BATCH_SIZE": "dynamic_instrumentation_upload_batch_size", + "DD_DEBUGGER_UPLOAD_FLUSH_INTERVAL": "dynamic_instrumentation_upload_interval", + "DD_DIAGNOSTIC_SOURCE_ENABLED": "trace_diagnostic_source_enabled", + "DD_DISABLED_INTEGRATIONS": "trace_disabled_integrations", + "DD_DOGSTATSD_ARGS": "agent_dogstatsd_executable_args", + "DD_DOGSTATSD_PATH": "agent_dogstatsd_executable_path", + "DD_DOGSTATSD_PIPE_NAME": "dogstatsd_named_pipe", + "DD_DOGSTATSD_PORT": "dogstatsd_port", + "DD_DOGSTATSD_SOCKET": "dogstatsd_socket", + "DD_DOGSTATSD_URL": "dogstatsd_url", + "DD_DOTNET_TRACER_CONFIG_FILE": "trace_config_file", + "DD_DYNAMIC_INSTRUMENTATION_DIAGNOSTICS_INTERVAL": "dynamic_instrumentation_diagnostics_interval", + "DD_DYNAMIC_INSTRUMENTATION_ENABLED": "dynamic_instrumentation_enabled", + "DD_DYNAMIC_INSTRUMENTATION_MAX_DEPTH_TO_SERIALIZE": "dynamic_instrumentation_serialization_max_depth", + "DD_DYNAMIC_INSTRUMENTATION_MAX_TIME_TO_SERIALIZE": "dynamic_instrumentation_serialization_max_duration", + "DD_DYNAMIC_INSTRUMENTATION_REDACTED_IDENTIFIERS": "dynamic_instrumentation_redacted_identifiers", + "DD_DYNAMIC_INSTRUMENTATION_REDACTED_TYPES": "dynamic_instrumentation_redacted_types", + "DD_DYNAMIC_INSTRUMENTATION_SYMBOL_DATABASE_BATCH_SIZE_BYTES": "dynamic_instrumentation_symbol_database_batch_size_bytes", + "DD_DYNAMIC_INSTRUMENTATION_SYMBOL_DATABASE_UPLOAD_ENABLED": "dynamic_instrumentation_symbol_database_upload_enabled", + "DD_DYNAMIC_INSTRUMENTATION_UPLOAD_BATCH_SIZE": "dynamic_instrumentation_upload_batch_size", + "DD_DYNAMIC_INSTRUMENTATION_UPLOAD_FLUSH_INTERVAL": "dynamic_instrumentation_upload_interval", + "DD_ENV": "env", + "DD_EXCEPTION_DEBUGGING_CAPTURE_FULL_CALLSTACK_ENABLED": "dd_exception_debugging_capture_full_callstack_enabled", + "DD_EXCEPTION_DEBUGGING_ENABLED": "dd_exception_debugging_enabled", + "DD_EXCEPTION_DEBUGGING_MAX_EXCEPTION_ANALYSIS_LIMIT": "dd_exception_debugging_max_exception_analysis_limit", + "DD_EXCEPTION_DEBUGGING_MAX_FRAMES_TO_CAPTURE": "dd_exception_debugging_max_frames_to_capture", + "DD_EXCEPTION_DEBUGGING_RATE_LIMIT_SECONDS": "dd_exception_debugging_rate_limit_seconds", + "DD_EXCEPTION_REPLAY_CAPTURE_FULL_CALLSTACK_ENABLED": "dd_exception_replay_capture_full_callstack_enabled", + "DD_EXCEPTION_REPLAY_ENABLED": "dd_exception_replay_enabled", + "DD_EXCEPTION_REPLAY_MAX_EXCEPTION_ANALYSIS_LIMIT": "dd_exception_replay_max_exception_analysis_limit", + "DD_EXCEPTION_REPLAY_MAX_FRAMES_TO_CAPTURE": "dd_exception_replay_max_frames_to_capture", + "DD_EXCEPTION_REPLAY_RATE_LIMIT_SECONDS": "dd_exception_replay_rate_limit_seconds", + "DD_EXPERIMENTAL_API_SECURITY_ENABLED": "experimental_api_security_enabled", + "DD_EXPERIMENTAL_APPSEC_STANDALONE_ENABLED": "experimental_appsec_standalone_enabled", + "DD_EXPERIMENTAL_APPSEC_USE_UNSAFE_ENCODER": "appsec_use_unsafe_encoder", + "DD_GIT_COMMIT_SHA": "commit_sha", + "DD_GIT_REPOSITORY_URL": "repository_url", + "DD_GRPC_CLIENT_ERROR_STATUSES": "trace_grpc_client_error_statuses", + "DD_GRPC_SERVER_ERROR_STATUSES": "trace_grpc_server_error_statuses", + "DD_HTTP_CLIENT_ERROR_STATUSES": "trace_http_client_error_statuses", + "DD_HTTP_SERVER_ERROR_STATUSES": "trace_http_server_error_statuses", + "DD_HTTP_SERVER_TAG_QUERY_STRING": "trace_http_server_tag_query_string_enabled", + "DD_HTTP_SERVER_TAG_QUERY_STRING_SIZE": "trace_http_server_tag_query_string_size", + "DD_IAST_COOKIE_FILTER_PATTERN": "iast_cookie_filter_pattern", + "DD_IAST_DB_ROWS_TO_TAINT": "iast_db_rows_to_taint", + "DD_IAST_DEDUPLICATION_ENABLED": "iast_deduplication_enabled", + "DD_IAST_ENABLED": "iast_enabled", + "DD_IAST_MAX_CONCURRENT_REQUESTS": "iast_max_concurrent_requests", + "DD_IAST_MAX_RANGE_COUNT": "iast_max_range_count", + "DD_IAST_REDACTION_ENABLED": "iast_redaction_enabled", + "DD_IAST_REDACTION_KEYS_REGEXP": "iast_redaction_keys_regexp", + "DD_IAST_REDACTION_NAME_PATTERN": "iast_redaction_name_pattern", + "DD_IAST_REDACTION_REGEXP_TIMEOUT": "iast_redaction_regexp_timeout", + "DD_IAST_REDACTION_VALUES_REGEXP": "iast_redaction_values_regexp", + "DD_IAST_REDACTION_VALUE_PATTERN": "iast_redaction_value_pattern", + "DD_IAST_REGEXP_TIMEOUT": "iast_regexp_timeout", + "DD_IAST_REQUEST_SAMPLING": "iast_request_sampling_percentage", + "DD_IAST_STACK_TRACE_ENABLED": "appsec_stack_trace_enabled", + "DD_IAST_TELEMETRY_VERBOSITY": "iast_telemetry_verbosity", + "DD_IAST_TRUNCATION_MAX_VALUE_LENGTH": "iast_truncation_max_value_length", + "DD_IAST_VULNERABILITIES_PER_REQUEST": "iast_vulnerability_per_request", + "DD_IAST_WEAK_CIPHER_ALGORITHMS": "iast_weak_cipher_algorithms", + "DD_IAST_WEAK_HASH_ALGORITHMS": "iast_weak_hash_algorithms", + "DD_INJECTION_ENABLED": "ssi_injection_enabled", + "DD_INJECT_FORCE": "ssi_forced_injection_enabled", + "DD_INJECT_FORCED": "dd_lib_injection_forced", + "DD_INSTRUMENTATION_TELEMETRY_AGENTLESS_ENABLED": "instrumentation_telemetry_agentless_enabled", + "DD_INSTRUMENTATION_TELEMETRY_AGENT_PROXY_ENABLED": "instrumentation_telemetry_agent_proxy_enabled", + "DD_INSTRUMENTATION_TELEMETRY_ENABLED": "instrumentation_telemetry_enabled", + "DD_INSTRUMENTATION_TELEMETRY_URL": "instrumentation_telemetry_agentless_url", + "DD_INTERAL_FORCE_SYMBOL_DATABASE_UPLOAD": "internal_force_symbol_database_upload", + "DD_INTERNAL_RCM_POLL_INTERVAL": "remote_config_poll_interval", + "DD_INTERNAL_TELEMETRY_DEBUG_ENABLED": "instrumentation_telemetry_debug_enabled", + "DD_INTERNAL_TELEMETRY_V2_ENABLED": "instrumentation_telemetry_v2_enabled", + "DD_INTERNAL_WAIT_FOR_DEBUGGER_ATTACH": "internal_wait_for_debugger_attach_enabled", + "DD_INTERNAL_WAIT_FOR_NATIVE_DEBUGGER_ATTACH": "internal_wait_for_native_debugger_attach_enabled", + "DD_LIB_INJECTED": "dd_lib_injected", + "DD_LIB_INJECTION_ATTEMPTED": "dd_lib_injection_attempted", + "DD_LOGS_DIRECT_SUBMISSION_BATCH_PERIOD_SECONDS": "logs_direct_submission_batch_period_seconds", + "DD_LOGS_DIRECT_SUBMISSION_HOST": "logs_direct_submission_host", + "DD_LOGS_DIRECT_SUBMISSION_INTEGRATIONS": "logs_direct_submission_integrations", + "DD_LOGS_DIRECT_SUBMISSION_MAX_BATCH_SIZE": "logs_direct_submission_max_batch_size", + "DD_LOGS_DIRECT_SUBMISSION_MAX_QUEUE_SIZE": "logs_direct_submission_max_queue_size", + "DD_LOGS_DIRECT_SUBMISSION_MINIMUM_LEVEL": "logs_direct_submission_minimum_level", + "DD_LOGS_DIRECT_SUBMISSION_SOURCE": "logs_direct_submission_source", + "DD_LOGS_DIRECT_SUBMISSION_TAGS": "logs_direct_submission_tags", + "DD_LOGS_DIRECT_SUBMISSION_URL": "logs_direct_submission_url", + "DD_LOGS_INJECTION": "logs_injection_enabled", + "DD_LOG_INJECTION": "logs_injection_enabled", + "DD_LOG_LEVEL": "agent_log_level", + "DD_MAX_LOGFILE_SIZE": "trace_log_file_max_size", + "DD_MAX_TRACES_PER_SECOND": "trace_rate_limit", + "DD_PROFILING_CODEHOTSPOTS_ENABLED": "profiling_codehotspots_enabled", + "DD_PROFILING_ENABLED": "profiling_enabled", + "DD_PROFILING_ENDPOINT_COLLECTION_ENABLED": "profiling_endpoint_collection_enabled", + "DD_PROPAGATION_STYLE_EXTRACT": "trace_propagation_style_extract", + "DD_PROPAGATION_STYLE_INJECT": "trace_propagation_style_inject", "DD_PROXY_HTTPS": "proxy_https", "DD_PROXY_NO_PROXY": "proxy_no_proxy", - "DD_TRACE_DEBUG_LOOKUP_MDTOKEN": "trace_lookup_mdtoken_enabled", + "DD_REMOTE_CONFIG_POLL_INTERVAL_SECONDS": "remote_config_poll_interval", + "DD_RUNTIME_METRICS_ENABLED": "runtime_metrics_enabled", + "DD_SERVICE": "service", + "DD_SERVICE_MAPPING": "dd_service_mapping", + "DD_SERVICE_NAME": "service", + "DD_SITE": "site", + "DD_SPAN_SAMPLING_RULES": "span_sample_rules", + "DD_SPAN_SAMPLING_RULES_FILE": "dd_span_sampling_rules_file", + "DD_SYMBOL_DATABASE_BATCH_SIZE_BYTES": "symbol_database_batch_size_bytes", + "DD_SYMBOL_DATABASE_THIRD_PARTY_DETECTION_EXCLUDES": "symbol_database_third_party_detection_excludes", + "DD_SYMBOL_DATABASE_THIRD_PARTY_DETECTION_INCLUDES": "symbol_database_third_party_detection_includes", + "DD_SYMBOL_DATABASE_UPLOAD_ENABLED": "symbol_database_upload_enabled", + "DD_SYMBOL_DATABASE_COMPRESSION_ENABLED": "symbol_database_compression_enabled", + "DD_TAGS": "agent_tags", + "DD_TELEMETRY_DEPENDENCY_COLLECTION_ENABLED": "instrumentation_telemetry_dependency_collection_enabled", + "DD_TELEMETRY_HEARTBEAT_INTERVAL": "instrumentation_telemetry_heartbeat_interval", + "DD_TELEMETRY_LOG_COLLECTION_ENABLED": "instrumentation_telemetry_log_collection_enabled", + "DD_TELEMETRY_METRICS_ENABLED": "instrumentation_telemetry_metrics_enabled", + "DD_TEST_SESSION_NAME": "test_session_name", + "DD_THIRD_PARTY_DETECTION_EXCLUDES": "third_party_detection_excludes", + "DD_THIRD_PARTY_DETECTION_INCLUDES": "third_party_detection_includes", + "DD_TRACE_128_BIT_TRACEID_GENERATION_ENABLED": "trace_128_bits_id_enabled", + "DD_TRACE_128_BIT_TRACEID_LOGGING_ENABLED": "trace_128_bits_id_logging_enabled", + "DD_TRACE_ACTIVITY_LISTENER_ENABLED": "trace_activity_listener_enabled", + "DD_TRACE_AGENT_ARGS": "agent_trace_agent_excecutable_args", + "DD_TRACE_AGENT_HOSTNAME": "agent_host", + "DD_TRACE_AGENT_PATH": "agent_trace_agent_excecutable_path", + "DD_TRACE_AGENT_PORT": "trace_agent_port", + "DD_TRACE_AGENT_URL": "trace_agent_url", + "DD_TRACE_ANALYTICS_ENABLED": "trace_analytics_enabled", + "DD_TRACE_BAGGAGE_MAX_BYTES": "trace_baggage_max_bytes", + "DD_TRACE_BAGGAGE_MAX_ITEMS": "trace_baggage_max_items", + "DD_TRACE_BATCH_INTERVAL": "trace_serialization_batch_interval", + "DD_TRACE_BUFFER_SIZE": "trace_serialization_buffer_size", + "DD_TRACE_CLIENT_IP_ENABLED": "trace_client_ip_enabled", + "DD_TRACE_CLIENT_IP_HEADER": "trace_client_ip_header", + "DD_TRACE_COMMANDS_COLLECTION_ENABLED": "trace_commands_collection_enabled", + "DD_TRACE_COMPUTE_STATS": "dd_trace_compute_stats", + "DD_TRACE_CONFIG_FILE": "trace_config_file", + "DD_TRACE_DEBUG": "trace_debug_enabled", "DD_TRACE_DEBUG_LOOKUP_FALLBACK": "trace_lookup_fallback_enabled", - "DD_TRACE_ROUTE_TEMPLATE_RESOURCE_NAMES_ENABLED": "trace_route_template_resource_names_enabled", + "DD_TRACE_DEBUG_LOOKUP_MDTOKEN": "trace_lookup_mdtoken_enabled", "DD_TRACE_DELAY_WCF_INSTRUMENTATION_ENABLED": "trace_delay_wcf_instrumentation_enabled", - "DD_TRACE_WCF_WEB_HTTP_RESOURCE_NAMES_ENABLED": "trace_wcf_web_http_resource_names_enabled", - "DD_TRACE_WCF_RESOURCE_OBFUSCATION_ENABLED": "trace_wcf_obfuscation_enabled", + "DD_TRACE_DELEGATE_SAMPLING": "trace_sample_delegation", + "DD_TRACE_DISABLED_ADONET_COMMAND_TYPES": "trace_disabled_adonet_command_types", + "DD_TRACE_ENABLED": "trace_enabled", + "DD_TRACE_EXPAND_ROUTE_TEMPLATES_ENABLED": "trace_route_template_expansion_enabled", + "DD_TRACE_GIT_METADATA_ENABLED": "git_metadata_enabled", + "DD_TRACE_GLOBAL_TAGS": "trace_tags", + "DD_TRACE_HEADER_TAGS": "trace_header_tags", "DD_TRACE_HEADER_TAG_NORMALIZATION_FIX_ENABLED": "trace_header_tag_normalization_fix_enabled", - "DD_TRACE_OTEL_ENABLED": "trace_otel_enabled", - "DD_TRACE_OTEL_LEGACY_OPERATION_NAME_ENABLED": "trace_otel_legacy_operation_name_enabled", - "DD_TRACE_ACTIVITY_LISTENER_ENABLED": "trace_activity_listener_enabled", - "DD_TRACE_128_BIT_TRACEID_GENERATION_ENABLED": "trace_128_bits_id_enabled", - "DD_TRACE_128_BIT_TRACEID_LOGGING_ENABLED": "trace_128_bits_id_logging_enabled", "DD_TRACE_HEALTH_METRICS_ENABLED": "dd_trace_health_metrics_enabled", - "DD_LIB_INJECTION_ATTEMPTED": "dd_lib_injection_attempted", - "DD_LIB_INJECTED": "dd_lib_injected", - "DD_INJECT_FORCED": "dd_lib_injection_forced", - "DD_SPAN_SAMPLING_RULES_FILE": "dd_span_sampling_rules_file", - "DD_TRACE_COMPUTE_STATS": "dd_trace_compute_stats", - "DD_EXCEPTION_DEBUGGING_ENABLED": "dd_exception_debugging_enabled", - "DD_EXCEPTION_DEBUGGING_MAX_FRAMES_TO_CAPTURE": "dd_exception_debugging_max_frames_to_capture", - "DD_EXCEPTION_DEBUGGING_CAPTURE_FULL_CALLSTACK_ENABLED": "dd_exception_debugging_capture_full_callstack_enabled", - "DD_EXCEPTION_DEBUGGING_RATE_LIMIT_SECONDS": "dd_exception_debugging_rate_limit_seconds", - "DD_EXCEPTION_DEBUGGING_MAX_EXCEPTION_ANALYSIS_LIMIT": "dd_exception_debugging_max_exception_analysis_limit", - "DD_EXCEPTION_REPLAY_ENABLED": "dd_exception_replay_enabled", - "DD_EXCEPTION_REPLAY_MAX_FRAMES_TO_CAPTURE": "dd_exception_replay_max_frames_to_capture", - "DD_EXCEPTION_REPLAY_CAPTURE_FULL_CALLSTACK_ENABLED": "dd_exception_replay_capture_full_callstack_enabled", - "DD_EXCEPTION_REPLAY_RATE_LIMIT_SECONDS": "dd_exception_replay_rate_limit_seconds", - "DD_EXCEPTION_REPLAY_MAX_EXCEPTION_ANALYSIS_LIMIT": "dd_exception_replay_max_exception_analysis_limit", - "exception_replay_capture_interval_seconds": "dd_exception_replay_capture_interval_seconds", - "exception_replay_capture_max_frames": "dd_exception_replay_capture_max_frames", - "exception_replay_enabled": "dd_exception_replay_enabled", + "DD_TRACE_HTTP_CLIENT_ERROR_STATUSES": "trace_http_client_error_statuses", + "DD_TRACE_HTTP_CLIENT_EXCLUDED_URL_SUBSTRINGS": "trace_http_client_excluded_urls", + "DD_TRACE_HTTP_CLIENT_TAG_QUERY_STRING": "trace_http_client_tag_query_string", + "DD_TRACE_HTTP_SERVER_ERROR_STATUSES": "trace_http_server_error_statuses", + "DD_TRACE_KAFKA_CREATE_CONSUMER_SCOPE_ENABLED": "trace_kafka_create_consumer_scope_enabled", + "DD_TRACE_LOGFILE_RETENTION_DAYS": "trace_log_file_retention_days", + "DD_TRACE_LOGGING_RATE": "trace_log_rate", + "DD_TRACE_LOG_DIRECTORY": "trace_log_directory", + "DD_TRACE_LOG_PATH": "trace_log_path", + "DD_TRACE_LOG_SINKS": "trace_log_sinks", + "DD_TRACE_METHODS": "trace_methods", + "DD_TRACE_METRICS_ENABLED": "trace_metrics_enabled", "DD_TRACE_OBFUSCATION_QUERY_STRING_PATTERN": "dd_trace_obfuscation_query_string_pattern", - "DD_CALL_BASIC_CONFIG": "dd_call_basic_config", - "DD_SERVICE_MAPPING": "dd_service_mapping", - "DD_INSTRUMENTATION_TELEMETRY_ENABLED": "instrumentation_telemetry_enabled", - "DD_INSTRUMENTATION_TELEMETRY_AGENTLESS_ENABLED": "instrumentation_telemetry_agentless_enabled", - "DD_INSTRUMENTATION_TELEMETRY_AGENT_PROXY_ENABLED": "instrumentation_telemetry_agent_proxy_enabled", - "DD_INSTRUMENTATION_TELEMETRY_URL": "instrumentation_telemetry_agentless_url", - "DD_TELEMETRY_HEARTBEAT_INTERVAL": "instrumentation_telemetry_heartbeat_interval", - "DD_TELEMETRY_DEPENDENCY_COLLECTION_ENABLED": "instrumentation_telemetry_dependency_collection_enabled", - "DD_TELEMETRY_LOG_COLLECTION_ENABLED": "instrumentation_telemetry_log_collection_enabled", - "DD_TELEMETRY_METRICS_ENABLED": "instrumentation_telemetry_metrics_enabled", - "DD_INTERNAL_TELEMETRY_V2_ENABLED": "instrumentation_telemetry_v2_enabled", - "DD_INTERNAL_TELEMETRY_DEBUG_ENABLED": "instrumentation_telemetry_debug_enabled", - "DD_APPSEC_ENABLED": "appsec_enabled", - "DD_APPSEC_RULES": "appsec_rules", - "DD_APPSEC_IPHEADER": "appsec_ip_header", - "DD_APPSEC_EXTRA_HEADERS": "appsec_extra_headers", - "DD_APPSEC_KEEP_TRACES": "appsec_force_keep_traces_enabled", - "DD_APPSEC_TRACE_RATE_LIMIT": "appsec_trace_rate_limit", - "DD_APPSEC_WAF_TIMEOUT": "appsec_waf_timeout", - "DD_APPSEC_OBFUSCATION_PARAMETER_KEY_REGEXP": "appsec_obfuscation_parameter_key_regexp", - "DD_APPSEC_OBFUSCATION_PARAMETER_VALUE_REGEXP": "appsec_obfuscation_parameter_value_regexp", - "DD_APPSEC_HTTP_BLOCKED_TEMPLATE_HTML": "appsec_blocked_template_html", - "DD_APPSEC_HTTP_BLOCKED_TEMPLATE_JSON": "appsec_blocked_template_json", - "DD_APPSEC_AUTOMATED_USER_EVENTS_TRACKING": "appsec_auto_user_events_tracking", - "DD_APPSEC_RASP_ENABLED": "appsec_rasp_enabled", - "DD_APPSEC_STACK_TRACE_ENABLED": "appsec_stack_trace_enabled", - "DD_APPSEC_MAX_STACK_TRACES": "appsec_max_stack_traces", - "DD_APPSEC_MAX_STACK_TRACE_DEPTH": "appsec_max_stack_trace_depth", - "DD_APPSEC_MAX_STACK_TRACE_DEPTH_TOP_PERCENT": "appsec_max_stack_trace_depth_top_percent", - "DD_APPSEC_SCA_ENABLED": "appsec_sca_enabled", - "DD_APPSEC_AUTO_USER_INSTRUMENTATION_MODE": "appsec_auto_user_instrumentation_mode", - "DD_EXPERIMENTAL_APPSEC_USE_UNSAFE_ENCODER": "appsec_use_unsafe_encoder", - "DD_API_SECURITY_REQUEST_SAMPLE_RATE":"api_security_request_sample_rate", - "DD_API_SECURITY_MAX_CONCURRENT_REQUESTS":"api_security_max_concurrent_requests", - "DD_API_SECURITY_SAMPLE_DELAY":"api_security_sample_delay", - "DD_API_SECURITY_ENABLED":"api_security_enabled", - "DD_EXPERIMENTAL_API_SECURITY_ENABLED":"experimental_api_security_enabled", - "DD_APPSEC_WAF_DEBUG": "appsec_waf_debug_enabled", - "DD_AZURE_APP_SERVICES": "aas_enabled", - "DD_AAS_DOTNET_EXTENSION_VERSION": "aas_site_extensions_version", - "WEBSITE_OWNER_NAME": "aas_website_owner_name", - "WEBSITE_RESOURCE_GROUP": "aas_website_resource_group", - "WEBSITE_SITE_NAME": "aas_website_site_name", + "DD_TRACE_OBFUSCATION_QUERY_STRING_REGEXP": "trace_obfuscation_query_string_regexp", + "DD_TRACE_OBFUSCATION_QUERY_STRING_REGEXP_TIMEOUT": "trace_obfuscation_query_string_regexp_timeout", + "DD_TRACE_OTEL_ENABLED": "trace_otel_enabled", + "DD_TRACE_OTEL_LEGACY_OPERATION_NAME_ENABLED": "trace_otel_legacy_operation_name_enabled", + "DD_TRACE_PARTIAL_FLUSH_ENABLED": "trace_partial_flush_enabled", + "DD_TRACE_PARTIAL_FLUSH_MIN_SPANS": "trace_partial_flush_min_spans", + "DD_TRACE_PEER_SERVICE_DEFAULTS_ENABLED": "trace_peer_service_defaults_enabled", + "DD_TRACE_PEER_SERVICE_MAPPING": "trace_peer_service_mapping", + "DD_TRACE_PIPE_NAME": "trace_agent_named_pipe", + "DD_TRACE_PIPE_TIMEOUT_MS": "trace_agent_named_pipe_timeout_ms", + "DD_TRACE_PROPAGATION_EXTRACT_FIRST": "trace_propagation_extract_first", + "DD_TRACE_PROPAGATION_STYLE": "trace_propagation_style", + "DD_TRACE_PROPAGATION_STYLE_EXTRACT": "trace_propagation_style_extract", + "DD_TRACE_PROPAGATION_STYLE_INJECT": "trace_propagation_style_inject", + "DD_TRACE_RATE_LIMIT": "trace_rate_limit", + "DD_TRACE_REMOVE_INTEGRATION_SERVICE_NAMES_ENABLED": "trace_remove_integration_service_names_enabled", + "DD_TRACE_ROUTE_TEMPLATE_RESOURCE_NAMES_ENABLED": "trace_route_template_resource_names_enabled", + "DD_TRACE_SAMPLING_RULES": "trace_sample_rules", + "DD_TRACE_SAMPLING_RULES_FORMAT": "trace_sampling_rules_format", + "DD_TRACE_SPAN_ATTRIBUTE_SCHEMA": "trace_span_attribute_schema", + "DD_TRACE_STARTUP_LOGS": "trace_startup_logs_enabled", + "DD_TRACE_STATS_COMPUTATION_ENABLED": "trace_stats_computation_enabled", + "DD_TRACE_WCF_RESOURCE_OBFUSCATION_ENABLED": "trace_wcf_obfuscation_enabled", + "DD_TRACE_WCF_WEB_HTTP_RESOURCE_NAMES_ENABLED": "trace_wcf_web_http_resource_names_enabled", + "DD_TRACE_X_DATADOG_TAGS_MAX_LENGTH": "trace_x_datadog_tags_max_length", + "DD_VERSION": "application_version", "FUNCTIONS_EXTENSION_VERSION": "aas_functions_runtime_version", "FUNCTIONS_WORKER_RUNTIME": "aas_functions_worker_runtime", - "COMPUTERNAME": "aas_instance_name", - "WEBSITE_INSTANCE_ID": "aas_website_instance_id", - "WEBSITE_OS": "aas_website_os", - "WEBSITE_SKU": "aas_website_sku", "FUNCTION_NAME": "gcp_deprecated_function_name", + "FUNCTION_TARGET": "gcp_function_target", "GCP_PROJECT": "gcp_deprecated_project", "K_SERVICE": "gcp_function_name", - "FUNCTION_TARGET": "gcp_function_target", - "DD_AAS_ENABLE_CUSTOM_TRACING": "aas_custom_tracing_enabled", - "DD_AAS_ENABLE_CUSTOM_METRICS": "aas_custom_metrics_enabled", - "DD_DYNAMIC_INSTRUMENTATION_ENABLED": "dynamic_instrumentation_enabled", - "DD_DEBUGGER_MAX_DEPTH_TO_SERIALIZE": "dynamic_instrumentation_serialization_max_depth", - "DD_DEBUGGER_MAX_TIME_TO_SERIALIZE": "dynamic_instrumentation_serialization_max_duration", - "DD_DEBUGGER_UPLOAD_BATCH_SIZE": "dynamic_instrumentation_upload_batch_size", - "DD_DEBUGGER_DIAGNOSTICS_INTERVAL": "dynamic_instrumentation_diagnostics_interval", - "DD_DEBUGGER_UPLOAD_FLUSH_INTERVAL": "dynamic_instrumentation_upload_interval", - "DD_DYNAMIC_INSTRUMENTATION_MAX_DEPTH_TO_SERIALIZE": "dynamic_instrumentation_serialization_max_depth", - "DD_DYNAMIC_INSTRUMENTATION_MAX_TIME_TO_SERIALIZE": "dynamic_instrumentation_serialization_max_duration", - "DD_DYNAMIC_INSTRUMENTATION_UPLOAD_BATCH_SIZE": "dynamic_instrumentation_upload_batch_size", - "DD_DYNAMIC_INSTRUMENTATION_DIAGNOSTICS_INTERVAL": "dynamic_instrumentation_diagnostics_interval", - "DD_DYNAMIC_INSTRUMENTATION_UPLOAD_FLUSH_INTERVAL": "dynamic_instrumentation_upload_interval", - "DD_DYNAMIC_INSTRUMENTATION_REDACTED_IDENTIFIERS": "dynamic_instrumentation_redacted_identifiers", - "DD_DYNAMIC_INSTRUMENTATION_REDACTED_TYPES": "dynamic_instrumentation_redacted_types", - "dynamic_instrumentation.redacted_types": "dynamic_instrumentation_redacted_types", + "OTEL_LOGS_EXPORTER": "otel_logs_exporter", + "OTEL_LOG_LEVEL": "otel_log_level", + "OTEL_METRICS_EXPORTER": "otel_metrics_exporter", + "OTEL_PROPAGATORS": "otel_propagators", + "OTEL_RESOURCE_ATTRIBUTES": "otel_resource_attributes", + "OTEL_SDK_DISABLED": "otel_sdk_disabled", + "OTEL_SERVICE_NAME": "otel_service_name", + "OTEL_TRACES_EXPORTER": "otel_traces_exporter", + "OTEL_TRACES_SAMPLER": "otel_traces_sampler", + "OTEL_TRACES_SAMPLER_ARG": "otel_traces_sampler_arg", + "WEBSITE_INSTANCE_ID": "aas_website_instance_id", + "WEBSITE_OS": "aas_website_os", + "WEBSITE_OWNER_NAME": "aas_website_owner_name", + "WEBSITE_RESOURCE_GROUP": "aas_website_resource_group", + "WEBSITE_SITE_NAME": "aas_website_site_name", + "WEBSITE_SKU": "aas_website_sku", + "_DD_TRACE_STATS_COMPUTATION_INTERVAL": "trace_stats_computation_interval", + "_dd_appsec_deduplication_enabled": "appsec_deduplication_enabled", + "_dd_iast_debug": "iast_debug_enabled", + "_dd_iast_lazy_taint": "iast_lazy_taint", + "_dd_iast_propagation_debug": "iast_propagation_debug", + "_dd_inject_was_attempted": "trace_inject_was_attempted", + "_dd_llmobs_evaluator_sampling_rules": "llmobs_evaluator_sampling_rules", + "aas_app_type": "aas_app_type", + "aas_configuration_error": "aas_configuration_error", + "aas_functions_runtime_version": "aas_functions_runtime_version", + "aas_siteextensions_version": "aas_site_extensions_version", + "activity_listener_enabled": "activity_listener_enabled", + "agent_feature_drop_p0s": "agent_feature_drop_p0s", + "agent_transport": "agent_transport", + "agent_url": "trace_agent_url", + "analytics_enabled": "analytics_enabled", + "appsec.apiSecurity.enabled": "api_security_enabled", + "appsec.apiSecurity.requestSampling": "api_security_request_sample_rate", + "appsec.apiSecurity.sampleDelay": "api_security_sample_delay", + "appsec.blockedTemplateGraphql": "appsec_blocked_template_graphql", + "appsec.blockedTemplateHtml": "appsec_blocked_template_html", + "appsec.blockedTemplateJson": "appsec_blocked_template_json", + "appsec.customRulesProvided": "appsec_rules_custom_provided", + "appsec.enabled": "appsec_enabled", + "appsec.eventTracking": "appsec_auto_user_events_tracking", + "appsec.eventTracking.mode": "appsec_auto_user_events_tracking", + "appsec.obfuscatorKeyRegex": "appsec_obfuscation_parameter_key_regexp", + "appsec.obfuscatorValueRegex": "appsec_obfuscation_parameter_value_regexp", + "appsec.rasp.enabled": "appsec_rasp_enabled", + "appsec.rateLimit": "appsec_rate_limit", + "appsec.rules": "appsec_rules", + "appsec.rules.metadata.rules_version": "appsec_rules_metadata_rules_version", + "appsec.rules.version": "appsec_rules_version", + "appsec.sca.enabled": "appsec_sca_enabled", + "appsec.sca_enabled": "appsec_sca_enabled", + "appsec.stackTrace.enabled": "appsec_stack_trace_enabled", + "appsec.stackTrace.maxDepth": "appsec_max_stack_trace_depth", + "appsec.stackTrace.maxStackTraces": "appsec_max_stack_traces", + "appsec.standalone.enabled": "experimental_appsec_standalone_enabled", + "appsec.testing": "appsec_testing", + "appsec.trace.rate.limit": "appsec_trace_rate_limit", + "appsec.waf.timeout": "appsec_waf_timeout", + "appsec.wafTimeout": "appsec_waf_timeout", + "autofinish_spans": "trace_auto_finish_spans_enabled", + "autoload_no_compile": "autoload_no_compile", + "aws.dynamoDb.tablePrimaryKeys": "aws_dynamodb_table_primary_keys", + "baggageMaxBytes": "trace_baggage_max_bytes", + "baggageMaxItems": "trace_baggage_max_items", + "ciVisAgentlessLogSubmissionEnabled": "ci_visibility_agentless_enabled", + "ciVisibilityTestSessionName": "test_session_name", + "civisibility.agentless.enabled": "ci_visibility_agentless_enabled", + "civisibility.enabled": "ci_visibility_enabled", + "clientIpEnabled": "trace_client_ip_enabled", + "clientIpHeader": "trace_client_ip_header", + "clientIpHeaderDisabled": "client_ip_header_disabled", + "cloudPayloadTagging.maxDepth": "cloud_payload_tagging_max_depth", + "cloudPayloadTagging.requestsEnabled": "cloud_payload_tagging_requests_enabled", + "cloudPayloadTagging.responsesEnabled": "cloud_payload_tagging_responses_enabled", + "cloudPayloadTagging.rules.aws.eventbridge.expand": "cloud_payload_tagging_rules_aws_eventbridge_expand", + "cloudPayloadTagging.rules.aws.eventbridge.request": "cloud_payload_tagging_rules_aws_eventbridge_request", + "cloudPayloadTagging.rules.aws.eventbridge.response": "cloud_payload_tagging_rules_aws_eventbridge_response", + "cloudPayloadTagging.rules.aws.kinesis.expand": "cloud_payload_tagging_rules_aws_kinesis_expand", + "cloudPayloadTagging.rules.aws.kinesis.request": "cloud_payload_tagging_rules_aws_kinesis_request", + "cloudPayloadTagging.rules.aws.kinesis.response": "cloud_payload_tagging_rules_aws_kinesis_response", + "cloudPayloadTagging.rules.aws.s3.expand": "cloud_payload_tagging_rules_aws_s3_expand", + "cloudPayloadTagging.rules.aws.s3.request": "cloud_payload_tagging_rules_aws_s3_request", + "cloudPayloadTagging.rules.aws.s3.response": "cloud_payload_tagging_rules_aws_s3_response", + "cloudPayloadTagging.rules.aws.sns.expand": "cloud_payload_tagging_rules_aws_sns_expand", + "cloudPayloadTagging.rules.aws.sns.request": "cloud_payload_tagging_rules_aws_sns_request", + "cloudPayloadTagging.rules.aws.sns.response": "cloud_payload_tagging_rules_aws_sns_response", + "cloudPayloadTagging.rules.aws.sqs.expand": "cloud_payload_tagging_rules_aws_sqs_expand", + "cloudPayloadTagging.rules.aws.sqs.request": "cloud_payload_tagging_rules_aws_sqs_request", + "cloudPayloadTagging.rules.aws.sqs.response": "cloud_payload_tagging_rules_aws_sqs_response", + "cloud_hosting": "cloud_hosting_provider", + "codeOriginForSpans.enabled": "code_origin_for_spans_enabled", + "code_hotspots_enabled": "code_hotspots_enabled", + "commitSHA": "commit_sha", + "crashtracking.enabled": "crashtracking_enabled", + "crashtracking_alt_stack": "crashtracking_alt_stack", + "crashtracking_available": "crashtracking_available", + "crashtracking_debug_url": "crashtracking_debug_url", + "crashtracking_enabled": "crashtracking_enabled", + "crashtracking_stacktrace_resolver": "crashtracking_stacktrace_resolver", + "crashtracking_started": "crashtracking_started", + "crashtracking_stderr_filename": "crashtracking_stderr_filename", + "crashtracking_stdout_filename": "crashtracking_stdout_filename", + "cws.enabled": "cws_enabled", + "data.streams.enabled": "data_streams_enabled", + "data_streams_enabled": "data_streams_enabled", + "dbmPropagationMode": "dbm_propagation_mode", + "dbm_propagation_mode": "dbm_propagation_mode", + "dd.trace.debug": "trace_debug_enabled", + "dd_agent_host": "agent_host", + "dd_agent_port": "trace_agent_port", + "dd_analytics_enabled": "analytics_enabled", + "dd_api_security_parse_response_body": "appsec_parse_response_body", + "dd_appsec_automated_user_events_tracking_enabled": "appsec_auto_user_events_tracking_enabled", + "dd_civisibility_log_level": "ci_visibility_log_level", + "dd_crashtracking_create_alt_stack": "crashtracking_create_alt_stack", + "dd_crashtracking_debug_url": "crashtracking_debug_url", + "dd_crashtracking_enabled": "crashtracking_enabled", + "dd_crashtracking_stacktrace_resolver": "crashtracking_stacktrace_resolver", + "dd_crashtracking_stderr_filename": "crashtracking_stderr_filename", + "dd_crashtracking_stdout_filename": "crashtracking_stdout_filename", + "dd_crashtracking_tags": "crashtracking_tags", + "dd_crashtracking_use_alt_stack": "crashtracking_alt_stack", + "dd_crashtracking_wait_for_receiver": "crashtracking_wait_for_receiver", + "dd_dynamic_instrumentation_max_payload_size": "dynamic_instrumentation_max_payload_size", + "dd_dynamic_instrumentation_metrics_enabled": "dynamic_instrumentation_metrics_enabled", + "dd_dynamic_instrumentation_upload_timeout": "dynamic_instrumentation_upload_timeout", + "dd_http_client_tag_query_string": "trace_http_client_tag_query_string", + "dd_iast_redaction_value_numeral": "iast_redaction_value_numeral", + "dd_instrumentation_install_id": "instrumentation_install_id", + "dd_instrumentation_install_type": "instrumentation_install_type", + "dd_llmobs_agentless_enabled": "llmobs_agentless_enabled", + "dd_llmobs_enabled": "llmobs_enabled", + "dd_llmobs_ml_app": "llmobs_ml_app", + "dd_llmobs_sample_rate": "llmobs_sample_rate", + "dd_priority_sampling": "trace_priority_sampling_enabled", + "dd_profiling_agentless": "profiling_agentless", + "dd_profiling_api_timeout": "profiling_api_timeout", + "dd_profiling_capture_pct": "profiling_capture_pct", + "dd_profiling_enable_asserts": "profiling_enable_asserts", + "dd_profiling_enable_code_provenance": "profiling_enable_code_provenance", + "dd_profiling_export_libdd_enabled": "profiling_export_libdd_enabled", + "dd_profiling_export_py_enabled": "profiling_export_py_enabled", + "dd_profiling_force_legacy_exporter": "profiling_force_legacy_exporter", + "dd_profiling_heap_enabled": "profiling_heap_enabled", + "dd_profiling_heap_sample_size": "profiling_heap_sample_size", + "dd_profiling_ignore_profiler": "profiling_ignore_profiler", + "dd_profiling_lock_enabled": "profiling_lock_enabled", + "dd_profiling_lock_name_inspect_dir": "profiling_lock_name_inspect_dir", + "dd_profiling_max_events": "profiling_max_events", + "dd_profiling_max_frames": "profiling_max_frames", + "dd_profiling_max_time_usage_pct": "profiling_max_time_usage_pct", + "dd_profiling_memory_enabled": "profiling_memory_enabled", + "dd_profiling_memory_events_buffer": "profiling_memory_events_buffer", + "dd_profiling_output_pprof": "profiling_output_pprof", + "dd_profiling_sample_pool_capacity": "profiling_sample_pool_capacity", + "dd_profiling_stack_enabled": "profiling_stack_enabled", + "dd_profiling_stack_v2_enabled": "profiling_stack_v2_enabled", + "dd_profiling_tags": "profiling_tags", + "dd_profiling_timeline_enabled": "profiling_timeline_enabled", + "dd_profiling_upload_interval": "profiling_upload_interval", + "dd_remote_configuration_enabled": "remote_config_enabled", + "dd_remoteconfig_poll_seconds": "remote_config_poll_interval", + "dd_symbol_database_includes": "symbol_database_includes", + "dd_testing_raise": "testing_raise", + "dd_trace_agent_timeout_seconds": "trace_agent_timeout", + "dd_trace_api_version": "trace_api_version", + "dd_trace_propagation_http_baggage_enabled": "trace_propagation_http_baggage_enabled", + "dd_trace_report_hostname": "trace_report_hostname", + "dd_trace_sample_rate": "trace_sample_rate", + "dd_trace_span_links_enabled": "trace_span_links_enabled", + "dd_trace_span_traceback_max_size": "trace_span_traceback_max_size", + "dd_trace_writer_buffer_size_bytes": "trace_serialization_buffer_size", + "dd_trace_writer_interval_seconds": "trace_agent_flush_interval", + "dd_trace_writer_max_payload_size_bytes": "trace_agent_max_payload_size", + "dd_trace_writer_reuse_connections": "trace_agent_reuse_connections", + "ddtrace_auto_used": "ddtrace_auto_used", + "ddtrace_bootstrapped": "ddtrace_bootstrapped", + "debug": "trace_debug_enabled", + "debug_stack_enabled": "debug_stack_enabled", + "discovery": "agent_discovery_enabled", + "distributed_tracing": "trace_distributed_trace_enabled", + "dogstatsd.hostname": "dogstatsd_hostname", + "dogstatsd.port": "dogstatsd_port", + "dogstatsd.start-delay": "dogstatsd_start_delay", + "dogstatsd_addr": "dogstatsd_url", + "dogstatsd_url": "dogstatsd_url", + "dsmEnabled": "data_streams_enabled", + "dynamic.instrumentation.classfile.dump.enabled": "dynamic_instrumentation_classfile_dump_enabled", + "dynamic.instrumentation.enabled": "dynamic_instrumentation_enabled", + "dynamic.instrumentation.metrics.enabled": "dynamic_instrumentation_metrics_enabled", + "dynamicInstrumentationEnabled": "dynamic_instrumentation_enabled", + "dynamic_instrumentation.enabled": "dynamic_instrumentation_enabled", "dynamic_instrumentation.redacted_identifiers": "dynamic_instrumentation_redacted_identifiers", - "DD_SYMBOL_DATABASE_BATCH_SIZE_BYTES": "symbol_database_batch_size_bytes", - "DD_DYNAMIC_INSTRUMENTATION_SYMBOL_DATABASE_BATCH_SIZE_BYTES": "dynamic_instrumentation_symbol_database_batch_size_bytes", - "DD_SYMBOL_DATABASE_UPLOAD_ENABLED": "symbol_database_upload_enabled", - "DD_DYNAMIC_INSTRUMENTATION_SYMBOL_DATABASE_UPLOAD_ENABLED": "dynamic_instrumentation_symbol_database_upload_enabled", - "DD_INTERAL_FORCE_SYMBOL_DATABASE_UPLOAD": "internal_force_symbol_database_upload", - "DD_THIRD_PARTY_DETECTION_INCLUDES": "third_party_detection_includes", - "DD_THIRD_PARTY_DETECTION_EXCLUDES": "third_party_detection_excludes", - "DD_SYMBOL_DATABASE_THIRD_PARTY_DETECTION_INCLUDES": "symbol_database_third_party_detection_includes", - "DD_SYMBOL_DATABASE_THIRD_PARTY_DETECTION_EXCLUDES": "symbol_database_third_party_detection_excludes", - "DD_CODE_ORIGIN_FOR_SPANS_ENABLED": "code_origin_for_spans_enabled", - "DD_CODE_ORIGIN_FOR_SPANS_MAX_USER_FRAMES": "code_origin_for_spans_max_user_frames", - "DD_LOGS_DIRECT_SUBMISSION_INTEGRATIONS": "logs_direct_submission_integrations", - "DD_LOGS_DIRECT_SUBMISSION_HOST": "logs_direct_submission_host", - "DD_LOGS_DIRECT_SUBMISSION_SOURCE": "logs_direct_submission_source", - "DD_LOGS_DIRECT_SUBMISSION_TAGS": "logs_direct_submission_tags", - "DD_LOGS_DIRECT_SUBMISSION_URL": "logs_direct_submission_url", - "DD_LOGS_DIRECT_SUBMISSION_MINIMUM_LEVEL": "logs_direct_submission_minimum_level", - "DD_LOGS_DIRECT_SUBMISSION_MAX_BATCH_SIZE": "logs_direct_submission_max_batch_size", - "DD_LOGS_DIRECT_SUBMISSION_MAX_QUEUE_SIZE": "logs_direct_submission_max_queue_size", - "DD_LOGS_DIRECT_SUBMISSION_BATCH_PERIOD_SECONDS": "logs_direct_submission_batch_period_seconds", - "DD_AGENT_HOST": "agent_host", - "DATADOG_TRACE_AGENT_HOSTNAME": "agent_host", - "DD_TRACE_AGENT_HOSTNAME": "agent_host", - "DD_TRACE_AGENT_PORT": "trace_agent_port", - "DATADOG_TRACE_AGENT_PORT": "trace_agent_port", - "DD_TRACE_PIPE_NAME": "trace_agent_named_pipe", - "DD_TRACE_PIPE_TIMEOUT_MS": "trace_agent_named_pipe_timeout_ms", - "DD_DOGSTATSD_PIPE_NAME": "dogstatsd_named_pipe", - "DD_APM_RECEIVER_PORT": "trace_agent_port", - "DD_TRACE_AGENT_URL": "trace_agent_url", - "DD_DOGSTATSD_PORT": "dogstatsd_port", - "DD_TRACE_PARTIAL_FLUSH_ENABLED": "trace_partial_flush_enabled", - "DD_TRACE_PARTIAL_FLUSH_MIN_SPANS": "trace_partial_flush_min_spans", - "DD_APM_RECEIVER_SOCKET": "trace_agent_socket", - "DD_DOGSTATSD_SOCKET": "dogstatsd_socket", - "DD_DOGSTATSD_URL": "dogstatsd_url", - "DD_IAST_ENABLED": "iast_enabled", - "DD_IAST_WEAK_HASH_ALGORITHMS": "iast_weak_hash_algorithms", - "DD_IAST_WEAK_CIPHER_ALGORITHMS": "iast_weak_cipher_algorithms", - "DD_IAST_DEDUPLICATION_ENABLED": "iast_deduplication_enabled", - "DD_IAST_REQUEST_SAMPLING": "iast_request_sampling_percentage", - "DD_IAST_MAX_CONCURRENT_REQUESTS": "iast_max_concurrent_requests", - "DD_IAST_MAX_RANGE_COUNT": "iast_max_range_count", - "DD_IAST_VULNERABILITIES_PER_REQUEST": "iast_vulnerability_per_request", - "DD_IAST_REDACTION_ENABLED": "iast_redaction_enabled", - "DD_IAST_REDACTION_KEYS_REGEXP": "iast_redaction_keys_regexp", - "DD_IAST_REDACTION_VALUES_REGEXP": "iast_redaction_values_regexp", - "DD_IAST_REDACTION_NAME_PATTERN": "iast_redaction_name_pattern", - "DD_IAST_REDACTION_VALUE_PATTERN": "iast_redaction_value_pattern", - "DD_IAST_REDACTION_REGEXP_TIMEOUT": "iast_redaction_regexp_timeout", - "DD_IAST_REGEXP_TIMEOUT": "iast_regexp_timeout", - "DD_IAST_TELEMETRY_VERBOSITY": "iast_telemetry_verbosity", - "DD_IAST_TRUNCATION_MAX_VALUE_LENGTH": "iast_truncation_max_value_length", - "DD_IAST_DB_ROWS_TO_TAINT": "iast_db_rows_to_taint", - "DD_IAST_COOKIE_FILTER_PATTERN": "iast_cookie_filter_pattern", - "DD_TRACE_STARTUP_LOGS": "trace_startup_logs_enabled", - "DD_TRACE_DISABLED_ADONET_COMMAND_TYPES": "trace_disabled_adonet_command_types", - "DD_MAX_LOGFILE_SIZE": "trace_log_file_max_size", - "DD_TRACE_LOGGING_RATE": "trace_log_rate", - "DD_TRACE_LOG_PATH": "trace_log_path", - "DD_TRACE_LOG_DIRECTORY": "trace_log_directory", - "DD_TRACE_LOGFILE_RETENTION_DAYS": "trace_log_file_retention_days", - "DD_TRACE_LOG_SINKS": "trace_log_sinks", - "DD_TRACE_COMMANDS_COLLECTION_ENABLED": "trace_commands_collection_enabled", - "DD_REMOTE_CONFIG_POLL_INTERVAL_SECONDS": "remote_config_poll_interval", + "dynamic_instrumentation.redacted_types": "dynamic_instrumentation_redacted_types", + "enabled": "trace_enabled", + "env": "env", + "environment_fulltrust_appdomain": "environment_fulltrust_appdomain_enabled", + "exception_replay_capture_interval_seconds": "dd_exception_replay_capture_interval_seconds", + "exception_replay_capture_max_frames": "dd_exception_replay_capture_max_frames", + "exception_replay_enabled": "dd_exception_replay_enabled", + "experimental.b3": "experimental_b3", + "experimental.enableGetRumData": "experimental_enable_get_rum_data", + "experimental.exporter": "experimental_exporter", + "experimental.runtimeId": "experimental_runtime_id", + "experimental.sampler.rateLimit": "experimental_sampler_rate_limit", + "experimental.sampler.sampleRate": "experimental_sampler_sample_rate", + "experimental.traceparent": "experimental_traceparent", + "flakyTestRetriesCount": "ci_visibility_flaky_retry_count", + "flushInterval": "flush_interval", + "flushMinSpans": "flush_min_spans", + "gitMetadataEnabled": "git_metadata_enabled", + "git_commit_sha": "commit_sha", + "git_repository_url": "repository_url", + "global_tag_version": "version", + "grpc.client.error.statuses": "trace_grpc_client_error_statuses", + "grpc.server.error.statuses": "trace_grpc_server_error_statuses", + "headerTags": "trace_header_tags", + "hostname": "agent_hostname", + "http.client.tag.query-string": "trace_http_client_tag_query_string", + "http.server.route-based-naming": "trace_http_server_route_based_naming_enabled", + "http.server.tag.query-string": "trace_http_server_tag_query_string", + "http_server_route_based_naming": "http_server_route_based_naming", + "hystrix.measured.enabled": "hystrix_measured_enabled", + "hystrix.tags.enabled": "hystrix_tags_enabled", + "iast.cookieFilterPattern": "iast_cookie_filter_pattern", + "iast.dbRowsToTaint": "iast_db_rows_to_taint", + "iast.debug.enabled": "iast_debug_enabled", + "iast.deduplication.enabled": "iast_deduplication_enabled", + "iast.deduplicationEnabled": "iast_deduplication_enabled", + "iast.enabled": "iast_enabled", + "iast.max-concurrent-requests": "iast_max_concurrent_requests", + "iast.maxConcurrentRequests": "iast_max_concurrent_requests", + "iast.maxContextOperations": "iast_max_context_operations", + "iast.redactionEnabled": "iast_redaction_enabled", + "iast.redactionNamePattern": "iast_redaction_name_pattern", + "iast.redactionValuePattern": "iast_redaction_value_pattern", + "iast.request-sampling": "iast_request_sampling", + "iast.requestSampling": "iast_request_sampling", + "iast.telemetryVerbosity": "iast_telemetry_verbosity", + "iast.vulnerabilities-per-request": "iast_vulnerability_per_request", + "ignite.cache.include_keys": "ignite_cache_include_keys_enabled", + "inferredProxyServicesEnabled": "inferred_proxy_services_enabled", + "inject_force": "ssi_forced_injection_enabled", + "injectionEnabled": "ssi_injection_enabled", + "instrumentation.telemetry.enabled": "instrumentation_telemetry_enabled", + "instrumentation_config_id": "instrumentation_config_id", + "integration_metrics_enabled": "integration_metrics_enabled", + "integrations.enabled": "trace_integrations_enabled", + "integrations_disabled": "trace_disabled_integrations", + "isAzureFunction": "azure_function", + "isCiVisibility": "ci_visibility_enabled", + "isEarlyFlakeDetectionEnabled": "ci_visibility_early_flake_detection_enabled", + "isFlakyTestRetriesEnabled": "ci_visibility_flaky_retry_enabled", + "isGCPFunction": "is_gcp_function", + "isGitUploadEnabled": "git_upload_enabled", + "isIntelligentTestRunnerEnabled": "intelligent_test_runner_enabled", + "isManualApiEnabled": "ci_visibility_manual_api_enabled", + "isTestDynamicInstrumentationEnabled": "ci_visibility_test_dynamic_instrumentation_enabled", + "jmxfetch.check-period": "jmxfetch_check_period", + "jmxfetch.enabled": "jmxfetch_enabled", + "jmxfetch.initial-refresh-beans-period": "jmxfetch_initial_refresh_beans_period", + "jmxfetch.multiple-runtime-services.enabled": "jmxfetch_multiple_runtime_services_enabled", + "jmxfetch.refresh-beans-period": "jmxfetch_initial_refresh_beans_period", + "jmxfetch.statsd.port": "jmxfetch_statsd_port", + "kafka.client.base64.decoding.enabled": "trace_kafka_client_base64_decoding_enabled", + "lambda_mode": "lambda_mode", + "langchain.spanCharLimit": "open_ai_span_char_limit", + "langchain.spanPromptCompletionSampleRate": "open_ai_span_prompt_completion_sample_rate", + "legacy.installer.enabled": "legacy_installer_enabled", + "legacyBaggageEnabled": "trace_legacy_baggage_enabled", + "llmobs.agentlessEnabled": "open_ai_agentless_enabled", + "llmobs.enabled": "open_ai_enabled", + "llmobs.mlApp": "open_ai_ml_app", + "logInjection": "logs_injection_enabled", + "logInjection_enabled": "logs_injection_enabled", + "logLevel": "trace_log_level", + "log_backtrace": "trace_log_backtrace_enabled", + "logger": "logger", + "logs.injection": "logs_injection_enabled", + "logs.mdc.tags.injection": "logs_mdc_tags_injection_enabled", + "lookup": "lookup", + "managed_tracer_framework": "managed_tracer_framework", + "memcachedCommandEnabled": "memchached_command_enabled", + "message.broker.split-by-destination": "message_broker_split_by_destination", + "native_tracer_version": "native_tracer_version", + "openAiLogsEnabled": "open_ai_logs_enabled", + "openaiSpanCharLimit": "open_ai_span_char_limit", + "openai_log_prompt_completion_sample_rate": "open_ai_log_prompt_completion_sample_rate", + "openai_logs_enabled": "open_ai_logs_enabled", + "openai_metrics_enabled": "open_ai_metrics_enabled", + "openai_service": "open_ai_service", + "openai_span_char_limit": "open_ai_span_char_limit", + "openai_span_prompt_completion_sample_rate": "open_ai_span_prompt_completion_sample_rate", + "orchestrion_enabled": "orchestrion_enabled", + "orchestrion_version": "orchestrion_version", + "os.name": "os_name", + "otel_enabled": "trace_otel_enabled", + "partialflush_enabled": "trace_partial_flush_enabled", + "partialflush_minspans": "trace_partial_flush_min_spans", + "peerServiceMapping": "trace_peer_service_mapping", + "platform": "platform", + "plugins": "plugins", + "port": "trace_agent_port", + "priority.sampling": "trace_priority_sample_enabled", + "priority_sampling": "trace_priority_sampling_enabled", + "profiler_loaded": "profiler_loaded", + "profiling.advanced.code_provenance_enabled": "profiling_enable_code_provenance", + "profiling.advanced.endpoint.collection.enabled": "profiling_endpoint_collection_enabled", + "profiling.allocation.enabled": "profiling_allocation_enabled", + "profiling.async.alloc.enabled": "profiling_async_alloc_enabled", + "profiling.async.cpu.enabled": "profiling_async_cpu_enabled", + "profiling.async.enabled": "profiling_async_enabled", + "profiling.async.memleak.enabled": "profiling_async_memleak_enabled", + "profiling.async.wall.enabled": "profiling_async_wall_enabled", + "profiling.ddprof.alloc.enabled": "profiling_ddprof_alloc_enabled", + "profiling.ddprof.cpu.enabled": "profiling_ddprof_cpu_enabled", + "profiling.ddprof.enabled": "profiling_ddprof_enabled", + "profiling.ddprof.memleak.enabled": "profiling_ddprof_memleak_enabled", + "profiling.ddprof.wall.enabled": "profiling_ddprof_wall_enabled", + "profiling.directallocation.enabled": "profiling_direct_allocation_enabled", + "profiling.enabled": "profiling_enabled", + "profiling.exporters": "profiling_exporters", + "profiling.heap.enabled": "profiling_heap_enabled", + "profiling.hotspots.enabled": "profiling_hotspots_enabled", + "profiling.legacy.tracing.integration": "profiling_legacy_tracing_integration_enabled", + "profiling.longLivedThreshold": "profiling_long_lived_threshold", + "profiling.sourceMap": "profiling_source_map_enabled", + "profiling.start-delay": "profiling_start_delay", + "profiling.start-force-first": "profiling_start_force_first", + "profiling.upload.period": "profiling_upload_period", + "profiling_endpoints_enabled": "profiling_endpoints_enabled", + "protocolVersion": "trace_agent_protocol_version", + "queryStringObfuscation": "trace_obfuscation_query_string_regexp", + "rcPollingInterval": "rc_polling_interval", + "remoteConfig.enabled": "remote_config_enabled", + "remoteConfig.pollInterval": "remote_config_poll_interval", + "remote_config.enabled": "remote_config_enabled", "remote_config_poll_interval_seconds": "remote_config_poll_interval", - "DD_INTERNAL_RCM_POLL_INTERVAL": "remote_config_poll_interval", + "reportHostname": "trace_report_hostname", + "repositoryUrl": "repository_url", + "resolver.outline.pool.enabled": "resolver_outline_pool_enabled", + "resolver.use.loadclass": "resolver_use_loadclass", + "retry_interval": "retry_interval", + "routetemplate_expansion_enabled": "trace_route_template_expansion_enabled", + "routetemplate_resourcenames_enabled": "trace_route_template_resource_names_enabled", + "runtime.metrics.enabled": "runtime_metrics_enabled", + "runtimeMetrics": "runtime_metrics_enabled", + "runtime_metrics.enabled": "runtime_metrics_enabled", + "runtime_metrics_v2_enabled": "runtime_metrics_v2_enabled", + "runtimemetrics_enabled": "runtime_metrics_enabled", + "sampleRate": "trace_sample_rate", + "sample_rate": "trace_sample_rate", + "sampler.rateLimit": "trace_rate_limit", + "sampler.rules": "trace_sample_rules", + "sampler.sampleRate": "trace_sample_rate", + "sampler.spanSamplingRules": "span_sample_rules", + "sampling_rules": "trace_sample_rules", + "scope": "scope", + "security_enabled": "appsec_enabled", + "send_retries": "trace_send_retries", + "service": "service", + "serviceMapping": "dd_service_mapping", + "site": "site", + "spanAttributeSchema": "trace_span_attribute_schema", + "spanComputePeerService": "trace_peer_service_defaults_enabled", + "spanLeakDebug": "span_leak_debug", + "spanRemoveIntegrationFromService": "trace_remove_integration_service_names_enabled", + "span_sampling_rules": "span_sample_rules", + "span_sampling_rules_file": "span_sample_rules_file", + "ssi_forced_injection_enabled": "ssi_forced_injection_enabled", + "ssi_injection_enabled": "ssi_injection_enabled", + "startupLogs": "trace_startup_logs_enabled", + "stats.enabled": "stats_enabled", + "stats_computation_enabled": "trace_stats_computation_enabled", + "tagsHeaderMaxLength": "trace_header_tags_max_length", + "telemetry.debug": "instrumentation_telemetry_debug_enabled", + "telemetry.dependencyCollection": "instrumentation_telemetry_dependency_collection_enabled", + "telemetry.enabled": "instrumentation_telemetry_enabled", + "telemetry.heartbeat.interval": "instrumentation_telemetry_heartbeat_interval", + "telemetry.heartbeatInterval": "instrumentation_telemetry_heartbeat_interval", + "telemetry.logCollection": "instrumentation_telemetry_log_collection_enabled", + "telemetry.metrics": "instrumentation_telemetry_metrics_enabled", + "telemetry.metricsInterval": "instrumentation_telemetry_metrics_interval", + "telemetryEnabled": "instrumentation_telemetry_enabled", + "telemetry_heartbeat_interval": "instrumentation_telemetry_heartbeat_interval", + "trace.128_bit_traceid_generation_enabled": "trace_128_bits_id_enabled", "trace.128_bit_traceid_logging_enabled": "trace_128_bits_id_logging_enabled", - "DD_PROFILING_ENABLED": "profiling_enabled", - "DD_PROFILING_CODEHOTSPOTS_ENABLED": "profiling_codehotspots_enabled", - "DD_PROFILING_ENDPOINT_COLLECTION_ENABLED": "profiling_endpoint_collection_enabled", - "DD_LOG_LEVEL": "agent_log_level", - "DD_TAGS": "agent_tags", - "DD_TRACE_GLOBAL_TAGS": "trace_tags", + "trace.agent.port": "trace_agent_port", + "trace.agent.timeout": "trace_agent_timeout", + "trace.agent.v0.5.enabled": "trace_agent_v0.5_enabled", + "trace.agent_attempt_retry_time_msec": "trace_agent_attempt_retry_time_msec", + "trace.agent_connect_timeout": "trace_agent_connect_timeout", + "trace.agent_debug_verbose_curl": "trace_agent_debug_verbose_curl_enabled", + "trace.agent_flush_after_n_requests": "trace_agent_flush_after_n_requests", + "trace.agent_flush_interval": "trace_agent_flush_interval", + "trace.agent_max_consecutive_failures": "trace_send_retries", + "trace.agent_max_payload_size": "trace_agent_max_payload_size", + "trace.agent_port": "trace_agent_port", + "trace.agent_retries": "trace_send_retries", + "trace.agent_stack_backlog": "trace_agent_stack_backlog", + "trace.agent_stack_initial_size": "trace_agent_stack_initial_size", + "trace.agent_test_session_token": "trace_agent_test_session_token", + "trace.agent_timeout": "trace_agent_timeout", "trace.agent_url": "trace_agent_url", + "trace.agentless": "trace_agentless", + "trace.analytics.enabled": "trace_analytics_enabled", + "trace.analytics_enabled": "trace_analytics_enabled", "trace.append_trace_ids_to_logs": "trace_append_trace_ids_to_logs", + "trace.auto_flush_enabled": "trace_auto_flush_enabled", + "trace.aws-sdk.legacy.tracing.enabled": "trace_aws_sdk_legacy_tracing_enabled", + "trace.aws-sdk.propagation.enabled": "trace_aws_sdk_propagation_enabled", + "trace.beta_high_memory_pressure_percent": "trace_beta_high_memory_pressure_percent", + "trace.bgs_connect_timeout": "trace_bgs_connect_timeout", + "trace.bgs_timeout": "trace_bgs_timeout", + "trace.buffer_size": "trace_serialization_buffer_size", + "trace.cli_enabled": "trace_cli_enabled", + "trace.client-ip.enabled": "trace_client_ip_enabled", + "trace.client-ip.resolver.enabled": "trace_client_ip_resolver_enabled", "trace.client_ip_enabled": "trace_client_ip_enabled", - "trace.analytics_enabled": "trace_analytics_enabled", - "trace.rate_limit": "trace_rate_limit", - "trace.report_hostname": "trace_report_hostname", - "trace.http_client_split_by_domain": "trace_http_client_split_by_domain", + "trace.client_ip_header": "client_ip_header", + "trace.db.client.split-by-instance": "trace_db_client_split_by_instance", + "trace.db.client.split-by-instance.type.suffix": "trace_db_client_split_by_instance_type_suffix", + "trace.db_client_split_by_instance": "trace_db_client_split_by_instance", "trace.debug": "trace_debug_enabled", - "trace.agent_timeout": "trace_agent_timeout", - "trace.agent_port": "trace_agent_port", - "trace.x_datadog_tags_max_length": "trace_x_datadog_tags_max_length", + "trace.debug_curl_output": "trace_debug_curl_output_enabled", + "trace.debug_prng_seed": "trace_debug_prng_seed", + "trace.enabled": "trace_enabled", + "trace.flush_collect_cycles": "trace_flush_collect_cycles_enabled", + "trace.forked_process": "trace_forked_process_enabled", + "trace.generate_root_span": "trace_generate_root_span_enabled", + "trace.git_metadata_enabled": "git_metadata_enabled", + "trace.grpc.server.trim-package-resource": "trace_grpc_server_trim_package_resource_enabled", + "trace.header.tags.legacy.parsing.enabled": "trace_header_tags_legacy_parsing_enabled", + "trace.health.metrics.enabled": "trace_health_metrics_enabled", + "trace.health.metrics.statsd.port": "trace_health_metrics_statsd_port", + "trace.health_metrics_enabled": "trace_health_metrics_enabled", + "trace.health_metrics_heartbeat_sample_rate": "trace_health_metrics_heartbeat_sample_rate", + "trace.hook_limit": "trace_hook_limit", + "trace.http.client.split-by-domain": "trace_http_client_split_by_domain", + "trace.http_client_split_by_domain": "trace_http_client_split_by_domain", + "trace.http_post_data_param_allowed": "trace_http_post_data_param_allowed", + "trace.http_url_query_param_allowed": "trace_http_url_query_param_allowed", + "trace.jms.propagation.enabled": "trace_jms_propagation_enabled", + "trace.jmxfetch.kafka.enabled": "trace_jmxfetch_kafka_enabled", + "trace.jmxfetch.tomcat.enabled": "trace_jmxfetch_tomcat_enabled", + "trace.kafka.client.propagation.enabled": "trace_kafka_client_propagation_enabled", + "trace.laravel_queue_distributed_tracing": "trace_laravel_queue_distributed_tracing", + "trace.log_file": "trace_log_file", + "trace.log_level": "trace_log_level", + "trace.measure_compile_time": "trace_measure_compile_time_enabled", + "trace.measure_peak_memory_usage": "trace_measure_peak_memory_usage_enabled", + "trace.memcached_obfuscation": "trace_memcached_obfuscation_enabled", + "trace.memory_limit": "trace_memory_limit", "trace.obfuscation_query_string_regexp": "trace_obfuscation_query_string_regexp", + "trace.once_logs": "trace_once_logs", + "trace.otel.enabled": "trace_otel_enabled", + "trace.otel_enabled": "trace_otel_enabled", + "trace.partial.flush.min.spans": "trace_partial_flush_min_spans", + "trace.peer.service.defaults.enabled": "trace_peer_service_defaults_enabled", + "trace.peer.service.mapping": "trace_peer_service_mapping", "trace.peer_service_defaults_enabled": "trace_peer_service_defaults_enabled", + "trace.peer_service_mapping": "trace_peer_service_mapping", + "trace.peerservicetaginterceptor.enabled": "trace_peer_service_tag_interceptor_enabled", + "trace.perf.metrics.enabled": "trace_perf_metrics_enabled", + "trace.play.report-http-status": "trace_play_report_http_status", "trace.propagate_service": "trace_propagate_service", + "trace.propagate_user_id_default": "trace_propagate_user_id_default_enabled", + "trace.propagation_extract_first": "trace_propagation_extract_first", + "trace.propagation_style": "trace_propagation_style", + "trace.propagation_style_extract": "trace_propagation_style_extract", + "trace.propagation_style_inject": "trace_propagation_style_inject", + "trace.rabbit.propagation.enabled": "trace_rabbit_propagation_enabled", + "trace.rate.limit": "trace_rate_limit", + "trace.rate_limit": "trace_rate_limit", + "trace.redis_client_split_by_host": "trace_redis_client_split_by_host_enabled", + "trace.remove.integration-service-names.enabled": "trace_remove_integration_service_names_enabled", + "trace.remove_autoinstrumentation_orphans": "trace_remove_auto_instrumentation_orphans_enabled", "trace.remove_integration_service_names_enabled": "trace_remove_integration_service_names_enabled", + "trace.remove_root_span_laravel_queue": "trace_remove_root_span_laravel_queue_enabled", + "trace.remove_root_span_symfony_messenger": "trace_remove_root_span_symfony_messenger_enabled", + "trace.report-hostname": "trace_report_hostname", + "trace.report_hostname": "trace_report_hostname", + "trace.request_init_hook": "trace_request_init_hook", + "trace.resource_uri_fragment_regex": "trace_resource_uri_fragment_regex", + "trace.resource_uri_mapping_incoming": "trace_resource_uri_mapping_incoming", + "trace.resource_uri_mapping_outgoing": "trace_resource_uri_mapping_outgoing", + "trace.resource_uri_query_param_allowed": "trace_resource_uri_query_param_allowed", + "trace.retain_thread_capabilities": "trace_retain_thread_capabilities_enabled", + "trace.sample.rate": "trace_sample_rate", "trace.sample_rate": "trace_sample_rate", - "trace.health_metrics_enabled": "trace_health_metrics_enabled", - "trace.telemetry_enabled": "instrumentation_telemetry_enabled", - "trace.cli_enabled": "trace_cli_enabled", - "trace.db_client_split_by_instance": "trace_db_client_split_by_instance", - "trace.startup_logs": "trace_startup_logs", - "http_server_route_based_naming": "http_server_route_based_naming", - "DD_TRACE_PEER_SERVICE_MAPPING": "trace_peer_service_mapping", - "peerServiceMapping": "trace_peer_service_mapping", - "trace.peer.service.mapping": "trace_peer_service_mapping", - "trace.peer_service_mapping": "trace_peer_service_mapping", - "spanComputePeerService": "trace_peer_service_defaults_enabled", - "spanLeakDebug": "span_leak_debug", - "trace.peer.service.defaults.enabled": "trace_peer_service_defaults_enabled", - "spanAttributeSchema": "trace_span_attribute_schema", + "trace.sampling_rules": "trace_sample_rules", + "trace.sampling_rules_format": "trace_sampling_rules_format", + "trace.scope.depth.limit": "trace_scope_depth_limit", + "trace.servlet.async-timeout.error": "trace_servlet_async_timeout_error_enabled", + "trace.servlet.principal.enabled": "trace_servlet_principal_enabled", + "trace.shutdown_timeout": "trace_shutdown_timeout", + "trace.sidecar_trace_sender": "trace_sidecar_trace_sender", + "trace.sources_path": "trace_sources_path", "trace.span.attribute.schema": "trace_span_attribute_schema", - "spanRemoveIntegrationFromService": "trace_remove_integration_service_names_enabled", - "trace.remove.integration-service-names.enabled": "trace_remove_integration_service_names_enabled", - "ddtrace_auto_used": "ddtrace_auto_used", - "ddtrace_bootstrapped": "ddtrace_bootstrapped", - "orchestrion_enabled": "orchestrion_enabled", - "orchestrion_version": "orchestrion_version", - "trace.once_logs": "trace_once_logs", + "trace.spans_limit": "trace_spans_limit", + "trace.sqs.propagation.enabled": "trace_sqs_propagation_enabled", + "trace.startup_logs": "trace_startup_logs", + "trace.status404decorator.enabled": "trace_status_404_decorator_enabled", + "trace.status404rule.enabled": "trace_status_404_rule_enabled", + "trace.symfony_messenger_distributed_tracing": "trace_symfony_messenger_distributed_tracing", + "trace.symfony_messenger_middlewares": "trace_symfony_messenger_middlewares", + "trace.telemetry_enabled": "instrumentation_telemetry_enabled", + "trace.traced_internal_functions": "trace_traced_internal_functions", + "trace.tracer.metrics.enabled": "trace_metrics_enabled", + "trace.url_as_resource_names_enabled": "trace_url_as_resource_names_enabled", + "trace.warn_legacy_dd_trace": "trace_warn_legacy_dd_trace_enabled", + "trace.wordpress_additional_actions": "trace_wordpress_additional_actions", "trace.wordpress_callbacks": "trace_wordpress_callbacks", "trace.wordpress_enhanced_integration": "trace_wordpress_enhanced_integration", - "trace.wordpress_additional_actions": "trace_wordpress_additional_actions", - "trace.sidecar_trace_sender": "trace_sidecar_trace_sender", - "trace.sampling_rules_format": "trace_sampling_rules_format", - "DD_TRACE_SAMPLING_RULES_FORMAT": "trace_sampling_rules_format", - "trace.agentless": "trace_agentless", - "dd_agent_port": "trace_agent_port", - "dd_priority_sampling": "trace_priority_sampling_enabled", - "dd_profiling_capture_pct": "profiling_capture_pct", - "dd_profiling_export_libdd_enabled": "profiling_export_libdd_enabled", - "dd_profiling_heap_enabled": "profiling_heap_enabled", - "dd_profiling_lock_enabled": "profiling_lock_enabled", - "dd_profiling_max_frames": "profiling_max_frames", - "dd_profiling_memory_enabled": "profiling_memory_enabled", - "dd_profiling_stack_enabled": "profiling_stack_enabled", - "dd_profiling_upload_interval": "profiling_upload_interval", - "dd_remote_configuration_enabled": "remote_config_enabled", - "dd_trace_agent_timeout_seconds": "trace_agent_timeout", - "dd_trace_api_version": "trace_api_version", - "dd_trace_writer_buffer_size_bytes": "trace_serialization_buffer_size", - "dd_trace_writer_interval_seconds": "trace_agent_flush_interval", - "dd_trace_writer_max_payload_size_bytes": "trace_agent_max_payload_size", - "dd_trace_writer_reuse_connections": "trace_agent_reuse_connections", - "tracing_enabled": "trace_enabled", - "ssi_injection_enabled": "ssi_injection_enabled", - "DD_INJECTION_ENABLED": "ssi_injection_enabled", - "ssi_forced_injection_enabled": "ssi_forced_injection_enabled", - "DD_INJECT_FORCE": "ssi_forced_injection_enabled", - "inject_force": "ssi_forced_injection_enabled", - "OTEL_LOGS_EXPORTER": "otel_logs_exporter", - "OTEL_LOG_LEVEL": "otel_log_level", - "OTEL_METRICS_EXPORTER": "otel_metrics_exporter", - "integration_metrics_enabled": "integration_metrics_enabled", - "OTEL_SDK_DISABLED": "otel_sdk_disabled", - "OTEL_SERVICE_NAME": "otel_service_name", - "OTEL_PROPAGATORS": "otel_propagators", - "OTEL_RESOURCE_ATTRIBUTES": "otel_resource_attributes", - "OTEL_TRACES_EXPORTER": "otel_traces_exporter", - "OTEL_TRACES_SAMPLER": "otel_traces_sampler", - "OTEL_TRACES_SAMPLER_ARG": "otel_traces_sampler_arg", - "crashtracking_enabled": "crashtracking_enabled", - "crashtracking_available": "crashtracking_available", - "crashtracking_started": "crashtracking_started", - "crashtracking_stdout_filename": "crashtracking_stdout_filename", - "crashtracking_stderr_filename": "crashtracking_stderr_filename", - "crashtracking_alt_stack": "crashtracking_alt_stack", - "crashtracking_stacktrace_resolver": "crashtracking_stacktrace_resolver", - "crashtracking_debug_url": "crashtracking_debug_url", - "debug_stack_enabled": "debug_stack_enabled", - "DD_TRACE_BAGGAGE_MAX_ITEMS": "trace_baggage_max_items", - "DD_TRACE_BAGGAGE_MAX_BYTES": "trace_baggage_max_bytes", - "appsec.apiSecurity.sampleDelay": "api_security_sample_delay", - "appsec.stackTrace.enabled": "appsec_stack_trace_enabled", - "appsec.stackTrace.maxDepth": "appsec_max_stack_trace_depth", - "appsec.stackTrace.maxStackTraces": "appsec_max_stack_traces", - "appsec.standalone.enabled": "experimental_appsec_standalone_enabled", - "baggageMaxBytes": "trace_baggage_max_bytes", - "baggageMaxItems": "trace_baggage_max_items", - "ciVisAgentlessLogSubmissionEnabled": "ci_visibility_agentless_enabled", - "ciVisibilityTestSessionName": "test_session_name", - "cloudPayloadTagging.maxDepth": "cloud_payload_tagging_max_depth", - "cloudPayloadTagging.requestsEnabled": "cloud_payload_tagging_requests_enabled", - "cloudPayloadTagging.responsesEnabled": "cloud_payload_tagging_responses_enabled", - "cloudPayloadTagging.rules.aws.eventbridge.expand": "cloud_payload_tagging_rules_aws_eventbridge_expand", - "cloudPayloadTagging.rules.aws.eventbridge.request": "cloud_payload_tagging_rules_aws_eventbridge_request", - "cloudPayloadTagging.rules.aws.eventbridge.response": "cloud_payload_tagging_rules_aws_eventbridge_response", - "cloudPayloadTagging.rules.aws.kinesis.expand": "cloud_payload_tagging_rules_aws_kinesis_expand", - "cloudPayloadTagging.rules.aws.kinesis.request": "cloud_payload_tagging_rules_aws_kinesis_request", - "cloudPayloadTagging.rules.aws.kinesis.response": "cloud_payload_tagging_rules_aws_kinesis_response", - "cloudPayloadTagging.rules.aws.s3.expand": "cloud_payload_tagging_rules_aws_s3_expand", - "cloudPayloadTagging.rules.aws.s3.request": "cloud_payload_tagging_rules_aws_s3_request", - "cloudPayloadTagging.rules.aws.s3.response": "cloud_payload_tagging_rules_aws_s3_response", - "cloudPayloadTagging.rules.aws.sns.expand": "cloud_payload_tagging_rules_aws_sns_expand", - "cloudPayloadTagging.rules.aws.sns.request": "cloud_payload_tagging_rules_aws_sns_request", - "cloudPayloadTagging.rules.aws.sns.response": "cloud_payload_tagging_rules_aws_sns_response", - "cloudPayloadTagging.rules.aws.sqs.expand": "cloud_payload_tagging_rules_aws_sqs_expand", - "cloudPayloadTagging.rules.aws.sqs.request": "cloud_payload_tagging_rules_aws_sqs_request", - "cloudPayloadTagging.rules.aws.sqs.response": "cloud_payload_tagging_rules_aws_sqs_response", - "codeOriginForSpans.enabled": "code_origin_for_spans_enabled", - "commitSHA": "commit_sha", - "crashtracking.enabled": "crashtracking_enabled", - "dynamicInstrumentationEnabled": "dynamic_instrumentation_enabled", - "flakyTestRetriesCount": "ci_visibility_flaky_retry_count", - "gitMetadataEnabled": "git_metadata_enabled", - "grpc.client.error.statuses": "trace_grpc_client_error_statuses", - "grpc.server.error.statuses": "trace_grpc_server_error_statuses", - "headerTags": "trace_header_tags", - "injectionEnabled": "ssi_injection_enabled", - "instrumentation_config_id": "instrumentation_config_id", - "isEarlyFlakeDetectionEnabled": "ci_visibility_early_flake_detection_enabled", - "isFlakyTestRetriesEnabled": "ci_visibility_flaky_retry_enabled", - "isManualApiEnabled": "ci_visibility_manual_api_enabled", - "isTestDynamicInstrumentationEnabled": "ci_visibility_test_dynamic_instrumentation_enabled", - "langchain.spanCharLimit": "langchain_span_char_limit", - "langchain.spanPromptCompletionSampleRate": "langchain_span_prompt_completion_sample_rate", - "legacyBaggageEnabled": "trace_legacy_baggage_enabled", - "llmobs.agentlessEnabled": "llmobs_agentless_enabled", - "llmobs.enabled": "llmobs_enabled", - "llmobs.mlApp": "llmobs_ml_app", - "profiling.longLivedThreshold": "profiling_long_lived_threshold", - "repositoryUrl": "repository_url", - "sampler.rules": "trace_sample_rules", - "sampler.spanSamplingRules": "span_sample_rules", - "telemetry.dependencyCollection": "instrumentation_telemetry_dependency_collection_enabled", - "telemetry.heartbeatInterval": "instrumentation_telemetry_heartbeat_interval", + "trace.x-datadog-tags.max.length": "trace_x_datadog_tags_max_length", + "trace.x_datadog_tags_max_length": "trace_x_datadog_tags_max_length", "traceEnabled": "trace_enabled", - "tracePropagationStyle.otelPropagators": "trace_propagation_style_otel_propagators" + "traceId128BitGenerationEnabled": "trace_128_bits_id_enabled", + "traceId128BitLoggingEnabled": "trace_128_bits_id_logging_enabled", + "tracePropagationExtractFirst": "trace_propagation_extract_first", + "tracePropagationStyle,otelPropagators": "trace_propagation_style_otel_propagators", + "tracePropagationStyle.extract": "trace_propagation_style_extract", + "tracePropagationStyle.inject": "trace_propagation_style_inject", + "tracePropagationStyle.otelPropagators": "trace_propagation_style_otel_propagators", + "trace_methods": "trace_methods", + "tracer_instance_count": "trace_instance_count", + "tracing": "trace_enabled", + "tracing.auto_instrument.enabled": "trace_auto_instrument_enabled", + "tracing.distributed_tracing.propagation_extract_style": "trace_propagation_style_extract", + "tracing.distributed_tracing.propagation_inject_style": "trace_propagation_style_inject", + "tracing.enabled": "trace_enabled", + "tracing.log_injection": "logs_injection_enabled", + "tracing.opentelemetry.enabled": "trace_otel_enabled", + "tracing.partial_flush.enabled": "trace_partial_flush_enabled", + "tracing.partial_flush.min_spans_threshold": "trace_partial_flush_min_spans", + "tracing.propagation_style_extract": "trace_propagation_style_extract", + "tracing.propagation_style_inject": "trace_propagation_style_inject", + "tracing.report_hostname": "trace_report_hostname", + "tracing.sampling.rate_limit": "trace_sample_rate", + "tracing_enabled": "trace_enabled", + "universal_version": "universal_version_enabled", + "url": "trace_agent_url", + "version": "application_version", + "wcf_obfuscation_enabled": "trace_wcf_obfuscation_enabled" } diff --git a/packages/dd-trace/test/plugins/externals.json b/packages/dd-trace/test/plugins/externals.json index 288fb9350c6..c3fc12fb176 100644 --- a/packages/dd-trace/test/plugins/externals.json +++ b/packages/dd-trace/test/plugins/externals.json @@ -416,6 +416,10 @@ { "name": "express", "versions": [">=4"] + }, + { + "name": "sqlite3", + "versions": ["^5.0.8"] } ] } From 6cda84792052ec3e866361d8723f7d803511439f Mon Sep 17 00:00:00 2001 From: Roberto Montero <108007532+robertomonteromiguel@users.noreply.github.com> Date: Wed, 18 Dec 2024 16:54:35 +0100 Subject: [PATCH 18/36] K8s tests: Run on parallel matrix (#5038) --- .gitlab-ci.yml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 6da75a763ac..dcf8a6c7772 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -22,8 +22,9 @@ onboarding_tests_installer: SCENARIO: [ SIMPLE_INSTALLER_AUTO_INJECTION, SIMPLE_AUTO_INJECTION_PROFILING ] onboarding_tests_k8s_injection: - variables: - WEBLOG_VARIANT: sample-app + parallel: + matrix: + - WEBLOG_VARIANT: sample-app requirements_json_test: rules: From 391ab8b6d313193725bad11b9af63ea388d22d64 Mon Sep 17 00:00:00 2001 From: Roch Devost Date: Wed, 18 Dec 2024 12:38:03 -0500 Subject: [PATCH 19/36] set node types minimum version to oldest (#5029) --- package.json | 2 +- yarn.lock | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/package.json b/package.json index cd540cb08a0..9b0abdb34db 100644 --- a/package.json +++ b/package.json @@ -118,7 +118,7 @@ "@eslint/eslintrc": "^3.1.0", "@eslint/js": "^9.11.1", "@stylistic/eslint-plugin-js": "^2.8.0", - "@types/node": "^16.18.103", + "@types/node": "^16.0.0", "autocannon": "^4.5.2", "aws-sdk": "^2.1446.0", "axios": "^1.7.4", diff --git a/yarn.lock b/yarn.lock index 49411da5f2f..a56218a0a45 100644 --- a/yarn.lock +++ b/yarn.lock @@ -956,10 +956,10 @@ dependencies: undici-types "~5.26.4" -"@types/node@^16.18.103": - version "16.18.103" - resolved "https://registry.yarnpkg.com/@types/node/-/node-16.18.103.tgz#5557c7c32a766fddbec4b933b1d5c365f89b20a4" - integrity sha512-gOAcUSik1nR/CRC3BsK8kr6tbmNIOTpvb1sT+v5Nmmys+Ho8YtnIHP90wEsVK4hTcHndOqPVIlehEGEA5y31bA== +"@types/node@^16.0.0": + version "16.18.122" + resolved "https://registry.yarnpkg.com/@types/node/-/node-16.18.122.tgz#54948ddbe2ddef8144ee16b37f160e3f99c32397" + integrity sha512-rF6rUBS80n4oK16EW8nE75U+9fw0SSUgoPtWSvHhPXdT7itbvmS7UjB/jyM8i3AkvI6yeSM5qCwo+xN0npGDHg== "@types/prop-types@*": version "15.7.5" From 216bf5d13b3d9e50a5055f096d93e73556fad515 Mon Sep 17 00:00:00 2001 From: Nicholas Hulston Date: Wed, 18 Dec 2024 13:02:43 -0500 Subject: [PATCH 20/36] [serverless] Add DynamoDB Span Pointers (#4912) * Add span pointer support for updateItem and deleteItem * putItem support * transactWriteItem support * batchWriteItem support * Add unit+integration tests (very large commit) * Move `DD_AWS_SDK_DYNAMODB_TABLE_PRIMARY_KEYS` parsing logic to config.js * Code refactoring * Move util functions to packages/datadog-plugin-aws-sdk/ * lint * log when encountering errors in `encodeValue`; fix test * Send config env var as string to telemetry; handle parsing logic in dynamodb.js * Update config_norm_rules.json * fix test * Add unit tests for DynamoDB generatePointerHash * better logging + checks --- .../src/services/dynamodb.js | 154 ++++ .../datadog-plugin-aws-sdk/src/services/s3.js | 2 +- packages/datadog-plugin-aws-sdk/src/util.js | 92 ++ .../test/dynamodb.spec.js | 831 ++++++++++++++++++ .../datadog-plugin-aws-sdk/test/util.spec.js | 213 +++++ packages/dd-trace/src/config.js | 3 + packages/dd-trace/src/constants.js | 1 + packages/dd-trace/src/util.js | 17 +- packages/dd-trace/test/plugins/externals.json | 4 + packages/dd-trace/test/util.spec.js | 18 - 10 files changed, 1300 insertions(+), 35 deletions(-) create mode 100644 packages/datadog-plugin-aws-sdk/src/util.js create mode 100644 packages/datadog-plugin-aws-sdk/test/dynamodb.spec.js create mode 100644 packages/datadog-plugin-aws-sdk/test/util.spec.js diff --git a/packages/datadog-plugin-aws-sdk/src/services/dynamodb.js b/packages/datadog-plugin-aws-sdk/src/services/dynamodb.js index 4097586b2c5..cbca2192ad6 100644 --- a/packages/datadog-plugin-aws-sdk/src/services/dynamodb.js +++ b/packages/datadog-plugin-aws-sdk/src/services/dynamodb.js @@ -1,6 +1,9 @@ 'use strict' const BaseAwsSdkPlugin = require('../base') +const log = require('../../../dd-trace/src/log') +const { DYNAMODB_PTR_KIND, SPAN_POINTER_DIRECTION } = require('../../../dd-trace/src/constants') +const { extractPrimaryKeys, generatePointerHash } = require('../util') class DynamoDb extends BaseAwsSdkPlugin { static get id () { return 'dynamodb' } @@ -48,6 +51,157 @@ class DynamoDb extends BaseAwsSdkPlugin { return tags } + + addSpanPointers (span, response) { + const request = response?.request + const operationName = request?.operation + + const hashes = [] + switch (operationName) { + case 'putItem': { + const hash = DynamoDb.calculatePutItemHash( + request?.params?.TableName, + request?.params?.Item, + this.getPrimaryKeyConfig() + ) + if (hash) hashes.push(hash) + break + } + case 'updateItem': + case 'deleteItem': { + const hash = DynamoDb.calculateHashWithKnownKeys(request?.params?.TableName, request?.params?.Key) + if (hash) hashes.push(hash) + break + } + case 'transactWriteItems': { + const transactItems = request?.params?.TransactItems || [] + for (const item of transactItems) { + if (item.Put) { + const hash = + DynamoDb.calculatePutItemHash(item.Put.TableName, item.Put.Item, this.getPrimaryKeyConfig()) + if (hash) hashes.push(hash) + } else if (item.Update || item.Delete) { + const operation = item.Update ? item.Update : item.Delete + const hash = DynamoDb.calculateHashWithKnownKeys(operation.TableName, operation.Key) + if (hash) hashes.push(hash) + } + } + break + } + case 'batchWriteItem': { + const requestItems = request?.params.RequestItems || {} + for (const [tableName, operations] of Object.entries(requestItems)) { + if (!Array.isArray(operations)) continue + for (const operation of operations) { + if (operation?.PutRequest) { + const hash = + DynamoDb.calculatePutItemHash(tableName, operation.PutRequest.Item, this.getPrimaryKeyConfig()) + if (hash) hashes.push(hash) + } else if (operation?.DeleteRequest) { + const hash = DynamoDb.calculateHashWithKnownKeys(tableName, operation.DeleteRequest.Key) + if (hash) hashes.push(hash) + } + } + } + break + } + } + + for (const hash of hashes) { + span.addSpanPointer(DYNAMODB_PTR_KIND, SPAN_POINTER_DIRECTION.DOWNSTREAM, hash) + } + } + + /** + * Parses primary key config from the `DD_AWS_SDK_DYNAMODB_TABLE_PRIMARY_KEYS` env var. + * Only runs when needed, and warns when missing or invalid config. + * @returns {Object|undefined} Parsed config from env var or undefined if empty/missing/invalid config. + */ + getPrimaryKeyConfig () { + if (this.dynamoPrimaryKeyConfig) { + // Return cached config if it exists + return this.dynamoPrimaryKeyConfig + } + + const configStr = this._tracerConfig?.aws?.dynamoDb?.tablePrimaryKeys + if (!configStr) { + log.warn('Missing DD_AWS_SDK_DYNAMODB_TABLE_PRIMARY_KEYS env variable. ' + + 'Please add your table\'s primary keys under this env variable.') + return + } + + try { + const parsedConfig = JSON.parse(configStr) + const config = {} + for (const [tableName, primaryKeys] of Object.entries(parsedConfig)) { + if (Array.isArray(primaryKeys) && primaryKeys.length > 0 && primaryKeys.length <= 2) { + config[tableName] = new Set(primaryKeys) + } else { + log.warn(`Invalid primary key configuration for table: ${tableName}.` + + 'Please fix the DD_AWS_SDK_DYNAMODB_TABLE_PRIMARY_KEYS env var.') + } + } + + this.dynamoPrimaryKeyConfig = config + return config + } catch (err) { + log.warn('Failed to parse DD_AWS_SDK_DYNAMODB_TABLE_PRIMARY_KEYS:', err.message) + } + } + + /** + * Calculates a hash for DynamoDB PutItem operations using table's configured primary keys. + * @param {string} tableName - Name of the DynamoDB table. + * @param {Object} item - Complete PutItem item parameter to be put. + * @param {Object.>} primaryKeyConfig - Mapping of table names to Sets of primary key names + * loaded from DD_AWS_SDK_DYNAMODB_TABLE_PRIMARY_KEYS. + * @returns {string|undefined} Hash combining table name and primary key/value pairs, or undefined if unable. + */ + static calculatePutItemHash (tableName, item, primaryKeyConfig) { + if (!tableName || !item) { + log.debug('Unable to calculate hash because missing required parameters') + return + } + if (!primaryKeyConfig) { + log.warn('Missing DD_AWS_SDK_DYNAMODB_TABLE_PRIMARY_KEYS env variable') + return + } + const primaryKeySet = primaryKeyConfig[tableName] + if (!primaryKeySet || !(primaryKeySet instanceof Set) || primaryKeySet.size === 0 || primaryKeySet.size > 2) { + log.warn( + `span pointers: failed to extract PutItem span pointer: table ${tableName} ` + + 'not found in primary key names or the DD_AWS_SDK_DYNAMODB_TABLE_PRIMARY_KEYS env var was invalid.' + + 'Please update the env var.' + ) + return + } + const keyValues = extractPrimaryKeys(primaryKeySet, item) + if (keyValues) { + return generatePointerHash([tableName, ...keyValues]) + } + } + + /** + * Calculates a hash for DynamoDB operations that have keys provided (UpdateItem, DeleteItem). + * @param {string} tableName - Name of the DynamoDB table. + * @param {Object} keysObject - Object containing primary key/value attributes in DynamoDB format. + * (e.g., { userId: { S: "123" }, sortKey: { N: "456" } }) + * @returns {string|undefined} Hash value combining table name and primary key/value pairs, or undefined if unable. + * + * @example + * calculateHashWithKnownKeys('UserTable', { userId: { S: "user123" }, timestamp: { N: "1234567" } }) + */ + static calculateHashWithKnownKeys (tableName, keysObject) { + if (!tableName || !keysObject) { + log.debug('Unable to calculate hash because missing parameters') + return + } + const keyNamesSet = new Set(Object.keys(keysObject)) + const keyValues = extractPrimaryKeys(keyNamesSet, keysObject) + if (keyValues) { + return generatePointerHash([tableName, ...keyValues]) + } + } } module.exports = DynamoDb diff --git a/packages/datadog-plugin-aws-sdk/src/services/s3.js b/packages/datadog-plugin-aws-sdk/src/services/s3.js index 5fcfb6ed165..d860223d67b 100644 --- a/packages/datadog-plugin-aws-sdk/src/services/s3.js +++ b/packages/datadog-plugin-aws-sdk/src/services/s3.js @@ -2,7 +2,7 @@ const BaseAwsSdkPlugin = require('../base') const log = require('../../../dd-trace/src/log') -const { generatePointerHash } = require('../../../dd-trace/src/util') +const { generatePointerHash } = require('../util') const { S3_PTR_KIND, SPAN_POINTER_DIRECTION } = require('../../../dd-trace/src/constants') class S3 extends BaseAwsSdkPlugin { diff --git a/packages/datadog-plugin-aws-sdk/src/util.js b/packages/datadog-plugin-aws-sdk/src/util.js new file mode 100644 index 00000000000..4bb7e86c8cd --- /dev/null +++ b/packages/datadog-plugin-aws-sdk/src/util.js @@ -0,0 +1,92 @@ +'use strict' + +const crypto = require('crypto') +const log = require('../../dd-trace/src/log') + +/** + * Generates a unique hash from an array of strings by joining them with | before hashing. + * Used to uniquely identify AWS requests for span pointers. + * @param {string[]} components - Array of strings to hash + * @returns {string} A 32-character hash uniquely identifying the components + */ +function generatePointerHash (components) { + // If passing S3's ETag as a component, make sure any quotes have already been removed! + const dataToHash = components.join('|') + const hash = crypto.createHash('sha256').update(dataToHash).digest('hex') + return hash.substring(0, 32) +} + +/** + * Encodes a DynamoDB attribute value to Buffer for span pointer hashing. + * @param {Object} valueObject - DynamoDB value in AWS format ({ S: string } or { N: string } or { B: Buffer }) + * @returns {Buffer|undefined} Encoded value as Buffer, or undefined if invalid input. + * + * @example + * encodeValue({ S: "user123" }) -> Buffer("user123") + * encodeValue({ N: "42" }) -> Buffer("42") + * encodeValue({ B: Buffer([1, 2, 3]) }) -> Buffer([1, 2, 3]) + */ +function encodeValue (valueObject) { + if (!valueObject) { + return + } + + try { + const type = Object.keys(valueObject)[0] + const value = valueObject[type] + + switch (type) { + case 'S': + return Buffer.from(value) + case 'N': + return Buffer.from(value.toString()) + case 'B': + return Buffer.isBuffer(value) ? value : Buffer.from(value) + default: + log.debug(`Found unknown type while trying to create DynamoDB span pointer: ${type}`) + } + } catch (err) { + log.debug(`Failed to encode value while trying to create DynamoDB span pointer: ${err.message}`) + } +} + +/** + * Extracts and encodes primary key values from a DynamoDB item. + * Handles tables with single-key and two-key scenarios. + * + * @param {Set} keySet - Set of primary key names. + * @param {Object} keyValuePairs - Object containing key/value pairs. + * @returns {Array|undefined} [key1Name, key1Value, key2Name, key2Value], or undefined if invalid input. + * key2 entries are empty strings in the single-key case. + * @example + * extractPrimaryKeys(new Set(['userId']), {userId: {S: "user123"}}) + * // Returns ["userId", Buffer("user123"), "", ""] + * extractPrimaryKeys(new Set(['userId', 'timestamp']), {userId: {S: "user123"}, timestamp: {N: "1234}}) + * // Returns ["timestamp", Buffer.from("1234"), "userId", Buffer.from("user123")] + */ +const extractPrimaryKeys = (keySet, keyValuePairs) => { + const keyNames = Array.from(keySet) + if (keyNames.length === 0) { + return + } + + if (keyNames.length === 1) { + const value = encodeValue(keyValuePairs[keyNames[0]]) + if (value) { + return [keyNames[0], value, '', ''] + } + } else { + const [key1, key2] = keyNames.sort() + const value1 = encodeValue(keyValuePairs[key1]) + const value2 = encodeValue(keyValuePairs[key2]) + if (value1 && value2) { + return [key1, value1, key2, value2] + } + } +} + +module.exports = { + generatePointerHash, + encodeValue, + extractPrimaryKeys +} diff --git a/packages/datadog-plugin-aws-sdk/test/dynamodb.spec.js b/packages/datadog-plugin-aws-sdk/test/dynamodb.spec.js new file mode 100644 index 00000000000..7fba9babfb0 --- /dev/null +++ b/packages/datadog-plugin-aws-sdk/test/dynamodb.spec.js @@ -0,0 +1,831 @@ +'use strict' + +const agent = require('../../dd-trace/test/plugins/agent') +const { setup } = require('./spec_helpers') +const axios = require('axios') +const { DYNAMODB_PTR_KIND, SPAN_POINTER_DIRECTION } = require('../../dd-trace/src/constants') +const DynamoDb = require('../src/services/dynamodb') +const { generatePointerHash } = require('../src/util') + +/* eslint-disable no-console */ +async function resetLocalStackDynamo () { + try { + await axios.post('http://localhost:4566/reset') + console.log('LocalStack Dynamo reset successful') + } catch (error) { + console.error('Error resetting LocalStack Dynamo:', error.message) + } +} + +describe('Plugin', () => { + describe('aws-sdk (dynamodb)', function () { + setup() + + withVersions('aws-sdk', ['aws-sdk', '@aws-sdk/smithy-client'], (version, moduleName) => { + let tracer + let AWS + let dynamo + + const dynamoClientName = moduleName === '@aws-sdk/smithy-client' ? '@aws-sdk/client-dynamodb' : 'aws-sdk' + + // Test both cases: tables with only partition key and with partition+sort key. + const oneKeyTableName = 'OneKeyTable' + const twoKeyTableName = 'TwoKeyTable' + + describe('with configuration', () => { + before(() => { + tracer = require('../../dd-trace') + tracer.init() + return agent.load('aws-sdk') + }) + + before(async () => { + AWS = require(`../../../versions/${dynamoClientName}@${version}`).get() + dynamo = new AWS.DynamoDB({ endpoint: 'http://127.0.0.1:4566', region: 'us-east-1' }) + + const deleteTable = async (tableName) => { + if (dynamoClientName === '@aws-sdk/client-dynamodb') { + try { + await dynamo.deleteTable({ TableName: tableName }) + await new Promise(resolve => setTimeout(resolve, 1000)) + } catch (err) { + if (err.name !== 'ResourceNotFoundException') { + throw err + } + } + } else { + try { + if (typeof dynamo.deleteTable({}).promise === 'function') { + await dynamo.deleteTable({ TableName: tableName }).promise() + await dynamo.waitFor('tableNotExists', { TableName: tableName }).promise() + } else { + await new Promise((resolve, reject) => { + dynamo.deleteTable({ TableName: tableName }, (err) => { + if (err && err.code !== 'ResourceNotFoundException') { + reject(err) + } else { + resolve() + } + }) + }) + } + } catch (err) { + if (err.code !== 'ResourceNotFoundException') { + throw err + } + } + } + } + + const createTable = async (params) => { + if (dynamoClientName === '@aws-sdk/client-dynamodb') { + await dynamo.createTable(params) + } else { + if (typeof dynamo.createTable({}).promise === 'function') { + await dynamo.createTable(params).promise() + } else { + await new Promise((resolve, reject) => { + dynamo.createTable(params, (err, data) => { + if (err) reject(err) + else resolve(data) + }) + }) + } + } + } + + // Delete existing tables + await deleteTable(oneKeyTableName) + await deleteTable(twoKeyTableName) + + // Create tables + await createTable({ + TableName: oneKeyTableName, + KeySchema: [{ AttributeName: 'name', KeyType: 'HASH' }], + AttributeDefinitions: [{ AttributeName: 'name', AttributeType: 'S' }], + ProvisionedThroughput: { ReadCapacityUnits: 5, WriteCapacityUnits: 5 } + }) + + await createTable({ + TableName: twoKeyTableName, + KeySchema: [ + { AttributeName: 'id', KeyType: 'HASH' }, + { AttributeName: 'binary', KeyType: 'RANGE' } + ], + AttributeDefinitions: [ + { AttributeName: 'id', AttributeType: 'N' }, + { AttributeName: 'binary', AttributeType: 'B' } + ], + ProvisionedThroughput: { ReadCapacityUnits: 5, WriteCapacityUnits: 5 } + }) + }) + + after(async () => { + await resetLocalStackDynamo() + return agent.close({ ritmReset: false }) + }) + + describe('span pointers', () => { + beforeEach(() => { + DynamoDb.dynamoPrimaryKeyConfig = null + delete process.env.DD_AWS_SDK_DYNAMODB_TABLE_PRIMARY_KEYS + }) + + function testSpanPointers ({ expectedHashes, operation }) { + let expectedLength = 0 + if (expectedHashes) { + expectedLength = Array.isArray(expectedHashes) ? expectedHashes.length : 1 + } + return (done) => { + operation((err) => { + if (err) { + return done(err) + } + + agent.use(traces => { + try { + const span = traces[0][0] + const links = JSON.parse(span.meta?.['_dd.span_links'] || '[]') + expect(links).to.have.lengthOf(expectedLength) + + if (expectedHashes) { + if (Array.isArray(expectedHashes)) { + expectedHashes.forEach((hash, i) => { + expect(links[i].attributes['ptr.hash']).to.equal(hash) + }) + } else { + expect(links[0].attributes).to.deep.equal({ + 'ptr.kind': DYNAMODB_PTR_KIND, + 'ptr.dir': SPAN_POINTER_DIRECTION.DOWNSTREAM, + 'ptr.hash': expectedHashes, + 'link.kind': 'span-pointer' + }) + } + } + return done() + } catch (error) { + return done(error) + } + }).catch(error => { + done(error) + }) + }) + } + } + + describe('1-key table', () => { + it('should add span pointer for putItem when config is valid', () => { + testSpanPointers({ + expectedHashes: '27f424c8202ab35efbf8b0b444b1928f', + operation: (callback) => { + process.env.DD_AWS_SDK_DYNAMODB_TABLE_PRIMARY_KEYS = + '{"OneKeyTable": ["name"]}' + dynamo.putItem({ + TableName: oneKeyTableName, + Item: { + name: { S: 'test1' }, + foo: { S: 'bar1' } + } + }, callback) + } + }) + }) + + it('should not add links or error for putItem when config is invalid', () => { + testSpanPointers({ + operation: (callback) => { + process.env.DD_AWS_SDK_DYNAMODB_TABLE_PRIMARY_KEYS = '{"DifferentTable": ["test"]}' + dynamo.putItem({ + TableName: oneKeyTableName, + Item: { + name: { S: 'test2' }, + foo: { S: 'bar2' } + } + }, callback) + } + }) + }) + + it('should not add links or error for putItem when config is missing', () => { + testSpanPointers({ + operation: (callback) => { + process.env.DD_AWS_SDK_DYNAMODB_TABLE_PRIMARY_KEYS = null + dynamo.putItem({ + TableName: oneKeyTableName, + Item: { + name: { S: 'test3' }, + foo: { S: 'bar3' } + } + }, callback) + } + }) + }) + + it('should add span pointer for updateItem', () => { + testSpanPointers({ + expectedHashes: '27f424c8202ab35efbf8b0b444b1928f', + operation: (callback) => { + dynamo.updateItem({ + TableName: oneKeyTableName, + Key: { name: { S: 'test1' } }, + AttributeUpdates: { + foo: { + Action: 'PUT', + Value: { S: 'bar4' } + } + } + }, callback) + } + }) + }) + + it('should add span pointer for deleteItem', () => { + testSpanPointers({ + expectedHashes: '27f424c8202ab35efbf8b0b444b1928f', + operation: (callback) => { + dynamo.deleteItem({ + TableName: oneKeyTableName, + Key: { name: { S: 'test1' } } + }, callback) + } + }) + }) + + it('should add span pointers for transactWriteItems', () => { + // Skip for older versions that don't support transactWriteItems + if (typeof dynamo.transactWriteItems !== 'function') { + return + } + testSpanPointers({ + expectedHashes: [ + '955ab85fc7d1d63fe4faf18696514f13', + '856c95a173d9952008a70283175041fc', + '9682c132f1900106a792f166d0619e0b' + ], + operation: (callback) => { + process.env.DD_AWS_SDK_DYNAMODB_TABLE_PRIMARY_KEYS = '{"OneKeyTable": ["name"]}' + dynamo.transactWriteItems({ + TransactItems: [ + { + Put: { + TableName: oneKeyTableName, + Item: { + name: { S: 'test4' }, + foo: { S: 'bar4' } + } + } + }, + { + Update: { + TableName: oneKeyTableName, + Key: { name: { S: 'test2' } }, + UpdateExpression: 'SET foo = :newfoo', + ExpressionAttributeValues: { + ':newfoo': { S: 'bar5' } + } + } + }, + { + Delete: { + TableName: oneKeyTableName, + Key: { name: { S: 'test3' } } + } + } + ] + }, callback) + } + }) + }) + + it('should add span pointers for batchWriteItem', () => { + // Skip for older versions that don't support batchWriteItem + if (typeof dynamo.batchWriteItem !== 'function') { + return + } + testSpanPointers({ + expectedHashes: [ + '955ab85fc7d1d63fe4faf18696514f13', + '9682c132f1900106a792f166d0619e0b' + ], + operation: (callback) => { + process.env.DD_AWS_SDK_DYNAMODB_TABLE_PRIMARY_KEYS = '{"OneKeyTable": ["name"]}' + dynamo.batchWriteItem({ + RequestItems: { + [oneKeyTableName]: [ + { + PutRequest: { + Item: { + name: { S: 'test4' }, + foo: { S: 'bar4' } + } + } + }, + { + DeleteRequest: { + Key: { + name: { S: 'test3' } + } + } + } + ] + } + }, callback) + } + }) + }) + }) + + describe('2-key table', () => { + it('should add span pointer for putItem when config is valid', () => { + testSpanPointers({ + expectedHashes: 'cc32f0e49ee05d3f2820ccc999bfe306', + operation: (callback) => { + process.env.DD_AWS_SDK_DYNAMODB_TABLE_PRIMARY_KEYS = '{"TwoKeyTable": ["id", "binary"]}' + dynamo.putItem({ + TableName: twoKeyTableName, + Item: { + id: { N: '1' }, + binary: { B: Buffer.from('Hello world 1') } + } + }, callback) + } + }) + }) + + it('should not add links or error for putItem when config is invalid', () => { + testSpanPointers({ + operation: (callback) => { + process.env.DD_AWS_SDK_DYNAMODB_TABLE_PRIMARY_KEYS = '{"DifferentTable": ["test"]}' + dynamo.putItem({ + TableName: twoKeyTableName, + Item: { + id: { N: '2' }, + binary: { B: Buffer.from('Hello world 2') } + } + }, callback) + } + }) + }) + + it('should not add links or error for putItem when config is missing', () => { + testSpanPointers({ + operation: (callback) => { + process.env.DD_AWS_SDK_DYNAMODB_TABLE_PRIMARY_KEYS = null + dynamo.putItem({ + TableName: twoKeyTableName, + Item: { + id: { N: '3' }, + binary: { B: Buffer.from('Hello world 3') } + } + }, callback) + } + }) + }) + + it('should add span pointer for updateItem', function (done) { + dynamo.putItem({ + TableName: twoKeyTableName, + Item: { + id: { N: '100' }, + binary: { B: Buffer.from('abc') } + } + }, async function (err) { + if (err) { + return done(err) + } + await new Promise(resolve => setTimeout(resolve, 100)) + testSpanPointers({ + expectedHashes: '5dac7d25254d596482a3c2c187e51046', + operation: (callback) => { + dynamo.updateItem({ + TableName: twoKeyTableName, + Key: { + id: { N: '100' }, + binary: { B: Buffer.from('abc') } + }, + AttributeUpdates: { + someOtherField: { + Action: 'PUT', + Value: { S: 'new value' } + } + } + }, callback) + } + })(done) + }) + }) + + it('should add span pointer for deleteItem', function (done) { + dynamo.putItem({ + TableName: twoKeyTableName, + Item: { + id: { N: '200' }, + binary: { B: Buffer.from('Hello world') } + } + }, async function (err) { + if (err) return done(err) + await new Promise(resolve => setTimeout(resolve, 100)) + testSpanPointers({ + expectedHashes: 'c356b0dd48c734d889e95122750c2679', + operation: (callback) => { + dynamo.deleteItem({ + TableName: twoKeyTableName, + Key: { + id: { N: '200' }, + binary: { B: Buffer.from('Hello world') } + } + }, callback) + } + })(done) + }) + }) + + it('should add span pointers for transactWriteItems', () => { + // Skip for older versions that don't support transactWriteItems + if (typeof dynamo.transactWriteItems !== 'function') { + return + } + testSpanPointers({ + expectedHashes: [ + 'dd071963cd90e4b3088043f0b9a9f53c', + '7794824f72d673ac7844353bc3ea25d9', + '8a6f801cc4e7d1d5e0dd37e0904e6316' + ], + operation: (callback) => { + process.env.DD_AWS_SDK_DYNAMODB_TABLE_PRIMARY_KEYS = '{"TwoKeyTable": ["id", "binary"]}' + dynamo.transactWriteItems({ + TransactItems: [ + { + Put: { + TableName: twoKeyTableName, + Item: { + id: { N: '4' }, + binary: { B: Buffer.from('Hello world 4') } + } + } + }, + { + Update: { + TableName: twoKeyTableName, + Key: { + id: { N: '2' }, + binary: { B: Buffer.from('Hello world 2') } + }, + AttributeUpdates: { + someOtherField: { + Action: 'PUT', + Value: { S: 'new value' } + } + } + } + }, + { + Delete: { + TableName: twoKeyTableName, + Key: { + id: { N: '3' }, + binary: { B: Buffer.from('Hello world 3') } + } + } + } + ] + }, callback) + } + }) + }) + + it('should add span pointers for batchWriteItem', () => { + // Skip for older versions that don't support batchWriteItem + if (typeof dynamo.batchWriteItem !== 'function') { + return + } + testSpanPointers({ + expectedHashes: [ + '1f64650acbe1ae4d8413049c6bd9bbe8', + '8a6f801cc4e7d1d5e0dd37e0904e6316' + ], + operation: (callback) => { + process.env.DD_AWS_SDK_DYNAMODB_TABLE_PRIMARY_KEYS = '{"TwoKeyTable": ["id", "binary"]}' + dynamo.batchWriteItem({ + RequestItems: { + [twoKeyTableName]: [ + { + PutRequest: { + Item: { + id: { N: '5' }, + binary: { B: Buffer.from('Hello world 5') } + } + } + }, + { + DeleteRequest: { + Key: { + id: { N: '3' }, + binary: { B: Buffer.from('Hello world 3') } + } + } + } + ] + } + }, callback) + } + }) + }) + }) + }) + }) + }) + + describe('getPrimaryKeyConfig', () => { + let dynamoDbInstance + + beforeEach(() => { + dynamoDbInstance = new DynamoDb() + dynamoDbInstance.dynamoPrimaryKeyConfig = null + dynamoDbInstance._tracerConfig = {} + }) + + it('should return cached config if available', () => { + const cachedConfig = { Table1: new Set(['key1']) } + dynamoDbInstance.dynamoPrimaryKeyConfig = cachedConfig + + const result = dynamoDbInstance.getPrimaryKeyConfig() + expect(result).to.equal(cachedConfig) + }) + + it('should return undefined when config str is missing', () => { + const result = dynamoDbInstance.getPrimaryKeyConfig() + expect(result).to.be.undefined + }) + + it('should parse valid config with single table', () => { + const configStr = '{"Table1": ["key1", "key2"]}' + dynamoDbInstance._tracerConfig = { aws: { dynamoDb: { tablePrimaryKeys: configStr } } } + + const result = dynamoDbInstance.getPrimaryKeyConfig() + expect(result).to.deep.equal({ + Table1: new Set(['key1', 'key2']) + }) + }) + + it('should parse valid config with multiple tables', () => { + const configStr = '{"Table1": ["key1"], "Table2": ["key2", "key3"]}' + dynamoDbInstance._tracerConfig = { aws: { dynamoDb: { tablePrimaryKeys: configStr } } } + + const result = dynamoDbInstance.getPrimaryKeyConfig() + expect(result).to.deep.equal({ + Table1: new Set(['key1']), + Table2: new Set(['key2', 'key3']) + }) + }) + }) + + describe('calculatePutItemHash', () => { + it('generates correct hash for single string key', () => { + const tableName = 'UserTable' + const item = { userId: { S: 'user123' }, name: { S: 'John' } } + const keyConfig = { UserTable: new Set(['userId']) } + + const actualHash = DynamoDb.calculatePutItemHash(tableName, item, keyConfig) + const expectedHash = generatePointerHash([tableName, 'userId', 'user123', '', '']) + expect(actualHash).to.equal(expectedHash) + }) + + it('generates correct hash for single number key', () => { + const tableName = 'OrderTable' + const item = { orderId: { N: '98765' }, total: { N: '50.00' } } + const keyConfig = { OrderTable: new Set(['orderId']) } + + const actualHash = DynamoDb.calculatePutItemHash(tableName, item, keyConfig) + const expectedHash = generatePointerHash([tableName, 'orderId', '98765', '', '']) + expect(actualHash).to.equal(expectedHash) + }) + + it('generates correct hash for single binary key', () => { + const tableName = 'BinaryTable' + const binaryData = Buffer.from([1, 2, 3]) + const item = { binaryId: { B: binaryData }, data: { S: 'test' } } + const keyConfig = { BinaryTable: new Set(['binaryId']) } + + const actualHash = DynamoDb.calculatePutItemHash(tableName, item, keyConfig) + const expectedHash = generatePointerHash([tableName, 'binaryId', binaryData, '', '']) + expect(actualHash).to.equal(expectedHash) + }) + + it('generates correct hash for string-string key', () => { + const tableName = 'UserEmailTable' + const item = { + userId: { S: 'user123' }, + email: { S: 'test@example.com' }, + verified: { BOOL: true } + } + const keyConfig = { UserEmailTable: new Set(['userId', 'email']) } + + const actualHash = DynamoDb.calculatePutItemHash(tableName, item, keyConfig) + const expectedHash = generatePointerHash([tableName, 'email', 'test@example.com', 'userId', 'user123']) + expect(actualHash).to.equal(expectedHash) + }) + + it('generates correct hash for string-number key', () => { + const tableName = 'UserActivityTable' + const item = { + userId: { S: 'user123' }, + timestamp: { N: '1234567' }, + action: { S: 'login' } + } + const keyConfig = { UserActivityTable: new Set(['userId', 'timestamp']) } + + const actualHash = DynamoDb.calculatePutItemHash(tableName, item, keyConfig) + const expectedHash = generatePointerHash([tableName, 'timestamp', '1234567', 'userId', 'user123']) + expect(actualHash).to.equal(expectedHash) + }) + + it('generates correct hash for binary-binary key', () => { + const tableName = 'BinaryTable' + const binary1 = Buffer.from('abc') + const binary2 = Buffer.from('1ef230') + const item = { + key1: { B: binary1 }, + key2: { B: binary2 }, + data: { S: 'test' } + } + const keyConfig = { BinaryTable: new Set(['key1', 'key2']) } + + const actualHash = DynamoDb.calculatePutItemHash(tableName, item, keyConfig) + const expectedHash = generatePointerHash([tableName, 'key1', binary1, 'key2', binary2]) + expect(actualHash).to.equal(expectedHash) + }) + + it('generates unique hashes for different tables', () => { + const item = { userId: { S: 'user123' } } + const keyConfig = { + Table1: new Set(['userId']), + Table2: new Set(['userId']) + } + + const hash1 = DynamoDb.calculatePutItemHash('Table1', item, keyConfig) + const hash2 = DynamoDb.calculatePutItemHash('Table2', item, keyConfig) + expect(hash1).to.not.equal(hash2) + }) + + describe('edge cases', () => { + it('returns undefined for unknown table', () => { + const tableName = 'UnknownTable' + const item = { userId: { S: 'user123' } } + const keyConfig = { KnownTable: new Set(['userId']) } + + const result = DynamoDb.calculatePutItemHash(tableName, item, keyConfig) + expect(result).to.be.undefined + }) + + it('returns undefined for empty primary key config', () => { + const tableName = 'UserTable' + const item = { userId: { S: 'user123' } } + + const result = DynamoDb.calculatePutItemHash(tableName, item, {}) + expect(result).to.be.undefined + }) + + it('returns undefined for invalid primary key config', () => { + const tableName = 'UserTable' + const item = { userId: { S: 'user123' } } + const invalidConfig = { UserTable: ['userId'] } // Array instead of Set + + const result = DynamoDb.calculatePutItemHash(tableName, item, invalidConfig) + expect(result).to.be.undefined + }) + + it('returns undefined when missing attributes in item', () => { + const tableName = 'UserTable' + const item = { someOtherField: { S: 'value' } } + const keyConfig = { UserTable: new Set(['userId']) } + + const actualHash = DynamoDb.calculatePutItemHash(tableName, item, keyConfig) + expect(actualHash).to.be.undefined + }) + + it('returns undefined for Set with more than 2 keys', () => { + const tableName = 'TestTable' + const item = { key1: { S: 'value1' }, key2: { S: 'value2' }, key3: { S: 'value3' } } + const keyConfig = { TestTable: new Set(['key1', 'key2', 'key3']) } + + const result = DynamoDb.calculatePutItemHash(tableName, item, keyConfig) + expect(result).to.be.undefined + }) + + it('returns undefined for empty keyConfig', () => { + const result = DynamoDb.calculatePutItemHash('TestTable', {}, {}) + expect(result).to.be.undefined + }) + + it('returns undefined for undefined keyConfig', () => { + const result = DynamoDb.calculatePutItemHash('TestTable', {}, undefined) + expect(result).to.be.undefined + }) + }) + }) + + describe('calculateHashWithKnownKeys', () => { + it('generates correct hash for single string key', () => { + const tableName = 'UserTable' + const keys = { userId: { S: 'user123' } } + const actualHash = DynamoDb.calculateHashWithKnownKeys(tableName, keys) + const expectedHash = generatePointerHash([tableName, 'userId', 'user123', '', '']) + expect(actualHash).to.equal(expectedHash) + }) + + it('generates correct hash for single number key', () => { + const tableName = 'OrderTable' + const keys = { orderId: { N: '98765' } } + const actualHash = DynamoDb.calculateHashWithKnownKeys(tableName, keys) + const expectedHash = generatePointerHash([tableName, 'orderId', '98765', '', '']) + expect(actualHash).to.equal(expectedHash) + }) + + it('generates correct hash for single binary key', () => { + const tableName = 'BinaryTable' + const binaryData = Buffer.from([1, 2, 3]) + const keys = { binaryId: { B: binaryData } } + const actualHash = DynamoDb.calculateHashWithKnownKeys(tableName, keys) + const expectedHash = generatePointerHash([tableName, 'binaryId', binaryData, '', '']) + expect(actualHash).to.equal(expectedHash) + }) + + it('generates correct hash for string-string key', () => { + const tableName = 'UserEmailTable' + const keys = { + userId: { S: 'user123' }, + email: { S: 'test@example.com' } + } + const actualHash = DynamoDb.calculateHashWithKnownKeys(tableName, keys) + const expectedHash = generatePointerHash([tableName, 'email', 'test@example.com', 'userId', 'user123']) + expect(actualHash).to.equal(expectedHash) + }) + + it('generates correct hash for string-number key', () => { + const tableName = 'UserActivityTable' + const keys = { + userId: { S: 'user123' }, + timestamp: { N: '1234567' } + } + const actualHash = DynamoDb.calculateHashWithKnownKeys(tableName, keys) + const expectedHash = generatePointerHash([tableName, 'timestamp', '1234567', 'userId', 'user123']) + expect(actualHash).to.equal(expectedHash) + }) + + it('generates correct hash for binary-binary key', () => { + const tableName = 'BinaryTable' + const binary1 = Buffer.from('abc') + const binary2 = Buffer.from('1ef230') + const keys = { + key1: { B: binary1 }, + key2: { B: binary2 } + } + const actualHash = DynamoDb.calculateHashWithKnownKeys(tableName, keys) + const expectedHash = generatePointerHash([tableName, 'key1', binary1, 'key2', binary2]) + expect(actualHash).to.equal(expectedHash) + }) + + it('generates unique hashes', () => { + const keys = { userId: { S: 'user123' } } + const hash1 = DynamoDb.calculateHashWithKnownKeys('Table1', keys) + const hash2 = DynamoDb.calculateHashWithKnownKeys('Table2', keys) + expect(hash1).to.not.equal(hash2) + }) + + describe('edge cases', () => { + it('handles empty keys object', () => { + const tableName = 'UserTable' + const hash = DynamoDb.calculateHashWithKnownKeys(tableName, {}) + expect(hash).to.be.undefined + }) + + it('handles invalid key types', () => { + const tableName = 'UserTable' + const keys = { userId: { INVALID: 'user123' } } + const hash = DynamoDb.calculateHashWithKnownKeys(tableName, keys) + expect(hash).to.be.undefined + }) + + it('handles null keys object', () => { + const hash = DynamoDb.calculateHashWithKnownKeys('TestTable', null) + expect(hash).to.be.undefined + }) + + it('handles undefined keys object', () => { + const hash = DynamoDb.calculateHashWithKnownKeys('TestTable', undefined) + expect(hash).to.be.undefined + }) + + it('handles mixed valid and invalid key types', () => { + const keys = { + validKey: { S: 'test' }, + invalidKey: { INVALID: 'value' } + } + const hash = DynamoDb.calculateHashWithKnownKeys('TestTable', keys) + expect(hash).to.be.undefined + }) + }) + }) + }) +}) diff --git a/packages/datadog-plugin-aws-sdk/test/util.spec.js b/packages/datadog-plugin-aws-sdk/test/util.spec.js new file mode 100644 index 00000000000..68bf57a7bfc --- /dev/null +++ b/packages/datadog-plugin-aws-sdk/test/util.spec.js @@ -0,0 +1,213 @@ +const { generatePointerHash, encodeValue, extractPrimaryKeys } = require('../src/util') + +describe('generatePointerHash', () => { + describe('should generate a valid hash for S3 object with', () => { + it('basic values', () => { + const hash = generatePointerHash(['some-bucket', 'some-key.data', 'ab12ef34']) + expect(hash).to.equal('e721375466d4116ab551213fdea08413') + }) + + it('non-ascii key', () => { + const hash = generatePointerHash(['some-bucket', 'some-key.你好', 'ab12ef34']) + expect(hash).to.equal('d1333a04b9928ab462b5c6cadfa401f4') + }) + + it('multipart-upload', () => { + const hash = generatePointerHash(['some-bucket', 'some-key.data', 'ab12ef34-5']) + expect(hash).to.equal('2b90dffc37ebc7bc610152c3dc72af9f') + }) + }) + + describe('should generate a valid hash for DynamoDB item with', () => { + it('one string primary key', () => { + const hash = generatePointerHash(['some-table', 'some-key', 'some-value', '', '']) + expect(hash).to.equal('7f1aee721472bcb48701d45c7c7f7821') + }) + + it('one buffered binary primary key', () => { + const hash = generatePointerHash(['some-table', 'some-key', Buffer.from('some-value'), '', '']) + expect(hash).to.equal('7f1aee721472bcb48701d45c7c7f7821') + }) + + it('one number primary key', () => { + const hash = generatePointerHash(['some-table', 'some-key', '123.456', '', '']) + expect(hash).to.equal('434a6dba3997ce4dbbadc98d87a0cc24') + }) + + it('one buffered number primary key', () => { + const hash = generatePointerHash(['some-table', 'some-key', Buffer.from('123.456'), '', '']) + expect(hash).to.equal('434a6dba3997ce4dbbadc98d87a0cc24') + }) + + it('string and number primary key', () => { + // sort primary keys lexicographically + const hash = generatePointerHash(['some-table', 'other-key', '123', 'some-key', 'some-value']) + expect(hash).to.equal('7aa1b80b0e49bd2078a5453399f4dd67') + }) + + it('buffered string and number primary key', () => { + const hash = generatePointerHash([ + 'some-table', + 'other-key', + Buffer.from('123'), + 'some-key', Buffer.from('some-value') + ]) + expect(hash).to.equal('7aa1b80b0e49bd2078a5453399f4dd67') + }) + }) +}) + +describe('encodeValue', () => { + describe('basic type handling', () => { + it('handles string (S) type correctly', () => { + const result = encodeValue({ S: 'hello world' }) + expect(Buffer.isBuffer(result)).to.be.true + expect(result).to.deep.equal(Buffer.from('hello world')) + }) + + it('handles number (N) as string type correctly', () => { + const result = encodeValue({ N: '123.45' }) + expect(Buffer.isBuffer(result)).to.be.true + expect(result).to.deep.equal(Buffer.from('123.45')) + }) + + it('handles number (N) as type string or number the same', () => { + const result1 = encodeValue({ N: 456.78 }) + const result2 = encodeValue({ N: '456.78' }) + expect(Buffer.isBuffer(result1)).to.be.true + expect(result1).to.deep.equal(result2) + }) + + it('handles binary (B) type correctly', () => { + const binaryData = Buffer.from([1, 2, 3]) + const result = encodeValue({ B: binaryData }) + expect(Buffer.isBuffer(result)).to.be.true + expect(result).to.deep.equal(binaryData) + }) + }) + + describe('edge cases', () => { + it('returns undefined for null input', () => { + const result = encodeValue(null) + expect(result).to.be.undefined + }) + + it('returns undefined for undefined input', () => { + const result = encodeValue(undefined) + expect(result).to.be.undefined + }) + + it('returns undefined for unsupported type', () => { + const result = encodeValue({ A: 'abc' }) + expect(result).to.be.undefined + }) + + it('returns undefined for malformed input', () => { + const result = encodeValue({}) + expect(result).to.be.undefined + }) + + it('handles empty string values', () => { + const result = encodeValue({ S: '' }) + expect(Buffer.isBuffer(result)).to.be.true + expect(result.length).to.equal(0) + }) + + it('handles empty buffer', () => { + const result = encodeValue({ B: Buffer.from([]) }) + expect(Buffer.isBuffer(result)).to.be.true + expect(result.length).to.equal(0) + }) + }) +}) + +describe('extractPrimaryKeys', () => { + describe('single key table', () => { + it('handles string key', () => { + const keySet = new Set(['userId']) + const item = { userId: { S: 'user123' } } + const result = extractPrimaryKeys(keySet, item) + expect(result).to.deep.equal(['userId', Buffer.from('user123'), '', '']) + }) + + it('handles number key', () => { + const keySet = new Set(['timestamp']) + const item = { timestamp: { N: '1234567' } } + const result = extractPrimaryKeys(keySet, item) + expect(result).to.deep.equal(['timestamp', Buffer.from('1234567'), '', '']) + }) + + it('handles binary key', () => { + const keySet = new Set(['binaryId']) + const binaryData = Buffer.from([1, 2, 3]) + const item = { binaryId: { B: binaryData } } + const result = extractPrimaryKeys(keySet, item) + expect(result).to.deep.equal(['binaryId', binaryData, '', '']) + }) + }) + + describe('double key table', () => { + it('handles and sorts string-string keys', () => { + const keySet = new Set(['userId', 'email']) + const item = { + userId: { S: 'user123' }, + email: { S: 'test@example.com' } + } + const result = extractPrimaryKeys(keySet, item) + expect(result).to.deep.equal(['email', Buffer.from('test@example.com'), 'userId', Buffer.from('user123')]) + }) + + it('handles and sorts string-number keys', () => { + const keySet = new Set(['timestamp', 'userId']) + const item = { + timestamp: { N: '1234567' }, + userId: { S: 'user123' } + } + const result = extractPrimaryKeys(keySet, item) + expect(result).to.deep.equal(['timestamp', Buffer.from('1234567'), 'userId', Buffer.from('user123')]) + }) + }) + + describe('edge cases', () => { + it('returns undefined when missing values', () => { + const keySet = new Set(['userId', 'timestamp']) + const item = { userId: { S: 'user123' } } // timestamp missing + const result = extractPrimaryKeys(keySet, item) + expect(result).to.be.undefined + }) + + it('returns undefined when invalid value types', () => { + const keySet = new Set(['userId', 'timestamp']) + const item = { + userId: { S: 'user123' }, + timestamp: { INVALID: '1234567' } + } + const result = extractPrimaryKeys(keySet, item) + expect(result).to.be.undefined + }) + + it('handles empty Set input', () => { + const result = extractPrimaryKeys(new Set([]), {}) + expect(result).to.be.undefined + }) + + it('returns undefined when null values in item', () => { + const keySet = new Set(['key1', 'key2']) + const item = { + key1: null, + key2: { S: 'value2' } + } + const result = extractPrimaryKeys(keySet, item) + expect(result).to.be.undefined + }) + + it('returns undefined when undefined values in item', () => { + const keySet = new Set(['key1', 'key2']) + const item = { + key2: { S: 'value2' } + } + const result = extractPrimaryKeys(keySet, item) + expect(result).to.be.undefined + }) + }) +}) diff --git a/packages/dd-trace/src/config.js b/packages/dd-trace/src/config.js index a46cc3153fc..a16df70ee07 100644 --- a/packages/dd-trace/src/config.js +++ b/packages/dd-trace/src/config.js @@ -566,6 +566,7 @@ class Config { this._setValue(defaults, 'url', undefined) this._setValue(defaults, 'version', pkg.version) this._setValue(defaults, 'instrumentation_config_id', undefined) + this._setValue(defaults, 'aws.dynamoDb.tablePrimaryKeys', undefined) } _applyEnvironment () { @@ -590,6 +591,7 @@ class Config { DD_APPSEC_RASP_ENABLED, DD_APPSEC_TRACE_RATE_LIMIT, DD_APPSEC_WAF_TIMEOUT, + DD_AWS_SDK_DYNAMODB_TABLE_PRIMARY_KEYS, DD_CRASHTRACKING_ENABLED, DD_CODE_ORIGIN_FOR_SPANS_ENABLED, DD_DATA_STREAMS_ENABLED, @@ -879,6 +881,7 @@ class Config { this._setBoolean(env, 'tracing', DD_TRACING_ENABLED) this._setString(env, 'version', DD_VERSION || tags.version) this._setBoolean(env, 'inferredProxyServicesEnabled', DD_TRACE_INFERRED_PROXY_SERVICES_ENABLED) + this._setString(env, 'aws.dynamoDb.tablePrimaryKeys', DD_AWS_SDK_DYNAMODB_TABLE_PRIMARY_KEYS) } _applyOptions (options) { diff --git a/packages/dd-trace/src/constants.js b/packages/dd-trace/src/constants.js index 4e7faf669d4..3c93480df9f 100644 --- a/packages/dd-trace/src/constants.js +++ b/packages/dd-trace/src/constants.js @@ -47,6 +47,7 @@ module.exports = { SCHEMA_NAME: 'schema.name', GRPC_CLIENT_ERROR_STATUSES: [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16], GRPC_SERVER_ERROR_STATUSES: [2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16], + DYNAMODB_PTR_KIND: 'aws.dynamodb.item', S3_PTR_KIND: 'aws.s3.object', SPAN_POINTER_DIRECTION: Object.freeze({ UPSTREAM: 'u', diff --git a/packages/dd-trace/src/util.js b/packages/dd-trace/src/util.js index 8cfa3d6f58c..de3618fcd27 100644 --- a/packages/dd-trace/src/util.js +++ b/packages/dd-trace/src/util.js @@ -1,6 +1,5 @@ 'use strict' -const crypto = require('crypto') const path = require('path') function isTrue (str) { @@ -78,25 +77,11 @@ function hasOwn (object, prop) { return Object.prototype.hasOwnProperty.call(object, prop) } -/** - * Generates a unique hash from an array of strings by joining them with | before hashing. - * Used to uniquely identify AWS requests for span pointers. - * @param {string[]} components - Array of strings to hash - * @returns {string} A 32-character hash uniquely identifying the components - */ -function generatePointerHash (components) { - // If passing S3's ETag as a component, make sure any quotes have already been removed! - const dataToHash = components.join('|') - const hash = crypto.createHash('sha256').update(dataToHash).digest('hex') - return hash.substring(0, 32) -} - module.exports = { isTrue, isFalse, isError, globMatch, calculateDDBasePath, - hasOwn, - generatePointerHash + hasOwn } diff --git a/packages/dd-trace/test/plugins/externals.json b/packages/dd-trace/test/plugins/externals.json index c3fc12fb176..73a61536476 100644 --- a/packages/dd-trace/test/plugins/externals.json +++ b/packages/dd-trace/test/plugins/externals.json @@ -30,6 +30,10 @@ "name": "@aws-sdk/client-s3", "versions": [">=3"] }, + { + "name": "@aws-sdk/client-dynamodb", + "versions": [">=3"] + }, { "name": "@aws-sdk/client-sfn", "versions": [">=3"] diff --git a/packages/dd-trace/test/util.spec.js b/packages/dd-trace/test/util.spec.js index 40b209a96cf..f32b47c0cee 100644 --- a/packages/dd-trace/test/util.spec.js +++ b/packages/dd-trace/test/util.spec.js @@ -3,7 +3,6 @@ require('./setup/tap') const { isTrue, isFalse, globMatch } = require('../src/util') -const { generatePointerHash } = require('../src/util') const TRUES = [ 1, @@ -69,20 +68,3 @@ describe('util', () => { }) }) }) - -describe('generatePointerHash', () => { - it('should generate a valid hash for a basic S3 object', () => { - const hash = generatePointerHash(['some-bucket', 'some-key.data', 'ab12ef34']) - expect(hash).to.equal('e721375466d4116ab551213fdea08413') - }) - - it('should generate a valid hash for an S3 object with a non-ascii key', () => { - const hash1 = generatePointerHash(['some-bucket', 'some-key.你好', 'ab12ef34']) - expect(hash1).to.equal('d1333a04b9928ab462b5c6cadfa401f4') - }) - - it('should generate a valid hash for multipart-uploaded S3 object', () => { - const hash1 = generatePointerHash(['some-bucket', 'some-key.data', 'ab12ef34-5']) - expect(hash1).to.equal('2b90dffc37ebc7bc610152c3dc72af9f') - }) -}) From bfe48c9d8987bb222abb57eb023f4eaddab6b8cf Mon Sep 17 00:00:00 2001 From: Roch Devost Date: Wed, 18 Dec 2024 15:26:55 -0500 Subject: [PATCH 21/36] update package size job to node 20 (#5040) --- .github/workflows/package-size.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/package-size.yml b/.github/workflows/package-size.yml index 628614c7dc5..b6fee75c4c4 100644 --- a/.github/workflows/package-size.yml +++ b/.github/workflows/package-size.yml @@ -17,9 +17,9 @@ jobs: steps: - uses: actions/checkout@v4 - name: Setup Node.js - uses: actions/setup-node@v2 + uses: actions/setup-node@v4 with: - node-version: '18' + node-version: '20' - run: yarn - name: Compute module size tree and report uses: qard/heaviest-objects-in-the-universe@v1 From 9bff311dc202b3c18cbe5ed517aa512c8df3caaa Mon Sep 17 00:00:00 2001 From: Roch Devost Date: Wed, 18 Dec 2024 15:27:40 -0500 Subject: [PATCH 22/36] fix runtime metrics test not waiting for gc observer to run (#5039) --- packages/dd-trace/src/runtime_metrics.js | 2 +- .../dd-trace/test/runtime_metrics.spec.js | 128 ++++++++++-------- 2 files changed, 70 insertions(+), 60 deletions(-) diff --git a/packages/dd-trace/src/runtime_metrics.js b/packages/dd-trace/src/runtime_metrics.js index a9036612a67..f16b227ca18 100644 --- a/packages/dd-trace/src/runtime_metrics.js +++ b/packages/dd-trace/src/runtime_metrics.js @@ -361,7 +361,7 @@ function startGCObserver () { gcObserver = new PerformanceObserver(list => { for (const entry of list.getEntries()) { - const type = gcType(entry.kind) + const type = gcType(entry.detail?.kind || entry.kind) runtimeMetrics.histogram('runtime.node.gc.pause.by.type', entry.duration, `gc_type:${type}`) runtimeMetrics.histogram('runtime.node.gc.pause', entry.duration) diff --git a/packages/dd-trace/test/runtime_metrics.spec.js b/packages/dd-trace/test/runtime_metrics.spec.js index f3f20464630..20ce93112ae 100644 --- a/packages/dd-trace/test/runtime_metrics.spec.js +++ b/packages/dd-trace/test/runtime_metrics.spec.js @@ -13,6 +13,7 @@ suiteDescribe('runtimeMetrics', () => { let runtimeMetrics let config let clock + let setImmediate let client let Client @@ -50,6 +51,7 @@ suiteDescribe('runtimeMetrics', () => { } } + setImmediate = globalThis.setImmediate clock = sinon.useFakeTimers() runtimeMetrics.start(config) @@ -91,71 +93,79 @@ suiteDescribe('runtimeMetrics', () => { }) }) - it('should start collecting runtimeMetrics every 10 seconds', () => { + it('should start collecting runtimeMetrics every 10 seconds', (done) => { runtimeMetrics.stop() runtimeMetrics.start(config) global.gc() - clock.tick(10000) - - expect(client.gauge).to.have.been.calledWith('runtime.node.cpu.user') - expect(client.gauge).to.have.been.calledWith('runtime.node.cpu.system') - expect(client.gauge).to.have.been.calledWith('runtime.node.cpu.total') - - expect(client.gauge).to.have.been.calledWith('runtime.node.mem.rss') - expect(client.gauge).to.have.been.calledWith('runtime.node.mem.heap_total') - expect(client.gauge).to.have.been.calledWith('runtime.node.mem.heap_used') - - expect(client.gauge).to.have.been.calledWith('runtime.node.process.uptime') - - expect(client.gauge).to.have.been.calledWith('runtime.node.heap.total_heap_size') - expect(client.gauge).to.have.been.calledWith('runtime.node.heap.total_heap_size_executable') - expect(client.gauge).to.have.been.calledWith('runtime.node.heap.total_physical_size') - expect(client.gauge).to.have.been.calledWith('runtime.node.heap.total_available_size') - expect(client.gauge).to.have.been.calledWith('runtime.node.heap.total_heap_size') - expect(client.gauge).to.have.been.calledWith('runtime.node.heap.heap_size_limit') - - expect(client.gauge).to.have.been.calledWith('runtime.node.heap.malloced_memory') - expect(client.gauge).to.have.been.calledWith('runtime.node.heap.peak_malloced_memory') - - expect(client.gauge).to.have.been.calledWith('runtime.node.event_loop.delay.max') - expect(client.gauge).to.have.been.calledWith('runtime.node.event_loop.delay.min') - expect(client.increment).to.have.been.calledWith('runtime.node.event_loop.delay.sum') - expect(client.gauge).to.have.been.calledWith('runtime.node.event_loop.delay.avg') - expect(client.gauge).to.have.been.calledWith('runtime.node.event_loop.delay.median') - expect(client.gauge).to.have.been.calledWith('runtime.node.event_loop.delay.95percentile') - expect(client.increment).to.have.been.calledWith('runtime.node.event_loop.delay.count') - - expect(client.gauge).to.have.been.calledWith('runtime.node.event_loop.utilization') - - expect(client.gauge).to.have.been.calledWith('runtime.node.gc.pause.max') - expect(client.gauge).to.have.been.calledWith('runtime.node.gc.pause.min') - expect(client.increment).to.have.been.calledWith('runtime.node.gc.pause.sum') - expect(client.gauge).to.have.been.calledWith('runtime.node.gc.pause.avg') - expect(client.gauge).to.have.been.calledWith('runtime.node.gc.pause.median') - expect(client.gauge).to.have.been.calledWith('runtime.node.gc.pause.95percentile') - expect(client.increment).to.have.been.calledWith('runtime.node.gc.pause.count') - - expect(client.gauge).to.have.been.calledWith('runtime.node.gc.pause.by.type.max') - expect(client.gauge).to.have.been.calledWith('runtime.node.gc.pause.by.type.min') - expect(client.increment).to.have.been.calledWith('runtime.node.gc.pause.by.type.sum') - expect(client.gauge).to.have.been.calledWith('runtime.node.gc.pause.by.type.avg') - expect(client.gauge).to.have.been.calledWith('runtime.node.gc.pause.by.type.median') - expect(client.gauge).to.have.been.calledWith('runtime.node.gc.pause.by.type.95percentile') - expect(client.increment).to.have.been.calledWith('runtime.node.gc.pause.by.type.count') - expect(client.increment).to.have.been.calledWith( - 'runtime.node.gc.pause.by.type.count', sinon.match.any, sinon.match(val => { - return val && /^gc_type:[a-z_]+$/.test(val[0]) - }) - ) - - expect(client.gauge).to.have.been.calledWith('runtime.node.heap.size.by.space') - expect(client.gauge).to.have.been.calledWith('runtime.node.heap.used_size.by.space') - expect(client.gauge).to.have.been.calledWith('runtime.node.heap.available_size.by.space') - expect(client.gauge).to.have.been.calledWith('runtime.node.heap.physical_size.by.space') + setImmediate(() => setImmediate(() => { // Wait for GC observer to trigger. + clock.tick(10000) - expect(client.flush).to.have.been.called + try { + expect(client.gauge).to.have.been.calledWith('runtime.node.cpu.user') + expect(client.gauge).to.have.been.calledWith('runtime.node.cpu.system') + expect(client.gauge).to.have.been.calledWith('runtime.node.cpu.total') + + expect(client.gauge).to.have.been.calledWith('runtime.node.mem.rss') + expect(client.gauge).to.have.been.calledWith('runtime.node.mem.heap_total') + expect(client.gauge).to.have.been.calledWith('runtime.node.mem.heap_used') + + expect(client.gauge).to.have.been.calledWith('runtime.node.process.uptime') + + expect(client.gauge).to.have.been.calledWith('runtime.node.heap.total_heap_size') + expect(client.gauge).to.have.been.calledWith('runtime.node.heap.total_heap_size_executable') + expect(client.gauge).to.have.been.calledWith('runtime.node.heap.total_physical_size') + expect(client.gauge).to.have.been.calledWith('runtime.node.heap.total_available_size') + expect(client.gauge).to.have.been.calledWith('runtime.node.heap.total_heap_size') + expect(client.gauge).to.have.been.calledWith('runtime.node.heap.heap_size_limit') + + expect(client.gauge).to.have.been.calledWith('runtime.node.heap.malloced_memory') + expect(client.gauge).to.have.been.calledWith('runtime.node.heap.peak_malloced_memory') + + expect(client.gauge).to.have.been.calledWith('runtime.node.event_loop.delay.max') + expect(client.gauge).to.have.been.calledWith('runtime.node.event_loop.delay.min') + expect(client.increment).to.have.been.calledWith('runtime.node.event_loop.delay.sum') + expect(client.gauge).to.have.been.calledWith('runtime.node.event_loop.delay.avg') + expect(client.gauge).to.have.been.calledWith('runtime.node.event_loop.delay.median') + expect(client.gauge).to.have.been.calledWith('runtime.node.event_loop.delay.95percentile') + expect(client.increment).to.have.been.calledWith('runtime.node.event_loop.delay.count') + + expect(client.gauge).to.have.been.calledWith('runtime.node.event_loop.utilization') + + expect(client.gauge).to.have.been.calledWith('runtime.node.gc.pause.max') + expect(client.gauge).to.have.been.calledWith('runtime.node.gc.pause.min') + expect(client.increment).to.have.been.calledWith('runtime.node.gc.pause.sum') + expect(client.gauge).to.have.been.calledWith('runtime.node.gc.pause.avg') + expect(client.gauge).to.have.been.calledWith('runtime.node.gc.pause.median') + expect(client.gauge).to.have.been.calledWith('runtime.node.gc.pause.95percentile') + expect(client.increment).to.have.been.calledWith('runtime.node.gc.pause.count') + + expect(client.gauge).to.have.been.calledWith('runtime.node.gc.pause.by.type.max') + expect(client.gauge).to.have.been.calledWith('runtime.node.gc.pause.by.type.min') + expect(client.increment).to.have.been.calledWith('runtime.node.gc.pause.by.type.sum') + expect(client.gauge).to.have.been.calledWith('runtime.node.gc.pause.by.type.avg') + expect(client.gauge).to.have.been.calledWith('runtime.node.gc.pause.by.type.median') + expect(client.gauge).to.have.been.calledWith('runtime.node.gc.pause.by.type.95percentile') + expect(client.increment).to.have.been.calledWith('runtime.node.gc.pause.by.type.count') + expect(client.increment).to.have.been.calledWith( + 'runtime.node.gc.pause.by.type.count', sinon.match.any, sinon.match(val => { + return val && /^gc_type:[a-z_]+$/.test(val[0]) + }) + ) + + expect(client.gauge).to.have.been.calledWith('runtime.node.heap.size.by.space') + expect(client.gauge).to.have.been.calledWith('runtime.node.heap.used_size.by.space') + expect(client.gauge).to.have.been.calledWith('runtime.node.heap.available_size.by.space') + expect(client.gauge).to.have.been.calledWith('runtime.node.heap.physical_size.by.space') + + expect(client.flush).to.have.been.called + + done() + } catch (e) { + done(e) + } + })) }) }) From c5dc10c9a399d8f85a675060ec3bbe8b585c2994 Mon Sep 17 00:00:00 2001 From: Thomas Hunter II Date: Wed, 18 Dec 2024 13:06:08 -0800 Subject: [PATCH 23/36] repo: ask for config details on bug creation (#5027) --- .github/ISSUE_TEMPLATE/bug_report.yaml | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/.github/ISSUE_TEMPLATE/bug_report.yaml b/.github/ISSUE_TEMPLATE/bug_report.yaml index 8fb53ba14fa..833243210ca 100644 --- a/.github/ISSUE_TEMPLATE/bug_report.yaml +++ b/.github/ISSUE_TEMPLATE/bug_report.yaml @@ -40,6 +40,13 @@ body: validations: required: false + - type: textarea + attributes: + label: Tracer Config + description: "Please provide the `tracer.init(config)` object and any applicable tracer environment variables" + validations: + required: false + - type: input attributes: label: Operating System From b7ccd40dc7469da79fa6fce1daba39a77eac4a48 Mon Sep 17 00:00:00 2001 From: Roch Devost Date: Wed, 18 Dec 2024 16:39:21 -0500 Subject: [PATCH 24/36] update type tests to typescript 4.9.4 (#5041) --- docs/package.json | 4 ++-- docs/yarn.lock | 22 +++++++++++----------- 2 files changed, 13 insertions(+), 13 deletions(-) diff --git a/docs/package.json b/docs/package.json index 0ec46d7584a..c68302e3eca 100644 --- a/docs/package.json +++ b/docs/package.json @@ -10,7 +10,7 @@ "license": "BSD-3-Clause", "private": true, "devDependencies": { - "typedoc": "^0.25.8", - "typescript": "^4.6" + "typedoc": "^0.25.13", + "typescript": "^4.9.4" } } diff --git a/docs/yarn.lock b/docs/yarn.lock index 4b011ed3db2..4c517dabb07 100644 --- a/docs/yarn.lock +++ b/docs/yarn.lock @@ -20,9 +20,9 @@ brace-expansion@^2.0.1: balanced-match "^1.0.0" jsonc-parser@^3.2.0: - version "3.2.1" - resolved "https://registry.yarnpkg.com/jsonc-parser/-/jsonc-parser-3.2.1.tgz#031904571ccf929d7670ee8c547545081cb37f1a" - integrity sha512-AilxAyFOAcK5wA1+LeaySVBrHsGQvUFCDWXKpZjzaL0PqW+xfBOttn8GNtWKFWqneyMZj41MWF9Kl6iPWLwgOA== + version "3.3.1" + resolved "https://registry.yarnpkg.com/jsonc-parser/-/jsonc-parser-3.3.1.tgz#f2a524b4f7fd11e3d791e559977ad60b98b798b4" + integrity sha512-HUgH65KyejrUFPvHFPbqOY0rsFip3Bo5wb4ngvdi1EpCYWUQDC5V+Y7mZws+DLkr4M//zQJoanu1SP+87Dv1oQ== lunr@^2.3.9: version "2.3.9" @@ -35,9 +35,9 @@ marked@^4.3.0: integrity sha512-PRsaiG84bK+AMvxziE/lCFss8juXjNaWzVbN5tXAm4XjeaS9NAHhop+PjQxz2A9h8Q4M/xGmzP8vqNwy6JeK0A== minimatch@^9.0.3: - version "9.0.3" - resolved "https://registry.yarnpkg.com/minimatch/-/minimatch-9.0.3.tgz#a6e00c3de44c3a542bfaae70abfc22420a6da825" - integrity sha512-RHiac9mvaRw0x3AYRgDC1CxAP7HTcNrrECeA8YYJeWnpo+2Q5CegtZjaotWTWxDG3UeGA1coE05iH1mPjT/2mg== + version "9.0.5" + resolved "https://registry.yarnpkg.com/minimatch/-/minimatch-9.0.5.tgz#d74f9dd6b57d83d8e98cfb82133b03978bc929e5" + integrity sha512-G6T0ZX48xgozx7587koeX9Ys2NYy6Gmv//P89sEte9V9whIapMNF4idKxnW2QtCcLiTWlb/wfCabAtAFWhhBow== dependencies: brace-expansion "^2.0.1" @@ -51,17 +51,17 @@ shiki@^0.14.7: vscode-oniguruma "^1.7.0" vscode-textmate "^8.0.0" -typedoc@^0.25.8: - version "0.25.8" - resolved "https://registry.yarnpkg.com/typedoc/-/typedoc-0.25.8.tgz#7d0e1bf12d23bf1c459fd4893c82cb855911ff12" - integrity sha512-mh8oLW66nwmeB9uTa0Bdcjfis+48bAjSH3uqdzSuSawfduROQLlXw//WSNZLYDdhmMVB7YcYZicq6e8T0d271A== +typedoc@^0.25.13: + version "0.25.13" + resolved "https://registry.yarnpkg.com/typedoc/-/typedoc-0.25.13.tgz#9a98819e3b2d155a6d78589b46fa4c03768f0922" + integrity sha512-pQqiwiJ+Z4pigfOnnysObszLiU3mVLWAExSPf+Mu06G/qsc3wzbuM56SZQvONhHLncLUhYzOVkjFFpFfL5AzhQ== dependencies: lunr "^2.3.9" marked "^4.3.0" minimatch "^9.0.3" shiki "^0.14.7" -typescript@^4.6: +typescript@^4.9.4: version "4.9.5" resolved "https://registry.yarnpkg.com/typescript/-/typescript-4.9.5.tgz#095979f9bcc0d09da324d58d03ce8f8374cbe65a" integrity sha512-1FXk9E2Hm+QzZQ7z+McJiHL4NW1F2EzMu9Nq9i3zAaGqibafqYwCVU6WyWAuyQRRzOlxou8xZSyXLEN8oKj24g== From a9a1b1d04a08d119b72833898df5896a6102c1db Mon Sep 17 00:00:00 2001 From: Thomas Watson Date: Thu, 19 Dec 2024 08:58:06 +0100 Subject: [PATCH 25/36] [DI] Add test for associating probes with 128 bit span ids (#5037) Includes a major refactor of basic.spec.js to allow for easier testing of different combinations of envrionment variables. --- integration-tests/debugger/basic.spec.js | 896 ++++++++++++----------- 1 file changed, 467 insertions(+), 429 deletions(-) diff --git a/integration-tests/debugger/basic.spec.js b/integration-tests/debugger/basic.spec.js index 6db68d0607d..4bb5d7b2fa6 100644 --- a/integration-tests/debugger/basic.spec.js +++ b/integration-tests/debugger/basic.spec.js @@ -9,514 +9,552 @@ const { ACKNOWLEDGED, ERROR } = require('../../packages/dd-trace/src/appsec/remo const { version } = require('../../package.json') describe('Dynamic Instrumentation', function () { - describe('DD_TRACING_ENABLED=true', function () { - testWithTracingEnabled() - }) - - describe('DD_TRACING_ENABLED=false', function () { - testWithTracingEnabled(false) - }) -}) - -function testWithTracingEnabled (tracingEnabled = true) { - const t = setup({ DD_TRACING_ENABLED: tracingEnabled }) + describe('Default env', function () { + const t = setup() - it('base case: target app should work as expected if no test probe has been added', async function () { - const response = await t.axios.get(t.breakpoint.url) - assert.strictEqual(response.status, 200) - assert.deepStrictEqual(response.data, { hello: 'bar' }) - }) + it('base case: target app should work as expected if no test probe has been added', async function () { + const response = await t.axios.get(t.breakpoint.url) + assert.strictEqual(response.status, 200) + assert.deepStrictEqual(response.data, { hello: 'bar' }) + }) - describe('diagnostics messages', function () { - it('should send expected diagnostics messages if probe is received and triggered', function (done) { - let receivedAckUpdate = false - const probeId = t.rcConfig.config.id - const expectedPayloads = [{ - ddsource: 'dd_debugger', - service: 'node', - debugger: { diagnostics: { probeId, probeVersion: 0, status: 'RECEIVED' } } - }, { - ddsource: 'dd_debugger', - service: 'node', - debugger: { diagnostics: { probeId, probeVersion: 0, status: 'INSTALLED' } } - }, { - ddsource: 'dd_debugger', - service: 'node', - debugger: { diagnostics: { probeId, probeVersion: 0, status: 'EMITTING' } } - }] - - t.agent.on('remote-config-ack-update', (id, version, state, error) => { - assert.strictEqual(id, t.rcConfig.id) - assert.strictEqual(version, 1) - assert.strictEqual(state, ACKNOWLEDGED) - assert.notOk(error) // falsy check since error will be an empty string, but that's an implementation detail - - receivedAckUpdate = true - endIfDone() - }) + describe('diagnostics messages', function () { + it('should send expected diagnostics messages if probe is received and triggered', function (done) { + let receivedAckUpdate = false + const probeId = t.rcConfig.config.id + const expectedPayloads = [{ + ddsource: 'dd_debugger', + service: 'node', + debugger: { diagnostics: { probeId, probeVersion: 0, status: 'RECEIVED' } } + }, { + ddsource: 'dd_debugger', + service: 'node', + debugger: { diagnostics: { probeId, probeVersion: 0, status: 'INSTALLED' } } + }, { + ddsource: 'dd_debugger', + service: 'node', + debugger: { diagnostics: { probeId, probeVersion: 0, status: 'EMITTING' } } + }] - t.agent.on('debugger-diagnostics', ({ payload }) => { - const expected = expectedPayloads.shift() - assertObjectContains(payload, expected) - assertUUID(payload.debugger.diagnostics.runtimeId) + t.agent.on('remote-config-ack-update', (id, version, state, error) => { + assert.strictEqual(id, t.rcConfig.id) + assert.strictEqual(version, 1) + assert.strictEqual(state, ACKNOWLEDGED) + assert.notOk(error) // falsy check since error will be an empty string, but that's an implementation detail - if (payload.debugger.diagnostics.status === 'INSTALLED') { - t.axios.get(t.breakpoint.url) - .then((response) => { - assert.strictEqual(response.status, 200) - assert.deepStrictEqual(response.data, { hello: 'bar' }) - }) - .catch(done) - } else { + receivedAckUpdate = true endIfDone() - } - }) + }) - t.agent.addRemoteConfig(t.rcConfig) + t.agent.on('debugger-diagnostics', ({ payload }) => { + const expected = expectedPayloads.shift() + assertObjectContains(payload, expected) + assertUUID(payload.debugger.diagnostics.runtimeId) + + if (payload.debugger.diagnostics.status === 'INSTALLED') { + t.axios.get(t.breakpoint.url) + .then((response) => { + assert.strictEqual(response.status, 200) + assert.deepStrictEqual(response.data, { hello: 'bar' }) + }) + .catch(done) + } else { + endIfDone() + } + }) - function endIfDone () { - if (receivedAckUpdate && expectedPayloads.length === 0) done() - } - }) + t.agent.addRemoteConfig(t.rcConfig) - it('should send expected diagnostics messages if probe is first received and then updated', function (done) { - let receivedAckUpdates = 0 - const probeId = t.rcConfig.config.id - const expectedPayloads = [{ - ddsource: 'dd_debugger', - service: 'node', - debugger: { diagnostics: { probeId, probeVersion: 0, status: 'RECEIVED' } } - }, { - ddsource: 'dd_debugger', - service: 'node', - debugger: { diagnostics: { probeId, probeVersion: 0, status: 'INSTALLED' } } - }, { - ddsource: 'dd_debugger', - service: 'node', - debugger: { diagnostics: { probeId, probeVersion: 1, status: 'RECEIVED' } } - }, { - ddsource: 'dd_debugger', - service: 'node', - debugger: { diagnostics: { probeId, probeVersion: 1, status: 'INSTALLED' } } - }] - const triggers = [ - () => { - t.rcConfig.config.version++ - t.agent.updateRemoteConfig(t.rcConfig.id, t.rcConfig.config) - }, - () => {} - ] - - t.agent.on('remote-config-ack-update', (id, version, state, error) => { - assert.strictEqual(id, t.rcConfig.id) - assert.strictEqual(version, ++receivedAckUpdates) - assert.strictEqual(state, ACKNOWLEDGED) - assert.notOk(error) // falsy check since error will be an empty string, but that's an implementation detail - - endIfDone() + function endIfDone () { + if (receivedAckUpdate && expectedPayloads.length === 0) done() + } }) - t.agent.on('debugger-diagnostics', ({ payload }) => { - const expected = expectedPayloads.shift() - assertObjectContains(payload, expected) - assertUUID(payload.debugger.diagnostics.runtimeId) - if (payload.debugger.diagnostics.status === 'INSTALLED') triggers.shift()() - endIfDone() - }) + it('should send expected diagnostics messages if probe is first received and then updated', function (done) { + let receivedAckUpdates = 0 + const probeId = t.rcConfig.config.id + const expectedPayloads = [{ + ddsource: 'dd_debugger', + service: 'node', + debugger: { diagnostics: { probeId, probeVersion: 0, status: 'RECEIVED' } } + }, { + ddsource: 'dd_debugger', + service: 'node', + debugger: { diagnostics: { probeId, probeVersion: 0, status: 'INSTALLED' } } + }, { + ddsource: 'dd_debugger', + service: 'node', + debugger: { diagnostics: { probeId, probeVersion: 1, status: 'RECEIVED' } } + }, { + ddsource: 'dd_debugger', + service: 'node', + debugger: { diagnostics: { probeId, probeVersion: 1, status: 'INSTALLED' } } + }] + const triggers = [ + () => { + t.rcConfig.config.version++ + t.agent.updateRemoteConfig(t.rcConfig.id, t.rcConfig.config) + }, + () => {} + ] - t.agent.addRemoteConfig(t.rcConfig) + t.agent.on('remote-config-ack-update', (id, version, state, error) => { + assert.strictEqual(id, t.rcConfig.id) + assert.strictEqual(version, ++receivedAckUpdates) + assert.strictEqual(state, ACKNOWLEDGED) + assert.notOk(error) // falsy check since error will be an empty string, but that's an implementation detail - function endIfDone () { - if (receivedAckUpdates === 2 && expectedPayloads.length === 0) done() - } - }) + endIfDone() + }) - it('should send expected diagnostics messages if probe is first received and then deleted', function (done) { - let receivedAckUpdate = false - let payloadsProcessed = false - const probeId = t.rcConfig.config.id - const expectedPayloads = [{ - ddsource: 'dd_debugger', - service: 'node', - debugger: { diagnostics: { probeId, probeVersion: 0, status: 'RECEIVED' } } - }, { - ddsource: 'dd_debugger', - service: 'node', - debugger: { diagnostics: { probeId, probeVersion: 0, status: 'INSTALLED' } } - }] - - t.agent.on('remote-config-ack-update', (id, version, state, error) => { - assert.strictEqual(id, t.rcConfig.id) - assert.strictEqual(version, 1) - assert.strictEqual(state, ACKNOWLEDGED) - assert.notOk(error) // falsy check since error will be an empty string, but that's an implementation detail - - receivedAckUpdate = true - endIfDone() - }) + t.agent.on('debugger-diagnostics', ({ payload }) => { + const expected = expectedPayloads.shift() + assertObjectContains(payload, expected) + assertUUID(payload.debugger.diagnostics.runtimeId) + if (payload.debugger.diagnostics.status === 'INSTALLED') triggers.shift()() + endIfDone() + }) - t.agent.on('debugger-diagnostics', ({ payload }) => { - const expected = expectedPayloads.shift() - assertObjectContains(payload, expected) - assertUUID(payload.debugger.diagnostics.runtimeId) + t.agent.addRemoteConfig(t.rcConfig) - if (payload.debugger.diagnostics.status === 'INSTALLED') { - t.agent.removeRemoteConfig(t.rcConfig.id) - // Wait a little to see if we get any follow-up `debugger-diagnostics` messages - setTimeout(() => { - payloadsProcessed = true - endIfDone() - }, pollInterval * 2 * 1000) // wait twice as long as the RC poll interval + function endIfDone () { + if (receivedAckUpdates === 2 && expectedPayloads.length === 0) done() } }) - t.agent.addRemoteConfig(t.rcConfig) - - function endIfDone () { - if (receivedAckUpdate && payloadsProcessed) done() - } - }) - - const unsupporedOrInvalidProbes = [[ - 'should send expected error diagnostics messages if probe doesn\'t conform to expected schema', - 'bad config!!!', - { status: 'ERROR' } - ], [ - 'should send expected error diagnostics messages if probe type isn\'t supported', - t.generateProbeConfig({ type: 'INVALID_PROBE' }) - ], [ - 'should send expected error diagnostics messages if it isn\'t a line-probe', - t.generateProbeConfig({ where: { foo: 'bar' } }) // TODO: Use valid schema for method probe instead - ]] - - for (const [title, config, customErrorDiagnosticsObj] of unsupporedOrInvalidProbes) { - it(title, function (done) { + it('should send expected diagnostics messages if probe is first received and then deleted', function (done) { let receivedAckUpdate = false - - t.agent.on('remote-config-ack-update', (id, version, state, error) => { - assert.strictEqual(id, `logProbe_${config.id}`) - assert.strictEqual(version, 1) - assert.strictEqual(state, ERROR) - assert.strictEqual(error.slice(0, 6), 'Error:') - - receivedAckUpdate = true - endIfDone() - }) - - const probeId = config.id + let payloadsProcessed = false + const probeId = t.rcConfig.config.id const expectedPayloads = [{ ddsource: 'dd_debugger', service: 'node', - debugger: { diagnostics: { status: 'RECEIVED' } } + debugger: { diagnostics: { probeId, probeVersion: 0, status: 'RECEIVED' } } }, { ddsource: 'dd_debugger', service: 'node', - debugger: { diagnostics: customErrorDiagnosticsObj ?? { probeId, probeVersion: 0, status: 'ERROR' } } + debugger: { diagnostics: { probeId, probeVersion: 0, status: 'INSTALLED' } } }] + t.agent.on('remote-config-ack-update', (id, version, state, error) => { + assert.strictEqual(id, t.rcConfig.id) + assert.strictEqual(version, 1) + assert.strictEqual(state, ACKNOWLEDGED) + assert.notOk(error) // falsy check since error will be an empty string, but that's an implementation detail + + receivedAckUpdate = true + endIfDone() + }) + t.agent.on('debugger-diagnostics', ({ payload }) => { const expected = expectedPayloads.shift() assertObjectContains(payload, expected) - const { diagnostics } = payload.debugger - assertUUID(diagnostics.runtimeId) - - if (diagnostics.status === 'ERROR') { - assert.property(diagnostics, 'exception') - assert.hasAllKeys(diagnostics.exception, ['message', 'stacktrace']) - assert.typeOf(diagnostics.exception.message, 'string') - assert.typeOf(diagnostics.exception.stacktrace, 'string') + assertUUID(payload.debugger.diagnostics.runtimeId) + + if (payload.debugger.diagnostics.status === 'INSTALLED') { + t.agent.removeRemoteConfig(t.rcConfig.id) + // Wait a little to see if we get any follow-up `debugger-diagnostics` messages + setTimeout(() => { + payloadsProcessed = true + endIfDone() + }, pollInterval * 2 * 1000) // wait twice as long as the RC poll interval } - - endIfDone() }) - t.agent.addRemoteConfig({ - product: 'LIVE_DEBUGGING', - id: `logProbe_${config.id}`, - config - }) + t.agent.addRemoteConfig(t.rcConfig) function endIfDone () { - if (receivedAckUpdate && expectedPayloads.length === 0) done() + if (receivedAckUpdate && payloadsProcessed) done() } }) - } - }) - describe('input messages', function () { - it('should capture and send expected payload when a log line probe is triggered', function (done) { - let traceId, spanId, dd + const unsupporedOrInvalidProbes = [[ + 'should send expected error diagnostics messages if probe doesn\'t conform to expected schema', + 'bad config!!!', + { status: 'ERROR' } + ], [ + 'should send expected error diagnostics messages if probe type isn\'t supported', + t.generateProbeConfig({ type: 'INVALID_PROBE' }) + ], [ + 'should send expected error diagnostics messages if it isn\'t a line-probe', + t.generateProbeConfig({ where: { foo: 'bar' } }) // TODO: Use valid schema for method probe instead + ]] + + for (const [title, config, customErrorDiagnosticsObj] of unsupporedOrInvalidProbes) { + it(title, function (done) { + let receivedAckUpdate = false + + t.agent.on('remote-config-ack-update', (id, version, state, error) => { + assert.strictEqual(id, `logProbe_${config.id}`) + assert.strictEqual(version, 1) + assert.strictEqual(state, ERROR) + assert.strictEqual(error.slice(0, 6), 'Error:') + + receivedAckUpdate = true + endIfDone() + }) - t.triggerBreakpoint() + const probeId = config.id + const expectedPayloads = [{ + ddsource: 'dd_debugger', + service: 'node', + debugger: { diagnostics: { status: 'RECEIVED' } } + }, { + ddsource: 'dd_debugger', + service: 'node', + debugger: { diagnostics: customErrorDiagnosticsObj ?? { probeId, probeVersion: 0, status: 'ERROR' } } + }] + + t.agent.on('debugger-diagnostics', ({ payload }) => { + const expected = expectedPayloads.shift() + assertObjectContains(payload, expected) + const { diagnostics } = payload.debugger + assertUUID(diagnostics.runtimeId) + + if (diagnostics.status === 'ERROR') { + assert.property(diagnostics, 'exception') + assert.hasAllKeys(diagnostics.exception, ['message', 'stacktrace']) + assert.typeOf(diagnostics.exception.message, 'string') + assert.typeOf(diagnostics.exception.stacktrace, 'string') + } - t.agent.on('message', ({ payload }) => { - const span = payload.find((arr) => arr[0].name === 'fastify.request')[0] - traceId = span.trace_id.toString() - spanId = span.span_id.toString() + endIfDone() + }) - assertDD() - }) + t.agent.addRemoteConfig({ + product: 'LIVE_DEBUGGING', + id: `logProbe_${config.id}`, + config + }) - t.agent.on('debugger-input', ({ payload }) => { - const expected = { - ddsource: 'dd_debugger', - hostname: os.hostname(), - service: 'node', - message: 'Hello World!', - logger: { - name: t.breakpoint.file, - method: 'fooHandler', - version, - thread_name: 'MainThread' - }, - 'debugger.snapshot': { - probe: { - id: t.rcConfig.config.id, - version: 0, - location: { file: t.breakpoint.file, lines: [String(t.breakpoint.line)] } - }, - language: 'javascript' + function endIfDone () { + if (receivedAckUpdate && expectedPayloads.length === 0) done() } - } + }) + } + }) + + describe('input messages', function () { + it( + 'should capture and send expected payload when a log line probe is triggered', + testBasicInputWithDD.bind(null, t) + ) - assertObjectContains(payload, expected) + it('should respond with updated message if probe message is updated', function (done) { + const expectedMessages = ['Hello World!', 'Hello Updated World!'] + const triggers = [ + async () => { + await t.axios.get(t.breakpoint.url) + t.rcConfig.config.version++ + t.rcConfig.config.template = 'Hello Updated World!' + t.agent.updateRemoteConfig(t.rcConfig.id, t.rcConfig.config) + }, + async () => { + await t.axios.get(t.breakpoint.url) + } + ] - assert.match(payload.logger.thread_id, /^pid:\d+$/) + t.agent.on('debugger-diagnostics', ({ payload }) => { + if (payload.debugger.diagnostics.status === 'INSTALLED') triggers.shift()().catch(done) + }) - if (tracingEnabled) { - assert.isObject(payload.dd) - assert.hasAllKeys(payload.dd, ['trace_id', 'span_id']) - assert.typeOf(payload.dd.trace_id, 'string') - assert.typeOf(payload.dd.span_id, 'string') - assert.isAbove(payload.dd.trace_id.length, 0) - assert.isAbove(payload.dd.span_id.length, 0) - dd = payload.dd - } else { - assert.doesNotHaveAnyKeys(payload, ['dd']) - } + t.agent.on('debugger-input', ({ payload }) => { + assert.strictEqual(payload.message, expectedMessages.shift()) + if (expectedMessages.length === 0) done() + }) - assertUUID(payload['debugger.snapshot'].id) - assert.isNumber(payload['debugger.snapshot'].timestamp) - assert.isTrue(payload['debugger.snapshot'].timestamp > Date.now() - 1000 * 60) - assert.isTrue(payload['debugger.snapshot'].timestamp <= Date.now()) - - assert.isArray(payload['debugger.snapshot'].stack) - assert.isAbove(payload['debugger.snapshot'].stack.length, 0) - for (const frame of payload['debugger.snapshot'].stack) { - assert.isObject(frame) - assert.hasAllKeys(frame, ['fileName', 'function', 'lineNumber', 'columnNumber']) - assert.isString(frame.fileName) - assert.isString(frame.function) - assert.isAbove(frame.lineNumber, 0) - assert.isAbove(frame.columnNumber, 0) - } - const topFrame = payload['debugger.snapshot'].stack[0] - // path seems to be prefeixed with `/private` on Mac - assert.match(topFrame.fileName, new RegExp(`${t.appFile}$`)) - assert.strictEqual(topFrame.function, 'fooHandler') - assert.strictEqual(topFrame.lineNumber, t.breakpoint.line) - assert.strictEqual(topFrame.columnNumber, 3) - - if (tracingEnabled) { - assertDD() - } else { - done() - } + t.agent.addRemoteConfig(t.rcConfig) }) - t.agent.addRemoteConfig(t.rcConfig) - - function assertDD () { - if (!traceId || !spanId || !dd) return - assert.strictEqual(dd.trace_id, traceId) - assert.strictEqual(dd.span_id, spanId) - done() - } - }) + it('should not trigger if probe is deleted', function (done) { + t.agent.on('debugger-diagnostics', ({ payload }) => { + if (payload.debugger.diagnostics.status === 'INSTALLED') { + t.agent.once('remote-confg-responded', async () => { + await t.axios.get(t.breakpoint.url) + // We want to wait enough time to see if the client triggers on the breakpoint so that the test can fail + // if it does, but not so long that the test times out. + // TODO: Is there some signal we can use instead of a timer? + setTimeout(done, pollInterval * 2 * 1000) // wait twice as long as the RC poll interval + }) - it('should respond with updated message if probe message is updated', function (done) { - const expectedMessages = ['Hello World!', 'Hello Updated World!'] - const triggers = [ - async () => { - await t.axios.get(t.breakpoint.url) - t.rcConfig.config.version++ - t.rcConfig.config.template = 'Hello Updated World!' - t.agent.updateRemoteConfig(t.rcConfig.id, t.rcConfig.config) - }, - async () => { - await t.axios.get(t.breakpoint.url) - } - ] + t.agent.removeRemoteConfig(t.rcConfig.id) + } + }) - t.agent.on('debugger-diagnostics', ({ payload }) => { - if (payload.debugger.diagnostics.status === 'INSTALLED') triggers.shift()().catch(done) - }) + t.agent.on('debugger-input', () => { + assert.fail('should not capture anything when the probe is deleted') + }) - t.agent.on('debugger-input', ({ payload }) => { - assert.strictEqual(payload.message, expectedMessages.shift()) - if (expectedMessages.length === 0) done() + t.agent.addRemoteConfig(t.rcConfig) }) - - t.agent.addRemoteConfig(t.rcConfig) }) - it('should not trigger if probe is deleted', function (done) { - t.agent.on('debugger-diagnostics', ({ payload }) => { - if (payload.debugger.diagnostics.status === 'INSTALLED') { - t.agent.once('remote-confg-responded', async () => { - await t.axios.get(t.breakpoint.url) - // We want to wait enough time to see if the client triggers on the breakpoint so that the test can fail - // if it does, but not so long that the test times out. - // TODO: Is there some signal we can use instead of a timer? - setTimeout(done, pollInterval * 2 * 1000) // wait twice as long as the RC poll interval - }) + describe('sampling', function () { + it('should respect sampling rate for single probe', function (done) { + let start, timer + let payloadsReceived = 0 + const rcConfig = t.generateRemoteConfig({ sampling: { snapshotsPerSecond: 1 } }) - t.agent.removeRemoteConfig(t.rcConfig.id) + function triggerBreakpointContinuously () { + t.axios.get(t.breakpoint.url).catch(done) + timer = setTimeout(triggerBreakpointContinuously, 10) } - }) - t.agent.on('debugger-input', () => { - assert.fail('should not capture anything when the probe is deleted') - }) + t.agent.on('debugger-diagnostics', ({ payload }) => { + if (payload.debugger.diagnostics.status === 'INSTALLED') triggerBreakpointContinuously() + }) - t.agent.addRemoteConfig(t.rcConfig) - }) - }) + t.agent.on('debugger-input', () => { + payloadsReceived++ + if (payloadsReceived === 1) { + start = Date.now() + } else if (payloadsReceived === 2) { + const duration = Date.now() - start + clearTimeout(timer) - describe('sampling', function () { - it('should respect sampling rate for single probe', function (done) { - let start, timer - let payloadsReceived = 0 - const rcConfig = t.generateRemoteConfig({ sampling: { snapshotsPerSecond: 1 } }) + // Allow for a variance of -5/+50ms (time will tell if this is enough) + assert.isAbove(duration, 995) + assert.isBelow(duration, 1050) - function triggerBreakpointContinuously () { - t.axios.get(t.breakpoint.url).catch(done) - timer = setTimeout(triggerBreakpointContinuously, 10) - } + // Wait at least a full sampling period, to see if we get any more payloads + timer = setTimeout(done, 1250) + } else { + clearTimeout(timer) + done(new Error('Too many payloads received!')) + } + }) - t.agent.on('debugger-diagnostics', ({ payload }) => { - if (payload.debugger.diagnostics.status === 'INSTALLED') triggerBreakpointContinuously() + t.agent.addRemoteConfig(rcConfig) }) - t.agent.on('debugger-input', () => { - payloadsReceived++ - if (payloadsReceived === 1) { - start = Date.now() - } else if (payloadsReceived === 2) { - const duration = Date.now() - start - clearTimeout(timer) - - // Allow for a variance of -5/+50ms (time will tell if this is enough) - assert.isAbove(duration, 995) - assert.isBelow(duration, 1050) - - // Wait at least a full sampling period, to see if we get any more payloads - timer = setTimeout(done, 1250) - } else { - clearTimeout(timer) - done(new Error('Too many payloads received!')) + it('should adhere to individual probes sample rate', function (done) { + const rcConfig1 = t.breakpoints[0].generateRemoteConfig({ sampling: { snapshotsPerSecond: 1 } }) + const rcConfig2 = t.breakpoints[1].generateRemoteConfig({ sampling: { snapshotsPerSecond: 1 } }) + const state = { + [rcConfig1.config.id]: { + payloadsReceived: 0, + tiggerBreakpointContinuously () { + t.axios.get(t.breakpoints[0].url).catch(done) + this.timer = setTimeout(this.tiggerBreakpointContinuously.bind(this), 10) + } + }, + [rcConfig2.config.id]: { + payloadsReceived: 0, + tiggerBreakpointContinuously () { + t.axios.get(t.breakpoints[1].url).catch(done) + this.timer = setTimeout(this.tiggerBreakpointContinuously.bind(this), 10) + } + } } - }) - t.agent.addRemoteConfig(rcConfig) - }) + t.agent.on('debugger-diagnostics', ({ payload }) => { + const { probeId, status } = payload.debugger.diagnostics + if (status === 'INSTALLED') state[probeId].tiggerBreakpointContinuously() + }) - it('should adhere to individual probes sample rate', function (done) { - const rcConfig1 = t.breakpoints[0].generateRemoteConfig({ sampling: { snapshotsPerSecond: 1 } }) - const rcConfig2 = t.breakpoints[1].generateRemoteConfig({ sampling: { snapshotsPerSecond: 1 } }) - const state = { - [rcConfig1.config.id]: { - payloadsReceived: 0, - tiggerBreakpointContinuously () { - t.axios.get(t.breakpoints[0].url).catch(done) - this.timer = setTimeout(this.tiggerBreakpointContinuously.bind(this), 10) - } - }, - [rcConfig2.config.id]: { - payloadsReceived: 0, - tiggerBreakpointContinuously () { - t.axios.get(t.breakpoints[1].url).catch(done) - this.timer = setTimeout(this.tiggerBreakpointContinuously.bind(this), 10) + t.agent.on('debugger-input', ({ payload }) => { + const _state = state[payload['debugger.snapshot'].probe.id] + _state.payloadsReceived++ + if (_state.payloadsReceived === 1) { + _state.start = Date.now() + } else if (_state.payloadsReceived === 2) { + const duration = Date.now() - _state.start + clearTimeout(_state.timer) + + // Allow for a variance of -5/+50ms (time will tell if this is enough) + assert.isAbove(duration, 995) + assert.isBelow(duration, 1050) + + // Wait at least a full sampling period, to see if we get any more payloads + _state.timer = setTimeout(doneWhenCalledTwice, 1250) + } else { + clearTimeout(_state.timer) + done(new Error('Too many payloads received!')) } - } - } + }) - t.agent.on('debugger-diagnostics', ({ payload }) => { - const { probeId, status } = payload.debugger.diagnostics - if (status === 'INSTALLED') state[probeId].tiggerBreakpointContinuously() - }) + t.agent.addRemoteConfig(rcConfig1) + t.agent.addRemoteConfig(rcConfig2) - t.agent.on('debugger-input', ({ payload }) => { - const _state = state[payload['debugger.snapshot'].probe.id] - _state.payloadsReceived++ - if (_state.payloadsReceived === 1) { - _state.start = Date.now() - } else if (_state.payloadsReceived === 2) { - const duration = Date.now() - _state.start - clearTimeout(_state.timer) - - // Allow for a variance of -5/+50ms (time will tell if this is enough) - assert.isAbove(duration, 995) - assert.isBelow(duration, 1050) - - // Wait at least a full sampling period, to see if we get any more payloads - _state.timer = setTimeout(doneWhenCalledTwice, 1250) - } else { - clearTimeout(_state.timer) - done(new Error('Too many payloads received!')) + function doneWhenCalledTwice () { + if (doneWhenCalledTwice.calledOnce) return done() + doneWhenCalledTwice.calledOnce = true } }) + }) - t.agent.addRemoteConfig(rcConfig1) - t.agent.addRemoteConfig(rcConfig2) + describe('race conditions', function () { + it('should remove the last breakpoint completely before trying to add a new one', function (done) { + const rcConfig2 = t.generateRemoteConfig() + + t.agent.on('debugger-diagnostics', ({ payload: { debugger: { diagnostics: { status, probeId } } } }) => { + if (status !== 'INSTALLED') return + + if (probeId === t.rcConfig.config.id) { + // First INSTALLED payload: Try to trigger the race condition. + t.agent.removeRemoteConfig(t.rcConfig.id) + t.agent.addRemoteConfig(rcConfig2) + } else { + // Second INSTALLED payload: Perform an HTTP request to see if we successfully handled the race condition. + let finished = false + + // If the race condition occurred, the debugger will have been detached from the main thread and the new + // probe will never trigger. If that's the case, the following timer will fire: + const timer = setTimeout(() => { + done(new Error('Race condition occurred!')) + }, 1000) + + // If we successfully handled the race condition, the probe will trigger, we'll get a probe result and the + // following event listener will be called: + t.agent.once('debugger-input', () => { + clearTimeout(timer) + finished = true + done() + }) - function doneWhenCalledTwice () { - if (doneWhenCalledTwice.calledOnce) return done() - doneWhenCalledTwice.calledOnce = true - } + // Perform HTTP request to try and trigger the probe + t.axios.get(t.breakpoint.url).catch((err) => { + // If the request hasn't fully completed by the time the tests ends and the target app is destroyed, Axios + // will complain with a "socket hang up" error. Hence this sanity check before calling `done(err)`. If we + // later add more tests below this one, this shouuldn't be an issue. + if (!finished) done(err) + }) + } + }) + + t.agent.addRemoteConfig(t.rcConfig) + }) }) }) - describe('race conditions', function () { - it('should remove the last breakpoint completely before trying to add a new one', function (done) { - const rcConfig2 = t.generateRemoteConfig() - - t.agent.on('debugger-diagnostics', ({ payload: { debugger: { diagnostics: { status, probeId } } } }) => { - if (status !== 'INSTALLED') return - - if (probeId === t.rcConfig.config.id) { - // First INSTALLED payload: Try to trigger the race condition. - t.agent.removeRemoteConfig(t.rcConfig.id) - t.agent.addRemoteConfig(rcConfig2) - } else { - // Second INSTALLED payload: Perform an HTTP request to see if we successfully handled the race condition. - let finished = false - - // If the race condition occurred, the debugger will have been detached from the main thread and the new - // probe will never trigger. If that's the case, the following timer will fire: - const timer = setTimeout(() => { - done(new Error('Race condition occurred!')) - }, 1000) - - // If we successfully handled the race condition, the probe will trigger, we'll get a probe result and the - // following event listener will be called: - t.agent.once('debugger-input', () => { - clearTimeout(timer) - finished = true - done() - }) + describe('DD_TRACING_ENABLED=true, DD_TRACE_128_BIT_TRACEID_GENERATION_ENABLED=true', function () { + const t = setup({ DD_TRACING_ENABLED: true, DD_TRACE_128_BIT_TRACEID_GENERATION_ENABLED: true }) - // Perform HTTP request to try and trigger the probe - t.axios.get(t.breakpoint.url).catch((err) => { - // If the request hasn't fully completed by the time the tests ends and the target app is destroyed, Axios - // will complain with a "socket hang up" error. Hence this sanity check before calling `done(err)`. If we - // later add more tests below this one, this shouuldn't be an issue. - if (!finished) done(err) - }) - } - }) + describe('input messages', function () { + it( + 'should capture and send expected payload when a log line probe is triggered', + testBasicInputWithDD.bind(null, t) + ) + }) + }) - t.agent.addRemoteConfig(t.rcConfig) + describe('DD_TRACING_ENABLED=true, DD_TRACE_128_BIT_TRACEID_GENERATION_ENABLED=false', function () { + const t = setup({ DD_TRACING_ENABLED: true, DD_TRACE_128_BIT_TRACEID_GENERATION_ENABLED: false }) + + describe('input messages', function () { + it( + 'should capture and send expected payload when a log line probe is triggered', + testBasicInputWithDD.bind(null, t) + ) }) }) + + describe('DD_TRACING_ENABLED=false', function () { + const t = setup({ DD_TRACING_ENABLED: false }) + + describe('input messages', function () { + it( + 'should capture and send expected payload when a log line probe is triggered', + testBasicInputWithoutDD.bind(null, t) + ) + }) + }) +}) + +function testBasicInputWithDD (t, done) { + let traceId, spanId, dd + + t.triggerBreakpoint() + + t.agent.on('message', ({ payload }) => { + const span = payload.find((arr) => arr[0].name === 'fastify.request')[0] + traceId = span.trace_id.toString() + spanId = span.span_id.toString() + + assertDD() + }) + + t.agent.on('debugger-input', ({ payload }) => { + assertBasicInputPayload(t, payload) + + assert.isObject(payload.dd) + assert.hasAllKeys(payload.dd, ['trace_id', 'span_id']) + assert.typeOf(payload.dd.trace_id, 'string') + assert.typeOf(payload.dd.span_id, 'string') + assert.isAbove(payload.dd.trace_id.length, 0) + assert.isAbove(payload.dd.span_id.length, 0) + dd = payload.dd + + assertDD() + }) + + t.agent.addRemoteConfig(t.rcConfig) + + function assertDD () { + if (!traceId || !spanId || !dd) return + assert.strictEqual(dd.trace_id, traceId) + assert.strictEqual(dd.span_id, spanId) + done() + } +} + +function testBasicInputWithoutDD (t, done) { + t.triggerBreakpoint() + + t.agent.on('debugger-input', ({ payload }) => { + assertBasicInputPayload(t, payload) + assert.doesNotHaveAnyKeys(payload, ['dd']) + done() + }) + + t.agent.addRemoteConfig(t.rcConfig) +} + +function assertBasicInputPayload (t, payload) { + const expected = { + ddsource: 'dd_debugger', + hostname: os.hostname(), + service: 'node', + message: 'Hello World!', + logger: { + name: t.breakpoint.file, + method: 'fooHandler', + version, + thread_name: 'MainThread' + }, + 'debugger.snapshot': { + probe: { + id: t.rcConfig.config.id, + version: 0, + location: { file: t.breakpoint.file, lines: [String(t.breakpoint.line)] } + }, + language: 'javascript' + } + } + + assertObjectContains(payload, expected) + + assert.match(payload.logger.thread_id, /^pid:\d+$/) + + assertUUID(payload['debugger.snapshot'].id) + assert.isNumber(payload['debugger.snapshot'].timestamp) + assert.isTrue(payload['debugger.snapshot'].timestamp > Date.now() - 1000 * 60) + assert.isTrue(payload['debugger.snapshot'].timestamp <= Date.now()) + + assert.isArray(payload['debugger.snapshot'].stack) + assert.isAbove(payload['debugger.snapshot'].stack.length, 0) + for (const frame of payload['debugger.snapshot'].stack) { + assert.isObject(frame) + assert.hasAllKeys(frame, ['fileName', 'function', 'lineNumber', 'columnNumber']) + assert.isString(frame.fileName) + assert.isString(frame.function) + assert.isAbove(frame.lineNumber, 0) + assert.isAbove(frame.columnNumber, 0) + } + const topFrame = payload['debugger.snapshot'].stack[0] + // path seems to be prefeixed with `/private` on Mac + assert.match(topFrame.fileName, new RegExp(`${t.appFile}$`)) + assert.strictEqual(topFrame.function, 'fooHandler') + assert.strictEqual(topFrame.lineNumber, t.breakpoint.line) + assert.strictEqual(topFrame.columnNumber, 3) } From 8ee2d0aef0bd246e635d09e146e2d018fdfd63a6 Mon Sep 17 00:00:00 2001 From: Ida Liu <119438987+ida613@users.noreply.github.com> Date: Thu, 19 Dec 2024 10:21:09 -0500 Subject: [PATCH 26/36] add logging for priority sampler (#5028) * add logging * update logging * update log traceChannel * update logging * make log.trace take callbacks * fix linter errors * update log.trace * Update packages/dd-trace/src/log/index.js Co-authored-by: Roch Devost * update log writter and add tests * fix linter error --------- Co-authored-by: Roch Devost --- packages/dd-trace/src/log/channels.js | 11 +++++++++-- packages/dd-trace/src/log/index.js | 12 +++++++++++- packages/dd-trace/src/log/writer.js | 17 ++++++++++++++--- packages/dd-trace/src/priority_sampler.js | 12 +++++++++++- packages/dd-trace/test/log.spec.js | 17 +++++++++++++++++ 5 files changed, 62 insertions(+), 7 deletions(-) diff --git a/packages/dd-trace/src/log/channels.js b/packages/dd-trace/src/log/channels.js index 545fef4195a..b3b10624705 100644 --- a/packages/dd-trace/src/log/channels.js +++ b/packages/dd-trace/src/log/channels.js @@ -3,7 +3,7 @@ const { channel } = require('dc-polyfill') const Level = { - trace: 20, + trace: 10, debug: 20, info: 30, warn: 40, @@ -12,6 +12,7 @@ const Level = { off: 100 } +const traceChannel = channel('datadog:log:trace') const debugChannel = channel('datadog:log:debug') const infoChannel = channel('datadog:log:info') const warnChannel = channel('datadog:log:warn') @@ -31,6 +32,9 @@ class LogChannel { } subscribe (logger) { + if (Level.trace >= this._level) { + traceChannel.subscribe(logger.trace) + } if (Level.debug >= this._level) { debugChannel.subscribe(logger.debug) } @@ -46,6 +50,9 @@ class LogChannel { } unsubscribe (logger) { + if (traceChannel.hasSubscribers) { + traceChannel.unsubscribe(logger.trace) + } if (debugChannel.hasSubscribers) { debugChannel.unsubscribe(logger.debug) } @@ -63,7 +70,7 @@ class LogChannel { module.exports = { LogChannel, - + traceChannel, debugChannel, infoChannel, warnChannel, diff --git a/packages/dd-trace/src/log/index.js b/packages/dd-trace/src/log/index.js index 3a5392340df..213b6ccc8e6 100644 --- a/packages/dd-trace/src/log/index.js +++ b/packages/dd-trace/src/log/index.js @@ -2,7 +2,7 @@ const coalesce = require('koalas') const { isTrue } = require('../util') -const { debugChannel, infoChannel, warnChannel, errorChannel } = require('./channels') +const { traceChannel, debugChannel, infoChannel, warnChannel, errorChannel } = require('./channels') const logWriter = require('./writer') const { Log } = require('./log') @@ -56,6 +56,16 @@ const log = { return this }, + trace (...args) { + if (traceChannel.hasSubscribers) { + const logRecord = {} + Error.captureStackTrace(logRecord, this.trace) + const stack = logRecord.stack.split('\n')[1].replace(/^\s+at ([^\s]) .+/, '$1') + traceChannel.publish(Log.parse('Trace', args, { stack })) + } + return this + }, + debug (...args) { if (debugChannel.hasSubscribers) { debugChannel.publish(Log.parse(...args)) diff --git a/packages/dd-trace/src/log/writer.js b/packages/dd-trace/src/log/writer.js index 4724253244b..a721f7f9e35 100644 --- a/packages/dd-trace/src/log/writer.js +++ b/packages/dd-trace/src/log/writer.js @@ -4,6 +4,7 @@ const { storage } = require('../../../datadog-core') const { LogChannel } = require('./channels') const { Log } = require('./log') const defaultLogger = { + trace: msg => console.trace(msg), /* eslint-disable-line no-console */ debug: msg => console.debug(msg), /* eslint-disable-line no-console */ info: msg => console.info(msg), /* eslint-disable-line no-console */ warn: msg => console.warn(msg), /* eslint-disable-line no-console */ @@ -23,7 +24,7 @@ function withNoop (fn) { } function unsubscribeAll () { - logChannel.unsubscribe({ debug: onDebug, info: onInfo, warn: onWarn, error: onError }) + logChannel.unsubscribe({ trace: onTrace, debug: onDebug, info: onInfo, warn: onWarn, error: onError }) } function toggleSubscription (enable, level) { @@ -31,7 +32,7 @@ function toggleSubscription (enable, level) { if (enable) { logChannel = new LogChannel(level) - logChannel.subscribe({ debug: onDebug, info: onInfo, warn: onWarn, error: onError }) + logChannel.subscribe({ trace: onTrace, debug: onDebug, info: onInfo, warn: onWarn, error: onError }) } } @@ -88,6 +89,12 @@ function onDebug (log) { if (cause) withNoop(() => logger.debug(cause)) } +function onTrace (log) { + const { formatted, cause } = getErrorLog(log) + if (formatted) withNoop(() => logger.trace(formatted)) + if (cause) withNoop(() => logger.trace(cause)) +} + function error (...args) { onError(Log.parse(...args)) } @@ -110,4 +117,8 @@ function debug (...args) { onDebug(Log.parse(...args)) } -module.exports = { use, toggle, reset, error, warn, info, debug } +function trace (...args) { + onTrace(Log.parse(...args)) +} + +module.exports = { use, toggle, reset, error, warn, info, debug, trace } diff --git a/packages/dd-trace/src/priority_sampler.js b/packages/dd-trace/src/priority_sampler.js index f9968a41194..3a89f71f664 100644 --- a/packages/dd-trace/src/priority_sampler.js +++ b/packages/dd-trace/src/priority_sampler.js @@ -1,5 +1,6 @@ 'use strict' +const log = require('./log') const RateLimiter = require('./rate_limiter') const Sampler = require('./sampler') const { setSamplingRules } = require('./startup-log') @@ -44,16 +45,19 @@ class PrioritySampler { this.update({}) } - configure (env, { sampleRate, provenance = undefined, rateLimit = 100, rules = [] } = {}) { + configure (env, opts = {}) { + const { sampleRate, provenance = undefined, rateLimit = 100, rules = [] } = opts this._env = env this._rules = this._normalizeRules(rules, sampleRate, rateLimit, provenance) this._limiter = new RateLimiter(rateLimit) + log.trace(env, opts) setSamplingRules(this._rules) } isSampled (span) { const priority = this._getPriorityFromAuto(span) + log.trace(span) return priority === USER_KEEP || priority === AUTO_KEEP } @@ -67,6 +71,8 @@ class PrioritySampler { if (context._sampling.priority !== undefined) return if (!root) return // noop span + log.trace(span, auto) + const tag = this._getPriorityFromTags(context._tags, context) if (this.validate(tag)) { @@ -94,6 +100,8 @@ class PrioritySampler { samplers[DEFAULT_KEY] = samplers[DEFAULT_KEY] || defaultSampler this._samplers = samplers + + log.trace(rates) } validate (samplingPriority) { @@ -117,6 +125,8 @@ class PrioritySampler { context._sampling.mechanism = mechanism const root = context._trace.started[0] + + log.trace(span, samplingPriority, mechanism) this._addDecisionMaker(root) } diff --git a/packages/dd-trace/test/log.spec.js b/packages/dd-trace/test/log.spec.js index a035c864f71..ac2feea9f7a 100644 --- a/packages/dd-trace/test/log.spec.js +++ b/packages/dd-trace/test/log.spec.js @@ -86,6 +86,7 @@ describe('log', () => { sinon.stub(console, 'error') sinon.stub(console, 'warn') sinon.stub(console, 'debug') + sinon.stub(console, 'trace') error = new Error() @@ -104,6 +105,7 @@ describe('log', () => { console.error.restore() console.warn.restore() console.debug.restore() + console.trace.restore() }) it('should support chaining', () => { @@ -139,6 +141,21 @@ describe('log', () => { }) }) + describe('trace', () => { + it('should not log to console by default', () => { + log.trace('trace') + + expect(console.trace).to.not.have.been.called + }) + + it('should log to console after setting log level to trace', () => { + log.toggle(true, 'trace') + log.trace('argument') + + expect(console.trace).to.have.been.calledTwice + }) + }) + describe('error', () => { it('should log to console by default', () => { log.error(error) From e36f26b03944d2ba73b1caff0a377fbdca0c76b5 Mon Sep 17 00:00:00 2001 From: ishabi Date: Thu, 19 Dec 2024 17:56:26 +0100 Subject: [PATCH 27/36] Exploit prevention command injection (#4966) * Exploit prevention command injection * fix spawnSync abort error test * add telemetry tests * fix sql injection tests on postgres * add different test * revert spawnSync changes * fix linter * add spawnSync tests * remove spawnSync not needed test * fix cmdi params * Revert "fix cmdi params" This reverts commit 4a3d76657d7ced365268c60d5146e86833eafb19. --- packages/dd-trace/src/appsec/addresses.js | 1 + .../src/appsec/rasp/command_injection.js | 19 ++- packages/dd-trace/src/appsec/rasp/lfi.js | 4 +- .../dd-trace/src/appsec/rasp/sql_injection.js | 4 +- packages/dd-trace/src/appsec/rasp/ssrf.js | 4 +- .../src/appsec/remote_config/capabilities.js | 3 +- .../src/appsec/remote_config/index.js | 2 + packages/dd-trace/src/appsec/reporter.js | 6 +- packages/dd-trace/src/appsec/telemetry.js | 9 +- packages/dd-trace/src/appsec/waf/index.js | 4 +- .../src/appsec/waf/waf_context_wrapper.js | 4 +- .../command_injection.express.plugin.spec.js | 104 ++++++------ .../command_injection.integration.spec.js | 71 +++++++-- .../appsec/rasp/command_injection.spec.js | 149 +++++++++++------- .../dd-trace/test/appsec/rasp/lfi.spec.js | 2 +- .../appsec/rasp/resources/rasp_rules.json | 51 +++++- .../appsec/rasp/resources/shi-app/index.js | 14 ++ .../rasp/sql_injection.pg.plugin.spec.js | 8 +- .../test/appsec/rasp/sql_injection.spec.js | 4 +- .../dd-trace/test/appsec/rasp/ssrf.spec.js | 2 +- .../test/appsec/remote_config/index.spec.js | 10 ++ .../dd-trace/test/appsec/reporter.spec.js | 8 +- 22 files changed, 328 insertions(+), 155 deletions(-) diff --git a/packages/dd-trace/src/appsec/addresses.js b/packages/dd-trace/src/appsec/addresses.js index a492a5e454f..20290baf9c4 100644 --- a/packages/dd-trace/src/appsec/addresses.js +++ b/packages/dd-trace/src/appsec/addresses.js @@ -31,6 +31,7 @@ module.exports = { DB_STATEMENT: 'server.db.statement', DB_SYSTEM: 'server.db.system', + EXEC_COMMAND: 'server.sys.exec.cmd', SHELL_COMMAND: 'server.sys.shell.cmd', LOGIN_SUCCESS: 'server.business_logic.users.login.success', diff --git a/packages/dd-trace/src/appsec/rasp/command_injection.js b/packages/dd-trace/src/appsec/rasp/command_injection.js index 8d6d977aace..62546e2b6a6 100644 --- a/packages/dd-trace/src/appsec/rasp/command_injection.js +++ b/packages/dd-trace/src/appsec/rasp/command_injection.js @@ -25,19 +25,26 @@ function disable () { } function analyzeCommandInjection ({ file, fileArgs, shell, abortController }) { - if (!file || !shell) return + if (!file) return const store = storage.getStore() const req = store?.req if (!req) return - const commandParams = fileArgs ? [file, ...fileArgs] : file - - const persistent = { - [addresses.SHELL_COMMAND]: commandParams + const persistent = {} + const raspRule = { type: RULE_TYPES.COMMAND_INJECTION } + const params = fileArgs ? [file, ...fileArgs] : file + + if (shell) { + persistent[addresses.SHELL_COMMAND] = params + raspRule.variant = 'shell' + } else { + const commandParams = Array.isArray(params) ? params : [params] + persistent[addresses.EXEC_COMMAND] = commandParams + raspRule.variant = 'exec' } - const result = waf.run({ persistent }, req, RULE_TYPES.COMMAND_INJECTION) + const result = waf.run({ persistent }, req, raspRule) const res = store?.res handleResult(result, req, res, abortController, config) diff --git a/packages/dd-trace/src/appsec/rasp/lfi.js b/packages/dd-trace/src/appsec/rasp/lfi.js index 1190734064d..657369ad0fd 100644 --- a/packages/dd-trace/src/appsec/rasp/lfi.js +++ b/packages/dd-trace/src/appsec/rasp/lfi.js @@ -58,7 +58,9 @@ function analyzeLfi (ctx) { [FS_OPERATION_PATH]: path } - const result = waf.run({ persistent }, req, RULE_TYPES.LFI) + const raspRule = { type: RULE_TYPES.LFI } + + const result = waf.run({ persistent }, req, raspRule) handleResult(result, req, res, ctx.abortController, config) }) } diff --git a/packages/dd-trace/src/appsec/rasp/sql_injection.js b/packages/dd-trace/src/appsec/rasp/sql_injection.js index d4a165d8615..157723258f7 100644 --- a/packages/dd-trace/src/appsec/rasp/sql_injection.js +++ b/packages/dd-trace/src/appsec/rasp/sql_injection.js @@ -72,7 +72,9 @@ function analyzeSqlInjection (query, dbSystem, abortController) { [addresses.DB_SYSTEM]: dbSystem } - const result = waf.run({ persistent }, req, RULE_TYPES.SQL_INJECTION) + const raspRule = { type: RULE_TYPES.SQL_INJECTION } + + const result = waf.run({ persistent }, req, raspRule) handleResult(result, req, res, abortController, config) } diff --git a/packages/dd-trace/src/appsec/rasp/ssrf.js b/packages/dd-trace/src/appsec/rasp/ssrf.js index 38a3c150d74..7d429d74549 100644 --- a/packages/dd-trace/src/appsec/rasp/ssrf.js +++ b/packages/dd-trace/src/appsec/rasp/ssrf.js @@ -29,7 +29,9 @@ function analyzeSsrf (ctx) { [addresses.HTTP_OUTGOING_URL]: outgoingUrl } - const result = waf.run({ persistent }, req, RULE_TYPES.SSRF) + const raspRule = { type: RULE_TYPES.SSRF } + + const result = waf.run({ persistent }, req, raspRule) const res = store?.res handleResult(result, req, res, ctx.abortController, config) diff --git a/packages/dd-trace/src/appsec/remote_config/capabilities.js b/packages/dd-trace/src/appsec/remote_config/capabilities.js index 16034f5f9ee..5057d38de43 100644 --- a/packages/dd-trace/src/appsec/remote_config/capabilities.js +++ b/packages/dd-trace/src/appsec/remote_config/capabilities.js @@ -25,5 +25,6 @@ module.exports = { ASM_AUTO_USER_INSTRUM_MODE: 1n << 31n, ASM_ENDPOINT_FINGERPRINT: 1n << 32n, ASM_NETWORK_FINGERPRINT: 1n << 34n, - ASM_HEADER_FINGERPRINT: 1n << 35n + ASM_HEADER_FINGERPRINT: 1n << 35n, + ASM_RASP_CMDI: 1n << 37n } diff --git a/packages/dd-trace/src/appsec/remote_config/index.js b/packages/dd-trace/src/appsec/remote_config/index.js index 7884175abb0..6bebe40e142 100644 --- a/packages/dd-trace/src/appsec/remote_config/index.js +++ b/packages/dd-trace/src/appsec/remote_config/index.js @@ -101,6 +101,7 @@ function enableWafUpdate (appsecConfig) { rc.updateCapabilities(RemoteConfigCapabilities.ASM_RASP_SSRF, true) rc.updateCapabilities(RemoteConfigCapabilities.ASM_RASP_LFI, true) rc.updateCapabilities(RemoteConfigCapabilities.ASM_RASP_SHI, true) + rc.updateCapabilities(RemoteConfigCapabilities.ASM_RASP_CMDI, true) } // TODO: delete noop handlers and kPreUpdate and replace with batched handlers @@ -133,6 +134,7 @@ function disableWafUpdate () { rc.updateCapabilities(RemoteConfigCapabilities.ASM_RASP_SSRF, false) rc.updateCapabilities(RemoteConfigCapabilities.ASM_RASP_LFI, false) rc.updateCapabilities(RemoteConfigCapabilities.ASM_RASP_SHI, false) + rc.updateCapabilities(RemoteConfigCapabilities.ASM_RASP_CMDI, false) rc.removeProductHandler('ASM_DATA') rc.removeProductHandler('ASM_DD') diff --git a/packages/dd-trace/src/appsec/reporter.js b/packages/dd-trace/src/appsec/reporter.js index 57519e5bc79..c2f9bac6cbc 100644 --- a/packages/dd-trace/src/appsec/reporter.js +++ b/packages/dd-trace/src/appsec/reporter.js @@ -101,7 +101,7 @@ function reportWafInit (wafVersion, rulesVersion, diagnosticsRules = {}) { incrementWafInitMetric(wafVersion, rulesVersion) } -function reportMetrics (metrics, raspRuleType) { +function reportMetrics (metrics, raspRule) { const store = storage.getStore() const rootSpan = store?.req && web.root(store.req) if (!rootSpan) return @@ -109,8 +109,8 @@ function reportMetrics (metrics, raspRuleType) { if (metrics.rulesVersion) { rootSpan.setTag('_dd.appsec.event_rules.version', metrics.rulesVersion) } - if (raspRuleType) { - updateRaspRequestsMetricTags(metrics, store.req, raspRuleType) + if (raspRule) { + updateRaspRequestsMetricTags(metrics, store.req, raspRule) } else { updateWafRequestsMetricTags(metrics, store.req) } diff --git a/packages/dd-trace/src/appsec/telemetry.js b/packages/dd-trace/src/appsec/telemetry.js index 8e9a2518f80..08f435b9c0e 100644 --- a/packages/dd-trace/src/appsec/telemetry.js +++ b/packages/dd-trace/src/appsec/telemetry.js @@ -79,7 +79,7 @@ function getOrCreateMetricTags (store, versionsTags) { return metricTags } -function updateRaspRequestsMetricTags (metrics, req, raspRuleType) { +function updateRaspRequestsMetricTags (metrics, req, raspRule) { if (!req) return const store = getStore(req) @@ -89,7 +89,12 @@ function updateRaspRequestsMetricTags (metrics, req, raspRuleType) { if (!enabled) return - const tags = { rule_type: raspRuleType, waf_version: metrics.wafVersion } + const tags = { rule_type: raspRule.type, waf_version: metrics.wafVersion } + + if (raspRule.variant) { + tags.rule_variant = raspRule.variant + } + appsecMetrics.count('rasp.rule.eval', tags).inc(1) if (metrics.wafTimeout) { diff --git a/packages/dd-trace/src/appsec/waf/index.js b/packages/dd-trace/src/appsec/waf/index.js index 3b2bc9e2a13..a14a5313a92 100644 --- a/packages/dd-trace/src/appsec/waf/index.js +++ b/packages/dd-trace/src/appsec/waf/index.js @@ -46,7 +46,7 @@ function update (newRules) { } } -function run (data, req, raspRuleType) { +function run (data, req, raspRule) { if (!req) { const store = storage.getStore() if (!store || !store.req) { @@ -59,7 +59,7 @@ function run (data, req, raspRuleType) { const wafContext = waf.wafManager.getWAFContext(req) - return wafContext.run(data, raspRuleType) + return wafContext.run(data, raspRule) } function disposeContext (req) { diff --git a/packages/dd-trace/src/appsec/waf/waf_context_wrapper.js b/packages/dd-trace/src/appsec/waf/waf_context_wrapper.js index 6a90b8f89bb..54dbd16e1be 100644 --- a/packages/dd-trace/src/appsec/waf/waf_context_wrapper.js +++ b/packages/dd-trace/src/appsec/waf/waf_context_wrapper.js @@ -21,7 +21,7 @@ class WAFContextWrapper { this.knownAddresses = knownAddresses } - run ({ persistent, ephemeral }, raspRuleType) { + run ({ persistent, ephemeral }, raspRule) { if (this.ddwafContext.disposed) { log.warn('[ASM] Calling run on a disposed context') return @@ -87,7 +87,7 @@ class WAFContextWrapper { blockTriggered, wafVersion: this.wafVersion, wafTimeout: result.timeout - }, raspRuleType) + }, raspRule) if (ruleTriggered) { Reporter.reportAttack(JSON.stringify(result.events)) diff --git a/packages/dd-trace/test/appsec/rasp/command_injection.express.plugin.spec.js b/packages/dd-trace/test/appsec/rasp/command_injection.express.plugin.spec.js index 3943bd0c3c3..d7609367ab9 100644 --- a/packages/dd-trace/test/appsec/rasp/command_injection.express.plugin.spec.js +++ b/packages/dd-trace/test/appsec/rasp/command_injection.express.plugin.spec.js @@ -5,42 +5,25 @@ const appsec = require('../../../src/appsec') const Config = require('../../../src/config') const path = require('path') const Axios = require('axios') -const { getWebSpan, checkRaspExecutedAndHasThreat, checkRaspExecutedAndNotThreat } = require('./utils') +const { checkRaspExecutedAndHasThreat, checkRaspExecutedAndNotThreat } = require('./utils') const { assert } = require('chai') describe('RASP - command_injection', () => { withVersions('express', 'express', expressVersion => { let app, server, axios + function testShellBlockingAndSafeRequests () { + it('should block the threat', async () => { + try { + await axios.get('/?dir=$(cat /etc/passwd 1>%262 ; echo .)') + } catch (e) { + if (!e.response) { + throw e + } - async function testBlockingRequest () { - try { - await axios.get('/?dir=$(cat /etc/passwd 1>%262 ; echo .)') - } catch (e) { - if (!e.response) { - throw e - } - - return checkRaspExecutedAndHasThreat(agent, 'rasp-command_injection-rule-id-3') - } - - assert.fail('Request should be blocked') - } - - function checkRaspNotExecutedAndNotThreat (agent, checkRuleEval = true) { - return agent.use((traces) => { - const span = getWebSpan(traces) - - assert.notProperty(span.meta, '_dd.appsec.json') - assert.notProperty(span.meta_struct || {}, '_dd.stack') - if (checkRuleEval) { - assert.notProperty(span.metrics, '_dd.appsec.rasp.rule.eval') + return checkRaspExecutedAndHasThreat(agent, 'rasp-command_injection-rule-id-3') } - }) - } - function testBlockingAndSafeRequests () { - it('should block the threat', async () => { - await testBlockingRequest() + assert.fail('Request should be blocked') }) it('should not block safe request', async () => { @@ -50,17 +33,25 @@ describe('RASP - command_injection', () => { }) } - function testSafeInNonShell () { - it('should not block the threat', async () => { - await axios.get('/?dir=$(cat /etc/passwd 1>%262 ; echo .)') + function testNonShellBlockingAndSafeRequests () { + it('should block the threat', async () => { + try { + await axios.get('/?command=/usr/bin/reboot') + } catch (e) { + if (!e.response) { + throw e + } - return checkRaspNotExecutedAndNotThreat(agent) + return checkRaspExecutedAndHasThreat(agent, 'rasp-command_injection-rule-id-4') + } + + assert.fail('Request should be blocked') }) it('should not block safe request', async () => { - await axios.get('/?dir=.') + await axios.get('/?command=.') - return checkRaspNotExecutedAndNotThreat(agent) + return checkRaspExecutedAndNotThreat(agent) }) } @@ -116,7 +107,7 @@ describe('RASP - command_injection', () => { } }) - testBlockingAndSafeRequests() + testShellBlockingAndSafeRequests() }) describe('with promise', () => { @@ -137,7 +128,7 @@ describe('RASP - command_injection', () => { } }) - testBlockingAndSafeRequests() + testShellBlockingAndSafeRequests() }) describe('with event emitter', () => { @@ -158,7 +149,7 @@ describe('RASP - command_injection', () => { } }) - testBlockingAndSafeRequests() + testShellBlockingAndSafeRequests() }) describe('execSync', () => { @@ -178,7 +169,7 @@ describe('RASP - command_injection', () => { } }) - testBlockingAndSafeRequests() + testShellBlockingAndSafeRequests() }) }) @@ -199,7 +190,7 @@ describe('RASP - command_injection', () => { } }) - testBlockingAndSafeRequests() + testShellBlockingAndSafeRequests() }) describe('with promise', () => { @@ -220,7 +211,7 @@ describe('RASP - command_injection', () => { } }) - testBlockingAndSafeRequests() + testShellBlockingAndSafeRequests() }) describe('with event emitter', () => { @@ -241,7 +232,7 @@ describe('RASP - command_injection', () => { } }) - testBlockingAndSafeRequests() + testShellBlockingAndSafeRequests() }) describe('execFileSync', () => { @@ -261,7 +252,7 @@ describe('RASP - command_injection', () => { } }) - testBlockingAndSafeRequests() + testShellBlockingAndSafeRequests() }) }) @@ -271,7 +262,7 @@ describe('RASP - command_injection', () => { app = (req, res) => { const childProcess = require('child_process') - childProcess.execFile('ls', [req.query.dir], function (e) { + childProcess.execFile(req.query.command, function (e) { if (e?.name === 'DatadogRaspAbortError') { res.writeHead(500) } @@ -281,7 +272,7 @@ describe('RASP - command_injection', () => { } }) - testSafeInNonShell() + testNonShellBlockingAndSafeRequests() }) describe('with promise', () => { @@ -291,7 +282,7 @@ describe('RASP - command_injection', () => { const execFile = util.promisify(require('child_process').execFile) try { - await execFile('ls', [req.query.dir]) + await execFile([req.query.command]) } catch (e) { if (e.name === 'DatadogRaspAbortError') { res.writeHead(500) @@ -302,15 +293,14 @@ describe('RASP - command_injection', () => { } }) - testSafeInNonShell() + testNonShellBlockingAndSafeRequests() }) describe('with event emitter', () => { beforeEach(() => { app = (req, res) => { const childProcess = require('child_process') - - const child = childProcess.execFile('ls', [req.query.dir]) + const child = childProcess.execFile(req.query.command) child.on('error', (e) => { if (e.name === 'DatadogRaspAbortError') { res.writeHead(500) @@ -323,7 +313,7 @@ describe('RASP - command_injection', () => { } }) - testSafeInNonShell() + testNonShellBlockingAndSafeRequests() }) describe('execFileSync', () => { @@ -332,7 +322,7 @@ describe('RASP - command_injection', () => { const childProcess = require('child_process') try { - childProcess.execFileSync('ls', [req.query.dir]) + childProcess.execFileSync([req.query.command]) } catch (e) { if (e.name === 'DatadogRaspAbortError') { res.writeHead(500) @@ -343,7 +333,7 @@ describe('RASP - command_injection', () => { } }) - testSafeInNonShell() + testNonShellBlockingAndSafeRequests() }) }) }) @@ -368,7 +358,7 @@ describe('RASP - command_injection', () => { } }) - testBlockingAndSafeRequests() + testShellBlockingAndSafeRequests() }) describe('spawnSync', () => { @@ -385,7 +375,7 @@ describe('RASP - command_injection', () => { } }) - testBlockingAndSafeRequests() + testShellBlockingAndSafeRequests() }) }) @@ -395,7 +385,7 @@ describe('RASP - command_injection', () => { app = (req, res) => { const childProcess = require('child_process') - const child = childProcess.spawn('ls', [req.query.dir]) + const child = childProcess.spawn(req.query.command) child.on('error', (e) => { if (e.name === 'DatadogRaspAbortError') { res.writeHead(500) @@ -408,7 +398,7 @@ describe('RASP - command_injection', () => { } }) - testSafeInNonShell() + testNonShellBlockingAndSafeRequests() }) describe('spawnSync', () => { @@ -416,7 +406,7 @@ describe('RASP - command_injection', () => { app = (req, res) => { const childProcess = require('child_process') - const child = childProcess.spawnSync('ls', [req.query.dir]) + const child = childProcess.spawnSync(req.query.command) if (child.error?.name === 'DatadogRaspAbortError') { res.writeHead(500) } @@ -425,7 +415,7 @@ describe('RASP - command_injection', () => { } }) - testSafeInNonShell() + testNonShellBlockingAndSafeRequests() }) }) }) diff --git a/packages/dd-trace/test/appsec/rasp/command_injection.integration.spec.js b/packages/dd-trace/test/appsec/rasp/command_injection.integration.spec.js index 4ebb8c4910a..d6fe4015202 100644 --- a/packages/dd-trace/test/appsec/rasp/command_injection.integration.spec.js +++ b/packages/dd-trace/test/appsec/rasp/command_injection.integration.spec.js @@ -42,6 +42,7 @@ describe('RASP - command_injection - integration', () => { APP_PORT: appPort, DD_APPSEC_ENABLED: 'true', DD_APPSEC_RASP_ENABLED: 'true', + DD_TELEMETRY_HEARTBEAT_INTERVAL: 1, DD_APPSEC_RULES: path.join(cwd, 'resources', 'rasp_rules.json') } }) @@ -52,7 +53,7 @@ describe('RASP - command_injection - integration', () => { await agent.stop() }) - async function testRequestBlocked (url) { + async function testRequestBlocked (url, ruleId = 3, variant = 'shell') { try { await axios.get(url) } catch (e) { @@ -61,28 +62,72 @@ describe('RASP - command_injection - integration', () => { } assert.strictEqual(e.response.status, 403) - return await agent.assertMessageReceived(({ headers, payload }) => { + + let appsecTelemetryReceived = false + + const checkMessages = await agent.assertMessageReceived(({ headers, payload }) => { assert.property(payload[0][0].meta, '_dd.appsec.json') - assert.include(payload[0][0].meta['_dd.appsec.json'], '"rasp-command_injection-rule-id-3"') + assert.include(payload[0][0].meta['_dd.appsec.json'], `"rasp-command_injection-rule-id-${ruleId}"`) }) + + const checkTelemetry = await agent.assertTelemetryReceived(({ headers, payload }) => { + const namespace = payload.payload.namespace + + // Only check telemetry received in appsec namespace and ignore others + if (namespace === 'appsec') { + appsecTelemetryReceived = true + const series = payload.payload.series + const evalSerie = series.find(s => s.metric === 'rasp.rule.eval') + const matchSerie = series.find(s => s.metric === 'rasp.rule.match') + + assert.exists(evalSerie, 'eval serie should exist') + assert.include(evalSerie.tags, 'rule_type:command_injection') + assert.include(evalSerie.tags, `rule_variant:${variant}`) + assert.strictEqual(evalSerie.type, 'count') + + assert.exists(matchSerie, 'match serie should exist') + assert.include(matchSerie.tags, 'rule_type:command_injection') + assert.include(matchSerie.tags, `rule_variant:${variant}`) + assert.strictEqual(matchSerie.type, 'count') + } + }, 30_000, 'generate-metrics', 2) + + const checks = await Promise.all([checkMessages, checkTelemetry]) + assert.equal(appsecTelemetryReceived, true) + + return checks } throw new Error('Request should be blocked') } - it('should block using execFileSync and exception handled by express', async () => { - await testRequestBlocked('/shi/execFileSync?dir=$(cat /etc/passwd 1>%262 ; echo .)') - }) + describe('with shell', () => { + it('should block using execFileSync and exception handled by express', async () => { + await testRequestBlocked('/shi/execFileSync?dir=$(cat /etc/passwd 1>%262 ; echo .)') + }) - it('should block using execFileSync and unhandled exception', async () => { - await testRequestBlocked('/shi/execFileSync/out-of-express-scope?dir=$(cat /etc/passwd 1>%262 ; echo .)') - }) + it('should block using execFileSync and unhandled exception', async () => { + await testRequestBlocked('/shi/execFileSync/out-of-express-scope?dir=$(cat /etc/passwd 1>%262 ; echo .)') + }) + + it('should block using execSync and exception handled by express', async () => { + await testRequestBlocked('/shi/execSync?dir=$(cat /etc/passwd 1>%262 ; echo .)') + }) - it('should block using execSync and exception handled by express', async () => { - await testRequestBlocked('/shi/execSync?dir=$(cat /etc/passwd 1>%262 ; echo .)') + it('should block using execSync and unhandled exception', async () => { + await testRequestBlocked('/shi/execSync/out-of-express-scope?dir=$(cat /etc/passwd 1>%262 ; echo .)') + }) }) - it('should block using execSync and unhandled exception', async () => { - await testRequestBlocked('/shi/execSync/out-of-express-scope?dir=$(cat /etc/passwd 1>%262 ; echo .)') + describe('without shell', () => { + it('should block using execFileSync and exception handled by express', async () => { + await testRequestBlocked('/cmdi/execFileSync?command=cat /etc/passwd 1>&2 ; echo .', 4, 'exec') + }) + + it('should block using execFileSync and unhandled exception', async () => { + await testRequestBlocked( + '/cmdi/execFileSync/out-of-express-scope?command=cat /etc/passwd 1>&2 ; echo .', 4, 'exec' + ) + }) }) }) diff --git a/packages/dd-trace/test/appsec/rasp/command_injection.spec.js b/packages/dd-trace/test/appsec/rasp/command_injection.spec.js index 785b155a113..bf920940c7a 100644 --- a/packages/dd-trace/test/appsec/rasp/command_injection.spec.js +++ b/packages/dd-trace/test/appsec/rasp/command_injection.spec.js @@ -49,49 +49,6 @@ describe('RASP - command_injection.js', () => { }) describe('analyzeCommandInjection', () => { - it('should analyze command_injection without arguments', () => { - const ctx = { - file: 'cmd', - shell: true - } - const req = {} - datadogCore.storage.getStore.returns({ req }) - - start.publish(ctx) - - const persistent = { [addresses.SHELL_COMMAND]: 'cmd' } - sinon.assert.calledOnceWithExactly(waf.run, { persistent }, req, 'command_injection') - }) - - it('should analyze command_injection with arguments', () => { - const ctx = { - file: 'cmd', - fileArgs: ['arg0', 'arg1'], - shell: true - } - const req = {} - datadogCore.storage.getStore.returns({ req }) - - start.publish(ctx) - - const persistent = { [addresses.SHELL_COMMAND]: ['cmd', 'arg0', 'arg1'] } - sinon.assert.calledOnceWithExactly(waf.run, { persistent }, req, 'command_injection') - }) - - it('should not analyze command_injection when it is not shell', () => { - const ctx = { - file: 'cmd', - fileArgs: ['arg0', 'arg1'], - shell: false - } - const req = {} - datadogCore.storage.getStore.returns({ req }) - - start.publish(ctx) - - sinon.assert.notCalled(waf.run) - }) - it('should not analyze command_injection if rasp is disabled', () => { commandInjection.disable() const ctx = { @@ -139,18 +96,102 @@ describe('RASP - command_injection.js', () => { sinon.assert.notCalled(waf.run) }) - it('should call handleResult', () => { - const abortController = { abort: 'abort' } - const ctx = { file: 'cmd', abortController, shell: true } - const wafResult = { waf: 'waf' } - const req = { req: 'req' } - const res = { res: 'res' } - waf.run.returns(wafResult) - datadogCore.storage.getStore.returns({ req, res }) - - start.publish(ctx) + describe('command_injection with shell', () => { + it('should analyze command_injection without arguments', () => { + const ctx = { + file: 'cmd', + shell: true + } + const req = {} + datadogCore.storage.getStore.returns({ req }) + + start.publish(ctx) + + const persistent = { [addresses.SHELL_COMMAND]: 'cmd' } + sinon.assert.calledOnceWithExactly( + waf.run, { persistent }, req, { type: 'command_injection', variant: 'shell' } + ) + }) + + it('should analyze command_injection with arguments', () => { + const ctx = { + file: 'cmd', + fileArgs: ['arg0', 'arg1'], + shell: true + } + const req = {} + datadogCore.storage.getStore.returns({ req }) + + start.publish(ctx) + + const persistent = { [addresses.SHELL_COMMAND]: ['cmd', 'arg0', 'arg1'] } + sinon.assert.calledOnceWithExactly( + waf.run, { persistent }, req, { type: 'command_injection', variant: 'shell' } + ) + }) + + it('should call handleResult', () => { + const abortController = { abort: 'abort' } + const ctx = { file: 'cmd', abortController, shell: true } + const wafResult = { waf: 'waf' } + const req = { req: 'req' } + const res = { res: 'res' } + waf.run.returns(wafResult) + datadogCore.storage.getStore.returns({ req, res }) + + start.publish(ctx) + + sinon.assert.calledOnceWithExactly(utils.handleResult, wafResult, req, res, abortController, config) + }) + }) - sinon.assert.calledOnceWithExactly(utils.handleResult, wafResult, req, res, abortController, config) + describe('command_injection without shell', () => { + it('should analyze command injection without arguments', () => { + const ctx = { + file: 'ls', + shell: false + } + const req = {} + datadogCore.storage.getStore.returns({ req }) + + start.publish(ctx) + + const persistent = { [addresses.EXEC_COMMAND]: ['ls'] } + sinon.assert.calledOnceWithExactly( + waf.run, { persistent }, req, { type: 'command_injection', variant: 'exec' } + ) + }) + + it('should analyze command injection with arguments', () => { + const ctx = { + file: 'ls', + fileArgs: ['-la', '/tmp'], + shell: false + } + const req = {} + datadogCore.storage.getStore.returns({ req }) + + start.publish(ctx) + + const persistent = { [addresses.EXEC_COMMAND]: ['ls', '-la', '/tmp'] } + sinon.assert.calledOnceWithExactly( + waf.run, { persistent }, req, { type: 'command_injection', variant: 'exec' } + ) + }) + + it('should call handleResult', () => { + const abortController = { abort: 'abort' } + const ctx = { file: 'cmd', abortController, shell: false } + const wafResult = { waf: 'waf' } + const req = { req: 'req' } + const res = { res: 'res' } + waf.run.returns(wafResult) + datadogCore.storage.getStore.returns({ req, res }) + + start.publish(ctx) + + sinon.assert.calledOnceWithExactly(utils.handleResult, wafResult, req, res, abortController, config) + }) }) }) }) diff --git a/packages/dd-trace/test/appsec/rasp/lfi.spec.js b/packages/dd-trace/test/appsec/rasp/lfi.spec.js index 405311ae0d3..0a1328e2c52 100644 --- a/packages/dd-trace/test/appsec/rasp/lfi.spec.js +++ b/packages/dd-trace/test/appsec/rasp/lfi.spec.js @@ -111,7 +111,7 @@ describe('RASP - lfi.js', () => { fsOperationStart.publish(ctx) const persistent = { [FS_OPERATION_PATH]: path } - sinon.assert.calledOnceWithExactly(waf.run, { persistent }, req, 'lfi') + sinon.assert.calledOnceWithExactly(waf.run, { persistent }, req, { type: 'lfi' }) }) it('should NOT analyze lfi for child fs operations', () => { diff --git a/packages/dd-trace/test/appsec/rasp/resources/rasp_rules.json b/packages/dd-trace/test/appsec/rasp/resources/rasp_rules.json index daca47d8d20..c0396bd9871 100644 --- a/packages/dd-trace/test/appsec/rasp/resources/rasp_rules.json +++ b/packages/dd-trace/test/appsec/rasp/resources/rasp_rules.json @@ -110,7 +110,7 @@ }, { "id": "rasp-command_injection-rule-id-3", - "name": "Command injection exploit", + "name": "Shell command injection exploit", "tags": { "type": "command_injection", "category": "vulnerability_trigger", @@ -156,6 +156,55 @@ "block", "stack_trace" ] + }, + { + "id": "rasp-command_injection-rule-id-4", + "name": "OS command injection exploit", + "tags": { + "type": "command_injection", + "category": "vulnerability_trigger", + "cwe": "77", + "capec": "1000/152/248/88", + "confidence": "0", + "module": "rasp" + }, + "conditions": [ + { + "parameters": { + "resource": [ + { + "address": "server.sys.exec.cmd" + } + ], + "params": [ + { + "address": "server.request.query" + }, + { + "address": "server.request.body" + }, + { + "address": "server.request.path_params" + }, + { + "address": "grpc.server.request.message" + }, + { + "address": "graphql.server.all_resolvers" + }, + { + "address": "graphql.server.resolver" + } + ] + }, + "operator": "cmdi_detector" + } + ], + "transformers": [], + "on_match": [ + "block", + "stack_trace" + ] } ] } diff --git a/packages/dd-trace/test/appsec/rasp/resources/shi-app/index.js b/packages/dd-trace/test/appsec/rasp/resources/shi-app/index.js index a6714bd2148..133c57dfb2b 100644 --- a/packages/dd-trace/test/appsec/rasp/resources/shi-app/index.js +++ b/packages/dd-trace/test/appsec/rasp/resources/shi-app/index.js @@ -39,6 +39,20 @@ app.get('/shi/execSync/out-of-express-scope', async (req, res) => { }) }) +app.get('/cmdi/execFileSync', async (req, res) => { + childProcess.execFileSync('sh', ['-c', req.query.command]) + + res.end('OK') +}) + +app.get('/cmdi/execFileSync/out-of-express-scope', async (req, res) => { + process.nextTick(() => { + childProcess.execFileSync('sh', ['-c', req.query.command]) + + res.end('OK') + }) +}) + app.listen(port, () => { process.send({ port }) }) diff --git a/packages/dd-trace/test/appsec/rasp/sql_injection.pg.plugin.spec.js b/packages/dd-trace/test/appsec/rasp/sql_injection.pg.plugin.spec.js index 8f05158c22d..2d4dd779c17 100644 --- a/packages/dd-trace/test/appsec/rasp/sql_injection.pg.plugin.spec.js +++ b/packages/dd-trace/test/appsec/rasp/sql_injection.pg.plugin.spec.js @@ -219,7 +219,7 @@ describe('RASP - sql_injection', () => { await axios.get('/') - assert.equal(run.args.filter(arg => arg[1] === 'sql_injection').length, 1) + assert.equal(run.args.filter(arg => arg[1]?.type === 'sql_injection').length, 1) }) it('should call to waf twice for sql injection with two different queries in pg Pool', async () => { @@ -232,7 +232,7 @@ describe('RASP - sql_injection', () => { await axios.get('/') - assert.equal(run.args.filter(arg => arg[1] === 'sql_injection').length, 2) + assert.equal(run.args.filter(arg => arg[1]?.type === 'sql_injection').length, 2) }) it('should call to waf twice for sql injection and same query when input address is updated', async () => { @@ -254,7 +254,7 @@ describe('RASP - sql_injection', () => { await axios.get('/') - assert.equal(run.args.filter(arg => arg[1] === 'sql_injection').length, 2) + assert.equal(run.args.filter(arg => arg[1]?.type === 'sql_injection').length, 2) }) it('should call to waf once for sql injection and same query when input address is updated', async () => { @@ -276,7 +276,7 @@ describe('RASP - sql_injection', () => { await axios.get('/') - assert.equal(run.args.filter(arg => arg[1] === 'sql_injection').length, 1) + assert.equal(run.args.filter(arg => arg[1]?.type === 'sql_injection').length, 1) }) }) }) diff --git a/packages/dd-trace/test/appsec/rasp/sql_injection.spec.js b/packages/dd-trace/test/appsec/rasp/sql_injection.spec.js index d713521e986..fe7c9af082d 100644 --- a/packages/dd-trace/test/appsec/rasp/sql_injection.spec.js +++ b/packages/dd-trace/test/appsec/rasp/sql_injection.spec.js @@ -57,7 +57,7 @@ describe('RASP - sql_injection', () => { [addresses.DB_STATEMENT]: 'SELECT 1', [addresses.DB_SYSTEM]: 'postgresql' } - sinon.assert.calledOnceWithExactly(waf.run, { persistent }, req, 'sql_injection') + sinon.assert.calledOnceWithExactly(waf.run, { persistent }, req, { type: 'sql_injection' }) }) it('should not analyze sql injection if rasp is disabled', () => { @@ -128,7 +128,7 @@ describe('RASP - sql_injection', () => { [addresses.DB_STATEMENT]: 'SELECT 1', [addresses.DB_SYSTEM]: 'mysql' } - sinon.assert.calledOnceWithExactly(waf.run, { persistent }, req, 'sql_injection') + sinon.assert.calledOnceWithExactly(waf.run, { persistent }, req, { type: 'sql_injection' }) }) it('should not analyze sql injection if rasp is disabled', () => { diff --git a/packages/dd-trace/test/appsec/rasp/ssrf.spec.js b/packages/dd-trace/test/appsec/rasp/ssrf.spec.js index c40867ea254..98d5c8a0104 100644 --- a/packages/dd-trace/test/appsec/rasp/ssrf.spec.js +++ b/packages/dd-trace/test/appsec/rasp/ssrf.spec.js @@ -54,7 +54,7 @@ describe('RASP - ssrf.js', () => { httpClientRequestStart.publish(ctx) const persistent = { [addresses.HTTP_OUTGOING_URL]: 'http://example.com' } - sinon.assert.calledOnceWithExactly(waf.run, { persistent }, req, 'ssrf') + sinon.assert.calledOnceWithExactly(waf.run, { persistent }, req, { type: 'ssrf' }) }) it('should not analyze ssrf if rasp is disabled', () => { diff --git a/packages/dd-trace/test/appsec/remote_config/index.spec.js b/packages/dd-trace/test/appsec/remote_config/index.spec.js index f3cc6a32dac..4d296d100d1 100644 --- a/packages/dd-trace/test/appsec/remote_config/index.spec.js +++ b/packages/dd-trace/test/appsec/remote_config/index.spec.js @@ -244,6 +244,8 @@ describe('Remote Config index', () => { .to.have.been.calledWithExactly(RemoteConfigCapabilities.ASM_RASP_LFI, true) expect(rc.updateCapabilities) .to.have.been.calledWithExactly(RemoteConfigCapabilities.ASM_RASP_SHI, true) + expect(rc.updateCapabilities) + .to.have.been.calledWithExactly(RemoteConfigCapabilities.ASM_RASP_CMDI, true) expect(rc.setProductHandler).to.have.been.calledWith('ASM_DATA') expect(rc.setProductHandler).to.have.been.calledWith('ASM_DD') @@ -288,6 +290,8 @@ describe('Remote Config index', () => { .to.have.been.calledWithExactly(RemoteConfigCapabilities.ASM_RASP_LFI, true) expect(rc.updateCapabilities) .to.have.been.calledWithExactly(RemoteConfigCapabilities.ASM_RASP_SHI, true) + expect(rc.updateCapabilities) + .to.have.been.calledWithExactly(RemoteConfigCapabilities.ASM_RASP_CMDI, true) expect(rc.setProductHandler).to.have.been.calledWith('ASM_DATA') expect(rc.setProductHandler).to.have.been.calledWith('ASM_DD') @@ -334,6 +338,8 @@ describe('Remote Config index', () => { .to.have.been.calledWithExactly(RemoteConfigCapabilities.ASM_RASP_LFI, true) expect(rc.updateCapabilities) .to.have.been.calledWithExactly(RemoteConfigCapabilities.ASM_RASP_SHI, true) + expect(rc.updateCapabilities) + .to.have.been.calledWithExactly(RemoteConfigCapabilities.ASM_RASP_CMDI, true) }) it('should not activate rasp capabilities if rasp is disabled', () => { @@ -375,6 +381,8 @@ describe('Remote Config index', () => { .to.not.have.been.calledWithExactly(RemoteConfigCapabilities.ASM_RASP_LFI) expect(rc.updateCapabilities) .to.not.have.been.calledWithExactly(RemoteConfigCapabilities.ASM_RASP_SHI) + expect(rc.updateCapabilities) + .to.not.have.been.calledWithExactly(RemoteConfigCapabilities.ASM_RASP_CMDI) }) }) @@ -416,6 +424,8 @@ describe('Remote Config index', () => { .to.have.been.calledWithExactly(RemoteConfigCapabilities.ASM_RASP_LFI, false) expect(rc.updateCapabilities) .to.have.been.calledWithExactly(RemoteConfigCapabilities.ASM_RASP_SHI, false) + expect(rc.updateCapabilities) + .to.have.been.calledWithExactly(RemoteConfigCapabilities.ASM_RASP_CMDI, false) expect(rc.removeProductHandler).to.have.been.calledWith('ASM_DATA') expect(rc.removeProductHandler).to.have.been.calledWith('ASM_DD') diff --git a/packages/dd-trace/test/appsec/reporter.spec.js b/packages/dd-trace/test/appsec/reporter.spec.js index cd7cc9a1581..a38092e728d 100644 --- a/packages/dd-trace/test/appsec/reporter.spec.js +++ b/packages/dd-trace/test/appsec/reporter.spec.js @@ -192,13 +192,15 @@ describe('reporter', () => { expect(telemetry.updateRaspRequestsMetricTags).to.not.have.been.called }) - it('should call updateRaspRequestsMetricTags when ruleType if provided', () => { + it('should call updateRaspRequestsMetricTags when raspRule is provided', () => { const metrics = { rulesVersion: '1.2.3' } const store = storage.getStore() - Reporter.reportMetrics(metrics, 'rule_type') + const raspRule = { type: 'rule_type', variant: 'rule_variant' } - expect(telemetry.updateRaspRequestsMetricTags).to.have.been.calledOnceWithExactly(metrics, store.req, 'rule_type') + Reporter.reportMetrics(metrics, raspRule) + + expect(telemetry.updateRaspRequestsMetricTags).to.have.been.calledOnceWithExactly(metrics, store.req, raspRule) expect(telemetry.updateWafRequestsMetricTags).to.not.have.been.called }) }) From 4e2e71663af1e4026968de7f97bb669b6a5dc1ab Mon Sep 17 00:00:00 2001 From: Attila Szegedi Date: Thu, 19 Dec 2024 18:04:19 +0100 Subject: [PATCH 28/36] Add filesystem events to the timeline (#4965) --- integration-tests/profiler/fstest.js | 40 ++++ integration-tests/profiler/profiler.spec.js | 175 ++++++++++++++---- packages/datadog-instrumentations/src/fs.js | 3 + .../profilers/event_plugins/event.js | 10 +- .../profiling/profilers/event_plugins/fs.js | 49 +++++ .../src/profiling/profilers/events.js | 25 ++- 6 files changed, 263 insertions(+), 39 deletions(-) create mode 100644 integration-tests/profiler/fstest.js create mode 100644 packages/dd-trace/src/profiling/profilers/event_plugins/fs.js diff --git a/integration-tests/profiler/fstest.js b/integration-tests/profiler/fstest.js new file mode 100644 index 00000000000..c65887c102e --- /dev/null +++ b/integration-tests/profiler/fstest.js @@ -0,0 +1,40 @@ +const fs = require('fs') +const os = require('os') +const path = require('path') + +const tracer = require('dd-trace').init() +tracer.profilerStarted().then(() => { + tracer.trace('x', (_, done) => { + setImmediate(() => { + // Generate 1MB of random data + const buffer = Buffer.alloc(1024 * 1024) + for (let i = 0; i < buffer.length; i++) { + buffer[i] = Math.floor(Math.random() * 256) + } + + // Create a temporary file + const tempFilePath = path.join(os.tmpdir(), 'tempfile.txt') + + fs.writeFile(tempFilePath, buffer, (err) => { + if (err) throw err + + // Read the data back + setImmediate(() => { + fs.readFile(tempFilePath, (err, readData) => { + setImmediate(() => { + // Delete the temporary file + fs.unlink(tempFilePath, (err) => { + if (err) throw err + }) + done() + }) + if (err) throw err + if (Buffer.compare(buffer, readData) !== 0) { + throw new Error('Data read from file is different from data written to file') + } + }) + }) + }) + }) + }) +}) diff --git a/integration-tests/profiler/profiler.spec.js b/integration-tests/profiler/profiler.spec.js index 80be4c8fd36..6c7f4942e1e 100644 --- a/integration-tests/profiler/profiler.spec.js +++ b/integration-tests/profiler/profiler.spec.js @@ -104,7 +104,108 @@ function expectTimeout (messagePromise, allowErrors = false) { ) } +class TimelineEventProcessor { + constructor (strings, encoded) { + this.strings = strings + this.encoded = encoded + } +} + +class NetworkEventProcessor extends TimelineEventProcessor { + constructor (strings, encoded) { + super(strings, encoded) + + this.hostKey = strings.dedup('host') + this.addressKey = strings.dedup('address') + this.portKey = strings.dedup('port') + } + + processLabel (label, processedLabels) { + switch (label.key) { + case this.hostKey: + processedLabels.host = label.str + return true + case this.addressKey: + processedLabels.address = label.str + return true + case this.portKey: + processedLabels.port = label.num + return true + default: + return false + } + } + + decorateEvent (ev, pl) { + // Exactly one of these is defined + assert.isTrue(!!pl.address !== !!pl.host, this.encoded) + if (pl.address) { + ev.address = this.strings.strings[pl.address] + } else { + ev.host = this.strings.strings[pl.host] + } + if (pl.port) { + ev.port = pl.port + } + } +} + async function gatherNetworkTimelineEvents (cwd, scriptFilePath, eventType, args) { + return gatherTimelineEvents(cwd, scriptFilePath, eventType, args, NetworkEventProcessor) +} + +class FilesystemEventProcessor extends TimelineEventProcessor { + constructor (strings, encoded) { + super(strings, encoded) + + this.fdKey = strings.dedup('fd') + this.fileKey = strings.dedup('file') + this.flagKey = strings.dedup('flag') + this.modeKey = strings.dedup('mode') + this.pathKey = strings.dedup('path') + } + + processLabel (label, processedLabels) { + switch (label.key) { + case this.fdKey: + processedLabels.fd = label.num + return true + case this.fileKey: + processedLabels.file = label.str + return true + case this.flagKey: + processedLabels.flag = label.str + return true + case this.modeKey: + processedLabels.mode = label.str + return true + case this.pathKey: + processedLabels.path = label.str + return true + default: + return false + } + } + + decorateEvent (ev, pl) { + ev.fd = pl.fd + ev.file = this.strings.strings[pl.file] + ev.flag = this.strings.strings[pl.flag] + ev.mode = this.strings.strings[pl.mode] + ev.path = this.strings.strings[pl.path] + for (const [k, v] of Object.entries(ev)) { + if (v === undefined) { + delete ev[k] + } + } + } +} + +async function gatherFilesystemTimelineEvents (cwd, scriptFilePath) { + return gatherTimelineEvents(cwd, scriptFilePath, 'fs', [], FilesystemEventProcessor) +} + +async function gatherTimelineEvents (cwd, scriptFilePath, eventType, args, Processor) { const procStart = BigInt(Date.now() * 1000000) const proc = fork(path.join(cwd, scriptFilePath), args, { cwd, @@ -123,36 +224,35 @@ async function gatherNetworkTimelineEvents (cwd, scriptFilePath, eventType, args const strings = profile.stringTable const tsKey = strings.dedup('end_timestamp_ns') const eventKey = strings.dedup('event') - const hostKey = strings.dedup('host') - const addressKey = strings.dedup('address') - const portKey = strings.dedup('port') - const nameKey = strings.dedup('operation') + const operationKey = strings.dedup('operation') const spanIdKey = strings.dedup('span id') const localRootSpanIdKey = strings.dedup('local root span id') const eventValue = strings.dedup(eventType) const events = [] + const processor = new Processor(strings, encoded) for (const sample of profile.sample) { - let ts, event, host, address, port, name, spanId, localRootSpanId + let ts, event, operation, spanId, localRootSpanId + const processedLabels = {} const unexpectedLabels = [] for (const label of sample.label) { switch (label.key) { case tsKey: ts = label.num; break - case nameKey: name = label.str; break + case operationKey: operation = label.str; break case eventKey: event = label.str; break - case hostKey: host = label.str; break - case addressKey: address = label.str; break - case portKey: port = label.num; break case spanIdKey: spanId = label.str; break case localRootSpanIdKey: localRootSpanId = label.str; break - default: unexpectedLabels.push(label.key) + default: + if (!processor.processLabel(label, processedLabels)) { + unexpectedLabels.push(label.key) + } } } - // Gather only DNS events; ignore sporadic GC events + // Timestamp must be defined and be between process start and end time + assert.isDefined(ts, encoded) + assert.isTrue(ts <= procEnd, encoded) + assert.isTrue(ts >= procStart, encoded) + // Gather only tested events if (event === eventValue) { - // Timestamp must be defined and be between process start and end time - assert.isDefined(ts, encoded) - assert.isTrue(ts <= procEnd, encoded) - assert.isTrue(ts >= procStart, encoded) if (process.platform !== 'win32') { assert.isDefined(spanId, encoded) assert.isDefined(localRootSpanId, encoded) @@ -160,23 +260,14 @@ async function gatherNetworkTimelineEvents (cwd, scriptFilePath, eventType, args assert.isUndefined(spanId, encoded) assert.isUndefined(localRootSpanId, encoded) } - assert.isDefined(name, encoded) + assert.isDefined(operation, encoded) if (unexpectedLabels.length > 0) { const labelsStr = JSON.stringify(unexpectedLabels) const labelsStrStr = unexpectedLabels.map(k => strings.strings[k]).join(',') assert.fail(`Unexpected labels: ${labelsStr}\n${labelsStrStr}\n${encoded}`) } - // Exactly one of these is defined - assert.isTrue(!!address !== !!host, encoded) - const ev = { name: strings.strings[name] } - if (address) { - ev.address = strings.strings[address] - } else { - ev.host = strings.strings[host] - } - if (port) { - ev.port = port - } + const ev = { operation: strings.strings[operation] } + processor.decorateEvent(ev, processedLabels) events.push(ev) } } @@ -323,14 +414,30 @@ describe('profiler', () => { assert.equal(endpoints.size, 3, encoded) }) + it('fs timeline events work', async () => { + const fsEvents = await gatherFilesystemTimelineEvents(cwd, 'profiler/fstest.js') + assert.equal(fsEvents.length, 6) + const path = fsEvents[0].path + const fd = fsEvents[1].fd + assert(path.endsWith('tempfile.txt')) + assert.sameDeepMembers(fsEvents, [ + { flag: 'w', mode: '', operation: 'open', path }, + { fd, operation: 'write' }, + { fd, operation: 'close' }, + { file: path, operation: 'writeFile' }, + { operation: 'readFile', path }, + { operation: 'unlink', path } + ]) + }) + it('dns timeline events work', async () => { const dnsEvents = await gatherNetworkTimelineEvents(cwd, 'profiler/dnstest.js', 'dns') assert.sameDeepMembers(dnsEvents, [ - { name: 'lookup', host: 'example.org' }, - { name: 'lookup', host: 'example.com' }, - { name: 'lookup', host: 'datadoghq.com' }, - { name: 'queryA', host: 'datadoghq.com' }, - { name: 'lookupService', address: '13.224.103.60', port: 80 } + { operation: 'lookup', host: 'example.org' }, + { operation: 'lookup', host: 'example.com' }, + { operation: 'lookup', host: 'datadoghq.com' }, + { operation: 'queryA', host: 'datadoghq.com' }, + { operation: 'lookupService', address: '13.224.103.60', port: 80 } ]) }) @@ -366,8 +473,8 @@ describe('profiler', () => { // The profiled program should have two TCP connection events to the two // servers. assert.sameDeepMembers(events, [ - { name: 'connect', host: '127.0.0.1', port: port1 }, - { name: 'connect', host: '127.0.0.1', port: port2 } + { operation: 'connect', host: '127.0.0.1', port: port1 }, + { operation: 'connect', host: '127.0.0.1', port: port2 } ]) } finally { server2.close() diff --git a/packages/datadog-instrumentations/src/fs.js b/packages/datadog-instrumentations/src/fs.js index 9ae201b9860..894c1b6ef33 100644 --- a/packages/datadog-instrumentations/src/fs.js +++ b/packages/datadog-instrumentations/src/fs.js @@ -13,6 +13,9 @@ const errorChannel = channel('apm:fs:operation:error') const ddFhSym = Symbol('ddFileHandle') let kHandle, kDirReadPromisified, kDirClosePromisified +// Update packages/dd-trace/src/profiling/profilers/event_plugins/fs.js if you make changes to param names in any of +// the following objects. + const paramsByMethod = { access: ['path', 'mode'], appendFile: ['path', 'data', 'options'], diff --git a/packages/dd-trace/src/profiling/profilers/event_plugins/event.js b/packages/dd-trace/src/profiling/profilers/event_plugins/event.js index 48e430ba607..eace600a9aa 100644 --- a/packages/dd-trace/src/profiling/profilers/event_plugins/event.js +++ b/packages/dd-trace/src/profiling/profilers/event_plugins/event.js @@ -32,11 +32,11 @@ class EventPlugin extends TracingPlugin { if (!store) return const { startEvent, startTime, error } = store - if (error) { - return // don't emit perf events for failed operations + if (error || this.ignoreEvent(startEvent)) { + return // don't emit perf events for failed operations or ignored events } - const duration = performance.now() - startTime + const duration = performance.now() - startTime const event = { entryType: this.entryType, startTime, @@ -53,6 +53,10 @@ class EventPlugin extends TracingPlugin { this.eventHandler(this.extendEvent(event, startEvent)) } + + ignoreEvent () { + return false + } } module.exports = EventPlugin diff --git a/packages/dd-trace/src/profiling/profilers/event_plugins/fs.js b/packages/dd-trace/src/profiling/profilers/event_plugins/fs.js new file mode 100644 index 00000000000..34eb7b52353 --- /dev/null +++ b/packages/dd-trace/src/profiling/profilers/event_plugins/fs.js @@ -0,0 +1,49 @@ +const EventPlugin = require('./event') + +// Values taken from parameter names in datadog-instrumentations/src/fs.js. +// Known param names that are disallowed because they can be strings and have arbitrary sizes: +// 'data' +// Known param names that are disallowed because they are never a string or number: +// 'buffer', 'buffers', 'listener' +const allowedParams = new Set([ + 'atime', 'dest', + 'existingPath', 'fd', 'file', + 'flag', 'gid', 'len', + 'length', 'mode', 'mtime', + 'newPath', 'offset', 'oldPath', + 'operation', 'options', 'path', + 'position', 'prefix', 'src', + 'target', 'type', 'uid' +]) + +class FilesystemPlugin extends EventPlugin { + static get id () { + return 'fs' + } + + static get operation () { + return 'operation' + } + + static get entryType () { + return 'fs' + } + + ignoreEvent (event) { + // Don't care about sync events, they show up in the event loop samples anyway + return event.operation?.endsWith('Sync') + } + + extendEvent (event, detail) { + const d = { ...detail } + Object.entries(d).forEach(([k, v]) => { + if (!(allowedParams.has(k) && (typeof v === 'string' || typeof v === 'number'))) { + delete d[k] + } + }) + event.detail = d + + return event + } +} +module.exports = FilesystemPlugin diff --git a/packages/dd-trace/src/profiling/profilers/events.js b/packages/dd-trace/src/profiling/profilers/events.js index 2200eaadd2e..8ff1748ceda 100644 --- a/packages/dd-trace/src/profiling/profilers/events.js +++ b/packages/dd-trace/src/profiling/profilers/events.js @@ -133,11 +133,32 @@ class NetDecorator { } } +class FilesystemDecorator { + constructor (stringTable) { + this.stringTable = stringTable + } + + decorateSample (sampleInput, item) { + const labels = sampleInput.label + const stringTable = this.stringTable + Object.entries(item.detail).forEach(([k, v]) => { + switch (typeof v) { + case 'string': + labels.push(labelFromStrStr(stringTable, k, v)) + break + case 'number': + labels.push(new Label({ key: stringTable.dedup(k), num: v })) + } + }) + } +} + // Keys correspond to PerformanceEntry.entryType, values are constructor // functions for type-specific decorators. const decoratorTypes = { - gc: GCDecorator, + fs: FilesystemDecorator, dns: DNSDecorator, + gc: GCDecorator, net: NetDecorator } @@ -255,7 +276,7 @@ class NodeApiEventSource { class DatadogInstrumentationEventSource { constructor (eventHandler, eventFilter) { - this.plugins = ['dns_lookup', 'dns_lookupservice', 'dns_resolve', 'dns_reverse', 'net'].map(m => { + this.plugins = ['dns_lookup', 'dns_lookupservice', 'dns_resolve', 'dns_reverse', 'fs', 'net'].map(m => { const Plugin = require(`./event_plugins/${m}`) return new Plugin(eventHandler, eventFilter) }) From 4f87373f4b64f8cf6521a2a0a6e4485cd3b0ceab Mon Sep 17 00:00:00 2001 From: Roch Devost Date: Thu, 19 Dec 2024 16:21:42 -0500 Subject: [PATCH 29/36] fix invalid output for log.trace (#5047) * fix invalid output for log.trace * move string formatting to logger and improve output * change depth --- packages/dd-trace/src/log/index.js | 14 ++++++++++++-- packages/dd-trace/src/log/writer.js | 7 ++++--- packages/dd-trace/src/opentracing/span.js | 2 +- packages/dd-trace/test/log.spec.js | 18 +++++++++++------- 4 files changed, 28 insertions(+), 13 deletions(-) diff --git a/packages/dd-trace/src/log/index.js b/packages/dd-trace/src/log/index.js index 213b6ccc8e6..3fb9afff6fa 100644 --- a/packages/dd-trace/src/log/index.js +++ b/packages/dd-trace/src/log/index.js @@ -1,6 +1,7 @@ 'use strict' const coalesce = require('koalas') +const { inspect } = require('util') const { isTrue } = require('../util') const { traceChannel, debugChannel, infoChannel, warnChannel, errorChannel } = require('./channels') const logWriter = require('./writer') @@ -59,9 +60,18 @@ const log = { trace (...args) { if (traceChannel.hasSubscribers) { const logRecord = {} + Error.captureStackTrace(logRecord, this.trace) - const stack = logRecord.stack.split('\n')[1].replace(/^\s+at ([^\s]) .+/, '$1') - traceChannel.publish(Log.parse('Trace', args, { stack })) + + const fn = logRecord.stack.split('\n')[1].replace(/^\s+at ([^\s]+) .+/, '$1') + const params = args.map(a => { + return a && a.hasOwnProperty('toString') && typeof a.toString === 'function' + ? a.toString() + : inspect(a, { depth: 3, breakLength: Infinity, compact: true }) + }).join(', ') + const formatted = logRecord.stack.replace('Error: ', `Trace: ${fn}(${params})`) + + traceChannel.publish(Log.parse(formatted)) } return this }, diff --git a/packages/dd-trace/src/log/writer.js b/packages/dd-trace/src/log/writer.js index a721f7f9e35..322c703b2b3 100644 --- a/packages/dd-trace/src/log/writer.js +++ b/packages/dd-trace/src/log/writer.js @@ -4,7 +4,6 @@ const { storage } = require('../../../datadog-core') const { LogChannel } = require('./channels') const { Log } = require('./log') const defaultLogger = { - trace: msg => console.trace(msg), /* eslint-disable-line no-console */ debug: msg => console.debug(msg), /* eslint-disable-line no-console */ info: msg => console.info(msg), /* eslint-disable-line no-console */ warn: msg => console.warn(msg), /* eslint-disable-line no-console */ @@ -91,8 +90,10 @@ function onDebug (log) { function onTrace (log) { const { formatted, cause } = getErrorLog(log) - if (formatted) withNoop(() => logger.trace(formatted)) - if (cause) withNoop(() => logger.trace(cause)) + // Using logger.debug() because not all loggers have trace level, + // and console.trace() has a completely different meaning. + if (formatted) withNoop(() => logger.debug(formatted)) + if (cause) withNoop(() => logger.debug(cause)) } function error (...args) { diff --git a/packages/dd-trace/src/opentracing/span.js b/packages/dd-trace/src/opentracing/span.js index 00fd51da027..23f885bbabd 100644 --- a/packages/dd-trace/src/opentracing/span.js +++ b/packages/dd-trace/src/opentracing/span.js @@ -107,7 +107,7 @@ class DatadogSpan { toString () { const spanContext = this.context() - const resourceName = spanContext._tags['resource.name'] + const resourceName = spanContext._tags['resource.name'] || '' const resource = resourceName.length > 100 ? `${resourceName.substring(0, 97)}...` : resourceName diff --git a/packages/dd-trace/test/log.spec.js b/packages/dd-trace/test/log.spec.js index ac2feea9f7a..16682f97db8 100644 --- a/packages/dd-trace/test/log.spec.js +++ b/packages/dd-trace/test/log.spec.js @@ -86,7 +86,6 @@ describe('log', () => { sinon.stub(console, 'error') sinon.stub(console, 'warn') sinon.stub(console, 'debug') - sinon.stub(console, 'trace') error = new Error() @@ -105,7 +104,6 @@ describe('log', () => { console.error.restore() console.warn.restore() console.debug.restore() - console.trace.restore() }) it('should support chaining', () => { @@ -145,14 +143,20 @@ describe('log', () => { it('should not log to console by default', () => { log.trace('trace') - expect(console.trace).to.not.have.been.called + expect(console.debug).to.not.have.been.called }) - it('should log to console after setting log level to trace', () => { + it('should log to console after setting log level to trace', function foo () { log.toggle(true, 'trace') - log.trace('argument') - - expect(console.trace).to.have.been.calledTwice + log.trace('argument', { hello: 'world' }, { + toString: () => 'string' + }, { foo: 'bar' }) + + expect(console.debug).to.have.been.calledOnce + expect(console.debug.firstCall.args[0]).to.match( + /^Trace: Test.foo\('argument', { hello: 'world' }, string, { foo: 'bar' }\)/ + ) + expect(console.debug.firstCall.args[0].split('\n').length).to.be.gte(3) }) }) From 3798033ebafff5a41ea9edb4a1724e5a64f7cab3 Mon Sep 17 00:00:00 2001 From: Thomas Watson Date: Fri, 20 Dec 2024 12:53:47 +0100 Subject: [PATCH 30/36] [DI] Attach ddtags to probe results (#5042) The following extra information is added to each probe result: - `env` - From the `DD_ENV` environment variable - `version` - From the `DD_VERSION` environment variable - `debugger_version` - The version of the tracing lib - `host_name` - The hostname that application is running on The `agent_version` is not added in this commit, but will be come later. --- integration-tests/debugger/basic.spec.js | 6 +- integration-tests/debugger/ddtags.spec.js | 56 +++++++++++++++++++ integration-tests/debugger/utils.js | 26 +++++---- integration-tests/helpers/fake-agent.js | 1 + .../src/debugger/devtools_client/send.js | 5 ++ 5 files changed, 79 insertions(+), 15 deletions(-) create mode 100644 integration-tests/debugger/ddtags.spec.js diff --git a/integration-tests/debugger/basic.spec.js b/integration-tests/debugger/basic.spec.js index 4bb5d7b2fa6..aa6a1881d33 100644 --- a/integration-tests/debugger/basic.spec.js +++ b/integration-tests/debugger/basic.spec.js @@ -428,7 +428,7 @@ describe('Dynamic Instrumentation', function () { }) describe('DD_TRACING_ENABLED=true, DD_TRACE_128_BIT_TRACEID_GENERATION_ENABLED=true', function () { - const t = setup({ DD_TRACING_ENABLED: true, DD_TRACE_128_BIT_TRACEID_GENERATION_ENABLED: true }) + const t = setup({ env: { DD_TRACING_ENABLED: true, DD_TRACE_128_BIT_TRACEID_GENERATION_ENABLED: true } }) describe('input messages', function () { it( @@ -439,7 +439,7 @@ describe('Dynamic Instrumentation', function () { }) describe('DD_TRACING_ENABLED=true, DD_TRACE_128_BIT_TRACEID_GENERATION_ENABLED=false', function () { - const t = setup({ DD_TRACING_ENABLED: true, DD_TRACE_128_BIT_TRACEID_GENERATION_ENABLED: false }) + const t = setup({ env: { DD_TRACING_ENABLED: true, DD_TRACE_128_BIT_TRACEID_GENERATION_ENABLED: false } }) describe('input messages', function () { it( @@ -450,7 +450,7 @@ describe('Dynamic Instrumentation', function () { }) describe('DD_TRACING_ENABLED=false', function () { - const t = setup({ DD_TRACING_ENABLED: false }) + const t = setup({ env: { DD_TRACING_ENABLED: false } }) describe('input messages', function () { it( diff --git a/integration-tests/debugger/ddtags.spec.js b/integration-tests/debugger/ddtags.spec.js new file mode 100644 index 00000000000..5f864d71123 --- /dev/null +++ b/integration-tests/debugger/ddtags.spec.js @@ -0,0 +1,56 @@ +'use strict' + +const os = require('os') + +const { assert } = require('chai') +const { setup } = require('./utils') +const { version } = require('../../package.json') + +describe('Dynamic Instrumentation', function () { + describe('ddtags', function () { + const t = setup({ + env: { + DD_ENV: 'test-env', + DD_VERSION: 'test-version', + DD_GIT_COMMIT_SHA: 'test-commit-sha', + DD_GIT_REPOSITORY_URL: 'test-repository-url' + }, + testApp: 'target-app/basic.js' + }) + + it('should add the expected ddtags as a query param to /debugger/v1/input', function (done) { + t.triggerBreakpoint() + + t.agent.on('debugger-input', ({ query }) => { + assert.property(query, 'ddtags') + + // Before: "a:b,c:d" + // After: { a: 'b', c: 'd' } + const ddtags = query.ddtags + .split(',') + .map((tag) => tag.split(':')) + .reduce((acc, [k, v]) => { acc[k] = v; return acc }, {}) + + assert.hasAllKeys(ddtags, [ + 'env', + 'version', + 'debugger_version', + 'host_name', + 'git.commit.sha', + 'git.repository_url' + ]) + + assert.strictEqual(ddtags.env, 'test-env') + assert.strictEqual(ddtags.version, 'test-version') + assert.strictEqual(ddtags.debugger_version, version) + assert.strictEqual(ddtags.host_name, os.hostname()) + assert.strictEqual(ddtags['git.commit.sha'], 'test-commit-sha') + assert.strictEqual(ddtags['git.repository_url'], 'test-repository-url') + + done() + }) + + t.agent.addRemoteConfig(t.rcConfig) + }) + }) +}) diff --git a/integration-tests/debugger/utils.js b/integration-tests/debugger/utils.js index b260e5eabe5..4f215723816 100644 --- a/integration-tests/debugger/utils.js +++ b/integration-tests/debugger/utils.js @@ -18,9 +18,9 @@ module.exports = { setup } -function setup (env) { +function setup ({ env, testApp } = {}) { let sandbox, cwd, appPort - const breakpoints = getBreakpointInfo(1) // `1` to disregard the `setup` function + const breakpoints = getBreakpointInfo({ file: testApp, stackIndex: 1 }) // `1` to disregard the `setup` function const t = { breakpoint: breakpoints[0], breakpoints, @@ -108,16 +108,18 @@ function setup (env) { return t } -function getBreakpointInfo (stackIndex = 0) { - // First, get the filename of file that called this function - const testFile = new Error().stack - .split('\n')[stackIndex + 2] // +2 to skip this function + the first line, which is the error message - .split(' (')[1] - .slice(0, -1) - .split(':')[0] - - // Then, find the corresponding file in which the breakpoint(s) exists - const file = join('target-app', basename(testFile).replace('.spec', '')) +function getBreakpointInfo ({ file, stackIndex = 0 }) { + if (!file) { + // First, get the filename of file that called this function + const testFile = new Error().stack + .split('\n')[stackIndex + 2] // +2 to skip this function + the first line, which is the error message + .split(' (')[1] + .slice(0, -1) + .split(':')[0] + + // Then, find the corresponding file in which the breakpoint(s) exists + file = join('target-app', basename(testFile).replace('.spec', '')) + } // Finally, find the line number(s) of the breakpoint(s) const lines = readFileSync(join(__dirname, file), 'utf8').split('\n') diff --git a/integration-tests/helpers/fake-agent.js b/integration-tests/helpers/fake-agent.js index 4902c80d9a1..317584a5670 100644 --- a/integration-tests/helpers/fake-agent.js +++ b/integration-tests/helpers/fake-agent.js @@ -326,6 +326,7 @@ function buildExpressServer (agent) { res.status(200).send() agent.emit('debugger-input', { headers: req.headers, + query: req.query, payload: req.body }) }) diff --git a/packages/dd-trace/src/debugger/devtools_client/send.js b/packages/dd-trace/src/debugger/devtools_client/send.js index 9d607b1ad1c..375afd7d47a 100644 --- a/packages/dd-trace/src/debugger/devtools_client/send.js +++ b/packages/dd-trace/src/debugger/devtools_client/send.js @@ -6,6 +6,7 @@ const { stringify } = require('querystring') const config = require('./config') const request = require('../../exporters/common/request') const { GIT_COMMIT_SHA, GIT_REPOSITORY_URL } = require('../../plugins/util/tags') +const { version } = require('../../../../../package.json') module.exports = send @@ -16,6 +17,10 @@ const hostname = getHostname() const service = config.service const ddtags = [ + ['env', process.env.DD_ENV], + ['version', process.env.DD_VERSION], + ['debugger_version', version], + ['host_name', hostname], [GIT_COMMIT_SHA, config.commitSHA], [GIT_REPOSITORY_URL, config.repositoryUrl] ].map((pair) => pair.join(':')).join(',') From 98ceacfd844cd587107928f71d4a2f0f06c023f3 Mon Sep 17 00:00:00 2001 From: Sam Brenner <106700075+sabrenner@users.noreply.github.com> Date: Fri, 20 Dec 2024 11:42:14 -0500 Subject: [PATCH 31/36] [MLOB-1942] fix(llmobs): auto-annotations for wrapped functions happen after manual annotations (#4960) * auto-annotation done before span finish * error cases * callback scoped consistently to apm * make clearer --- packages/dd-trace/src/llmobs/sdk.js | 116 +++++++++--- .../dd-trace/test/llmobs/sdk/index.spec.js | 177 ++++++++++++++++++ 2 files changed, 267 insertions(+), 26 deletions(-) diff --git a/packages/dd-trace/src/llmobs/sdk.js b/packages/dd-trace/src/llmobs/sdk.js index 91fe1e8f70a..2a6d548d656 100644 --- a/packages/dd-trace/src/llmobs/sdk.js +++ b/packages/dd-trace/src/llmobs/sdk.js @@ -1,12 +1,12 @@ 'use strict' -const { SPAN_KIND, OUTPUT_VALUE } = require('./constants/tags') +const { SPAN_KIND, OUTPUT_VALUE, INPUT_VALUE } = require('./constants/tags') const { getFunctionArguments, validateKind } = require('./util') -const { isTrue } = require('../util') +const { isTrue, isError } = require('../util') const { storage } = require('./storage') @@ -134,29 +134,63 @@ class LLMObs extends NoopLLMObs { function wrapped () { const span = llmobs._tracer.scope().active() - - const result = llmobs._activate(span, { kind, options: llmobsOptions }, () => { - if (!['llm', 'embedding'].includes(kind)) { - llmobs.annotate(span, { inputData: getFunctionArguments(fn, arguments) }) + const fnArgs = arguments + + const lastArgId = fnArgs.length - 1 + const cb = fnArgs[lastArgId] + const hasCallback = typeof cb === 'function' + + if (hasCallback) { + const scopeBoundCb = llmobs._bind(cb) + fnArgs[lastArgId] = function () { + // it is standard practice to follow the callback signature (err, result) + // however, we try to parse the arguments to determine if the first argument is an error + // if it is not, and is not undefined, we will use that for the output value + const maybeError = arguments[0] + const maybeResult = arguments[1] + + llmobs._autoAnnotate( + span, + kind, + getFunctionArguments(fn, fnArgs), + isError(maybeError) || maybeError == null ? maybeResult : maybeError + ) + + return scopeBoundCb.apply(this, arguments) } + } - return fn.apply(this, arguments) - }) + try { + const result = llmobs._activate(span, { kind, options: llmobsOptions }, () => fn.apply(this, fnArgs)) + + if (result && typeof result.then === 'function') { + return result.then( + value => { + if (!hasCallback) { + llmobs._autoAnnotate(span, kind, getFunctionArguments(fn, fnArgs), value) + } + return value + }, + err => { + llmobs._autoAnnotate(span, kind, getFunctionArguments(fn, fnArgs)) + throw err + } + ) + } - if (result && typeof result.then === 'function') { - return result.then(value => { - if (value && !['llm', 'retrieval'].includes(kind) && !LLMObsTagger.tagMap.get(span)?.[OUTPUT_VALUE]) { - llmobs.annotate(span, { outputData: value }) - } - return value - }) - } + // it is possible to return a value and have a callback + // however, since the span finishes when the callback is called, it is possible that + // the callback is called before the function returns (although unlikely) + // we do not want to throw for "annotating a finished span" in this case + if (!hasCallback) { + llmobs._autoAnnotate(span, kind, getFunctionArguments(fn, fnArgs), result) + } - if (result && !['llm', 'retrieval'].includes(kind) && !LLMObsTagger.tagMap.get(span)?.[OUTPUT_VALUE]) { - llmobs.annotate(span, { outputData: result }) + return result + } catch (e) { + llmobs._autoAnnotate(span, kind, getFunctionArguments(fn, fnArgs)) + throw e } - - return result } return this._tracer.wrap(name, spanOptions, wrapped) @@ -333,20 +367,34 @@ class LLMObs extends NoopLLMObs { flushCh.publish() } + _autoAnnotate (span, kind, input, output) { + const annotations = {} + if (input && !['llm', 'embedding'].includes(kind) && !LLMObsTagger.tagMap.get(span)?.[INPUT_VALUE]) { + annotations.inputData = input + } + + if (output && !['llm', 'retrieval'].includes(kind) && !LLMObsTagger.tagMap.get(span)?.[OUTPUT_VALUE]) { + annotations.outputData = output + } + + this.annotate(span, annotations) + } + _active () { const store = storage.getStore() return store?.span } - _activate (span, { kind, options } = {}, fn) { + _activate (span, options, fn) { const parent = this._active() if (this.enabled) storage.enterWith({ span }) - this._tagger.registerLLMObsSpan(span, { - ...options, - parent, - kind - }) + if (options) { + this._tagger.registerLLMObsSpan(span, { + ...options, + parent + }) + } try { return fn() @@ -355,6 +403,22 @@ class LLMObs extends NoopLLMObs { } } + // bind function to active LLMObs span + _bind (fn) { + if (typeof fn !== 'function') return fn + + const llmobs = this + const activeSpan = llmobs._active() + + const bound = function () { + return llmobs._activate(activeSpan, null, () => { + return fn.apply(this, arguments) + }) + } + + return bound + } + _extractOptions (options) { const { modelName, diff --git a/packages/dd-trace/test/llmobs/sdk/index.spec.js b/packages/dd-trace/test/llmobs/sdk/index.spec.js index 69dad1d60c4..e7cfb81a47d 100644 --- a/packages/dd-trace/test/llmobs/sdk/index.spec.js +++ b/packages/dd-trace/test/llmobs/sdk/index.spec.js @@ -17,6 +17,7 @@ describe('sdk', () => { let LLMObsSDK let llmobs let tracer + let clock before(() => { tracer = require('../../../../dd-trace') @@ -43,6 +44,8 @@ describe('sdk', () => { // remove max listener warnings, we don't care about the writer anyways process.removeAllListeners('beforeExit') + + clock = sinon.useFakeTimers() }) afterEach(() => { @@ -435,6 +438,180 @@ describe('sdk', () => { }) }) + it('does not crash for auto-annotation values that are overriden', () => { + const circular = {} + circular.circular = circular + + let span + function myWorkflow (input) { + span = llmobs._active() + llmobs.annotate({ + inputData: 'circular', + outputData: 'foo' + }) + return '' + } + + const wrappedMyWorkflow = llmobs.wrap({ kind: 'workflow' }, myWorkflow) + wrappedMyWorkflow(circular) + + expect(LLMObsTagger.tagMap.get(span)).to.deep.equal({ + '_ml_obs.meta.span.kind': 'workflow', + '_ml_obs.meta.ml_app': 'mlApp', + '_ml_obs.llmobs_parent_id': 'undefined', + '_ml_obs.meta.input.value': 'circular', + '_ml_obs.meta.output.value': 'foo' + }) + }) + + it('only auto-annotates input on error', () => { + let span + function myTask (foo, bar) { + span = llmobs._active() + throw new Error('error') + } + + const wrappedMyTask = llmobs.wrap({ kind: 'task' }, myTask) + + expect(() => wrappedMyTask('foo', 'bar')).to.throw() + + expect(LLMObsTagger.tagMap.get(span)).to.deep.equal({ + '_ml_obs.meta.span.kind': 'task', + '_ml_obs.meta.ml_app': 'mlApp', + '_ml_obs.llmobs_parent_id': 'undefined', + '_ml_obs.meta.input.value': JSON.stringify({ foo: 'foo', bar: 'bar' }) + }) + }) + + it('only auto-annotates input on error for promises', () => { + let span + function myTask (foo, bar) { + span = llmobs._active() + return Promise.reject(new Error('error')) + } + + const wrappedMyTask = llmobs.wrap({ kind: 'task' }, myTask) + + return wrappedMyTask('foo', 'bar') + .catch(() => { + expect(LLMObsTagger.tagMap.get(span)).to.deep.equal({ + '_ml_obs.meta.span.kind': 'task', + '_ml_obs.meta.ml_app': 'mlApp', + '_ml_obs.llmobs_parent_id': 'undefined', + '_ml_obs.meta.input.value': JSON.stringify({ foo: 'foo', bar: 'bar' }) + }) + }) + }) + + it('auto-annotates the inputs of the callback function as the outputs for the span', () => { + let span + function myWorkflow (input, cb) { + span = llmobs._active() + setTimeout(() => { + cb(null, 'output') + }, 1000) + } + + const wrappedMyWorkflow = llmobs.wrap({ kind: 'workflow' }, myWorkflow) + wrappedMyWorkflow('input', (err, res) => { + expect(err).to.not.exist + expect(res).to.equal('output') + }) + + clock.tick(1000) + + expect(LLMObsTagger.tagMap.get(span)).to.deep.equal({ + '_ml_obs.meta.span.kind': 'workflow', + '_ml_obs.meta.ml_app': 'mlApp', + '_ml_obs.llmobs_parent_id': 'undefined', + '_ml_obs.meta.input.value': JSON.stringify({ input: 'input' }), + '_ml_obs.meta.output.value': 'output' + }) + }) + + it('ignores the error portion of the callback for auto-annotation', () => { + let span + function myWorkflow (input, cb) { + span = llmobs._active() + setTimeout(() => { + cb(new Error('error'), 'output') + }, 1000) + } + + const wrappedMyWorkflow = llmobs.wrap({ kind: 'workflow' }, myWorkflow) + wrappedMyWorkflow('input', (err, res) => { + expect(err).to.exist + expect(res).to.equal('output') + }) + + clock.tick(1000) + + expect(LLMObsTagger.tagMap.get(span)).to.deep.equal({ + '_ml_obs.meta.span.kind': 'workflow', + '_ml_obs.meta.ml_app': 'mlApp', + '_ml_obs.llmobs_parent_id': 'undefined', + '_ml_obs.meta.input.value': JSON.stringify({ input: 'input' }), + '_ml_obs.meta.output.value': 'output' + }) + }) + + it('auto-annotates the first argument of the callback as the output if it is not an error', () => { + let span + function myWorkflow (input, cb) { + span = llmobs._active() + setTimeout(() => { + cb('output', 'ignore') + }, 1000) + } + + const wrappedMyWorkflow = llmobs.wrap({ kind: 'workflow' }, myWorkflow) + wrappedMyWorkflow('input', (res, irrelevant) => { + expect(res).to.equal('output') + expect(irrelevant).to.equal('ignore') + }) + + clock.tick(1000) + + expect(LLMObsTagger.tagMap.get(span)).to.deep.equal({ + '_ml_obs.meta.span.kind': 'workflow', + '_ml_obs.meta.ml_app': 'mlApp', + '_ml_obs.llmobs_parent_id': 'undefined', + '_ml_obs.meta.input.value': JSON.stringify({ input: 'input' }), + '_ml_obs.meta.output.value': 'output' + }) + }) + + it('maintains context consistent with the tracer', () => { + let llmSpan, workflowSpan, taskSpan + + function myLlm (input, cb) { + llmSpan = llmobs._active() + setTimeout(() => { + cb(null, 'output') + }, 1000) + } + const myWrappedLlm = llmobs.wrap({ kind: 'llm' }, myLlm) + + llmobs.trace({ kind: 'workflow', name: 'myWorkflow' }, _workflow => { + workflowSpan = _workflow + tracer.trace('apmOperation', () => { + myWrappedLlm('input', (err, res) => { + expect(err).to.not.exist + expect(res).to.equal('output') + llmobs.trace({ kind: 'task', name: 'afterLlmTask' }, _task => { + taskSpan = _task + + const llmParentId = LLMObsTagger.tagMap.get(llmSpan)['_ml_obs.llmobs_parent_id'] + expect(llmParentId).to.equal(workflowSpan.context().toSpanId()) + + const taskParentId = LLMObsTagger.tagMap.get(taskSpan)['_ml_obs.llmobs_parent_id'] + expect(taskParentId).to.equal(workflowSpan.context().toSpanId()) + }) + }) + }) + }) + }) + // TODO: need span kind optional for this test it.skip('sets the span name to "unnamed-anonymous-function" if no name is provided', () => { let span From 4d6a8e3fe8edee52a9e28796d6138f776cb4a767 Mon Sep 17 00:00:00 2001 From: ishabi Date: Tue, 24 Dec 2024 17:35:20 +0100 Subject: [PATCH 32/36] support aerospike 6 (#5057) --- .github/workflows/plugins.yml | 4 ++++ packages/datadog-instrumentations/src/aerospike.js | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/.github/workflows/plugins.yml b/.github/workflows/plugins.yml index 4822539ecab..79650e6d473 100644 --- a/.github/workflows/plugins.yml +++ b/.github/workflows/plugins.yml @@ -35,6 +35,10 @@ jobs: range: '>=5.12.1' aerospike-image: ce-6.4.0.3 test-image: ubuntu-latest + - node-version: 22 + range: '>=6.0.0' + aerospike-image: ce-6.4.0.3 + test-image: ubuntu-latest runs-on: ${{ matrix.test-image }} services: aerospike: diff --git a/packages/datadog-instrumentations/src/aerospike.js b/packages/datadog-instrumentations/src/aerospike.js index 497a64aaf80..ba310b6e2de 100644 --- a/packages/datadog-instrumentations/src/aerospike.js +++ b/packages/datadog-instrumentations/src/aerospike.js @@ -40,7 +40,7 @@ function wrapProcess (process) { addHook({ name: 'aerospike', file: 'lib/commands/command.js', - versions: ['4', '5'] + versions: ['4', '5', '6'] }, commandFactory => { return shimmer.wrapFunction(commandFactory, f => wrapCreateCommand(f)) From 330e973219497fe3b494a96a7ce11cba719eb3ea Mon Sep 17 00:00:00 2001 From: Thomas Watson Date: Thu, 2 Jan 2025 10:55:47 +0100 Subject: [PATCH 33/36] [DI] Clean up snapshot integration test (#5050) The comment about the breakpoint line number being hardcoded is no longer true. Since this is no longer the case, this commit removes the hack used to avoid changing the line number when adding new variables to the captured snapshot. --- integration-tests/debugger/snapshot.spec.js | 30 +++++----- .../debugger/target-app/snapshot.js | 59 +++++++++---------- 2 files changed, 42 insertions(+), 47 deletions(-) diff --git a/integration-tests/debugger/snapshot.spec.js b/integration-tests/debugger/snapshot.spec.js index e3d17b225c4..e2f9d9eb047 100644 --- a/integration-tests/debugger/snapshot.spec.js +++ b/integration-tests/debugger/snapshot.spec.js @@ -16,10 +16,10 @@ describe('Dynamic Instrumentation', function () { assert.deepEqual(Object.keys(captures.lines), [String(t.breakpoint.line)]) const { locals } = captures.lines[t.breakpoint.line] - const { request, fastify, getSomeData } = locals + const { request, fastify, getUndefined } = locals delete locals.request delete locals.fastify - delete locals.getSomeData + delete locals.getUndefined // from block scope assert.deepEqual(locals, { @@ -67,19 +67,19 @@ describe('Dynamic Instrumentation', function () { } }, emptyObj: { type: 'Object', fields: {} }, - fn: { - type: 'Function', - fields: { - length: { type: 'number', value: '0' }, - name: { type: 'string', value: 'fn' } - } - }, p: { type: 'Promise', fields: { '[[PromiseState]]': { type: 'string', value: 'fulfilled' }, '[[PromiseResult]]': { type: 'undefined' } } + }, + arrowFn: { + type: 'Function', + fields: { + length: { type: 'number', value: '0' }, + name: { type: 'string', value: 'arrowFn' } + } } }) @@ -99,11 +99,11 @@ describe('Dynamic Instrumentation', function () { assert.equal(fastify.type, 'Object') assert.typeOf(fastify.fields, 'Object') - assert.deepEqual(getSomeData, { + assert.deepEqual(getUndefined, { type: 'Function', fields: { length: { type: 'number', value: '0' }, - name: { type: 'string', value: 'getSomeData' } + name: { type: 'string', value: 'getUndefined' } } }) @@ -118,7 +118,7 @@ describe('Dynamic Instrumentation', function () { const { locals } = captures.lines[t.breakpoint.line] delete locals.request delete locals.fastify - delete locals.getSomeData + delete locals.getUndefined assert.deepEqual(locals, { nil: { type: 'null', isNull: true }, @@ -139,8 +139,8 @@ describe('Dynamic Instrumentation', function () { arr: { type: 'Array', notCapturedReason: 'depth' }, obj: { type: 'Object', notCapturedReason: 'depth' }, emptyObj: { type: 'Object', notCapturedReason: 'depth' }, - fn: { type: 'Function', notCapturedReason: 'depth' }, - p: { type: 'Promise', notCapturedReason: 'depth' } + p: { type: 'Promise', notCapturedReason: 'depth' }, + arrowFn: { type: 'Function', notCapturedReason: 'depth' } }) done() @@ -212,7 +212,7 @@ describe('Dynamic Instrumentation', function () { // Up to 3 properties from the local scope 'request', 'nil', 'undef', // Up to 3 properties from the closure scope - 'fastify', 'getSomeData' + 'fastify', 'getUndefined' ]) assert.strictEqual(locals.request.type, 'Request') diff --git a/integration-tests/debugger/target-app/snapshot.js b/integration-tests/debugger/target-app/snapshot.js index 03cfc758556..63cc6f3d33b 100644 --- a/integration-tests/debugger/target-app/snapshot.js +++ b/integration-tests/debugger/target-app/snapshot.js @@ -5,12 +5,33 @@ const Fastify = require('fastify') const fastify = Fastify() -// Since line probes have hardcoded line numbers, we want to try and keep the line numbers from changing within the -// `handler` function below when making changes to this file. This is achieved by calling `getSomeData` and keeping all -// variable names on the same line as much as possible. fastify.get('/:name', function handler (request) { - // eslint-disable-next-line no-unused-vars - const { nil, undef, bool, num, bigint, str, lstr, sym, regex, arr, obj, emptyObj, fn, p } = getSomeData() + /* eslint-disable no-unused-vars */ + const nil = null + const undef = getUndefined() + const bool = true + const num = 42 + const bigint = 42n + const str = 'foo' + // eslint-disable-next-line @stylistic/js/max-len + const lstr = 'Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam, quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo consequat. Duis aute irure dolor in reprehenderit in voluptate velit esse cillum dolore eu fugiat nulla pariatur. Excepteur sint occaecat cupidatat non proident, sunt in culpa qui officia deserunt mollit anim id est laborum.' + const sym = Symbol('foo') + const regex = /bar/i + const arr = [1, 2, 3, 4, 5] + const obj = { + foo: { + baz: 42, + nil: null, + undef: undefined, + deep: { nested: { obj: { that: { goes: { on: { forever: true } } } } } } + }, + bar: true + } + const emptyObj = {} + const p = Promise.resolve() + const arrowFn = () => {} + /* eslint-enable no-unused-vars */ + return { hello: request.params.name } // BREAKPOINT: /foo }) @@ -22,30 +43,4 @@ fastify.listen({ port: process.env.APP_PORT }, (err) => { process.send({ port: process.env.APP_PORT }) }) -function getSomeData () { - return { - nil: null, - undef: undefined, - bool: true, - num: 42, - bigint: 42n, - str: 'foo', - // eslint-disable-next-line @stylistic/js/max-len - lstr: 'Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam, quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo consequat. Duis aute irure dolor in reprehenderit in voluptate velit esse cillum dolore eu fugiat nulla pariatur. Excepteur sint occaecat cupidatat non proident, sunt in culpa qui officia deserunt mollit anim id est laborum.', - sym: Symbol('foo'), - regex: /bar/i, - arr: [1, 2, 3, 4, 5], - obj: { - foo: { - baz: 42, - nil: null, - undef: undefined, - deep: { nested: { obj: { that: { goes: { on: { forever: true } } } } } } - }, - bar: true - }, - emptyObj: {}, - fn: () => {}, - p: Promise.resolve() - } -} +function getUndefined () {} From 8981beb6c71a4a665c6d8da87ca37e9a25fa7919 Mon Sep 17 00:00:00 2001 From: Thomas Watson Date: Thu, 2 Jan 2025 11:02:33 +0100 Subject: [PATCH 34/36] [DI] Add TODO comment (#5054) --- packages/dd-trace/src/debugger/devtools_client/index.js | 2 ++ 1 file changed, 2 insertions(+) diff --git a/packages/dd-trace/src/debugger/devtools_client/index.js b/packages/dd-trace/src/debugger/devtools_client/index.js index 9634003bf61..df158b7d2da 100644 --- a/packages/dd-trace/src/debugger/devtools_client/index.js +++ b/packages/dd-trace/src/debugger/devtools_client/index.js @@ -141,6 +141,8 @@ function highestOrUndefined (num, max) { } async function getDD (callFrameId) { + // TODO: Consider if an `objectGroup` should be used, so it can be explicitly released using + // `Runtime.releaseObjectGroup` const { result } = await session.post('Debugger.evaluateOnCallFrame', { callFrameId, expression, From f813f43d201b3bf552eacfcedfcc85fb0e43c15b Mon Sep 17 00:00:00 2001 From: ishabi Date: Thu, 2 Jan 2025 16:37:48 +0100 Subject: [PATCH 35/36] upgrade mocha@9 to mocha@10 (#5065) --- package.json | 2 +- yarn.lock | 280 +++++++++++++++++++++++++-------------------------- 2 files changed, 137 insertions(+), 145 deletions(-) diff --git a/package.json b/package.json index 9b0abdb34db..3e4582e9438 100644 --- a/package.json +++ b/package.json @@ -145,7 +145,7 @@ "jszip": "^3.5.0", "knex": "^2.4.2", "mkdirp": "^3.0.1", - "mocha": "^9", + "mocha": "^10", "msgpack-lite": "^0.1.26", "multer": "^1.4.5-lts.1", "nock": "^11.3.3", diff --git a/yarn.lock b/yarn.lock index a56218a0a45..fc37d5e7016 100644 --- a/yarn.lock +++ b/yarn.lock @@ -1012,11 +1012,6 @@ resolved "https://registry.npmjs.org/@types/yoga-layout/-/yoga-layout-1.9.2.tgz" integrity sha512-S9q47ByT2pPvD65IvrWp7qppVMpk9WGMbVq9wbWZOHg6tnXSD4vyhao6nOSBwwfDdV2p3Kx9evA9vI+XWTfDvw== -"@ungap/promise-all-settled@1.1.2": - version "1.1.2" - resolved "https://registry.npmjs.org/@ungap/promise-all-settled/-/promise-all-settled-1.1.2.tgz" - integrity sha512-sL/cEvJWAnClXw0wHk85/2L0G6Sj8UB0Ctc1TEMbKSsmpRosqhwj9gWgFRZSrBr2f9tiXISwNhCPmlfqUqyb9Q== - "@ungap/structured-clone@^1.2.0": version "1.2.0" resolved "https://registry.yarnpkg.com/@ungap/structured-clone/-/structured-clone-1.2.0.tgz#756641adb587851b5ccb3e095daf27ae581c8406" @@ -1063,10 +1058,10 @@ ajv@^6.12.4: json-schema-traverse "^0.4.1" uri-js "^4.2.2" -ansi-colors@4.1.1: - version "4.1.1" - resolved "https://registry.npmjs.org/ansi-colors/-/ansi-colors-4.1.1.tgz" - integrity sha512-JoX0apGbHaUJBNl6yF+p6JAFYZ666/hhCGKN5t9QFjbJQKUU/g8MNbFDbvfrgKXvI1QpZplPOnwIo99lX/AAmA== +ansi-colors@^4.1.3: + version "4.1.3" + resolved "https://registry.yarnpkg.com/ansi-colors/-/ansi-colors-4.1.3.tgz#37611340eb2243e70cc604cad35d63270d48781b" + integrity sha512-/6w/C21Pm1A7aZitlI5Ni/2J6FFQN8i1Cvz3kHABAAbw93v/NlvKdVOqz7CCWz/3iv/JplRSEEZ83XION15ovw== ansi-escapes@^4.2.1: version "4.3.2" @@ -1355,6 +1350,13 @@ brace-expansion@^1.1.7: balanced-match "^1.0.0" concat-map "0.0.1" +brace-expansion@^2.0.1: + version "2.0.1" + resolved "https://registry.yarnpkg.com/brace-expansion/-/brace-expansion-2.0.1.tgz#1edc459e0f0c548486ecf9fc99f2221364b9a0ae" + integrity sha512-XnAIvQ8eM+kC6aULx6wuQiwVsnzsi9d3WxzV3FpWTGA19F621kwdbsAcFKXgKUHZWsy+mY6iL1sHTxWEFCytDA== + dependencies: + balanced-match "^1.0.0" + braces@~3.0.2: version "3.0.3" resolved "https://registry.yarnpkg.com/braces/-/braces-3.0.3.tgz#490332f40919452272d55a8480adc0c441358789" @@ -1362,9 +1364,9 @@ braces@~3.0.2: dependencies: fill-range "^7.1.1" -browser-stdout@1.3.1: +browser-stdout@^1.3.1: version "1.3.1" - resolved "https://registry.npmjs.org/browser-stdout/-/browser-stdout-1.3.1.tgz" + resolved "https://registry.yarnpkg.com/browser-stdout/-/browser-stdout-1.3.1.tgz#baa559ee14ced73452229bad7326467c61fabd60" integrity sha512-qhAVI1+Av2X7qelOfAIYwXONood6XlZE/fXaBSmW/T5SzLAmCgzi+eiWE7fUvbHaeNBQH13UftjpXxsfLkMpgw== browserslist@^4.21.9: @@ -1533,7 +1535,7 @@ checksum@^1.0.0: dependencies: optimist "~0.3.5" -chokidar@3.5.3, chokidar@^3.3.0: +chokidar@^3.3.0: version "3.5.3" resolved "https://registry.npmjs.org/chokidar/-/chokidar-3.5.3.tgz" integrity sha512-Dr3sfKRP6oTcjf2JmUmFJfeVMvXBdegxB0iVQ5eb2V10uFJUCAS8OByZdVAyVb8xXNz3GjjTgj9kLWsZTqE6kw== @@ -1548,6 +1550,21 @@ chokidar@3.5.3, chokidar@^3.3.0: optionalDependencies: fsevents "~2.3.2" +chokidar@^3.5.3: + version "3.6.0" + resolved "https://registry.yarnpkg.com/chokidar/-/chokidar-3.6.0.tgz#197c6cc669ef2a8dc5e7b4d97ee4e092c3eb0d5b" + integrity sha512-7VT13fmjotKpGipCW9JEQAusEPE+Ei8nl6/g4FBAmIm0GOOLMua9NDDo/DWp0ZAxCr3cPq5ZpBqmPAQgDda2Pw== + dependencies: + anymatch "~3.1.2" + braces "~3.0.2" + glob-parent "~5.1.2" + is-binary-path "~2.1.0" + is-glob "~4.0.1" + normalize-path "~3.0.0" + readdirp "~3.6.0" + optionalDependencies: + fsevents "~2.3.2" + ci-info@^2.0.0: version "2.0.0" resolved "https://registry.npmjs.org/ci-info/-/ci-info-2.0.0.tgz" @@ -1818,13 +1835,6 @@ debug@2.6.9: dependencies: ms "2.0.0" -debug@4.3.3: - version "4.3.3" - resolved "https://registry.yarnpkg.com/debug/-/debug-4.3.3.tgz#04266e0b70a98d4462e6e288e38259213332b664" - integrity sha512-/zxw5+vh1Tfv+4Qn7a5nsbcJKPaSvCDhojn6FEl9vupwK2VCSDtEiEtqr8DFtzYFOdz63LBkxec7DYuc2jon6Q== - dependencies: - ms "2.1.2" - debug@4.3.4: version "4.3.4" resolved "https://registry.npmjs.org/debug/-/debug-4.3.4.tgz" @@ -1846,6 +1856,13 @@ debug@^4.1.0, debug@^4.1.1, debug@^4.3.1, debug@^4.3.2: dependencies: ms "2.1.2" +debug@^4.3.5: + version "4.4.0" + resolved "https://registry.yarnpkg.com/debug/-/debug-4.4.0.tgz#2b3f2aea2ffeb776477460267377dc8710faba8a" + integrity sha512-6WTZ/IxCY/T6BALoZHaE4ctp9xm+Z5kY/pzYaCHRFeyVhojxlrm+46y68HA6hr0TcwEssoxNiDEUJQjfPZ/RYA== + dependencies: + ms "^2.1.3" + decamelize@^1.2.0: version "1.2.0" resolved "https://registry.npmjs.org/decamelize/-/decamelize-1.2.0.tgz" @@ -1918,17 +1935,12 @@ detect-newline@^3.0.0: resolved "https://registry.npmjs.org/detect-newline/-/detect-newline-3.1.0.tgz" integrity "sha1-V29d/GOuGhkv8ZLYrTr2MImRtlE= sha512-TLz+x/vEXm/Y7P7wn1EJFNLxYpUD4TgMosxY6fAVJUnJMbupHBOncxyWUG9OpTaH9EBD7uFI5LfEgmMOc54DsA==" -diff@5.0.0: - version "5.0.0" - resolved "https://registry.npmjs.org/diff/-/diff-5.0.0.tgz" - integrity sha512-/VTCrvm5Z0JGty/BWHljh+BAiw3IK+2j87NGMu8Nwc/f48WoDAC395uomO9ZD117ZOBaHmkX1oyLvkVM/aIT3w== - diff@^4.0.1, diff@^4.0.2: version "4.0.2" resolved "https://registry.npmjs.org/diff/-/diff-4.0.2.tgz" integrity sha512-58lmxKSA4BNyLz+HHMUzlOEpg09FV+ev6ZMe3vJihgdxzgcwZ8VoEEPmALCZG9LmqfVoNMMKpttIYTVG6uDY7A== -diff@^5.1.0: +diff@^5.1.0, diff@^5.2.0: version "5.2.0" resolved "https://registry.yarnpkg.com/diff/-/diff-5.2.0.tgz#26ded047cd1179b78b9537d5ef725503ce1ae531" integrity sha512-uIFDxqpRZGZ6ThOk84hEfqWoHx2devRFvpTZcTHur85vImfaxUbTW9Ryh4CpCuDnToOP1CEtXKIgytHBPVff5A== @@ -2116,11 +2128,6 @@ escape-html@~1.0.3: resolved "https://registry.npmjs.org/escape-html/-/escape-html-1.0.3.tgz" integrity sha512-NiSupZ4OeuGwr68lGIeym/ksIZMJodUGOSCZ/FSnTxcrekbvqrgdUxlJOMpijaKZVjAJrWrGs/6Jy8OMuyj9ow== -escape-string-regexp@4.0.0, escape-string-regexp@^4.0.0: - version "4.0.0" - resolved "https://registry.npmjs.org/escape-string-regexp/-/escape-string-regexp-4.0.0.tgz" - integrity sha512-TtpcNJ3XAzx3Gq8sWRzJaVajRs0uVxA2YAkdb1jm2YkPz4G6egUFAyA3n5vtEIZefPk5Wa4UXbKuS5fKkJWdgA== - escape-string-regexp@^1.0.5: version "1.0.5" resolved "https://registry.npmjs.org/escape-string-regexp/-/escape-string-regexp-1.0.5.tgz" @@ -2131,6 +2138,11 @@ escape-string-regexp@^2.0.0: resolved "https://registry.npmjs.org/escape-string-regexp/-/escape-string-regexp-2.0.0.tgz" integrity sha512-UpzcLCXolUWcNu5HtVMHYdXJjArjsF9C0aNnquZYY4uW/Vu0miy5YoWvbV345HauVvcAUnpRuhMMcqTcGOY2+w== +escape-string-regexp@^4.0.0: + version "4.0.0" + resolved "https://registry.npmjs.org/escape-string-regexp/-/escape-string-regexp-4.0.0.tgz" + integrity sha512-TtpcNJ3XAzx3Gq8sWRzJaVajRs0uVxA2YAkdb1jm2YkPz4G6egUFAyA3n5vtEIZefPk5Wa4UXbKuS5fKkJWdgA== + eslint-compat-utils@^0.5.1: version "0.5.1" resolved "https://registry.yarnpkg.com/eslint-compat-utils/-/eslint-compat-utils-0.5.1.tgz#7fc92b776d185a70c4070d03fd26fde3d59652e4" @@ -2471,14 +2483,6 @@ find-cache-dir@^3.2.0: make-dir "^3.0.2" pkg-dir "^4.1.0" -find-up@5.0.0, find-up@^5.0.0: - version "5.0.0" - resolved "https://registry.npmjs.org/find-up/-/find-up-5.0.0.tgz" - integrity sha512-78/PXT1wlLLDgTzDs7sjq9hzz0vXD+zn+7wypEe4fXQxCmdmqfGsEPQxmiCSQI3ajFV91bVSsvNtrJRiW6nGng== - dependencies: - locate-path "^6.0.0" - path-exists "^4.0.0" - find-up@^4.0.0, find-up@^4.1.0: version "4.1.0" resolved "https://registry.npmjs.org/find-up/-/find-up-4.1.0.tgz" @@ -2487,6 +2491,14 @@ find-up@^4.0.0, find-up@^4.1.0: locate-path "^5.0.0" path-exists "^4.0.0" +find-up@^5.0.0: + version "5.0.0" + resolved "https://registry.npmjs.org/find-up/-/find-up-5.0.0.tgz" + integrity sha512-78/PXT1wlLLDgTzDs7sjq9hzz0vXD+zn+7wypEe4fXQxCmdmqfGsEPQxmiCSQI3ajFV91bVSsvNtrJRiW6nGng== + dependencies: + locate-path "^6.0.0" + path-exists "^4.0.0" + findit@^2.0.0: version "2.0.0" resolved "https://registry.npmjs.org/findit/-/findit-2.0.0.tgz" @@ -2674,29 +2686,28 @@ glob-parent@~5.1.2: dependencies: is-glob "^4.0.1" -glob@7.2.0: - version "7.2.0" - resolved "https://registry.yarnpkg.com/glob/-/glob-7.2.0.tgz#d15535af7732e02e948f4c41628bd910293f6023" - integrity sha512-lmLf6gtyrPq8tTjSmrO94wBeQbFR3HbLHbuyD69wuyQkImp2hWqMGB47OX65FBkPffO641IP9jWa1z4ivqG26Q== +glob@^7.0.5, glob@^7.1.3, glob@^7.1.4, glob@^7.1.6, glob@^7.2.3: + version "7.2.3" + resolved "https://registry.npmjs.org/glob/-/glob-7.2.3.tgz" + integrity sha512-nFR0zLpU2YCaRxwoCJvL6UvCH2JFyFVIvwTLsIf21AuHlMskA1hhTdk+LlYJtOlYt9v6dvszD2BGRqBL+iQK9Q== dependencies: fs.realpath "^1.0.0" inflight "^1.0.4" inherits "2" - minimatch "^3.0.4" + minimatch "^3.1.1" once "^1.3.0" path-is-absolute "^1.0.0" -glob@^7.0.5, glob@^7.1.3, glob@^7.1.4, glob@^7.1.6, glob@^7.2.3: - version "7.2.3" - resolved "https://registry.npmjs.org/glob/-/glob-7.2.3.tgz" - integrity sha512-nFR0zLpU2YCaRxwoCJvL6UvCH2JFyFVIvwTLsIf21AuHlMskA1hhTdk+LlYJtOlYt9v6dvszD2BGRqBL+iQK9Q== +glob@^8.1.0: + version "8.1.0" + resolved "https://registry.yarnpkg.com/glob/-/glob-8.1.0.tgz#d388f656593ef708ee3e34640fdfb99a9fd1c33e" + integrity sha512-r8hpEjiQEYlF2QU0df3dS+nxxSIreXQS1qRhMJM0Q5NDdR386C7jb7Hwwod8Fgiuex+k0GFjgft18yvxm5XoCQ== dependencies: fs.realpath "^1.0.0" inflight "^1.0.4" inherits "2" - minimatch "^3.1.1" + minimatch "^5.0.1" once "^1.3.0" - path-is-absolute "^1.0.0" globals@^11.1.0: version "11.12.0" @@ -2752,11 +2763,6 @@ graphql@0.13.2: dependencies: iterall "^1.2.1" -growl@1.10.5: - version "1.10.5" - resolved "https://registry.npmjs.org/growl/-/growl-1.10.5.tgz" - integrity sha512-qBr4OuELkhPenW6goKVXiv47US3clb3/IbuWF9KNKEijAy9oeHxU9IgzjvJhHkUzhaj7rOUD7+YGWqUjLp5oSA== - has-async-hooks@^1.0.0: version "1.0.0" resolved "https://registry.npmjs.org/has-async-hooks/-/has-async-hooks-1.0.0.tgz" @@ -2831,9 +2837,9 @@ hdr-histogram-percentiles-obj@^2.0.0: dependencies: hdr-histogram-js "^1.0.0" -he@1.2.0: +he@^1.2.0: version "1.2.0" - resolved "https://registry.npmjs.org/he/-/he-1.2.0.tgz" + resolved "https://registry.yarnpkg.com/he/-/he-1.2.0.tgz#84ae65fa7eafb165fddb61566ae14baf05664f0f" integrity sha512-F/1DnUGPopORZi0ni+CvrCgHQ5FyEAHRLSApuYWMmrbSwoN2Mn/7k+Gl38gJnR7yyDZk6WLXwiGod1JOWNDKGw== html-escaper@^2.0.0: @@ -3292,13 +3298,6 @@ jmespath@0.16.0: resolved "https://registry.npmjs.org/js-tokens/-/js-tokens-4.0.0.tgz" integrity sha512-RdJUflcE3cUzKiMqQgsCu06FPu9UdIJO0beYbPhHN4k6apgJtifcoCtT9bcxOpYBtpD2kCM6Sbzg4CausW/PKQ== -js-yaml@4.1.0, js-yaml@^4.1.0: - version "4.1.0" - resolved "https://registry.npmjs.org/js-yaml/-/js-yaml-4.1.0.tgz" - integrity sha512-wpxZs9NoxZaJESJGIZTyDEaYpl0FKSA+FB9aJiyemKhMwkxQg63h4T1KJgUGHpTqPDNRcmmYLugrRjJlBtWvRA== - dependencies: - argparse "^2.0.1" - js-yaml@^3.13.1: version "3.14.1" resolved "https://registry.npmjs.org/js-yaml/-/js-yaml-3.14.1.tgz" @@ -3307,6 +3306,13 @@ js-yaml@^3.13.1: argparse "^1.0.7" esprima "^4.0.0" +js-yaml@^4.1.0: + version "4.1.0" + resolved "https://registry.npmjs.org/js-yaml/-/js-yaml-4.1.0.tgz" + integrity sha512-wpxZs9NoxZaJESJGIZTyDEaYpl0FKSA+FB9aJiyemKhMwkxQg63h4T1KJgUGHpTqPDNRcmmYLugrRjJlBtWvRA== + dependencies: + argparse "^2.0.1" + jsesc@^2.5.1: version "2.5.2" resolved "https://registry.npmjs.org/jsesc/-/jsesc-2.5.2.tgz" @@ -3457,7 +3463,7 @@ lodash@^4.17.13, lodash@^4.17.20, lodash@^4.17.21, lodash@^4.17.4: resolved "https://registry.npmjs.org/lodash/-/lodash-4.17.21.tgz" integrity sha512-v2kDEe57lecTulaDIuNTPy3Ry4gLGJ6Z1O3vE1krgXZNrsQ+LFTGHVxVjcXPs17LhbZVGedAJv8XZ1tvj5FvSg== -log-symbols@4.1.0: +log-symbols@^4.1.0: version "4.1.0" resolved "https://registry.yarnpkg.com/log-symbols/-/log-symbols-4.1.0.tgz#3fbdbb95b4683ac9fc785111e792e558d4abd503" integrity sha512-8XPvpAA8uyhfteu8pIvQxpJZ7SYYdpUivZpGy6sFsBuKRY/7rQGavedeB8aK+Zkyq6upMFVL/9AW6vOYzfRyLg== @@ -3560,13 +3566,6 @@ mimic-fn@^2.1.0: resolved "https://registry.npmjs.org/mimic-fn/-/mimic-fn-2.1.0.tgz" integrity sha512-OqbOk5oEQeAZ8WXWydlu9HJjz9WVdEIvamMCcXmuqUYjTknH/sqsWvhQ3vgwKFRR1HpjvNBKQ37nbJgYzGqGcg== -minimatch@4.2.1: - version "4.2.1" - resolved "https://registry.yarnpkg.com/minimatch/-/minimatch-4.2.1.tgz#40d9d511a46bdc4e563c22c3080cde9c0d8299b4" - integrity sha512-9Uq1ChtSZO+Mxa/CL1eGizn2vRn3MlLgzhT0Iz8zaY8NdvxvB0d5QdPFmCKf7JKA9Lerx5vRrnwO03jsSfGG9g== - dependencies: - brace-expansion "^1.1.7" - minimatch@^3.0.4, minimatch@^3.0.5, minimatch@^3.1.1, minimatch@^3.1.2: version "3.1.2" resolved "https://registry.npmjs.org/minimatch/-/minimatch-3.1.2.tgz" @@ -3574,6 +3573,13 @@ minimatch@^3.0.4, minimatch@^3.0.5, minimatch@^3.1.1, minimatch@^3.1.2: dependencies: brace-expansion "^1.1.7" +minimatch@^5.0.1, minimatch@^5.1.6: + version "5.1.6" + resolved "https://registry.yarnpkg.com/minimatch/-/minimatch-5.1.6.tgz#1cfcb8cf5522ea69952cd2af95ae09477f122a96" + integrity sha512-lKwV/1brpG6mBUFHtb7NUmtABCb2WZZmm2wNiOA5hAb8VdCS4B3dtMWyvcoViccwAW/COERjXLt0zP1zXUN26g== + dependencies: + brace-expansion "^2.0.1" + minimist@^1.2.0, minimist@^1.2.6: version "1.2.7" resolved "https://registry.npmjs.org/minimist/-/minimist-1.2.7.tgz" @@ -3603,35 +3609,31 @@ mkdirp@^3.0.1: resolved "https://registry.npmjs.org/mkdirp/-/mkdirp-3.0.1.tgz" integrity "sha1-5E5MVgf7J5wWgkFxPMbg/qmty1A= sha512-+NsyUUAZDmo6YVHzL/stxSu3t9YS1iljliy3BSDrXJ/dkn1KYdmtZODGGjLcc9XLgVVpH4KshHB8XmZgMhaBXg==" -mocha@^9: - version "9.2.2" - resolved "https://registry.yarnpkg.com/mocha/-/mocha-9.2.2.tgz#d70db46bdb93ca57402c809333e5a84977a88fb9" - integrity sha512-L6XC3EdwT6YrIk0yXpavvLkn8h+EU+Y5UcCHKECyMbdUIxyMuZj4bX4U9e1nvnvUUvQVsV2VHQr5zLdcUkhW/g== - dependencies: - "@ungap/promise-all-settled" "1.1.2" - ansi-colors "4.1.1" - browser-stdout "1.3.1" - chokidar "3.5.3" - debug "4.3.3" - diff "5.0.0" - escape-string-regexp "4.0.0" - find-up "5.0.0" - glob "7.2.0" - growl "1.10.5" - he "1.2.0" - js-yaml "4.1.0" - log-symbols "4.1.0" - minimatch "4.2.1" - ms "2.1.3" - nanoid "3.3.8" - serialize-javascript "6.0.0" - strip-json-comments "3.1.1" - supports-color "8.1.1" - which "2.0.2" - workerpool "6.2.0" - yargs "16.2.0" - yargs-parser "20.2.4" - yargs-unparser "2.0.0" +mocha@^10: + version "10.8.2" + resolved "https://registry.yarnpkg.com/mocha/-/mocha-10.8.2.tgz#8d8342d016ed411b12a429eb731b825f961afb96" + integrity sha512-VZlYo/WE8t1tstuRmqgeyBgCbJc/lEdopaa+axcKzTBJ+UIdlAB9XnmvTCAH4pwR4ElNInaedhEBmZD8iCSVEg== + dependencies: + ansi-colors "^4.1.3" + browser-stdout "^1.3.1" + chokidar "^3.5.3" + debug "^4.3.5" + diff "^5.2.0" + escape-string-regexp "^4.0.0" + find-up "^5.0.0" + glob "^8.1.0" + he "^1.2.0" + js-yaml "^4.1.0" + log-symbols "^4.1.0" + minimatch "^5.1.6" + ms "^2.1.3" + serialize-javascript "^6.0.2" + strip-json-comments "^3.1.1" + supports-color "^8.1.1" + workerpool "^6.5.1" + yargs "^16.2.0" + yargs-parser "^20.2.9" + yargs-unparser "^2.0.0" module-details-from-path@^1.0.3: version "1.0.3" @@ -3653,7 +3655,7 @@ ms@2.1.2: resolved "https://registry.npmjs.org/ms/-/ms-2.1.2.tgz" integrity sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w== -ms@2.1.3, ms@^2.1.1, ms@^2.1.2: +ms@2.1.3, ms@^2.1.1, ms@^2.1.2, ms@^2.1.3: version "2.1.3" resolved "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz" integrity sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA== @@ -3681,11 +3683,6 @@ multer@^1.4.5-lts.1: type-is "^1.6.4" xtend "^4.0.0" -nanoid@3.3.8: - version "3.3.8" - resolved "https://registry.yarnpkg.com/nanoid/-/nanoid-3.3.8.tgz#b1be3030bee36aaff18bacb375e5cce521684baf" - integrity sha512-WNLf5Sd8oZxOm+TzppcYk8gVOgP+l58xNy58D0nbUnOxOWRWvlcCV4kUF7ltmI6PsrLl/BgKEyS4mqsGChFN0w== - natural-compare@^1.4.0: version "1.4.0" resolved "https://registry.npmjs.org/natural-compare/-/natural-compare-1.4.0.tgz" @@ -4438,10 +4435,10 @@ send@0.19.0: range-parser "~1.2.1" statuses "2.0.1" -serialize-javascript@6.0.0: - version "6.0.0" - resolved "https://registry.yarnpkg.com/serialize-javascript/-/serialize-javascript-6.0.0.tgz#efae5d88f45d7924141da8b5c3a7a7e663fefeb8" - integrity sha512-Qr3TosvguFt8ePWqsvRfrKyQXIiW+nGbYpy8XK24NQHE83caxWt+mIymTT19DGFbNWNLfEwsrkSmN64lVWB9ag== +serialize-javascript@^6.0.2: + version "6.0.2" + resolved "https://registry.yarnpkg.com/serialize-javascript/-/serialize-javascript-6.0.2.tgz#defa1e055c83bf6d59ea805d8da862254eb6a6c2" + integrity sha512-Saa1xPByTTq2gdeFZYLLo+RFE35NHZkAbqZeWNd3BpzppeVisAqpDjcp8dyf6uIvEqJRd46jemmyA4iFIeVk8g== dependencies: randombytes "^2.1.0" @@ -4695,18 +4692,11 @@ strip-bom@^4.0.0: resolved "https://registry.npmjs.org/strip-bom/-/strip-bom-4.0.0.tgz" integrity sha512-3xurFv5tEgii33Zi8Jtp55wEIILR9eh34FAW00PZf+JnSsTmV/ioewSgQl97JHvgjoRGwPShsWm+IdrxB35d0w== -strip-json-comments@3.1.1, strip-json-comments@^3.1.1: +strip-json-comments@^3.1.1: version "3.1.1" resolved "https://registry.npmjs.org/strip-json-comments/-/strip-json-comments-3.1.1.tgz" integrity sha512-6fPc+R4ihwqP6N/aIv2f1gMH8lOVtWQHoqC4yK6oSDVVocumAsfCqjkXnqiYMhmMwS/mEHLp7Vehlt3ql6lEig== -supports-color@8.1.1: - version "8.1.1" - resolved "https://registry.npmjs.org/supports-color/-/supports-color-8.1.1.tgz" - integrity sha512-MpUEN2OodtUzxvKQl72cUF7RQ5EiHsGvSsVG0ia9c5RbWGL2CI4C7EpPS8UTBIplnlzZiNuV56w+FuNxy3ty2Q== - dependencies: - has-flag "^4.0.0" - supports-color@^5.3.0: version "5.5.0" resolved "https://registry.npmjs.org/supports-color/-/supports-color-5.5.0.tgz" @@ -4721,6 +4711,13 @@ supports-color@^7.1.0, supports-color@^7.2.0: dependencies: has-flag "^4.0.0" +supports-color@^8.1.1: + version "8.1.1" + resolved "https://registry.yarnpkg.com/supports-color/-/supports-color-8.1.1.tgz#cd6fc17e28500cff56c1b86c0a7fd4a54a73005c" + integrity sha512-MpUEN2OodtUzxvKQl72cUF7RQ5EiHsGvSsVG0ia9c5RbWGL2CI4C7EpPS8UTBIplnlzZiNuV56w+FuNxy3ty2Q== + dependencies: + has-flag "^4.0.0" + supports-preserve-symlinks-flag@^1.0.0: version "1.0.0" resolved "https://registry.npmjs.org/supports-preserve-symlinks-flag/-/supports-preserve-symlinks-flag-1.0.0.tgz" @@ -5142,7 +5139,7 @@ which-typed-array@^1.1.14, which-typed-array@^1.1.15, which-typed-array@^1.1.2: gopd "^1.0.1" has-tostringtag "^1.0.2" -which@2.0.2, which@^2.0.1, which@^2.0.2: +which@^2.0.1, which@^2.0.2: version "2.0.2" resolved "https://registry.npmjs.org/which/-/which-2.0.2.tgz" integrity sha512-BLI3Tl1TW3Pvl70l3yq3Y64i+awpwXqsGBYWkkqMtnbXgrMD+yj7rhW0kuEDxzJaYXGjEW5ogapKNMEKNMjibA== @@ -5166,10 +5163,10 @@ wordwrap@~0.0.2: resolved "https://registry.npmjs.org/wordwrap/-/wordwrap-0.0.3.tgz" integrity sha512-1tMA907+V4QmxV7dbRvb4/8MaRALK6q9Abid3ndMYnbyo8piisCmeONVqVSXqQA3KaP4SLt5b7ud6E2sqP8TFw== -workerpool@6.2.0: - version "6.2.0" - resolved "https://registry.yarnpkg.com/workerpool/-/workerpool-6.2.0.tgz#827d93c9ba23ee2019c3ffaff5c27fccea289e8b" - integrity sha512-Rsk5qQHJ9eowMH28Jwhe8HEbmdYDX4lwoMWshiCXugjtHqMD9ZbiqSDLxcsfdqsETPzVUtX5s1Z5kStiIM6l4A== +workerpool@^6.5.1: + version "6.5.1" + resolved "https://registry.yarnpkg.com/workerpool/-/workerpool-6.5.1.tgz#060f73b39d0caf97c6db64da004cd01b4c099544" + integrity sha512-Fs4dNYcsdpYSAfVxhnl1L5zTksjvOJxtC5hzMNl+1t9B8hTJTdKDyZ5ju7ztgPy+ft9tBFXoOlDNiOT9WUXZlA== wrap-ansi@^6.2.0: version "6.2.0" @@ -5257,11 +5254,6 @@ yaml@^2.5.0: resolved "https://registry.yarnpkg.com/yaml/-/yaml-2.5.0.tgz#c6165a721cf8000e91c36490a41d7be25176cf5d" integrity sha512-2wWLbGbYDiSqqIKoPjar3MPgB94ErzCtrNE1FdqGuaO0pi2JGjmE8aW8TDZwzU7vuxcGRdL/4gPQwQ7hD5AMSw== -yargs-parser@20.2.4: - version "20.2.4" - resolved "https://registry.npmjs.org/yargs-parser/-/yargs-parser-20.2.4.tgz" - integrity sha512-WOkpgNhPTlE73h4VFAFsOnomJVaovO8VqLDzy5saChRBFQFBoMYirowyW+Q9HB4HFF4Z7VZTiG3iSzJJA29yRA== - yargs-parser@^18.1.2: version "18.1.3" resolved "https://registry.npmjs.org/yargs-parser/-/yargs-parser-18.1.3.tgz" @@ -5270,14 +5262,14 @@ yargs-parser@^18.1.2: camelcase "^5.0.0" decamelize "^1.2.0" -yargs-parser@^20.2.2: +yargs-parser@^20.2.2, yargs-parser@^20.2.9: version "20.2.9" resolved "https://registry.npmjs.org/yargs-parser/-/yargs-parser-20.2.9.tgz" integrity sha512-y11nGElTIV+CT3Zv9t7VKl+Q3hTQoT9a1Qzezhhl6Rp21gJ/IVTW7Z3y9EWXhuUBC2Shnf+DX0antecpAwSP8w== -yargs-unparser@2.0.0: +yargs-unparser@^2.0.0: version "2.0.0" - resolved "https://registry.npmjs.org/yargs-unparser/-/yargs-unparser-2.0.0.tgz" + resolved "https://registry.yarnpkg.com/yargs-unparser/-/yargs-unparser-2.0.0.tgz#f131f9226911ae5d9ad38c432fe809366c2325eb" integrity sha512-7pRTIA9Qc1caZ0bZ6RYRGbHJthJWuakf+WmHK0rVeLkNrrGhfoabBNdue6kdINI6r4if7ocq9aD/n7xwKOdzOA== dependencies: camelcase "^6.0.0" @@ -5285,19 +5277,6 @@ yargs-unparser@2.0.0: flat "^5.0.2" is-plain-obj "^2.1.0" -yargs@16.2.0: - version "16.2.0" - resolved "https://registry.npmjs.org/yargs/-/yargs-16.2.0.tgz" - integrity sha512-D1mvvtDG0L5ft/jGWkLpG1+m0eQxOfaBvTNELraWj22wSVUMWxZUvYgJYcKh6jGGIkJFhH4IZPQhR4TKpc8mBw== - dependencies: - cliui "^7.0.2" - escalade "^3.1.1" - get-caller-file "^2.0.5" - require-directory "^2.1.1" - string-width "^4.2.0" - y18n "^5.0.5" - yargs-parser "^20.2.2" - yargs@^15.0.2: version "15.4.1" resolved "https://registry.npmjs.org/yargs/-/yargs-15.4.1.tgz" @@ -5315,6 +5294,19 @@ yargs@^15.0.2: y18n "^4.0.0" yargs-parser "^18.1.2" +yargs@^16.2.0: + version "16.2.0" + resolved "https://registry.yarnpkg.com/yargs/-/yargs-16.2.0.tgz#1c82bf0f6b6a66eafce7ef30e376f49a12477f66" + integrity sha512-D1mvvtDG0L5ft/jGWkLpG1+m0eQxOfaBvTNELraWj22wSVUMWxZUvYgJYcKh6jGGIkJFhH4IZPQhR4TKpc8mBw== + dependencies: + cliui "^7.0.2" + escalade "^3.1.1" + get-caller-file "^2.0.5" + require-directory "^2.1.1" + string-width "^4.2.0" + y18n "^5.0.5" + yargs-parser "^20.2.2" + yocto-queue@^0.1.0: version "0.1.0" resolved "https://registry.npmjs.org/yocto-queue/-/yocto-queue-0.1.0.tgz" From 12f24185e76af18d18b41a7e53f360ae7de04c8a Mon Sep 17 00:00:00 2001 From: ishabi Date: Thu, 2 Jan 2025 19:28:37 +0100 Subject: [PATCH 36/36] Update native-appsec to 8.4.0 (#5064) --- package.json | 2 +- yarn.lock | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/package.json b/package.json index 3e4582e9438..fedd38e7312 100644 --- a/package.json +++ b/package.json @@ -82,7 +82,7 @@ }, "dependencies": { "@datadog/libdatadog": "^0.3.0", - "@datadog/native-appsec": "8.3.0", + "@datadog/native-appsec": "8.4.0", "@datadog/native-iast-rewriter": "2.6.1", "@datadog/native-iast-taint-tracking": "3.2.0", "@datadog/native-metrics": "^3.1.0", diff --git a/yarn.lock b/yarn.lock index fc37d5e7016..4d8e42d2abc 100644 --- a/yarn.lock +++ b/yarn.lock @@ -406,10 +406,10 @@ resolved "https://registry.yarnpkg.com/@datadog/libdatadog/-/libdatadog-0.3.0.tgz#2fc1e2695872840bc8c356f66acf675da428d6f0" integrity sha512-TbP8+WyXfh285T17FnLeLUOPl4SbkRYMqKgcmknID2mXHNrbt5XJgW9bnDgsrrtu31Q7FjWWw2WolgRLWyzLRA== -"@datadog/native-appsec@8.3.0": - version "8.3.0" - resolved "https://registry.yarnpkg.com/@datadog/native-appsec/-/native-appsec-8.3.0.tgz#91afd89d18d386be4da8a1b0e04500f2f8b5eb66" - integrity sha512-RYHbSJ/MwJcJaLzaCaZvUyNLUKFbMshayIiv4ckpFpQJDiq1T8t9iM2k7008s75g1vRuXfsRNX7MaLn4aoFuWA== +"@datadog/native-appsec@8.4.0": + version "8.4.0" + resolved "https://registry.yarnpkg.com/@datadog/native-appsec/-/native-appsec-8.4.0.tgz#5c44d949ff8f40a94c334554db79c1c470653bae" + integrity sha512-LC47AnpVLpQFEUOP/nIIs+i0wLb8XYO+et3ACaJlHa2YJM3asR4KZTqQjDQNy08PTAUbVvYWKwfSR1qVsU/BeA== dependencies: node-gyp-build "^3.9.0"