Skip to content

Commit

Permalink
Merge branch 'sabrenner/llmobs-sdk-release' into sabrenner/llmobs-wri…
Browse files Browse the repository at this point in the history 
…ters
  • Loading branch information
@sabrenner
sabrenner authored Sep 18, 2024
2 parents 185fbe3 + 54c8eec commit 4a85292
Show file tree
Hide file tree
Showing 2 changed files with 113 additions and 4 deletions.
22 changes: 22 additions & 0 deletions packages/dd-trace/src/config.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -497,6 +497,10 @@ class Config {
this._setValue(defaults, 'isGitUploadEnabled', false)
this._setValue(defaults, 'isIntelligentTestRunnerEnabled', false)
this._setValue(defaults, 'isManualApiEnabled', false)
this._setValue(defaults, 'llmobs.agentlessEnabled', false)
this._setValue(defaults, 'llmobs.apiKey', undefined)
this._setValue(defaults, 'llmobs.enabled', false)
this._setValue(defaults, 'llmobs.mlApp', undefined)
this._setValue(defaults, 'ciVisibilitySessionName', '')
this._setValue(defaults, 'logInjection', false)
this._setValue(defaults, 'lookup', undefined)
Expand Down Expand Up @@ -594,6 +598,9 @@ class Config {
DD_INSTRUMENTATION_TELEMETRY_ENABLED,
DD_INSTRUMENTATION_CONFIG_ID,
DD_LOGS_INJECTION,
DD_LLMOBS_AGENTLESS_ENABLED,
DD_LLMOBS_ENABLED,
DD_LLMOBS_ML_APP,
DD_OPENAI_LOGS_ENABLED,
DD_OPENAI_SPAN_CHAR_LIMIT,
DD_PROFILING_ENABLED,
Expand Down Expand Up @@ -734,6 +741,9 @@ class Config {
this._setArray(env, 'injectionEnabled', DD_INJECTION_ENABLED)
this._setBoolean(env, 'isAzureFunction', getIsAzureFunction())
this._setBoolean(env, 'isGCPFunction', getIsGCPFunction())
this._setBoolean(env, 'llmobs.agentlessEnabled', DD_LLMOBS_AGENTLESS_ENABLED)
this._setBoolean(env, 'llmobs.enabled', DD_LLMOBS_ENABLED)
this._setString(env, 'llmobs.mlApp', DD_LLMOBS_ML_APP)
this._setBoolean(env, 'logInjection', DD_LOGS_INJECTION)
// Requires an accompanying DD_APM_OBFUSCATION_MEMCACHED_KEEP_COMMAND=true in the agent
this._setBoolean(env, 'memcachedCommandEnabled', DD_TRACE_MEMCACHED_COMMAND_ENABLED)
Expand Down Expand Up @@ -902,6 +912,9 @@ class Config {
}
this._setString(opts, 'iast.telemetryVerbosity', options.iast && options.iast.telemetryVerbosity)
this._setBoolean(opts, 'isCiVisibility', options.isCiVisibility)
this._setBoolean(opts, 'llmobs.agentlessEnabled', options.llmobs?.agentlessEnabled)
this._setString(opts, 'llmobs.apiKey', options.llmobs?.apiKey)
this._setString(opts, 'llmobs.mlApp', options.llmobs?.mlApp)
this._setBoolean(opts, 'logInjection', options.logInjection)
this._setString(opts, 'lookup', options.lookup)
this._setBoolean(opts, 'openAiLogsEnabled', options.openAiLogsEnabled)
Expand Down Expand Up @@ -937,6 +950,15 @@ class Config {
this._setBoolean(opts, 'traceId128BitGenerationEnabled', options.traceId128BitGenerationEnabled)
this._setBoolean(opts, 'traceId128BitLoggingEnabled', options.traceId128BitLoggingEnabled)
this._setString(opts, 'version', options.version || tags.version)

// For LLMObs, we want the environment variable to take precedence over the options.
// This is reliant on environment config being set before options.
// This is to make sure the origins of each value are tracked appropriately for telemetry.
// We'll only set `llmobs.enabled` on the opts when it's not set on the environment, and options.llmobs is provided.
const llmobsEnabledEnv = this._env['llmobs.enabled']
if (llmobsEnabledEnv == null && options.llmobs) {
this._setBoolean(opts, 'llmobs.enabled', !!options.llmobs)
}
}

_isCiVisibility () {
Expand Down
95 changes: 91 additions & 4 deletions packages/dd-trace/test/config.spec.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -261,6 +261,10 @@ describe('Config', () => {
expect(config).to.have.nested.property('installSignature.id', null)
expect(config).to.have.nested.property('installSignature.time', null)
expect(config).to.have.nested.property('installSignature.type', null)
expect(config).to.have.nested.property('llmobs.mlApp', undefined)
expect(config).to.have.nested.property('llmobs.agentlessEnabled', false)
expect(config).to.have.nested.property('llmobs.apiKey', undefined)
expect(config).to.have.nested.property('llmobs.enabled', false)

expect(updateConfig).to.be.calledOnce

Expand Down Expand Up @@ -323,6 +327,9 @@ describe('Config', () => {
{ name: 'isGitUploadEnabled', value: false, origin: 'default' },
{ name: 'isIntelligentTestRunnerEnabled', value: false, origin: 'default' },
{ name: 'isManualApiEnabled', value: false, origin: 'default' },
{ name: 'llmobs.agentlessEnabled', value: false, origin: 'default' },
{ name: 'llmobs.apiKey', value: undefined, origin: 'default' },
{ name: 'llmobs.mlApp', value: undefined, origin: 'default' },
{ name: 'ciVisibilitySessionName', value: '', origin: 'default' },
{ name: 'logInjection', value: false, origin: 'default' },
{ name: 'lookup', value: undefined, origin: 'default' },
Expand Down Expand Up @@ -490,6 +497,8 @@ describe('Config', () => {
process.env.DD_INSTRUMENTATION_INSTALL_TYPE = 'k8s_single_step'
process.env.DD_INSTRUMENTATION_INSTALL_TIME = '1703188212'
process.env.DD_INSTRUMENTATION_CONFIG_ID = 'abcdef123'
process.env.DD_LLMOBS_AGENTLESS_ENABLED = 'true'
process.env.DD_LLMOBS_ML_APP = 'myMlApp'

// required if we want to check updates to config.debug and config.logLevel which is fetched from logger
reloadLoggerAndConfig()
Expand Down Expand Up @@ -584,6 +593,8 @@ describe('Config', () => {
type: 'k8s_single_step',
time: '1703188212'
})
expect(config).to.have.nested.property('llmobs.mlApp', 'myMlApp')
expect(config).to.have.nested.property('llmobs.agentlessEnabled', true)

expect(updateConfig).to.be.calledOnce

Expand Down Expand Up @@ -647,7 +658,9 @@ describe('Config', () => {
{ name: 'traceId128BitGenerationEnabled', value: true, origin: 'env_var' },
{ name: 'traceId128BitLoggingEnabled', value: true, origin: 'env_var' },
{ name: 'tracing', value: false, origin: 'env_var' },
{ name: 'version', value: '1.0.0', origin: 'env_var' }
{ name: 'version', value: '1.0.0', origin: 'env_var' },
{ name: 'llmobs.mlApp', value: 'myMlApp', origin: 'env_var' },
{ name: 'llmobs.agentlessEnabled', value: true, origin: 'env_var' }
])
})

Expand Down Expand Up @@ -793,7 +806,12 @@ describe('Config', () => {
pollInterval: 42
},
traceId128BitGenerationEnabled: true,
traceId128BitLoggingEnabled: true
traceId128BitLoggingEnabled: true,
llmobs: {
mlApp: 'myMlApp',
agentlessEnabled: true,
apiKey: 'myApiKey'
}
})

expect(config).to.have.property('protocolVersion', '0.5')
Expand Down Expand Up @@ -866,6 +884,9 @@ describe('Config', () => {
a: 'aa',
b: 'bb'
})
expect(config).to.have.nested.property('llmobs.mlApp', 'myMlApp')
expect(config).to.have.nested.property('llmobs.agentlessEnabled', true)
expect(config).to.have.nested.property('llmobs.apiKey', 'myApiKey')

expect(updateConfig).to.be.calledOnce

Expand Down Expand Up @@ -911,7 +932,10 @@ describe('Config', () => {
{ name: 'stats.enabled', value: false, origin: 'calculated' },
{ name: 'traceId128BitGenerationEnabled', value: true, origin: 'code' },
{ name: 'traceId128BitLoggingEnabled', value: true, origin: 'code' },
{ name: 'version', value: '0.1.0', origin: 'code' }
{ name: 'version', value: '0.1.0', origin: 'code' },
{ name: 'llmobs.mlApp', value: 'myMlApp', origin: 'code' },
{ name: 'llmobs.agentlessEnabled', value: true, origin: 'code' },
{ name: 'llmobs.apiKey', value: 'myApiKey', origin: 'code' }
])
})

Expand Down Expand Up @@ -1085,6 +1109,8 @@ describe('Config', () => {
process.env.DD_IAST_REDACTION_VALUE_PATTERN = 'value_pattern_to_be_overriden_by_options'
process.env.DD_TRACE_128_BIT_TRACEID_GENERATION_ENABLED = 'true'
process.env.DD_TRACE_128_BIT_TRACEID_LOGGING_ENABLED = 'true'
process.env.DD_LLMOBS_ML_APP = 'myMlApp'
process.env.DD_LLMOBS_AGENTLESS_ENABLED = 'true'

const config = new Config({
protocolVersion: '0.5',
Expand Down Expand Up @@ -1162,7 +1188,11 @@ describe('Config', () => {
pollInterval: 42
},
traceId128BitGenerationEnabled: false,
traceId128BitLoggingEnabled: false
traceId128BitLoggingEnabled: false,
llmobs: {
mlApp: 'myOtherMlApp',
agentlessEnabled: false
}
})

expect(config).to.have.property('protocolVersion', '0.5')
Expand Down Expand Up @@ -1221,6 +1251,8 @@ describe('Config', () => {
expect(config).to.have.nested.property('iast.redactionEnabled', true)
expect(config).to.have.nested.property('iast.redactionNamePattern', 'REDACTION_NAME_PATTERN')
expect(config).to.have.nested.property('iast.redactionValuePattern', 'REDACTION_VALUE_PATTERN')
expect(config).to.have.nested.property('llmobs.mlApp', 'myOtherMlApp')
expect(config).to.have.nested.property('llmobs.agentlessEnabled', false)
})

it('should give priority to non-experimental options', () => {
Expand Down Expand Up @@ -1979,6 +2011,61 @@ describe('Config', () => {
})
})

context('llmobs config', () => {
it('should disable llmobs by default', () => {
const config = new Config()
expect(config.llmobs.enabled).to.be.false

// check origin computation
expect(updateConfig.getCall(0).args[0]).to.deep.include({
name: 'llmobs.enabled', value: false, origin: 'default'
})
})

it('should enable llmobs if DD_LLMOBS_ENABLED is set to true', () => {
process.env.DD_LLMOBS_ENABLED = 'true'
const config = new Config()
expect(config.llmobs.enabled).to.be.true

// check origin computation
expect(updateConfig.getCall(0).args[0]).to.deep.include({
name: 'llmobs.enabled', value: true, origin: 'env_var'
})
})

it('should disable llmobs if DD_LLMOBS_ENABLED is set to false', () => {
process.env.DD_LLMOBS_ENABLED = 'false'
const config = new Config()
expect(config.llmobs.enabled).to.be.false

// check origin computation
expect(updateConfig.getCall(0).args[0]).to.deep.include({
name: 'llmobs.enabled', value: false, origin: 'env_var'
})
})

it('should enable llmobs with options and DD_LLMOBS_ENABLED is not set', () => {
const config = new Config({ llmobs: {} })
expect(config.llmobs.enabled).to.be.true

// check origin computation
expect(updateConfig.getCall(0).args[0]).to.deep.include({
name: 'llmobs.enabled', value: true, origin: 'code'
})
})

it('should have DD_LLMOBS_ENABLED take priority over options', () => {
process.env.DD_LLMOBS_ENABLED = 'false'
const config = new Config({ llmobs: {} })
expect(config.llmobs.enabled).to.be.false

// check origin computation
expect(updateConfig.getCall(0).args[0]).to.deep.include({
name: 'llmobs.enabled', value: false, origin: 'env_var'
})
})
})

it('should sanitize values for API Security sampling between 0 and 1', () => {
expect(new Config({
appsec: {
Expand Down

0 comments on commit 4a85292

Please sign in to comment.