Skip to content

Commit

Permalink
faulty threshold
Browse files Browse the repository at this point in the history
  • Loading branch information
@juan-fernandez
juan-fernandez committed Sep 26, 2024
1 parent 8fafdce commit 5da15bd
Show file tree
Hide file tree
Showing 3 changed files with 166 additions and 6 deletions.
136 changes: 132 additions & 4 deletions integration-tests/mocha/mocha.spec.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -34,7 +34,8 @@ const {
TEST_SOURCE_START,
TEST_CODE_OWNERS,
TEST_SESSION_NAME,
TEST_LEVEL_EVENT_TYPES
TEST_LEVEL_EVENT_TYPES,
TEST_EARLY_FLAKE_ABORT_REASON
} = require('../../packages/dd-trace/src/plugins/util/test')
const { DD_HOST_CPU_COUNT } = require('../../packages/dd-trace/src/plugins/util/env')
const { ERROR_MESSAGE } = require('../../packages/dd-trace/src/constants')
Expand Down Expand Up @@ -1166,6 +1167,7 @@ describe('mocha CommonJS', function () {
}).catch(done)
})
})

it('handles parameterized tests as a single unit', (done) => {
// Tests from ci-visibility/test-early-flake-detection/test-parameterized.js will be considered new
receiver.setKnownTests({
Expand Down Expand Up @@ -1243,6 +1245,7 @@ describe('mocha CommonJS', function () {
}).catch(done)
})
})

it('is disabled if DD_CIVISIBILITY_EARLY_FLAKE_DETECTION_ENABLED is false', (done) => {
// Tests from ci-visibility/test/ci-visibility-test-2.js will be considered new
receiver.setKnownTests({
Expand Down Expand Up @@ -1298,6 +1301,7 @@ describe('mocha CommonJS', function () {
}).catch(done)
})
})

it('retries flaky tests', (done) => {
// Tests from ci-visibility/test/occasionally-failing-test will be considered new
receiver.setKnownTests({})
Expand Down Expand Up @@ -1366,6 +1370,7 @@ describe('mocha CommonJS', function () {
}).catch(done)
})
})

it('does not retry new tests that are skipped', (done) => {
// Tests from ci-visibility/test/skipped-and-todo-test will be considered new
receiver.setKnownTests({})
Expand Down Expand Up @@ -1420,6 +1425,7 @@ describe('mocha CommonJS', function () {
}).catch(done)
})
})

it('handles spaces in test names', (done) => {
receiver.setSettings({
itr_enabled: false,
Expand Down Expand Up @@ -1485,6 +1491,7 @@ describe('mocha CommonJS', function () {
}).catch(done)
})
})

it('does not run EFD if the known tests request fails', (done) => {
receiver.setKnownTestsResponseCode(500)

Expand Down Expand Up @@ -1537,6 +1544,7 @@ describe('mocha CommonJS', function () {
eventsPromise.then(() => done()).catch(done)
})
})

it('retries flaky tests and sets exit code to 0 as long as one attempt passes', (done) => {
// Tests from ci-visibility/test/occasionally-failing-test will be considered new
receiver.setKnownTests({})
Expand Down Expand Up @@ -1611,6 +1619,68 @@ describe('mocha CommonJS', function () {
}).catch(done)
})
})

it('bails out of EFD if the percentage of new tests is too high', (done) => {
const NUM_RETRIES_EFD = 5

receiver.setSettings({
itr_enabled: false,
code_coverage: false,
tests_skipping: false,
early_flake_detection: {
enabled: true,
slow_test_retries: {
'5s': NUM_RETRIES_EFD
},
faulty_session_threshold: 0
}
})
// Tests from ci-visibility/test/ci-visibility-test-2.js will be considered new
receiver.setKnownTests({
mocha: {
'ci-visibility/test/ci-visibility-test.js': ['ci visibility can report tests']
}
})

const eventsPromise = receiver
.gatherPayloadsMaxTimeout(({ url }) => url.endsWith('/api/v2/citestcycle'), (payloads) => {
const events = payloads.flatMap(({ payload }) => payload.events)

const testSession = events.find(event => event.type === 'test_session_end').content
assert.notProperty(testSession.meta, TEST_EARLY_FLAKE_ENABLED)
assert.propertyVal(testSession.meta, TEST_EARLY_FLAKE_ABORT_REASON, 'faulty')

const tests = events.filter(event => event.type === 'test').map(event => event.content)

const newTests = tests.filter(test => test.meta[TEST_IS_NEW] === 'true')
assert.equal(newTests.length, 0)

const retriedTests = newTests.filter(test => test.meta[TEST_IS_RETRY] === 'true')
assert.equal(retriedTests.length, 0)
})

childProcess = exec(
runTestsWithCoverageCommand,
{
cwd,
env: {
...getCiVisAgentlessConfig(receiver.port),
TESTS_TO_RUN: JSON.stringify([
'./test/ci-visibility-test.js',
'./test/ci-visibility-test-2.js'
])
},
stdio: 'inherit'
}
)

childProcess.on('exit', () => {
eventsPromise.then(() => {
done()
}).catch(done)
})
})

context('parallel mode', () => {
it('retries new tests', (done) => {
// Tests from ci-visibility/test/occasionally-failing-test will be considered new
Expand Down Expand Up @@ -1741,10 +1811,68 @@ describe('mocha CommonJS', function () {
}).catch(done)
})
})
})
// TODO: faulty threshold
it('bails out of EFD if the percentage of new tests is too high', (done) => {
const NUM_RETRIES_EFD = 5

receiver.setSettings({
itr_enabled: false,
code_coverage: false,
tests_skipping: false,
early_flake_detection: {
enabled: true,
slow_test_retries: {
'5s': NUM_RETRIES_EFD
},
faulty_session_threshold: 0
}
})
// Tests from ci-visibility/test/ci-visibility-test-2.js will be considered new
receiver.setKnownTests({
mocha: {
'ci-visibility/test/ci-visibility-test.js': ['ci visibility can report tests']
}
})

const eventsPromise = receiver
.gatherPayloadsMaxTimeout(({ url }) => url.endsWith('/api/v2/citestcycle'), (payloads) => {
const events = payloads.flatMap(({ payload }) => payload.events)

// TODO: duration threshold
const testSession = events.find(event => event.type === 'test_session_end').content
assert.notProperty(testSession.meta, TEST_EARLY_FLAKE_ENABLED)
assert.propertyVal(testSession.meta, TEST_EARLY_FLAKE_ABORT_REASON, 'faulty')

const tests = events.filter(event => event.type === 'test').map(event => event.content)

const newTests = tests.filter(test => test.meta[TEST_IS_NEW] === 'true')
assert.equal(newTests.length, 0)

const retriedTests = newTests.filter(test => test.meta[TEST_IS_RETRY] === 'true')
assert.equal(retriedTests.length, 0)
})

childProcess = exec(
runTestsWithCoverageCommand,
{
cwd,
env: {
...getCiVisAgentlessConfig(receiver.port),
RUN_IN_PARALLEL: true,
TESTS_TO_RUN: JSON.stringify([
'./test/ci-visibility-test.js',
'./test/ci-visibility-test-2.js'
])
},
stdio: 'inherit'
}
)

childProcess.on('exit', () => {
eventsPromise.then(() => {
done()
}).catch(done)
})
})
})
})

context('flaky test retries', () => {
Expand Down
31 changes: 29 additions & 2 deletions packages/datadog-instrumentations/src/mocha/main.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,8 @@ const {
fromCoverageMapToCoverage,
getCoveredFilenamesFromCoverage,
mergeCoverage,
resetCoverage
resetCoverage,
getIsFaultyEarlyFlakeDetection
} = require('../../../dd-trace/src/plugins/util/test')

const {
Expand Down Expand Up @@ -163,6 +164,7 @@ function getOnEndHandler (isParallel) {
hasUnskippableSuites: !!unskippableSuites.length,
error,
isEarlyFlakeDetectionEnabled: config.isEarlyFlakeDetectionEnabled,
isEarlyFlakeDetectionFaulty: config.isEarlyFlakeDetectionFaulty,
isParallel
})
})
Expand Down Expand Up @@ -219,6 +221,7 @@ function getExecutionConfiguration (runner, isParallel, onFinishRequest) {

config.isEarlyFlakeDetectionEnabled = libraryConfig.isEarlyFlakeDetectionEnabled
config.earlyFlakeDetectionNumRetries = libraryConfig.earlyFlakeDetectionNumRetries
config.earlyFlakeDetectionFaultyThreshold = libraryConfig.earlyFlakeDetectionFaultyThreshold
// ITR and auto test retries are not supported in parallel mode yet
config.isSuitesSkippingEnabled = !isParallel && libraryConfig.isSuitesSkippingEnabled
config.isFlakyTestRetriesEnabled = !isParallel && libraryConfig.isFlakyTestRetriesEnabled
Expand Down Expand Up @@ -270,6 +273,18 @@ addHook({
})

getExecutionConfiguration(runner, false, () => {
if (config.isEarlyFlakeDetectionEnabled) {
const testSuites = this.files.map(file => getTestSuitePath(file, process.cwd()))
const isFaulty = getIsFaultyEarlyFlakeDetection(
testSuites,
config.knownTests.mocha,
config.earlyFlakeDetectionFaultyThreshold
)
if (isFaulty) {
config.isEarlyFlakeDetectionEnabled = false
config.isEarlyFlakeDetectionFaulty = true
}
}
if (getCodeCoverageCh.hasSubscribers) {
getCodeCoverageCh.publish({
onDone: (receivedCodeCoverage) => {
Expand Down Expand Up @@ -513,7 +528,7 @@ addHook({
versions: ['>=8.0.0'],
file: 'lib/nodejs/parallel-buffered-runner.js'
}, (ParallelBufferedRunner, frameworkVersion) => {
shimmer.wrap(ParallelBufferedRunner.prototype, 'run', run => function () {
shimmer.wrap(ParallelBufferedRunner.prototype, 'run', run => function (cb, { files }) {
if (!testStartCh.hasSubscribers) {
return run.apply(this, arguments)
}
Expand All @@ -522,6 +537,18 @@ addHook({
this.once('end', getOnEndHandler(true))

getExecutionConfiguration(this, true, () => {
if (config.isEarlyFlakeDetectionEnabled) {
const testSuites = files.map(file => getTestSuitePath(file, process.cwd()))
const isFaulty = getIsFaultyEarlyFlakeDetection(
testSuites,
config.knownTests.mocha,
config.earlyFlakeDetectionFaultyThreshold
)
if (isFaulty) {
config.isEarlyFlakeDetectionEnabled = false
config.isEarlyFlakeDetectionFaulty = true
}
}
run.apply(this, arguments)
})

Expand Down
5 changes: 5 additions & 0 deletions packages/datadog-plugin-mocha/src/index.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,7 @@ const {
TEST_IS_NEW,
TEST_IS_RETRY,
TEST_EARLY_FLAKE_ENABLED,
TEST_EARLY_FLAKE_ABORT_REASON,
TEST_SESSION_ID,
TEST_MODULE_ID,
TEST_MODULE,
Expand Down Expand Up @@ -280,6 +281,7 @@ class MochaPlugin extends CiPlugin {
hasUnskippableSuites,
error,
isEarlyFlakeDetectionEnabled,
isEarlyFlakeDetectionFaulty,
isParallel
}) => {
if (this.testSessionSpan) {
Expand Down Expand Up @@ -314,6 +316,9 @@ class MochaPlugin extends CiPlugin {
if (isEarlyFlakeDetectionEnabled) {
this.testSessionSpan.setTag(TEST_EARLY_FLAKE_ENABLED, 'true')
}
if (isEarlyFlakeDetectionFaulty) {
this.testSessionSpan.setTag(TEST_EARLY_FLAKE_ABORT_REASON, 'faulty')
}

this.testModuleSpan.finish()
this.telemetry.ciVisEvent(TELEMETRY_EVENT_FINISHED, 'module')
Expand Down

0 comments on commit 5da15bd

Please sign in to comment.