Skip to content

Commit

Permalink
faulty threshold test
Browse files Browse the repository at this point in the history
  • Loading branch information
@juan-fernandez
juan-fernandez committed Sep 24, 2024
1 parent 27ed0b2 commit 7cacba3
Show file tree
Hide file tree
Showing 4 changed files with 214 additions and 22 deletions.
6 changes: 5 additions & 1 deletion integration-tests/ci-visibility/vitest-tests/early-flake-detection.mjs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,11 @@ describe('early flake detection', () => {
})

test('can retry tests that always pass', () => {
expect(sum(1, 2)).to.equal(3)
if (process.env.ALWAYS_FAIL) {
expect(sum(1, 2)).to.equal(4)
} else {
expect(sum(1, 2)).to.equal(3)
}
})

test('does not retry if it is not new', () => {
Expand Down
157 changes: 150 additions & 7 deletions integration-tests/vitest/vitest.spec.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,9 @@ const {
TEST_SOURCE_FILE,
TEST_SOURCE_START,
TEST_IS_NEW,
TEST_NAME
TEST_NAME,
TEST_EARLY_FLAKE_ENABLED,
TEST_EARLY_FLAKE_ABORT_REASON
} = require('../../packages/dd-trace/src/plugins/util/test')
const { DD_HOST_CPU_COUNT } = require('../../packages/dd-trace/src/plugins/util/env')

Expand Down Expand Up @@ -410,7 +412,7 @@ versions.forEach((version) => {
})
}
// maybe only latest version?
context('early flake detection', () => {
context.only('early flake detection', () => {

Check warning on line 415 in integration-tests/vitest/vitest.spec.js

View workflow job for this annotation

GitHub Actions / lint 

Unexpected exclusive mocha test
it('retries new tests', (done) => {
receiver.setSettings({
itr_enabled: false,
Expand All @@ -428,7 +430,7 @@ versions.forEach((version) => {
vitest: {
'ci-visibility/vitest-tests/early-flake-detection.mjs': [
// 'early flake detection can retry tests that eventually pass', // will be considered new
// 'early flake detection can retry tests that never pass', // will be considered new
// 'early flake detection can retry tests that always pass', // will be considered new
// 'early flake detection does not retry if the test is skipped', // skipped so not retried
'early flake detection does not retry if it is not new'
]
Expand Down Expand Up @@ -466,7 +468,8 @@ versions.forEach((version) => {
const failedTests = tests.filter(test => test.meta[TEST_STATUS] === 'fail')
assert.equal(failedTests.length, 2)
const testSessionEvent = events.find(event => event.type === 'test_session_end').content
assert.equal(testSessionEvent.meta[TEST_STATUS], 'pass')
assert.propertyVal(testSessionEvent.meta, TEST_STATUS, 'pass')
assert.propertyVal(testSessionEvent.meta, TEST_EARLY_FLAKE_ENABLED, 'true')
})

childProcess = exec(
Expand All @@ -490,6 +493,145 @@ versions.forEach((version) => {
})
})

it('fails if all the attempts fail', (done) => {
receiver.setSettings({
itr_enabled: false,
code_coverage: false,
tests_skipping: false,
early_flake_detection: {
enabled: true,
slow_test_retries: {
'5s': NUM_RETRIES_EFD
}
}
})

receiver.setKnownTests({
vitest: {
'ci-visibility/vitest-tests/early-flake-detection.mjs': [
// 'early flake detection can retry tests that eventually pass', // will be considered new
// 'early flake detection can retry tests that always pass', // will be considered new
// 'early flake detection does not retry if the test is skipped', // skipped so not retried
'early flake detection does not retry if it is not new'
]
}
})

const eventsPromise = receiver
.gatherPayloadsMaxTimeout(({ url }) => url === '/api/v2/citestcycle', payloads => {
const events = payloads.flatMap(({ payload }) => payload.events)

const tests = events.filter(event => event.type === 'test').map(test => test.content)

assert.equal(tests.length, 10)

assert.includeMembers(tests.map(test => test.meta[TEST_NAME]), [
'early flake detection can retry tests that eventually pass',
'early flake detection can retry tests that eventually pass',
'early flake detection can retry tests that eventually pass',
'early flake detection can retry tests that eventually pass',
'early flake detection can retry tests that always pass',
'early flake detection can retry tests that always pass',
'early flake detection can retry tests that always pass',
'early flake detection can retry tests that always pass',
'early flake detection does not retry if it is not new',
'early flake detection does not retry if the test is skipped'
])
const newTests = tests.filter(test => test.meta[TEST_IS_NEW] === 'true')
assert.equal(newTests.length, 8) // 4 executions of the two new tests

const retriedTests = tests.filter(test => test.meta[TEST_IS_RETRY] === 'true')
assert.equal(retriedTests.length, 6) // 3 retries of the two new tests

// the multiple attempts did not result in a single pass,
// so the test session should be reported as failed
const failedTests = tests.filter(test => test.meta[TEST_STATUS] === 'fail')
assert.equal(failedTests.length, 6)
const testSessionEvent = events.find(event => event.type === 'test_session_end').content
assert.propertyVal(testSessionEvent.meta, TEST_STATUS, 'fail')
assert.propertyVal(testSessionEvent.meta, TEST_EARLY_FLAKE_ENABLED, 'true')
})

childProcess = exec(
'./node_modules/.bin/vitest run', // TODO: change tests we run
{
cwd,
env: {
...getCiVisAgentlessConfig(receiver.port),
TEST_DIR: 'ci-visibility/vitest-tests/early-flake-detection*',
NODE_OPTIONS: '--import dd-trace/register.js -r dd-trace/ci/init',
ALWAYS_FAIL: 'true'
},
stdio: 'pipe'
}
)

childProcess.on('exit', (exitCode) => {
eventsPromise.then(() => {
assert.equal(exitCode, 1)
done()
}).catch(done)
})
})

it('bails out of EFD if the percentage of new tests is too high', (done) => {
receiver.setSettings({
itr_enabled: false,
code_coverage: false,
tests_skipping: false,
early_flake_detection: {
enabled: true,
slow_test_retries: {
'5s': NUM_RETRIES_EFD
},
faulty_session_threshold: 0
}
})

receiver.setKnownTests({
vitest: {}
}) // tests from ci-visibility/vitest-tests/early-flake-detection.mjs will be new

const eventsPromise = receiver
.gatherPayloadsMaxTimeout(({ url }) => url.endsWith('/api/v2/citestcycle'), (payloads) => {
const events = payloads.flatMap(({ payload }) => payload.events)

const testSession = events.find(event => event.type === 'test_session_end').content
assert.propertyVal(testSession.meta, TEST_EARLY_FLAKE_ABORT_REASON, 'faulty')

const tests = events.filter(event => event.type === 'test').map(event => event.content)
assert.equal(tests.length, 4)

const newTests = tests.filter(
test => test.meta[TEST_IS_NEW] === 'true'
)
// no new tests
assert.equal(newTests.length, 0)
})

childProcess = exec(
'./node_modules/.bin/vitest run',
{
cwd,
env: {
...getCiVisAgentlessConfig(receiver.port),
TEST_DIR: 'ci-visibility/vitest-tests/early-flake-detection*',
NODE_OPTIONS: '--import dd-trace/register.js -r dd-trace/ci/init',
DD_TRACE_DEBUG: '1',
DD_TRACE_LOG_LEVEL: 'error'
},
stdio: 'pipe'
}
)

childProcess.on('exit', (exitCode) => {
eventsPromise.then(() => {
assert.equal(exitCode, 1)
done()
}).catch(done)
})
})

it('is disabled if DD_CIVISIBILITY_EARLY_FLAKE_DETECTION_ENABLED is false', (done) => {
receiver.setSettings({
itr_enabled: false,
Expand All @@ -507,7 +649,8 @@ versions.forEach((version) => {
vitest: {
'ci-visibility/vitest-tests/early-flake-detection.mjs': [
// 'early flake detection can retry tests that eventually pass', // will be considered new
// 'early flake detection can retry tests that never pass', // will be considered new
// 'early flake detection can retry tests that always pass', // will be considered new
// 'early flake detection does not retry if the test is skipped', // skipped so not retried
'early flake detection does not retry if it is not new'
]
}
Expand Down Expand Up @@ -540,7 +683,7 @@ versions.forEach((version) => {
})

childProcess = exec(
'./node_modules/.bin/vitest run', // TODO: change tests we run
'./node_modules/.bin/vitest run',
{
cwd,
env: {
Expand Down Expand Up @@ -604,7 +747,7 @@ versions.forEach((version) => {
})

childProcess = exec(
'./node_modules/.bin/vitest run', // TODO: change tests we run
'./node_modules/.bin/vitest run',
{
cwd,
env: {
Expand Down
40 changes: 28 additions & 12 deletions packages/datadog-instrumentations/src/vitest.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,7 @@ const testSessionStartCh = channel('ci:vitest:session:start')
const testSessionFinishCh = channel('ci:vitest:session:finish')
const libraryConfigurationCh = channel('ci:vitest:library-configuration')
const knownTestsCh = channel('ci:vitest:known-tests')
const isEarlyFlakeDetectionFaultyCh = channel('ci:vitest:is-early-flake-detection-faulty')

const taskToAsync = new WeakMap()
const taskToStatuses = new WeakMap()
Expand Down Expand Up @@ -113,6 +114,7 @@ function getSortWrapper (sort) {
let flakyTestRetriesCount = 0
let isEarlyFlakeDetectionEnabled = false
let earlyFlakeDetectionNumRetries = 0
let isEarlyFlakeDetectionFaulty = false
let knownTests = {}

try {
Expand All @@ -135,22 +137,34 @@ function getSortWrapper (sort) {
if (isEarlyFlakeDetectionEnabled) {
const knownTestsResponse = await getChannelPromise(knownTestsCh)
if (!knownTestsResponse.err) {
// TODO: check if there's a big difference between the known tests and the current tests
// to disable the feature
knownTests = knownTestsResponse.knownTests
const testFilepaths = await this.ctx.getTestFilepaths()

isEarlyFlakeDetectionFaultyCh.publish({
knownTests: knownTests.vitest || {},
testFilepaths,
onDone: (isFaulty) => {
isEarlyFlakeDetectionFaulty = isFaulty
}
})
if (isEarlyFlakeDetectionFaulty) {
isEarlyFlakeDetectionEnabled = false
log.warn('Early flake detection is disabled because the number of new tests is too high.')
} else {
// TODO: use this to pass session and module IDs to the worker, instead of polluting process.env
// Note: setting this.ctx.config.provide directly does not work because it's cached
try {
const workspaceProject = this.ctx.getCoreWorkspaceProject()
workspaceProject._provided._ddKnownTests = knownTests.vitest
workspaceProject._provided._ddIsEarlyFlakeDetectionEnabled = isEarlyFlakeDetectionEnabled
workspaceProject._provided._ddEarlyFlakeDetectionNumRetries = earlyFlakeDetectionNumRetries
} catch (e) {
log.warn('Could not send known tests to workers so Early Flake Detection will not work.')
}
}
} else {
isEarlyFlakeDetectionEnabled = false
}
// TODO: use this to pass session and module IDs to the worker, instead of polluting process.env
// Note: setting this.ctx.config.provide directly does not work because it's cached
try {
const workspaceProject = this.ctx.getCoreWorkspaceProject()
workspaceProject._provided._ddKnownTests = knownTests.vitest
workspaceProject._provided._ddIsEarlyFlakeDetectionEnabled = isEarlyFlakeDetectionEnabled
workspaceProject._provided._ddEarlyFlakeDetectionNumRetries = earlyFlakeDetectionNumRetries
} catch (e) {
log.error('Could not send known tests to workers so Early Flake Detection will not work.')
}
}

let testCodeCoverageLinesTotal
Expand Down Expand Up @@ -185,6 +199,8 @@ function getSortWrapper (sort) {
status: getSessionStatus(this.state),
testCodeCoverageLinesTotal,
error,
isEarlyFlakeDetectionEnabled,
isEarlyFlakeDetectionFaulty,
onFinish
})
})
Expand Down
33 changes: 31 additions & 2 deletions packages/datadog-plugin-vitest/src/index.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,14 +7,17 @@ const {
getTestSuitePath,
getTestSuiteCommonTags,
getTestSessionName,
getIsFaultyEarlyFlakeDetection,
TEST_SOURCE_FILE,
TEST_IS_RETRY,
TEST_CODE_COVERAGE_LINES_PCT,
TEST_CODE_OWNERS,
TEST_LEVEL_EVENT_TYPES,
TEST_SESSION_NAME,
TEST_SOURCE_START,
TEST_IS_NEW
TEST_IS_NEW,
TEST_EARLY_FLAKE_ENABLED,
TEST_EARLY_FLAKE_ABORT_REASON
} = require('../../dd-trace/src/plugins/util/test')
const { COMPONENT } = require('../../dd-trace/src/constants')
const {
Expand All @@ -38,6 +41,19 @@ class VitestPlugin extends CiPlugin {

this.taskToFinishTime = new WeakMap()

this.addSub('ci:vitest:is-early-flake-detection-faulty', ({
knownTests,
testFilepaths,
onDone
}) => {
const isFaulty = getIsFaultyEarlyFlakeDetection(
testFilepaths.map(testFilepath => getTestSuitePath(testFilepath, this.repositoryRoot)),
knownTests,
this.libraryConfig.earlyFlakeDetectionFaultyThreshold
)
onDone(isFaulty)
})

this.addSub('ci:vitest:test:start', ({ testName, testSuiteAbsolutePath, isRetry, isNew }) => {
const testSuite = getTestSuitePath(testSuiteAbsolutePath, this.repositoryRoot)
const store = storage.getStore()
Expand Down Expand Up @@ -199,7 +215,14 @@ class VitestPlugin extends CiPlugin {
}
})

this.addSub('ci:vitest:session:finish', ({ status, onFinish, error, testCodeCoverageLinesTotal }) => {
this.addSub('ci:vitest:session:finish', ({
status,
error,
testCodeCoverageLinesTotal,
isEarlyFlakeDetectionEnabled,
isEarlyFlakeDetectionFaulty,
onFinish
}) => {
this.testSessionSpan.setTag(TEST_STATUS, status)
this.testModuleSpan.setTag(TEST_STATUS, status)
if (error) {
Expand All @@ -210,6 +233,12 @@ class VitestPlugin extends CiPlugin {
this.testModuleSpan.setTag(TEST_CODE_COVERAGE_LINES_PCT, testCodeCoverageLinesTotal)
this.testSessionSpan.setTag(TEST_CODE_COVERAGE_LINES_PCT, testCodeCoverageLinesTotal)
}
if (isEarlyFlakeDetectionEnabled) {
this.testSessionSpan.setTag(TEST_EARLY_FLAKE_ENABLED, 'true')
}
if (isEarlyFlakeDetectionFaulty) {
this.testSessionSpan.setTag(TEST_EARLY_FLAKE_ABORT_REASON, 'faulty')
}
this.testModuleSpan.finish()
this.telemetry.ciVisEvent(TELEMETRY_EVENT_FINISHED, 'module')
this.testSessionSpan.finish()
Expand Down

0 comments on commit 7cacba3

Please sign in to comment.